diff --git a/cluster/gce/cloud-init/master.yaml b/cluster/gce/cloud-init/master.yaml index b7f2692fb..23b2e3491 100644 --- a/cluster/gce/cloud-init/master.yaml +++ b/cluster/gce/cloud-init/master.yaml @@ -38,7 +38,8 @@ write_files: enable_tls_streaming = true [plugins.cri.cni] bin_dir = "/home/containerd/opt/cni/bin" - conf_dir = "/home/containerd/etc/cni/net.d" + conf_dir = "/etc/cni/net.d" + conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template" [plugins.cri.registry.mirrors."docker.io"] endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] diff --git a/cluster/gce/cloud-init/node.yaml b/cluster/gce/cloud-init/node.yaml index 2428fd85f..e44a4a6a5 100644 --- a/cluster/gce/cloud-init/node.yaml +++ b/cluster/gce/cloud-init/node.yaml @@ -35,8 +35,9 @@ write_files: [plugins.cri] enable_tls_streaming = true [plugins.cri.cni] - bin_dir = "/home/kubernetes/bin" + bin_dir = "/home/containerd/opt/cni/bin" conf_dir = "/etc/cni/net.d" + conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template" [plugins.cri.registry.mirrors."docker.io"] endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] diff --git a/cluster/gce/cni.template b/cluster/gce/cni.template new file mode 100644 index 000000000..50a2ed424 --- /dev/null +++ b/cluster/gce/cni.template @@ -0,0 +1,24 @@ +{ + "name": "k8s-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "ptp", + "mtu": 1460, + "ipam": { + "type": "host-local", + "subnet": "{{.PodCIDR}}", + "routes": [ + {"dst": "0.0.0.0/0"} + ] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + }, + "noSnat": true + } + ] +} diff --git a/cluster/gce/env b/cluster/gce/env index 3d197933a..201bc58f1 100644 --- a/cluster/gce/env +++ b/cluster/gce/env @@ -13,7 +13,7 @@ export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,conta export KUBE_CONTAINER_RUNTIME="remote" export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock" export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr cri load" -export NETWORK_POLICY_PROVIDER="calico" +export NETWORK_PROVIDER="" export NON_MASQUERADE_CIDR="0.0.0.0/0" export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/system.slice/containerd.service" export KUBE_FEATURE_GATES="ExperimentalCriticalPodAnnotation=true,CRIContainerLogRotation=true" diff --git a/docs/config.md b/docs/config.md index 560619622..0c0f6b157 100644 --- a/docs/config.md +++ b/docs/config.md @@ -63,6 +63,13 @@ The explanation and default value of each configuration item are as follows: # conf_dir is the directory in which the admin places a CNI conf. conf_dir = "/etc/cni/net.d" + # conf_template is the file path of golang template used to generate + # cni config. + # If this is set, containerd will generate a cni config file from the + # template. Otherwise, containerd will wait for the system admin or pod + # network addon to drop the config file into the conf_dir. + conf_template = "" + # "plugins.cri.registry" contains config related to the registry [plugins.cri.registry] diff --git a/pkg/config/config.go b/pkg/config/config.go index 1f5fc8511..47c4371cb 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -45,6 +45,15 @@ type CniConfig struct { NetworkPluginBinDir string `toml:"bin_dir" json:"binDir"` // NetworkPluginConfDir is the directory in which the admin places a CNI conf. NetworkPluginConfDir string `toml:"conf_dir" json:"confDir"` + // NetworkPluginConfTemplate is the file path of golang template used to generate + // cni config. + // Usually the cni config should be placed by system admin or pod network + // addons like calico, weaveworks etc. However, in some cases only a simple cni + // config is needed with pod cidr dynamically set. + // NetworkPluginConfTemplate can be used in those cases. When it is set, + // containerd will get cidr from kubelet to replace {{.PodCIDR}} in the template, + // and write the config into NetworkPluginConfDir. + NetworkPluginConfTemplate string `toml:"conf_template" json:"confTemplate"` } // Mirror contains the config related to the registry mirror @@ -106,8 +115,9 @@ type Config struct { func DefaultConfig() PluginConfig { return PluginConfig{ CniConfig: CniConfig{ - NetworkPluginBinDir: "/opt/cni/bin", - NetworkPluginConfDir: "/etc/cni/net.d", + NetworkPluginBinDir: "/opt/cni/bin", + NetworkPluginConfDir: "/etc/cni/net.d", + NetworkPluginConfTemplate: "", }, ContainerdConfig: ContainerdConfig{ Snapshotter: containerd.DefaultSnapshotter, diff --git a/pkg/server/testing/fake_cni_plugin.go b/pkg/server/testing/fake_cni_plugin.go index b098bba62..651d7690b 100644 --- a/pkg/server/testing/fake_cni_plugin.go +++ b/pkg/server/testing/fake_cni_plugin.go @@ -21,7 +21,9 @@ import ( ) // FakeCNIPlugin is a fake plugin used for test. -type FakeCNIPlugin struct{} +type FakeCNIPlugin struct { + StatusErr error +} // NewFakeCNIPlugin create a FakeCNIPlugin. func NewFakeCNIPlugin() *FakeCNIPlugin { @@ -40,7 +42,7 @@ func (f *FakeCNIPlugin) Remove(id, path string, opts ...cni.NamespaceOpts) error // Status get the status of the plugin. func (f *FakeCNIPlugin) Status() error { - return nil + return f.StatusErr } // Load loads the network config. diff --git a/pkg/server/update_runtime_config.go b/pkg/server/update_runtime_config.go index f53550c77..55e298b66 100644 --- a/pkg/server/update_runtime_config.go +++ b/pkg/server/update_runtime_config.go @@ -17,13 +17,60 @@ limitations under the License. package server import ( - "golang.org/x/net/context" + "os" + "path/filepath" + "text/template" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/net/context" runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" ) +// cniConfigTemplate contains the values containerd will overwrite +// in the cni config template. +type cniConfigTemplate struct { + // PodCIDR is the cidr for pods on the node. + PodCIDR string +} + +// cniConfigFileName is the name of cni config file generated by containerd. +const cniConfigFileName = "10-containerd-net.conflist" + // UpdateRuntimeConfig updates the runtime config. Currently only handles podCIDR updates. -// TODO(random-liu): Figure out how to handle pod cidr in the cri plugin. func (c *criService) UpdateRuntimeConfig(ctx context.Context, r *runtime.UpdateRuntimeConfigRequest) (*runtime.UpdateRuntimeConfigResponse, error) { + podCIDR := r.GetRuntimeConfig().GetNetworkConfig().GetPodCidr() + if podCIDR == "" { + return &runtime.UpdateRuntimeConfigResponse{}, nil + } + confTemplate := c.config.NetworkPluginConfTemplate + if err := c.netPlugin.Status(); err == nil { + if confTemplate != "" { + logrus.Infof("Network plugin is ready, skip generating cni config from template %q", confTemplate) + } + return &runtime.UpdateRuntimeConfigResponse{}, nil + } + if confTemplate == "" { + logrus.Info("No cni config template is specified, wait for other system components to drop the config.") + return &runtime.UpdateRuntimeConfigResponse{}, nil + } + logrus.Infof("Generating cni config from template %q", confTemplate) + // generate cni config file from the template with updated pod cidr. + t, err := template.ParseFiles(confTemplate) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse cni config template %q", confTemplate) + } + if err := os.MkdirAll(c.config.NetworkPluginConfDir, 0755); err != nil { + return nil, errors.Wrapf(err, "failed to create cni config directory: %q", c.config.NetworkPluginConfDir) + } + confFile := filepath.Join(c.config.NetworkPluginConfDir, cniConfigFileName) + f, err := os.OpenFile(confFile, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return nil, errors.Wrapf(err, "failed to open cni config file %q", confFile) + } + defer f.Close() + if err := t.Execute(f, cniConfigTemplate{PodCIDR: podCIDR}); err != nil { + return nil, errors.Wrapf(err, "failed to generate cni config file %q", confFile) + } return &runtime.UpdateRuntimeConfigResponse{}, nil } diff --git a/pkg/server/update_runtime_config_test.go b/pkg/server/update_runtime_config_test.go new file mode 100644 index 000000000..b9e6ed90a --- /dev/null +++ b/pkg/server/update_runtime_config_test.go @@ -0,0 +1,141 @@ +/* +Copyright 2018 The containerd Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/context" + runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" + + criconfig "github.com/containerd/cri/pkg/config" + servertesting "github.com/containerd/cri/pkg/server/testing" +) + +func TestUpdateRuntimeConfig(t *testing.T) { + const ( + testTemplate = ` +{ + "name": "test-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "ptp", + "mtu": 1460, + "ipam": { + "type": "host-local", + "subnet": "{{.PodCIDR}}", + "routes": [ + {"dst": "0.0.0.0/0"} + ] + } + }, + ] +}` + testCIDR = "10.0.0.0/24" + expected = ` +{ + "name": "test-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "ptp", + "mtu": 1460, + "ipam": { + "type": "host-local", + "subnet": "10.0.0.0/24", + "routes": [ + {"dst": "0.0.0.0/0"} + ] + } + }, + ] +}` + ) + + for name, test := range map[string]struct { + noTemplate bool + emptyCIDR bool + networkReady bool + expectCNIConfig bool + }{ + "should not generate cni config if cidr is empty": { + emptyCIDR: true, + expectCNIConfig: false, + }, + "should not generate cni config if template file is not specified": { + noTemplate: true, + expectCNIConfig: false, + }, + "should not generate cni config if network is ready": { + networkReady: true, + expectCNIConfig: false, + }, + "should generate cni config if template is specified and cidr is provided": { + expectCNIConfig: true, + }, + } { + t.Run(name, func(t *testing.T) { + testDir, err := ioutil.TempDir(os.TempDir(), "test-runtime-config") + require.NoError(t, err) + defer os.RemoveAll(testDir) + templateName := filepath.Join(testDir, "template") + err = ioutil.WriteFile(templateName, []byte(testTemplate), 0666) + require.NoError(t, err) + confDir := filepath.Join(testDir, "net.d") + confName := filepath.Join(confDir, cniConfigFileName) + + c := newTestCRIService() + c.config.CniConfig = criconfig.CniConfig{ + NetworkPluginConfDir: confDir, + NetworkPluginConfTemplate: templateName, + } + req := &runtime.UpdateRuntimeConfigRequest{ + RuntimeConfig: &runtime.RuntimeConfig{ + NetworkConfig: &runtime.NetworkConfig{ + PodCidr: testCIDR, + }, + }, + } + if test.noTemplate { + c.config.CniConfig.NetworkPluginConfTemplate = "" + } + if test.emptyCIDR { + req.RuntimeConfig.NetworkConfig.PodCidr = "" + } + if !test.networkReady { + c.netPlugin.(*servertesting.FakeCNIPlugin).StatusErr = errors.New("random error") + } + _, err = c.UpdateRuntimeConfig(context.Background(), req) + assert.NoError(t, err) + if !test.expectCNIConfig { + _, err := os.Stat(confName) + assert.Error(t, err) + } else { + got, err := ioutil.ReadFile(confName) + assert.NoError(t, err) + assert.Equal(t, expected, string(got)) + } + }) + } +}