github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Add cni config template support.

Browse Source 
Signed-off-by: Lantao Liu <lantaol@google.com>
This commit is contained in:
Lantao Liu 2018-04-07 00:50:08 +00:00
parent 68ef2c338e
commit b2099c2061
9 changed files with 242 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cluster/gce/cloud-init/master.yaml
View File

@ -38,7 +38,8 @@ write_files:
enable_tls_streaming = true
[plugins.cri.cni]
bin_dir = "/home/containerd/opt/cni/bin"
conf_dir = "/home/containerd/etc/cni/net.d"
conf_dir = "/etc/cni/net.d"
conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template"
[plugins.cri.registry.mirrors."docker.io"]
endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]

3
cluster/gce/cloud-init/node.yaml
View File

@ -35,8 +35,9 @@ write_files:
[plugins.cri]
enable_tls_streaming = true
[plugins.cri.cni]
bin_dir = "/home/kubernetes/bin"
bin_dir = "/home/containerd/opt/cni/bin"
conf_dir = "/etc/cni/net.d"
conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template"
[plugins.cri.registry.mirrors."docker.io"]
endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]

24
cluster/gce/cni.template Normal file
View File

@ -0,0 +1,24 @@
{
"name": "k8s-pod-network",
"cniVersion": "0.3.1",
"plugins": [
{
"type": "ptp",
"mtu": 1460,
"ipam": {
"type": "host-local",
"subnet": "{{.PodCIDR}}",
"routes": [
{"dst": "0.0.0.0/0"}
]
}
},
{
"type": "portmap",
"capabilities": {
"portMappings": true
},
"noSnat": true
}
]
}

2
cluster/gce/env
View File

@ -13,7 +13,7 @@ export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,conta
export KUBE_CONTAINER_RUNTIME="remote"
export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock"
export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr cri load"
export NETWORK_POLICY_PROVIDER="calico"
export NETWORK_PROVIDER=""
export NON_MASQUERADE_CIDR="0.0.0.0/0"
export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/system.slice/containerd.service"
export KUBE_FEATURE_GATES="ExperimentalCriticalPodAnnotation=true,CRIContainerLogRotation=true"

7
docs/config.md
View File

@ -63,6 +63,13 @@ The explanation and default value of each configuration item are as follows:
# conf_dir is the directory in which the admin places a CNI conf.
conf_dir = "/etc/cni/net.d"
# conf_template is the file path of golang template used to generate
# cni config.
# If this is set, containerd will generate a cni config file from the
# template. Otherwise, containerd will wait for the system admin or pod
# network addon to drop the config file into the conf_dir.
conf_template = ""
# "plugins.cri.registry" contains config related to the registry
[plugins.cri.registry]

10
pkg/config/config.go
View File

@ -45,6 +45,15 @@ type CniConfig struct {
NetworkPluginBinDir string `toml:"bin_dir" json:"binDir"`
// NetworkPluginConfDir is the directory in which the admin places a CNI conf.
NetworkPluginConfDir string `toml:"conf_dir" json:"confDir"`
// NetworkPluginConfTemplate is the file path of golang template used to generate
// cni config.
// Usually the cni config should be placed by system admin or pod network
// addons like calico, weaveworks etc. However, in some cases only a simple cni
// config is needed with pod cidr dynamically set.
// NetworkPluginConfTemplate can be used in those cases. When it is set,
// containerd will get cidr from kubelet to replace {{.PodCIDR}} in the template,
// and write the config into NetworkPluginConfDir.
NetworkPluginConfTemplate string `toml:"conf_template" json:"confTemplate"`
}
// Mirror contains the config related to the registry mirror
@ -108,6 +117,7 @@ func DefaultConfig() PluginConfig {
CniConfig: CniConfig{
NetworkPluginBinDir: "/opt/cni/bin",
NetworkPluginConfDir: "/etc/cni/net.d",
NetworkPluginConfTemplate: "",
},
ContainerdConfig: ContainerdConfig{
Snapshotter: containerd.DefaultSnapshotter,

6
pkg/server/testing/fake_cni_plugin.go
View File

@ -21,7 +21,9 @@ import (
)
// FakeCNIPlugin is a fake plugin used for test.
type FakeCNIPlugin struct{}
type FakeCNIPlugin struct {
StatusErr error
}
// NewFakeCNIPlugin create a FakeCNIPlugin.
func NewFakeCNIPlugin() *FakeCNIPlugin {
@ -40,7 +42,7 @@ func (f *FakeCNIPlugin) Remove(id, path string, opts ...cni.NamespaceOpts) error
// Status get the status of the plugin.
func (f *FakeCNIPlugin) Status() error {
return nil
return f.StatusErr
}
// Load loads the network config.

51
pkg/server/update_runtime_config.go
View File

@ -17,13 +17,60 @@ limitations under the License.
package server
import (
"golang.org/x/net/context"
"os"
"path/filepath"
"text/template"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
)
// cniConfigTemplate contains the values containerd will overwrite
// in the cni config template.
type cniConfigTemplate struct {
// PodCIDR is the cidr for pods on the node.
PodCIDR string
}
// cniConfigFileName is the name of cni config file generated by containerd.
const cniConfigFileName = "10-containerd-net.conflist"
// UpdateRuntimeConfig updates the runtime config. Currently only handles podCIDR updates.
// TODO(random-liu): Figure out how to handle pod cidr in the cri plugin.
func (c *criService) UpdateRuntimeConfig(ctx context.Context, r *runtime.UpdateRuntimeConfigRequest) (*runtime.UpdateRuntimeConfigResponse, error) {
podCIDR := r.GetRuntimeConfig().GetNetworkConfig().GetPodCidr()
if podCIDR == "" {
return &runtime.UpdateRuntimeConfigResponse{}, nil
}
confTemplate := c.config.NetworkPluginConfTemplate
if err := c.netPlugin.Status(); err == nil {
if confTemplate != "" {
logrus.Infof("Network plugin is ready, skip generating cni config from template %q", confTemplate)
}
return &runtime.UpdateRuntimeConfigResponse{}, nil
}
if confTemplate == "" {
logrus.Info("No cni config template is specified, wait for other system components to drop the config.")
return &runtime.UpdateRuntimeConfigResponse{}, nil
}
logrus.Infof("Generating cni config from template %q", confTemplate)
// generate cni config file from the template with updated pod cidr.
t, err := template.ParseFiles(confTemplate)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse cni config template %q", confTemplate)
}
if err := os.MkdirAll(c.config.NetworkPluginConfDir, 0755); err != nil {
return nil, errors.Wrapf(err, "failed to create cni config directory: %q", c.config.NetworkPluginConfDir)
}
confFile := filepath.Join(c.config.NetworkPluginConfDir, cniConfigFileName)
f, err := os.OpenFile(confFile, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return nil, errors.Wrapf(err, "failed to open cni config file %q", confFile)
}
defer f.Close()
if err := t.Execute(f, cniConfigTemplate{PodCIDR: podCIDR}); err != nil {
return nil, errors.Wrapf(err, "failed to generate cni config file %q", confFile)
}
return &runtime.UpdateRuntimeConfigResponse{}, nil
}

141
pkg/server/update_runtime_config_test.go Normal file
View File

@ -0,0 +1,141 @@
/*
Copyright 2018 The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package server
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/net/context"
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
criconfig "github.com/containerd/cri/pkg/config"
servertesting "github.com/containerd/cri/pkg/server/testing"
)
func TestUpdateRuntimeConfig(t *testing.T) {
const (
testTemplate = `
{
"name": "test-pod-network",
"cniVersion": "0.3.1",
"plugins": [
{
"type": "ptp",
"mtu": 1460,
"ipam": {
"type": "host-local",
"subnet": "{{.PodCIDR}}",
"routes": [
{"dst": "0.0.0.0/0"}
]
}
},
]
}`
testCIDR = "10.0.0.0/24"
expected = `
{
"name": "test-pod-network",
"cniVersion": "0.3.1",
"plugins": [
{
"type": "ptp",
"mtu": 1460,
"ipam": {
"type": "host-local",
"subnet": "10.0.0.0/24",
"routes": [
{"dst": "0.0.0.0/0"}
]
}
},
]
}`
)
for name, test := range map[string]struct {
noTemplate bool
emptyCIDR bool
networkReady bool
expectCNIConfig bool
}{
"should not generate cni config if cidr is empty": {
emptyCIDR: true,
expectCNIConfig: false,
},
"should not generate cni config if template file is not specified": {
noTemplate: true,
expectCNIConfig: false,
},
"should not generate cni config if network is ready": {
networkReady: true,
expectCNIConfig: false,
},
"should generate cni config if template is specified and cidr is provided": {
expectCNIConfig: true,
},
} {
t.Run(name, func(t *testing.T) {
testDir, err := ioutil.TempDir(os.TempDir(), "test-runtime-config")
require.NoError(t, err)
defer os.RemoveAll(testDir)
templateName := filepath.Join(testDir, "template")
err = ioutil.WriteFile(templateName, []byte(testTemplate), 0666)
require.NoError(t, err)
confDir := filepath.Join(testDir, "net.d")
confName := filepath.Join(confDir, cniConfigFileName)
c := newTestCRIService()
c.config.CniConfig = criconfig.CniConfig{
NetworkPluginConfDir: confDir,
NetworkPluginConfTemplate: templateName,
}
req := &runtime.UpdateRuntimeConfigRequest{
RuntimeConfig: &runtime.RuntimeConfig{
NetworkConfig: &runtime.NetworkConfig{
PodCidr: testCIDR,
},
},
}
if test.noTemplate {
c.config.CniConfig.NetworkPluginConfTemplate = ""
}
if test.emptyCIDR {
req.RuntimeConfig.NetworkConfig.PodCidr = ""
}
if !test.networkReady {
c.netPlugin.(*servertesting.FakeCNIPlugin).StatusErr = errors.New("random error")
}
_, err = c.UpdateRuntimeConfig(context.Background(), req)
assert.NoError(t, err)
if !test.expectCNIConfig {
_, err := os.Stat(confName)
assert.Error(t, err)
} else {
got, err := ioutil.ReadFile(confName)
assert.NoError(t, err)
assert.Equal(t, expected, string(got))
}
})
}
}