diff --git a/cluster/gce/cloud-init/master.yaml b/cluster/gce/cloud-init/master.yaml index b7f2692fb..23b2e3491 100644 --- a/cluster/gce/cloud-init/master.yaml +++ b/cluster/gce/cloud-init/master.yaml @@ -38,7 +38,8 @@ write_files: enable_tls_streaming = true [plugins.cri.cni] bin_dir = "/home/containerd/opt/cni/bin" - conf_dir = "/home/containerd/etc/cni/net.d" + conf_dir = "/etc/cni/net.d" + conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template" [plugins.cri.registry.mirrors."docker.io"] endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] diff --git a/cluster/gce/cloud-init/node.yaml b/cluster/gce/cloud-init/node.yaml index 2428fd85f..e44a4a6a5 100644 --- a/cluster/gce/cloud-init/node.yaml +++ b/cluster/gce/cloud-init/node.yaml @@ -35,8 +35,9 @@ write_files: [plugins.cri] enable_tls_streaming = true [plugins.cri.cni] - bin_dir = "/home/kubernetes/bin" + bin_dir = "/home/containerd/opt/cni/bin" conf_dir = "/etc/cni/net.d" + conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template" [plugins.cri.registry.mirrors."docker.io"] endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] diff --git a/cluster/gce/cni.template b/cluster/gce/cni.template new file mode 100644 index 000000000..50a2ed424 --- /dev/null +++ b/cluster/gce/cni.template @@ -0,0 +1,24 @@ +{ + "name": "k8s-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "ptp", + "mtu": 1460, + "ipam": { + "type": "host-local", + "subnet": "{{.PodCIDR}}", + "routes": [ + {"dst": "0.0.0.0/0"} + ] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + }, + "noSnat": true + } + ] +} diff --git a/cluster/gce/env b/cluster/gce/env index 3d197933a..201bc58f1 100644 --- a/cluster/gce/env +++ b/cluster/gce/env @@ -13,7 +13,7 @@ export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,conta export KUBE_CONTAINER_RUNTIME="remote" export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock" export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr cri load" -export NETWORK_POLICY_PROVIDER="calico" +export NETWORK_PROVIDER="" export NON_MASQUERADE_CIDR="0.0.0.0/0" export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/system.slice/containerd.service" export KUBE_FEATURE_GATES="ExperimentalCriticalPodAnnotation=true,CRIContainerLogRotation=true" diff --git a/docs/config.md b/docs/config.md index 560619622..d5c481ff2 100644 --- a/docs/config.md +++ b/docs/config.md @@ -63,6 +63,16 @@ The explanation and default value of each configuration item are as follows: # conf_dir is the directory in which the admin places a CNI conf. conf_dir = "/etc/cni/net.d" + # conf_template is the file path of golang template used to generate + # cni config. + # If this is set, containerd will generate a cni config file from the + # template. Otherwise, containerd will wait for the system admin or cni + # daemon to drop the config file into the conf_dir. + # This is a temporary backward-compatible solution for kubenet users + # who don't have a cni daemonset in production yet. + # This will be deprecated when kubenet is deprecated. + conf_template = "" + # "plugins.cri.registry" contains config related to the registry [plugins.cri.registry] diff --git a/pkg/config/config.go b/pkg/config/config.go index 1f5fc8511..d9e417d75 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -45,6 +45,17 @@ type CniConfig struct { NetworkPluginBinDir string `toml:"bin_dir" json:"binDir"` // NetworkPluginConfDir is the directory in which the admin places a CNI conf. NetworkPluginConfDir string `toml:"conf_dir" json:"confDir"` + // NetworkPluginConfTemplate is the file path of golang template used to generate + // cni config. + // When it is set, containerd will get cidr from kubelet to replace {{.PodCIDR}} in + // the template, and write the config into NetworkPluginConfDir. + // Ideally the cni config should be placed by system admin or cni daemon like calico, + // weaveworks etc. However, there are still users using kubenet + // (https://kubernetes.io/docs/concepts/cluster-administration/network-plugins/#kubenet) + // today, who don't have a cni daemonset in production. NetworkPluginConfTemplate is + // a temporary backward-compatible solution for them. + // TODO(random-liu): Deprecate this option when kubenet is deprecated. + NetworkPluginConfTemplate string `toml:"conf_template" json:"confTemplate"` } // Mirror contains the config related to the registry mirror @@ -106,8 +117,9 @@ type Config struct { func DefaultConfig() PluginConfig { return PluginConfig{ CniConfig: CniConfig{ - NetworkPluginBinDir: "/opt/cni/bin", - NetworkPluginConfDir: "/etc/cni/net.d", + NetworkPluginBinDir: "/opt/cni/bin", + NetworkPluginConfDir: "/etc/cni/net.d", + NetworkPluginConfTemplate: "", }, ContainerdConfig: ContainerdConfig{ Snapshotter: containerd.DefaultSnapshotter, diff --git a/pkg/server/testing/fake_cni_plugin.go b/pkg/server/testing/fake_cni_plugin.go index b098bba62..b88896fc9 100644 --- a/pkg/server/testing/fake_cni_plugin.go +++ b/pkg/server/testing/fake_cni_plugin.go @@ -21,7 +21,10 @@ import ( ) // FakeCNIPlugin is a fake plugin used for test. -type FakeCNIPlugin struct{} +type FakeCNIPlugin struct { + StatusErr error + LoadErr error +} // NewFakeCNIPlugin create a FakeCNIPlugin. func NewFakeCNIPlugin() *FakeCNIPlugin { @@ -40,10 +43,10 @@ func (f *FakeCNIPlugin) Remove(id, path string, opts ...cni.NamespaceOpts) error // Status get the status of the plugin. func (f *FakeCNIPlugin) Status() error { - return nil + return f.StatusErr } // Load loads the network config. func (f *FakeCNIPlugin) Load(opts ...cni.LoadOption) error { - return nil + return f.LoadErr } diff --git a/pkg/server/update_runtime_config.go b/pkg/server/update_runtime_config.go index f53550c77..8682da66d 100644 --- a/pkg/server/update_runtime_config.go +++ b/pkg/server/update_runtime_config.go @@ -17,13 +17,62 @@ limitations under the License. package server import ( - "golang.org/x/net/context" + "os" + "path/filepath" + "text/template" + cni "github.com/containerd/go-cni" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/net/context" runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" ) +// cniConfigTemplate contains the values containerd will overwrite +// in the cni config template. +type cniConfigTemplate struct { + // PodCIDR is the cidr for pods on the node. + PodCIDR string +} + +// cniConfigFileName is the name of cni config file generated by containerd. +const cniConfigFileName = "10-containerd-net.conflist" + // UpdateRuntimeConfig updates the runtime config. Currently only handles podCIDR updates. -// TODO(random-liu): Figure out how to handle pod cidr in the cri plugin. func (c *criService) UpdateRuntimeConfig(ctx context.Context, r *runtime.UpdateRuntimeConfigRequest) (*runtime.UpdateRuntimeConfigResponse, error) { + podCIDR := r.GetRuntimeConfig().GetNetworkConfig().GetPodCidr() + if podCIDR == "" { + return &runtime.UpdateRuntimeConfigResponse{}, nil + } + confTemplate := c.config.NetworkPluginConfTemplate + if confTemplate == "" { + logrus.Info("No cni config template is specified, wait for other system components to drop the config.") + return &runtime.UpdateRuntimeConfigResponse{}, nil + } + if err := c.netPlugin.Status(); err == nil { + logrus.Infof("Network plugin is ready, skip generating cni config from template %q", confTemplate) + return &runtime.UpdateRuntimeConfigResponse{}, nil + } else if err := c.netPlugin.Load(cni.WithLoNetwork(), cni.WithDefaultConf()); err == nil { + logrus.Infof("CNI config is successfully loaded, skip generating cni config from template %q", confTemplate) + return &runtime.UpdateRuntimeConfigResponse{}, nil + } + logrus.Infof("Generating cni config from template %q", confTemplate) + // generate cni config file from the template with updated pod cidr. + t, err := template.ParseFiles(confTemplate) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse cni config template %q", confTemplate) + } + if err := os.MkdirAll(c.config.NetworkPluginConfDir, 0755); err != nil { + return nil, errors.Wrapf(err, "failed to create cni config directory: %q", c.config.NetworkPluginConfDir) + } + confFile := filepath.Join(c.config.NetworkPluginConfDir, cniConfigFileName) + f, err := os.OpenFile(confFile, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return nil, errors.Wrapf(err, "failed to open cni config file %q", confFile) + } + defer f.Close() + if err := t.Execute(f, cniConfigTemplate{PodCIDR: podCIDR}); err != nil { + return nil, errors.Wrapf(err, "failed to generate cni config file %q", confFile) + } return &runtime.UpdateRuntimeConfigResponse{}, nil } diff --git a/pkg/server/update_runtime_config_test.go b/pkg/server/update_runtime_config_test.go new file mode 100644 index 000000000..a929927c1 --- /dev/null +++ b/pkg/server/update_runtime_config_test.go @@ -0,0 +1,142 @@ +/* +Copyright 2018 The containerd Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/context" + runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" + + criconfig "github.com/containerd/cri/pkg/config" + servertesting "github.com/containerd/cri/pkg/server/testing" +) + +func TestUpdateRuntimeConfig(t *testing.T) { + const ( + testTemplate = ` +{ + "name": "test-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "ptp", + "mtu": 1460, + "ipam": { + "type": "host-local", + "subnet": "{{.PodCIDR}}", + "routes": [ + {"dst": "0.0.0.0/0"} + ] + } + }, + ] +}` + testCIDR = "10.0.0.0/24" + expected = ` +{ + "name": "test-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "ptp", + "mtu": 1460, + "ipam": { + "type": "host-local", + "subnet": "10.0.0.0/24", + "routes": [ + {"dst": "0.0.0.0/0"} + ] + } + }, + ] +}` + ) + + for name, test := range map[string]struct { + noTemplate bool + emptyCIDR bool + networkReady bool + expectCNIConfig bool + }{ + "should not generate cni config if cidr is empty": { + emptyCIDR: true, + expectCNIConfig: false, + }, + "should not generate cni config if template file is not specified": { + noTemplate: true, + expectCNIConfig: false, + }, + "should not generate cni config if network is ready": { + networkReady: true, + expectCNIConfig: false, + }, + "should generate cni config if template is specified and cidr is provided": { + expectCNIConfig: true, + }, + } { + t.Run(name, func(t *testing.T) { + testDir, err := ioutil.TempDir(os.TempDir(), "test-runtime-config") + require.NoError(t, err) + defer os.RemoveAll(testDir) + templateName := filepath.Join(testDir, "template") + err = ioutil.WriteFile(templateName, []byte(testTemplate), 0666) + require.NoError(t, err) + confDir := filepath.Join(testDir, "net.d") + confName := filepath.Join(confDir, cniConfigFileName) + + c := newTestCRIService() + c.config.CniConfig = criconfig.CniConfig{ + NetworkPluginConfDir: confDir, + NetworkPluginConfTemplate: templateName, + } + req := &runtime.UpdateRuntimeConfigRequest{ + RuntimeConfig: &runtime.RuntimeConfig{ + NetworkConfig: &runtime.NetworkConfig{ + PodCidr: testCIDR, + }, + }, + } + if test.noTemplate { + c.config.CniConfig.NetworkPluginConfTemplate = "" + } + if test.emptyCIDR { + req.RuntimeConfig.NetworkConfig.PodCidr = "" + } + if !test.networkReady { + c.netPlugin.(*servertesting.FakeCNIPlugin).StatusErr = errors.New("random error") + c.netPlugin.(*servertesting.FakeCNIPlugin).LoadErr = errors.New("random error") + } + _, err = c.UpdateRuntimeConfig(context.Background(), req) + assert.NoError(t, err) + if !test.expectCNIConfig { + _, err := os.Stat(confName) + assert.Error(t, err) + } else { + got, err := ioutil.ReadFile(confName) + assert.NoError(t, err) + assert.Equal(t, expected, string(got)) + } + }) + } +} diff --git a/vendor.conf b/vendor.conf index 6ff4fb6a2..643c6e701 100644 --- a/vendor.conf +++ b/vendor.conf @@ -4,7 +4,7 @@ github.com/boltdb/bolt e9cf4fae01b5a8ff89d0ec6b32f0d9c9f79aefdd github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895 github.com/containerd/cgroups fe281dd265766145e943a034aa41086474ea6130 github.com/containerd/console cb7008ab3d8359b78c5f464cb7cf160107ad5925 -github.com/containerd/containerd d1b3ea406130fdb7284f14a8754b2272f2537c4c +github.com/containerd/containerd v1.1.0-rc.1 github.com/containerd/continuity 3e8f2ea4b190484acb976a5b378d373429639a1a github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c github.com/containerd/go-runc bcb223a061a3dd7de1a89c0b402a60f4dd9bd307 diff --git a/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run.go b/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run.go index ad3a8f7aa..28944aaed 100644 --- a/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run.go +++ b/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run.go @@ -68,6 +68,10 @@ var ContainerFlags = []cli.Flag{ Name: "net-host", Usage: "enable host networking for the container", }, + cli.BoolFlag{ + Name: "privileged", + Usage: "run privileged container", + }, cli.BoolFlag{ Name: "read-only", Usage: "set the containers filesystem as readonly", diff --git a/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run_unix.go b/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run_unix.go index b6826b7cb..0a3a5db68 100644 --- a/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run_unix.go +++ b/vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run_unix.go @@ -103,6 +103,9 @@ func NewContainer(ctx gocontext.Context, client *containerd.Client, context *cli if context.Bool("tty") { opts = append(opts, oci.WithTTY) } + if context.Bool("privileged") { + opts = append(opts, oci.WithPrivileged) + } if context.Bool("net-host") { opts = append(opts, oci.WithHostNamespace(specs.NetworkNamespace), oci.WithHostHostsFile, oci.WithHostResolvconf) } diff --git a/vendor/github.com/containerd/containerd/oci/spec_opts.go b/vendor/github.com/containerd/containerd/oci/spec_opts.go index 675be3970..96f6b3496 100644 --- a/vendor/github.com/containerd/containerd/oci/spec_opts.go +++ b/vendor/github.com/containerd/containerd/oci/spec_opts.go @@ -27,6 +27,18 @@ import ( // SpecOpts sets spec specific information to a newly generated OCI spec type SpecOpts func(context.Context, Client, *containers.Container, *specs.Spec) error +// Compose converts a sequence of spec operations into a single operation +func Compose(opts ...SpecOpts) SpecOpts { + return func(ctx context.Context, client Client, c *containers.Container, s *specs.Spec) error { + for _, o := range opts { + if err := o(ctx, client, c, s); err != nil { + return err + } + } + return nil + } +} + // setProcess sets Process to empty if unset func setProcess(s *specs.Spec) { if s.Process == nil { diff --git a/vendor/github.com/containerd/containerd/oci/spec_opts_unix.go b/vendor/github.com/containerd/containerd/oci/spec_opts_unix.go index 27d78178b..46053aa55 100644 --- a/vendor/github.com/containerd/containerd/oci/spec_opts_unix.go +++ b/vendor/github.com/containerd/containerd/oci/spec_opts_unix.go @@ -443,20 +443,23 @@ func WithUsername(username string) SpecOpts { } } -// WithAllCapabilities set all linux capabilities for the process -func WithAllCapabilities(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { - setCapabilities(s) +// WithCapabilities sets Linux capabilities on the process +func WithCapabilities(caps []string) SpecOpts { + return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { + setCapabilities(s) - caps := getAllCapabilities() + s.Process.Capabilities.Bounding = caps + s.Process.Capabilities.Effective = caps + s.Process.Capabilities.Permitted = caps + s.Process.Capabilities.Inheritable = caps - s.Process.Capabilities.Bounding = caps - s.Process.Capabilities.Effective = caps - s.Process.Capabilities.Permitted = caps - s.Process.Capabilities.Inheritable = caps - - return nil + return nil + } } +// WithAllCapabilities sets all linux capabilities for the process +var WithAllCapabilities = WithCapabilities(getAllCapabilities()) + func getAllCapabilities() []string { last := capability.CAP_LAST_CAP // hack for RHEL6 which has no /proc/sys/kernel/cap_last_cap @@ -512,3 +515,93 @@ func getGIDFromPath(root string, filter func(user.Group) bool) (gid uint32, err func isRootfsAbs(root string) bool { return filepath.IsAbs(root) } + +// WithMaskedPaths sets the masked paths option +func WithMaskedPaths(paths []string) SpecOpts { + return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { + setLinux(s) + s.Linux.MaskedPaths = paths + return nil + } +} + +// WithReadonlyPaths sets the read only paths option +func WithReadonlyPaths(paths []string) SpecOpts { + return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { + setLinux(s) + s.Linux.ReadonlyPaths = paths + return nil + } +} + +// WithWriteableSysfs makes any sysfs mounts writeable +func WithWriteableSysfs(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { + for i, m := range s.Mounts { + if m.Type == "sysfs" { + var options []string + for _, o := range m.Options { + if o == "ro" { + o = "rw" + } + options = append(options, o) + } + s.Mounts[i].Options = options + } + } + return nil +} + +// WithWriteableCgroupfs makes any cgroup mounts writeable +func WithWriteableCgroupfs(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { + for i, m := range s.Mounts { + if m.Type == "cgroup" { + var options []string + for _, o := range m.Options { + if o == "ro" { + o = "rw" + } + options = append(options, o) + } + s.Mounts[i].Options = options + } + } + return nil +} + +// WithSelinuxLabel sets the process SELinux label +func WithSelinuxLabel(label string) SpecOpts { + return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { + setProcess(s) + s.Process.SelinuxLabel = label + return nil + } +} + +// WithApparmorProfile sets the Apparmor profile for the process +func WithApparmorProfile(profile string) SpecOpts { + return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { + setProcess(s) + s.Process.ApparmorProfile = profile + return nil + } +} + +// WithSeccompUnconfined clears the seccomp profile +func WithSeccompUnconfined(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { + setLinux(s) + s.Linux.Seccomp = nil + return nil +} + +// WithPrivileged sets up options for a privileged container +// TODO(justincormack) device handling +var WithPrivileged = Compose( + WithAllCapabilities, + WithMaskedPaths(nil), + WithReadonlyPaths(nil), + WithWriteableSysfs, + WithWriteableCgroupfs, + WithSelinuxLabel(""), + WithApparmorProfile(""), + WithSeccompUnconfined, +) diff --git a/vendor/github.com/containerd/containerd/vendor.conf b/vendor/github.com/containerd/containerd/vendor.conf index 16ce52a21..3ff6bb41d 100644 --- a/vendor/github.com/containerd/containerd/vendor.conf +++ b/vendor/github.com/containerd/containerd/vendor.conf @@ -43,7 +43,7 @@ github.com/gotestyourself/gotestyourself 44dbf532bbf5767611f6f2a61bded572e337010 github.com/google/go-cmp v0.1.0 # cri dependencies -github.com/containerd/cri v1.0.0-rc.0 +github.com/containerd/cri v1.0.0-rc.1 github.com/containerd/go-cni f2d7272f12d045b16ed924f50e91f9f9cecc55a7 github.com/blang/semver v3.1.0 github.com/containernetworking/cni v0.6.0 @@ -68,11 +68,11 @@ golang.org/x/crypto 49796115aa4b964c318aad4f3084fdb41e9aa067 golang.org/x/time f51c12702a4d776e4c1fa9b0fabab841babae631 gopkg.in/inf.v0 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4 gopkg.in/yaml.v2 53feefa2559fb8dfa8d81baad31be332c97d6c77 -k8s.io/api 5584376ceeffeb13a2e98b5e9f0e9dab37de4bab +k8s.io/api 7e796de92438aede7cb5d6bcf6c10f4fa65db560 k8s.io/apimachinery fcb9a12f7875d01f8390b28faedc37dcf2e713b9 -k8s.io/apiserver 837069aa36757a586e4a8165f1ff5ca06170aa4a -k8s.io/client-go 484f27892430b961df38fe6715cc396409207d9f -k8s.io/kubernetes v1.10.0-rc.1 +k8s.io/apiserver 4a8377c547bbff4576a35b5b5bf4026d9b5aa763 +k8s.io/client-go b9a0cf870f239c4a4ecfd3feb075a50e7cbe1473 +k8s.io/kubernetes v1.10.0 k8s.io/utils 258e2a2fa64568210fbd6267cf1d8fd87c3cb86e # zfs dependencies diff --git a/vendor/github.com/containerd/containerd/version/version.go b/vendor/github.com/containerd/containerd/version/version.go index 504809128..c0efe0deb 100644 --- a/vendor/github.com/containerd/containerd/version/version.go +++ b/vendor/github.com/containerd/containerd/version/version.go @@ -21,7 +21,7 @@ var ( Package = "github.com/containerd/containerd" // Version holds the complete version number. Filled in at linking time. - Version = "1.1.0-rc.0+unknown" + Version = "1.1.0-rc.1+unknown" // Revision is filled with the VCS (e.g. git) revision being used to build // the program at linking time.