diff --git a/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go b/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go index 3c880da1f3b..410cc5639fb 100644 --- a/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go +++ b/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go @@ -82,7 +82,7 @@ func runCleanupNode(c workflow.RunData) error { klog.Errorf("[reset] Failed to remove containers: %v", err) } - r.AddDirsToClean("/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes") + r.AddDirsToClean("/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes", "/var/lib/cni") // Remove contents from the config and pki directories klog.V(1).Infoln("[reset] Removing contents from the config and pki directories") diff --git a/cmd/kubelet/app/options/container_runtime.go b/cmd/kubelet/app/options/container_runtime.go index a811577d958..fba438a6196 100644 --- a/cmd/kubelet/app/options/container_runtime.go +++ b/cmd/kubelet/app/options/container_runtime.go @@ -54,7 +54,8 @@ func NewContainerRuntimeOptions() *config.ContainerRuntimeOptions { ExperimentalDockershim: false, //Alpha feature - CNIBinDir: "/opt/cni/bin", - CNIConfDir: "/etc/cni/net.d", + CNIBinDir: "/opt/cni/bin", + CNIConfDir: "/etc/cni/net.d", + CNICacheDir: "/var/lib/cni/cache", } } diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 131d4310160..807d742cb9d 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -1251,6 +1251,7 @@ func RunDockershim(f *options.KubeletFlags, c *kubeletconfiginternal.KubeletConf PluginName: r.NetworkPluginName, PluginConfDir: r.CNIConfDir, PluginBinDirString: r.CNIBinDir, + PluginCacheDir: r.CNICacheDir, MTU: int(r.NetworkPluginMTU), } diff --git a/pkg/kubelet/config/flags.go b/pkg/kubelet/config/flags.go index 4b5494e7866..bb5a97122f0 100644 --- a/pkg/kubelet/config/flags.go +++ b/pkg/kubelet/config/flags.go @@ -73,6 +73,9 @@ type ContainerRuntimeOptions struct { // CNIBinDir is the full path of the directory in which to search for // CNI plugin binaries CNIBinDir string + // CNICacheDir is the full path of the directory in which CNI should store + // cache files + CNICacheDir string } func (s *ContainerRuntimeOptions) AddFlags(fs *pflag.FlagSet) { @@ -96,5 +99,6 @@ func (s *ContainerRuntimeOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.NetworkPluginName, "network-plugin", s.NetworkPluginName, fmt.Sprintf(" The name of the network plugin to be invoked for various events in kubelet/pod lifecycle. %s", dockerOnlyWarning)) fs.StringVar(&s.CNIConfDir, "cni-conf-dir", s.CNIConfDir, fmt.Sprintf(" The full path of the directory in which to search for CNI config files. %s", dockerOnlyWarning)) fs.StringVar(&s.CNIBinDir, "cni-bin-dir", s.CNIBinDir, fmt.Sprintf(" A comma-separated list of full paths of directories in which to search for CNI plugin binaries. %s", dockerOnlyWarning)) + fs.StringVar(&s.CNICacheDir, "cni-cache-dir", s.CNICacheDir, fmt.Sprintf(" The full path of the directory in which CNI should store cache files. %s", dockerOnlyWarning)) fs.Int32Var(&s.NetworkPluginMTU, "network-plugin-mtu", s.NetworkPluginMTU, fmt.Sprintf(" The MTU to be passed to the network plugin, to override the default. Set to 0 to use the default 1460 MTU. %s", dockerOnlyWarning)) } diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index ba1d17a89ac..16cd9fda9b8 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -123,6 +123,8 @@ type NetworkPluginSettings struct { // Depending on the plugin, this may be an optional field, eg: kubenet // generates its own plugin conf. PluginConfDir string + // PluginCacheDir is the directory in which CNI should store cache files. + PluginCacheDir string // MTU is the desired MTU for network devices created by the plugin. MTU int } @@ -239,8 +241,8 @@ func NewDockerService(config *ClientConfig, podSandboxImage string, streamingCon // dockershim currently only supports CNI plugins. pluginSettings.PluginBinDirs = cni.SplitDirs(pluginSettings.PluginBinDirString) - cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginBinDirs) - cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDirs)) + cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginCacheDir, pluginSettings.PluginBinDirs) + cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDirs, pluginSettings.PluginCacheDir)) netHost := &dockerNetworkHost{ &namespaceGetter{ds}, &portMappingGetter{ds}, diff --git a/pkg/kubelet/dockershim/network/cni/cni.go b/pkg/kubelet/dockershim/network/cni/cni.go index f391ae68181..3e79c16c93f 100644 --- a/pkg/kubelet/dockershim/network/cni/cni.go +++ b/pkg/kubelet/dockershim/network/cni/cni.go @@ -60,6 +60,7 @@ type cniNetworkPlugin struct { nsenterPath string confDir string binDirs []string + cacheDir string podCidr string } @@ -116,7 +117,7 @@ func SplitDirs(dirs string) []string { return strings.Split(dirs, ",") } -func ProbeNetworkPlugins(confDir string, binDirs []string) []network.NetworkPlugin { +func ProbeNetworkPlugins(confDir, cacheDir string, binDirs []string) []network.NetworkPlugin { old := binDirs binDirs = make([]string, 0, len(binDirs)) for _, dir := range old { @@ -131,6 +132,7 @@ func ProbeNetworkPlugins(confDir string, binDirs []string) []network.NetworkPlug execer: utilexec.New(), confDir: confDir, binDirs: binDirs, + cacheDir: cacheDir, } // sync NetworkConfig in best effort during probing. @@ -362,6 +364,7 @@ func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string ContainerID: podSandboxID.ID, NetNS: podNetnsPath, IfName: network.DefaultInterfaceName, + CacheDir: plugin.cacheDir, Args: [][2]string{ {"IgnoreUnknown", "1"}, {"K8S_POD_NAMESPACE", podNs}, diff --git a/pkg/kubelet/dockershim/network/cni/cni_test.go b/pkg/kubelet/dockershim/network/cni/cni_test.go index 18593a3abe7..aa8fe117d21 100644 --- a/pkg/kubelet/dockershim/network/cni/cni_test.go +++ b/pkg/kubelet/dockershim/network/cni/cni_test.go @@ -194,6 +194,7 @@ func TestCNIPlugin(t *testing.T) { testConfDir := path.Join(tmpDir, "etc", "cni", "net.d") testBinDir := path.Join(tmpDir, "opt", "cni", "bin") testDataDir := path.Join(tmpDir, "output") + testCacheDir := path.Join(tmpDir, "var", "lib", "cni", "cache") defer tearDownPlugin(tmpDir) inputFile, outputFile, outputEnv := installPluginUnderTest(t, testBinDir, testConfDir, testDataDir, binName, netName, podIP) @@ -207,7 +208,7 @@ func TestCNIPlugin(t *testing.T) { NetnsPath: "/proc/12345/ns/net", }} - plugins := ProbeNetworkPlugins(testConfDir, []string{testBinDir}) + plugins := ProbeNetworkPlugins(testConfDir, testCacheDir, []string{testBinDir}) if len(plugins) != 1 { t.Fatalf("Expected only one network plugin, got %d", len(plugins)) } diff --git a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go index d1b7a871532..a22719cf45d 100644 --- a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go +++ b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go @@ -96,9 +96,10 @@ type kubenetNetworkPlugin struct { nonMasqueradeCIDR string podCidr string gateway net.IP + cacheDir string } -func NewPlugin(networkPluginDirs []string) network.NetworkPlugin { +func NewPlugin(networkPluginDirs []string, cacheDir string) network.NetworkPlugin { protocol := utiliptables.ProtocolIpv4 execer := utilexec.New() dbus := utildbus.New() @@ -113,6 +114,7 @@ func NewPlugin(networkPluginDirs []string) network.NetworkPlugin { hostportSyncer: hostport.NewHostportSyncer(iptInterface), hostportManager: hostport.NewHostportManager(iptInterface), nonMasqueradeCIDR: "10.0.0.0/8", + cacheDir: cacheDir, } } @@ -558,6 +560,7 @@ func (plugin *kubenetNetworkPlugin) buildCNIRuntimeConf(ifName string, id kubeco ContainerID: id.ID, NetNS: netnsPath, IfName: ifName, + CacheDir: plugin.cacheDir, }, nil } diff --git a/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go b/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go index d2a59126b3e..5b7ef00cb26 100644 --- a/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go +++ b/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go @@ -30,7 +30,7 @@ type kubenetNetworkPlugin struct { network.NoopNetworkPlugin } -func NewPlugin(networkPluginDirs []string) network.NetworkPlugin { +func NewPlugin(networkPluginDirs []string, cacheDir string) network.NetworkPlugin { return &kubenetNetworkPlugin{} } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index fad4d0f6af2..acdbf364451 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -607,6 +607,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, PluginName: crOptions.NetworkPluginName, PluginConfDir: crOptions.CNIConfDir, PluginBinDirString: crOptions.CNIBinDir, + PluginCacheDir: crOptions.CNICacheDir, MTU: int(crOptions.NetworkPluginMTU), } diff --git a/test/e2e_node/services/kubelet.go b/test/e2e_node/services/kubelet.go index 5392c8b657b..f3188202ea4 100644 --- a/test/e2e_node/services/kubelet.go +++ b/test/e2e_node/services/kubelet.go @@ -289,10 +289,16 @@ func (e *E2EServices) startKubelet() (*server, error) { return nil, err } + cniCacheDir, err := getCNICacheDirectory() + if err != nil { + return nil, err + } + cmdArgs = append(cmdArgs, "--network-plugin=kubenet", "--cni-bin-dir", cniBinDir, - "--cni-conf-dir", cniConfDir) + "--cni-conf-dir", cniConfDir, + "--cni-cache-dir", cniCacheDir) // Keep hostname override for convenience. if framework.TestContext.NodeName != "" { // If node name is specified, set hostname override. @@ -467,6 +473,15 @@ func getCNIConfDirectory() (string, error) { return filepath.Join(cwd, "cni", "net.d"), nil } +// getCNICacheDirectory returns CNI Cache directory. +func getCNICacheDirectory() (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", err + } + return filepath.Join(cwd, "cni", "cache"), nil +} + // getDynamicConfigDir returns the directory for dynamic Kubelet configuration func getDynamicConfigDir() (string, error) { cwd, err := os.Getwd()