kubelet: add CNI cache dir option and plumb through to CNI and kubenet

libcni 0.7.0 caches ADD operation results and allows the runtime to
retrieve these from the cache. In case the user wants a different
cache directory than the defaul, plumb that through like we do
for --cni-bin-dir and --cni-conf-dir.

cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go

@@ -82,7 +82,7 @@ func runCleanupNode(c workflow.RunData) error {
 		klog.Errorf("[reset] Failed to remove containers: %v", err)
 	}
 
-	r.AddDirsToClean("/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes")
+	r.AddDirsToClean("/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes", "/var/lib/cni")
 
 	// Remove contents from the config and pki directories
 	klog.V(1).Infoln("[reset] Removing contents from the config and pki directories")

cmd/kubelet/app/options/container_runtime.go

@@ -56,5 +56,6 @@ func NewContainerRuntimeOptions() *config.ContainerRuntimeOptions {
 		//Alpha feature
 		CNIBinDir:   "/opt/cni/bin",
 		CNIConfDir:  "/etc/cni/net.d",
+		CNICacheDir: "/var/lib/cni/cache",
 	}
 }

cmd/kubelet/app/server.go

@@ -1251,6 +1251,7 @@ func RunDockershim(f *options.KubeletFlags, c *kubeletconfiginternal.KubeletConf
 		PluginName:         r.NetworkPluginName,
 		PluginConfDir:      r.CNIConfDir,
 		PluginBinDirString: r.CNIBinDir,
+		PluginCacheDir:     r.CNICacheDir,
 		MTU:                int(r.NetworkPluginMTU),
 	}
 

pkg/kubelet/config/flags.go

@@ -73,6 +73,9 @@ type ContainerRuntimeOptions struct {
 	// CNIBinDir is the full path of the directory in which to search for
 	// CNI plugin binaries
 	CNIBinDir string
+	// CNICacheDir is the full path of the directory in which CNI should store
+	// cache files
+	CNICacheDir string
 }
 
 func (s *ContainerRuntimeOptions) AddFlags(fs *pflag.FlagSet) {
@@ -96,5 +99,6 @@ func (s *ContainerRuntimeOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.NetworkPluginName, "network-plugin", s.NetworkPluginName, fmt.Sprintf("<Warning: Alpha feature> The name of the network plugin to be invoked for various events in kubelet/pod lifecycle. %s", dockerOnlyWarning))
 	fs.StringVar(&s.CNIConfDir, "cni-conf-dir", s.CNIConfDir, fmt.Sprintf("<Warning: Alpha feature> The full path of the directory in which to search for CNI config files. %s", dockerOnlyWarning))
 	fs.StringVar(&s.CNIBinDir, "cni-bin-dir", s.CNIBinDir, fmt.Sprintf("<Warning: Alpha feature> A comma-separated list of full paths of directories in which to search for CNI plugin binaries. %s", dockerOnlyWarning))
+	fs.StringVar(&s.CNICacheDir, "cni-cache-dir", s.CNICacheDir, fmt.Sprintf("<Warning: Alpha feature> The full path of the directory in which CNI should store cache files. %s", dockerOnlyWarning))
 	fs.Int32Var(&s.NetworkPluginMTU, "network-plugin-mtu", s.NetworkPluginMTU, fmt.Sprintf("<Warning: Alpha feature> The MTU to be passed to the network plugin, to override the default. Set to 0 to use the default 1460 MTU. %s", dockerOnlyWarning))
 }

pkg/kubelet/dockershim/docker_service.go

@@ -123,6 +123,8 @@ type NetworkPluginSettings struct {
 	// Depending on the plugin, this may be an optional field, eg: kubenet
 	// generates its own plugin conf.
 	PluginConfDir string
+	// PluginCacheDir is the directory in which CNI should store cache files.
+	PluginCacheDir string
 	// MTU is the desired MTU for network devices created by the plugin.
 	MTU int
 }
@@ -239,8 +241,8 @@ func NewDockerService(config *ClientConfig, podSandboxImage string, streamingCon
 
 	// dockershim currently only supports CNI plugins.
 	pluginSettings.PluginBinDirs = cni.SplitDirs(pluginSettings.PluginBinDirString)
-	cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginBinDirs)
-	cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDirs))
+	cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginCacheDir, pluginSettings.PluginBinDirs)
+	cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDirs, pluginSettings.PluginCacheDir))
 	netHost := &dockerNetworkHost{
 		&namespaceGetter{ds},
 		&portMappingGetter{ds},

pkg/kubelet/dockershim/network/cni/cni.go

@@ -60,6 +60,7 @@ type cniNetworkPlugin struct {
 	nsenterPath string
 	confDir     string
 	binDirs     []string
+	cacheDir    string
 	podCidr     string
 }
 
@@ -116,7 +117,7 @@ func SplitDirs(dirs string) []string {
 	return strings.Split(dirs, ",")
 }
 
-func ProbeNetworkPlugins(confDir string, binDirs []string) []network.NetworkPlugin {
+func ProbeNetworkPlugins(confDir, cacheDir string, binDirs []string) []network.NetworkPlugin {
 	old := binDirs
 	binDirs = make([]string, 0, len(binDirs))
 	for _, dir := range old {
@@ -131,6 +132,7 @@ func ProbeNetworkPlugins(confDir string, binDirs []string) []network.NetworkPlug
 		execer:         utilexec.New(),
 		confDir:        confDir,
 		binDirs:        binDirs,
+		cacheDir:       cacheDir,
 	}
 
 	// sync NetworkConfig in best effort during probing.
@@ -362,6 +364,7 @@ func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string
 		ContainerID: podSandboxID.ID,
 		NetNS:       podNetnsPath,
 		IfName:      network.DefaultInterfaceName,
+		CacheDir:    plugin.cacheDir,
 		Args: [][2]string{
 			{"IgnoreUnknown", "1"},
 			{"K8S_POD_NAMESPACE", podNs},

pkg/kubelet/dockershim/network/cni/cni_test.go

@@ -194,6 +194,7 @@ func TestCNIPlugin(t *testing.T) {
 	testConfDir := path.Join(tmpDir, "etc", "cni", "net.d")
 	testBinDir := path.Join(tmpDir, "opt", "cni", "bin")
 	testDataDir := path.Join(tmpDir, "output")
+	testCacheDir := path.Join(tmpDir, "var", "lib", "cni", "cache")
 	defer tearDownPlugin(tmpDir)
 	inputFile, outputFile, outputEnv := installPluginUnderTest(t, testBinDir, testConfDir, testDataDir, binName, netName, podIP)
 
@@ -207,7 +208,7 @@ func TestCNIPlugin(t *testing.T) {
 		NetnsPath: "/proc/12345/ns/net",
 	}}
 
-	plugins := ProbeNetworkPlugins(testConfDir, []string{testBinDir})
+	plugins := ProbeNetworkPlugins(testConfDir, testCacheDir, []string{testBinDir})
 	if len(plugins) != 1 {
 		t.Fatalf("Expected only one network plugin, got %d", len(plugins))
 	}

pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go

@@ -96,9 +96,10 @@ type kubenetNetworkPlugin struct {
 	nonMasqueradeCIDR string
 	podCidr           string
 	gateway           net.IP
+	cacheDir          string
 }
 
-func NewPlugin(networkPluginDirs []string) network.NetworkPlugin {
+func NewPlugin(networkPluginDirs []string, cacheDir string) network.NetworkPlugin {
 	protocol := utiliptables.ProtocolIpv4
 	execer := utilexec.New()
 	dbus := utildbus.New()
@@ -113,6 +114,7 @@ func NewPlugin(networkPluginDirs []string) network.NetworkPlugin {
 		hostportSyncer:    hostport.NewHostportSyncer(iptInterface),
 		hostportManager:   hostport.NewHostportManager(iptInterface),
 		nonMasqueradeCIDR: "10.0.0.0/8",
+		cacheDir:          cacheDir,
 	}
 }
 
@@ -558,6 +560,7 @@ func (plugin *kubenetNetworkPlugin) buildCNIRuntimeConf(ifName string, id kubeco
 		ContainerID: id.ID,
 		NetNS:       netnsPath,
 		IfName:      ifName,
+		CacheDir:    plugin.cacheDir,
 	}, nil
 }
 

pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go

@@ -30,7 +30,7 @@ type kubenetNetworkPlugin struct {
 	network.NoopNetworkPlugin
 }
 
-func NewPlugin(networkPluginDirs []string) network.NetworkPlugin {
+func NewPlugin(networkPluginDirs []string, cacheDir string) network.NetworkPlugin {
 	return &kubenetNetworkPlugin{}
 }
 

pkg/kubelet/kubelet.go

@@ -607,6 +607,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		PluginName:         crOptions.NetworkPluginName,
 		PluginConfDir:      crOptions.CNIConfDir,
 		PluginBinDirString: crOptions.CNIBinDir,
+		PluginCacheDir:     crOptions.CNICacheDir,
 		MTU:                int(crOptions.NetworkPluginMTU),
 	}
 

test/e2e_node/services/kubelet.go

@@ -289,10 +289,16 @@ func (e *E2EServices) startKubelet() (*server, error) {
 		return nil, err
 	}
 
+	cniCacheDir, err := getCNICacheDirectory()
+	if err != nil {
+		return nil, err
+	}
+
 	cmdArgs = append(cmdArgs,
 		"--network-plugin=kubenet",
 		"--cni-bin-dir", cniBinDir,
-		"--cni-conf-dir", cniConfDir)
+		"--cni-conf-dir", cniConfDir,
+		"--cni-cache-dir", cniCacheDir)
 
 	// Keep hostname override for convenience.
 	if framework.TestContext.NodeName != "" { // If node name is specified, set hostname override.
@@ -467,6 +473,15 @@ func getCNIConfDirectory() (string, error) {
 	return filepath.Join(cwd, "cni", "net.d"), nil
 }
 
+// getCNICacheDirectory returns CNI Cache directory.
+func getCNICacheDirectory() (string, error) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(cwd, "cni", "cache"), nil
+}
+
 // getDynamicConfigDir returns the directory for dynamic Kubelet configuration
 func getDynamicConfigDir() (string, error) {
 	cwd, err := os.Getwd()