Dynamic Kubelet Configuration

Alpha implementation of the Dynamic Kubelet Configuration feature.
See the proposal doc in #29459.

test/e2e_node/BUILD

@@ -54,6 +54,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",

test/e2e_node/dynamic_kubelet_configuration_test.go

@@ -30,7 +30,7 @@ import (
 var _ = framework.KubeDescribe("DynamicKubeletConfiguration [Feature:DynamicKubeletConfig] [Serial] [Disruptive]", func() {
 	f := framework.NewDefaultFramework("dynamic-kubelet-configuration-test")
 
-	Context("When a configmap called `kubelet-{node-name}` is added to the `kube-system` namespace", func() {
+	Context("When the config source on a Node is updated to point to new config", func() {
 		It("The Kubelet on that node should restart to take up the new config", func() {
 			// Get the current KubeletConfiguration (known to be valid) by
 			// querying the configz endpoint for the current node.

test/e2e_node/services/BUILD

@@ -25,6 +25,7 @@ go_library(
         "//cmd/kube-apiserver/app/options:go_default_library",
         "//pkg/api:go_default_library",
         "//pkg/controller/namespace:go_default_library",
+        "//pkg/features:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/e2e_node/builder:go_default_library",
         "//vendor/github.com/coreos/etcd/etcdserver:go_default_library",

test/e2e_node/services/kubelet.go

@@ -28,6 +28,8 @@ import (
 
 	"github.com/golang/glog"
 
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e_node/builder"
 )
@@ -93,6 +95,9 @@ const (
 func (e *E2EServices) startKubelet() (*server, error) {
 	glog.Info("Starting kubelet")
 
+	// set feature gates so we can check which features are enabled and pass the appropriate flags
+	utilfeature.DefaultFeatureGate.Set(framework.TestContext.FeatureGates)
+
 	// Build kubeconfig
 	kubeconfigPath, err := createKubeconfigCWD()
 	if err != nil {
@@ -164,6 +169,16 @@ func (e *E2EServices) startKubelet() (*server, error) {
 		"--eviction-minimum-reclaim", "nodefs.available=5%,nodefs.inodesFree=5%", // The minimum reclaimed resources after eviction.
 		"--v", LOG_VERBOSITY_LEVEL, "--logtostderr",
 	)
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
+		// Enable dynamic config if the feature gate is enabled
+		dynamicConfigDir, err := getDynamicConfigDir()
+		if err != nil {
+			return nil, err
+		}
+		cmdArgs = append(cmdArgs, "--dynamic-config-dir", dynamicConfigDir)
+	}
+
 	// Enable kubenet by default.
 	cniBinDir, err := getCNIBinDirectory()
 	if err != nil {
@@ -294,6 +309,15 @@ func getCNIConfDirectory() (string, error) {
 	return filepath.Join(cwd, "cni", "net.d"), nil
 }
 
+// getDynamicConfigDir returns the directory for dynamic Kubelet configuration
+func getDynamicConfigDir() (string, error) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(cwd, "dynamic-kubelet-config"), nil
+}
+
 // adjustArgsForSystemd escape special characters in kubelet arguments for systemd. Systemd
 // may try to do auto expansion without escaping.
 func adjustArgsForSystemd(args []string) {

test/e2e_node/util.go

@@ -29,8 +29,9 @@ import (
 	"github.com/golang/glog"
 
 	"k8s.io/api/core/v1"
-	k8serr "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/kubernetes/pkg/api"
@@ -128,20 +129,16 @@ func isKubeletConfigEnabled(f *framework.Framework) (bool, error) {
 	return strings.Contains(cfgz.FeatureGates, "DynamicKubeletConfig=true"), nil
 }
 
-// Queries the API server for a Kubelet configuration for the node described by framework.TestContext.NodeName
-func getCurrentKubeletConfigMap(f *framework.Framework) (*v1.ConfigMap, error) {
-	return f.ClientSet.Core().ConfigMaps("kube-system").Get(fmt.Sprintf("kubelet-%s", framework.TestContext.NodeName), metav1.GetOptions{})
-}
-
 // Creates or updates the configmap for KubeletConfiguration, waits for the Kubelet to restart
-// with the new configuration. Returns an error if the configuration after waiting 40 seconds
+// with the new configuration. Returns an error if the configuration after waiting for restartGap
 // doesn't match what you attempted to set, or if the dynamic configuration feature is disabled.
 func setKubeletConfiguration(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) error {
 	const (
-		restartGap = 30 * time.Second
+		restartGap   = 40 * time.Second
+		pollInterval = 5 * time.Second
 	)
 
-	// Make sure Dynamic Kubelet Configuration feature is enabled on the Kubelet we are about to reconfigure
+	// make sure Dynamic Kubelet Configuration feature is enabled on the Kubelet we are about to reconfigure
 	configEnabled, err := isKubeletConfigEnabled(f)
 	if err != nil {
 		return fmt.Errorf("could not determine whether 'DynamicKubeletConfig' feature is enabled, err: %v", err)
@@ -152,37 +149,47 @@ func setKubeletConfiguration(f *framework.Framework, kubeCfg *componentconfig.Ku
 			"For `make test-e2e-node`, you can set `TEST_ARGS='--feature-gates=DynamicKubeletConfig=true'`.")
 	}
 
-	// Check whether a configmap for KubeletConfiguration already exists
-	_, err = getCurrentKubeletConfigMap(f)
+	nodeclient := f.ClientSet.CoreV1().Nodes()
 
-	if k8serr.IsNotFound(err) {
-		_, err := createConfigMap(f, kubeCfg)
-		if err != nil {
-			return err
-		}
-	} else if err != nil {
-		return err
-	} else {
-		// The configmap exists, update it instead of creating it.
-		_, err := updateConfigMap(f, kubeCfg)
-		if err != nil {
-			return err
-		}
-	}
-
-	// Wait for the Kubelet to restart.
-	time.Sleep(restartGap)
-
-	// Retrieve the new config and compare it to the one we attempted to set
-	newKubeCfg, err := getCurrentKubeletConfig()
+	// create the ConfigMap with the new configuration
+	cm, err := createConfigMap(f, kubeCfg)
 	if err != nil {
 		return err
 	}
 
-	// Return an error if the desired config is not in use by now
-	if !reflect.DeepEqual(*kubeCfg, *newKubeCfg) {
-		return fmt.Errorf("either the Kubelet did not restart or it did not present the modified configuration via /configz after restarting.")
+	// create the correct reference object
+	src := &v1.NodeConfigSource{
+		ConfigMapRef: &v1.ObjectReference{
+			Namespace: "kube-system",
+			Name:      cm.Name,
+			UID:       cm.UID,
+		},
 	}
+
+	// serialize the new node config source
+	raw, err := json.Marshal(src)
+	framework.ExpectNoError(err)
+	data := []byte(fmt.Sprintf(`{"spec":{"configSource":%s}}`, raw))
+
+	// patch the node
+	_, err = nodeclient.Patch(framework.TestContext.NodeName,
+		types.StrategicMergePatchType,
+		data)
+	framework.ExpectNoError(err)
+
+	// poll for new config, for a maximum wait of restartGap
+	Eventually(func() error {
+		newKubeCfg, err := getCurrentKubeletConfig()
+		if err != nil {
+			return fmt.Errorf("failed trying to get current Kubelet config, will retry, error: %v", err)
+		}
+		if !reflect.DeepEqual(*kubeCfg, *newKubeCfg) {
+			return fmt.Errorf("still waiting for new configuration to take effect, will continue to watch /configz")
+		}
+		glog.Infof("new configuration has taken effect")
+		return nil
+	}, restartGap, pollInterval).Should(BeNil())
+
 	return nil
 }
 
@@ -239,28 +246,9 @@ func decodeConfigz(resp *http.Response) (*componentconfig.KubeletConfiguration,
 	return &kubeCfg, nil
 }
 
-// Constructs a Kubelet ConfigMap targeting the current node running the node e2e tests
-func makeKubeletConfigMap(nodeName string, kubeCfg *componentconfig.KubeletConfiguration) *v1.ConfigMap {
-	kubeCfgExt := v1alpha1.KubeletConfiguration{}
-	api.Scheme.Convert(kubeCfg, &kubeCfgExt, nil)
-
-	bytes, err := json.Marshal(kubeCfgExt)
-	framework.ExpectNoError(err)
-
-	cmap := &v1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: fmt.Sprintf("kubelet-%s", nodeName),
-		},
-		Data: map[string]string{
-			"kubelet.config": string(bytes),
-		},
-	}
-	return cmap
-}
-
-// Uses KubeletConfiguration to create a `kubelet-<node-name>` ConfigMap in the "kube-system" namespace.
-func createConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) (*v1.ConfigMap, error) {
-	cmap := makeKubeletConfigMap(framework.TestContext.NodeName, kubeCfg)
+// creates a configmap containing kubeCfg in kube-system namespace
+func createConfigMap(f *framework.Framework, internalKC *componentconfig.KubeletConfiguration) (*v1.ConfigMap, error) {
+	cmap := makeKubeletConfigMap(internalKC)
 	cmap, err := f.ClientSet.Core().ConfigMaps("kube-system").Create(cmap)
 	if err != nil {
 		return nil, err
@@ -268,14 +256,25 @@ func createConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletCon
 	return cmap, nil
 }
 
-// Similar to createConfigMap, except this updates an existing ConfigMap.
-func updateConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) (*v1.ConfigMap, error) {
-	cmap := makeKubeletConfigMap(framework.TestContext.NodeName, kubeCfg)
-	cmap, err := f.ClientSet.Core().ConfigMaps("kube-system").Update(cmap)
-	if err != nil {
-		return nil, err
+// constructs a ConfigMap, populating one of its keys with the KubeletConfiguration. Uses GenerateName.
+func makeKubeletConfigMap(internalKC *componentconfig.KubeletConfiguration) *v1.ConfigMap {
+	externalKC := &v1alpha1.KubeletConfiguration{}
+	api.Scheme.Convert(internalKC, externalKC, nil)
+
+	encoder, err := newJSONEncoder(componentconfig.GroupName)
+	framework.ExpectNoError(err)
+
+	data, err := runtime.Encode(encoder, externalKC)
+	framework.ExpectNoError(err)
+
+	cmap := &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{GenerateName: "testcfg"},
+		Data: map[string]string{
+			"kubelet": string(data),
+		},
 	}
-	return cmap, nil
+
+	return cmap
 }
 
 func logPodEvents(f *framework.Framework) {
@@ -309,3 +308,20 @@ func logKubeletMetrics(metricKeys ...string) {
 		framework.Logf("Kubelet Metrics: %+v", framework.GetKubeletMetrics(metric, metricSet))
 	}
 }
+
+func newJSONEncoder(groupName string) (runtime.Encoder, error) {
+	// encode to json
+	mediaType := "application/json"
+	info, ok := runtime.SerializerInfoForMediaType(api.Codecs.SupportedMediaTypes(), mediaType)
+	if !ok {
+		return nil, fmt.Errorf("unsupported media type %q", mediaType)
+	}
+
+	versions := api.Registry.EnabledVersionsForGroup(groupName)
+	if len(versions) == 0 {
+		return nil, fmt.Errorf("no enabled versions for group %q", groupName)
+	}
+
+	// the "best" version supposedly comes first in the list returned from api.Registry.EnabledVersionsForGroup
+	return api.Codecs.EncoderForVersion(info.Serializer, versions[0]), nil
+}

test/integration/etcd/etcd_storage_path_test.go

@@ -348,6 +348,7 @@ var ephemeralWhiteList = createEphemeralWhiteList(
 	gvr("", "v1", "rangeallocations"),     // stored in various places in etcd but cannot be directly created
 	gvr("", "v1", "componentstatuses"),    // status info not stored in etcd
 	gvr("", "v1", "serializedreferences"), // used for serilization, not stored in etcd
+	gvr("", "v1", "nodeconfigsources"),    // subfield of node.spec, but shouldn't be directly created
 	gvr("", "v1", "podstatusresults"),     // wrapper object not stored in etcd
 	// --