kubeadm: Write kubelet config file to disk and persist in-cluster. Also write runtime environment file and fixup the kubelet phases command

cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go

@@ -68,9 +68,16 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 					StaticPodPath: "foo",
 					ClusterDNS:    []string{"foo"},
 					ClusterDomain: "foo",
-					Authorization: kubeletconfigv1beta1.KubeletAuthorization{Mode: "foo"},
+					Authorization: kubeletconfigv1beta1.KubeletAuthorization{
+						Mode: "Webhook",
+					},
 					Authentication: kubeletconfigv1beta1.KubeletAuthentication{
-						X509: kubeletconfigv1beta1.KubeletX509Authentication{ClientCAFile: "foo"},
+						X509: kubeletconfigv1beta1.KubeletX509Authentication{
+							ClientCAFile: "/etc/kubernetes/pki/ca.crt",
+						},
+						Anonymous: kubeletconfigv1beta1.KubeletAnonymousAuthentication{
+							Enabled: utilpointer.BoolPtr(false),
+						},
 					},
 				},
 			}

cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go

@@ -24,11 +24,11 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
-	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
 	kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
 	kubeproxyscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme"
 	kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
+	utilpointer "k8s.io/kubernetes/pkg/util/pointer"
 )
 
 const (
@@ -143,9 +143,7 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) {
 	}
 
 	SetDefaultsEtcdSelfHosted(obj)
-	if features.Enabled(obj.FeatureGates, features.DynamicKubeletConfig) {
 	SetDefaults_KubeletConfiguration(obj)
-	}
 	SetDefaults_ProxyConfiguration(obj)
 	SetDefaults_AuditPolicyConfiguration(obj)
 }
@@ -235,14 +233,30 @@ func SetDefaults_KubeletConfiguration(obj *MasterConfiguration) {
 		}
 	}
 	if obj.KubeletConfiguration.BaseConfig.ClusterDomain == "" {
-		obj.KubeletConfiguration.BaseConfig.ClusterDomain = DefaultServiceDNSDomain
+		obj.KubeletConfiguration.BaseConfig.ClusterDomain = obj.Networking.DNSDomain
 	}
-	if obj.KubeletConfiguration.BaseConfig.Authorization.Mode == "" {
+
-		obj.KubeletConfiguration.BaseConfig.Authorization.Mode = kubeletconfigv1beta1.KubeletAuthorizationModeWebhook
+	// Enforce security-related kubelet options
-	}
+
-	if obj.KubeletConfiguration.BaseConfig.Authentication.X509.ClientCAFile == "" {
+	// Require all clients to the kubelet API to have client certs signed by the cluster CA
 	obj.KubeletConfiguration.BaseConfig.Authentication.X509.ClientCAFile = DefaultCACertPath
-	}
+	obj.KubeletConfiguration.BaseConfig.Authentication.Anonymous.Enabled = utilpointer.BoolPtr(false)
+
+	// On every client request to the kubelet API, execute a webhook (SubjectAccessReview request) to the API server
+	// and ask it whether the client is authorized to access the kubelet API
+	obj.KubeletConfiguration.BaseConfig.Authorization.Mode = kubeletconfigv1beta1.KubeletAuthorizationModeWebhook
+
+	// Let clients using other authentication methods like ServiceAccount tokens also access the kubelet API
+	// TODO: Enable in a future PR
+	// obj.KubeletConfiguration.BaseConfig.Authentication.Webhook.Enabled = utilpointer.BoolPtr(true)
+
+	// Disable the readonly port of the kubelet, in order to not expose unnecessary information
+	// TODO: Enable in a future PR
+	// obj.KubeletConfiguration.BaseConfig.ReadOnlyPort = 0
+
+	// Serve a /healthz webserver on localhost:10248 that kubeadm can talk to
+	obj.KubeletConfiguration.BaseConfig.HealthzBindAddress = "127.0.0.1"
+	obj.KubeletConfiguration.BaseConfig.HealthzPort = utilpointer.Int32Ptr(10248)
 
 	scheme, _, _ := kubeletscheme.NewSchemeAndCodecs()
 	if scheme != nil {

cmd/kubeadm/app/apis/kubeadm/v1alpha2/defaults.go

@@ -23,11 +23,11 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
-	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
 	kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
 	kubeproxyscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme"
 	kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
+	utilpointer "k8s.io/kubernetes/pkg/util/pointer"
 )
 
 const (
@@ -127,9 +127,7 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) {
 		obj.ClusterName = DefaultClusterName
 	}
 
-	if features.Enabled(obj.FeatureGates, features.DynamicKubeletConfig) {
 	SetDefaults_KubeletConfiguration(obj)
-	}
 	SetDefaults_ProxyConfiguration(obj)
 	SetDefaults_AuditPolicyConfiguration(obj)
 }
@@ -198,14 +196,30 @@ func SetDefaults_KubeletConfiguration(obj *MasterConfiguration) {
 		}
 	}
 	if obj.KubeletConfiguration.BaseConfig.ClusterDomain == "" {
-		obj.KubeletConfiguration.BaseConfig.ClusterDomain = DefaultServiceDNSDomain
+		obj.KubeletConfiguration.BaseConfig.ClusterDomain = obj.Networking.DNSDomain
 	}
-	if obj.KubeletConfiguration.BaseConfig.Authorization.Mode == "" {
+
-		obj.KubeletConfiguration.BaseConfig.Authorization.Mode = kubeletconfigv1beta1.KubeletAuthorizationModeWebhook
+	// Enforce security-related kubelet options
-	}
+
-	if obj.KubeletConfiguration.BaseConfig.Authentication.X509.ClientCAFile == "" {
+	// Require all clients to the kubelet API to have client certs signed by the cluster CA
 	obj.KubeletConfiguration.BaseConfig.Authentication.X509.ClientCAFile = DefaultCACertPath
-	}
+	obj.KubeletConfiguration.BaseConfig.Authentication.Anonymous.Enabled = utilpointer.BoolPtr(false)
+
+	// On every client request to the kubelet API, execute a webhook (SubjectAccessReview request) to the API server
+	// and ask it whether the client is authorized to access the kubelet API
+	obj.KubeletConfiguration.BaseConfig.Authorization.Mode = kubeletconfigv1beta1.KubeletAuthorizationModeWebhook
+
+	// Let clients using other authentication methods like ServiceAccount tokens also access the kubelet API
+	// TODO: Enable in a future PR
+	// obj.KubeletConfiguration.BaseConfig.Authentication.Webhook.Enabled = utilpointer.BoolPtr(true)
+
+	// Disable the readonly port of the kubelet, in order to not expose unnecessary information
+	// TODO: Enable in a future PR
+	// obj.KubeletConfiguration.BaseConfig.ReadOnlyPort = 0
+
+	// Serve a /healthz webserver on localhost:10248 that kubeadm can talk to
+	obj.KubeletConfiguration.BaseConfig.HealthzBindAddress = "127.0.0.1"
+	obj.KubeletConfiguration.BaseConfig.HealthzPort = utilpointer.Int32Ptr(10248)
 
 	scheme, _, _ := kubeletscheme.NewSchemeAndCodecs()
 	if scheme != nil {

cmd/kubeadm/app/cmd/init.go

@@ -259,11 +259,7 @@ func NewInit(cfgPath string, externalcfg *kubeadmapiv1alpha2.MasterConfiguration
 		return nil, err
 	}
 
-	// Try to start the kubelet service in case it's inactive
+	return &Init{cfg: cfg, skipTokenPrint: skipTokenPrint, dryRun: dryRun, ignorePreflightErrors: ignorePreflightErrors}, nil
-	glog.V(1).Infof("Starting kubelet")
-	preflight.TryStartKubelet(ignorePreflightErrors)
-
-	return &Init{cfg: cfg, skipTokenPrint: skipTokenPrint, dryRun: dryRun}, nil
 }
 
 // Init defines struct used by "kubeadm init" command
@@ -271,10 +267,21 @@ type Init struct {
 	cfg                   *kubeadmapi.MasterConfiguration
 	skipTokenPrint        bool
 	dryRun                bool
+	ignorePreflightErrors sets.String
 }
 
 // Run executes master node provisioning, including certificates, needed static pod manifests, etc.
 func (i *Init) Run(out io.Writer) error {
+
+	// Write env file with flags for the kubelet to use
+	if err := kubeletphase.WriteKubeletDynamicEnvFile(i.cfg); err != nil {
+		return err
+	}
+
+	// Try to start the kubelet service in case it's inactive
+	glog.V(1).Infof("Starting kubelet")
+	preflight.TryStartKubelet(i.ignorePreflightErrors)
+
 	// Get directories to write files to; can be faked if we're dry-running
 	glog.V(1).Infof("[init] Getting certificates directory from configuration")
 	realCertsDir := i.cfg.CertificatesDir
@@ -346,14 +353,14 @@ func (i *Init) Run(out io.Writer) error {
 		return fmt.Errorf("error printing files on dryrun: %v", err)
 	}
 
-	// NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+	kubeletVersion, err := preflight.GetKubeletVersion(utilsexec.New())
-	if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) {
+	if err != nil {
-		glog.V(1).Infof("[init] feature --dynamic-config-dir is enabled")
+		return err
-		glog.V(1).Infof("[init] writing base kubelet configuration to disk on master")
-		// Write base kubelet configuration for dynamic kubelet configuration feature.
-		if err := kubeletphase.WriteInitKubeletConfigToDiskOnMaster(i.cfg); err != nil {
-			return fmt.Errorf("error writing base kubelet configuration to disk: %v", err)
 	}
+
+	// Write the kubelet configuration to disk.
+	if err := kubeletphase.WriteConfigToDisk(i.cfg.KubeletConfiguration.BaseConfig, kubeletVersion); err != nil {
+		return fmt.Errorf("error writing kubelet configuration to disk: %v", err)
 	}
 
 	// Create a kubernetes client and wait for the API server to be healthy (if not dryrunning)
@@ -381,15 +388,6 @@ func (i *Init) Run(out io.Writer) error {
 		return fmt.Errorf("couldn't initialize a Kubernetes cluster")
 	}
 
-	// NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
-	if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) {
-		// Create base kubelet configuration for dynamic kubelet configuration feature.
-		glog.V(1).Infof("[init] creating base kubelet configuration")
-		if err := kubeletphase.CreateBaseKubeletConfiguration(i.cfg, client); err != nil {
-			return fmt.Errorf("error creating base kubelet configuration: %v", err)
-		}
-	}
-
 	// Upload currently used configuration to the cluster
 	// Note: This is done right in the beginning of cluster initialization; as we might want to make other phases
 	// depend on centralized information from this source in the future
@@ -398,12 +396,26 @@ func (i *Init) Run(out io.Writer) error {
 		return fmt.Errorf("error uploading configuration: %v", err)
 	}
 
+	glog.V(1).Infof("[init] creating kubelet configuration configmap")
+	if err := kubeletphase.CreateConfigMap(i.cfg, client); err != nil {
+		return fmt.Errorf("error creating kubelet configuration ConfigMap: %v", err)
+	}
+
 	// PHASE 4: Mark the master with the right label/taint
 	glog.V(1).Infof("[init] marking the master with right label")
 	if err := markmasterphase.MarkMaster(client, i.cfg.NodeName, !i.cfg.NoTaintMaster); err != nil {
 		return fmt.Errorf("error marking master: %v", err)
 	}
 
+	// NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+	// This feature is disabled by default, as it is alpha still
+	if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) {
+		// Enable dynamic kubelet configuration for the node.
+		if err := kubeletphase.EnableDynamicConfigForNode(client, i.cfg.NodeName, kubeletVersion); err != nil {
+			return fmt.Errorf("error enabling dynamic kubelet configuration: %v", err)
+		}
+	}
+
 	// PHASE 5: Set up the node bootstrap tokens
 	if !i.skipTokenPrint {
 		glog.Infof("[bootstraptoken] using token: %s\n", i.cfg.Token)

cmd/kubeadm/app/cmd/join.go

@@ -252,10 +252,28 @@ func (j *Join) Run(out io.Writer) error {
 		return fmt.Errorf("couldn't save the CA certificate to disk: %v", err)
 	}
 
-	// NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+	kubeletVersion, err := preflight.GetKubeletVersion(utilsexec.New())
-	glog.V(1).Infoln("[join] consuming base kubelet configuration")
+	if err != nil {
+		return err
+	}
+
+	// Write the configuration for the kubelet down to disk so the kubelet can start
+	if err := kubeletphase.DownloadConfig(kubeconfigFile, kubeletVersion); err != nil {
+		return err
+	}
+
+	// Now the kubelet will perform the TLS Bootstrap, transforming bootstrap-kubeconfig.conf to kubeconfig.conf in /etc/kubernetes
+
+	// NOTE: the "--dynamic-config-dir" flag should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf for this to work
+	// This feature is disabled by default, as it is alpha still
+	glog.V(1).Infoln("[join] enabling dynamic kubelet configuration")
 	if features.Enabled(j.cfg.FeatureGates, features.DynamicKubeletConfig) {
-		if err := kubeletphase.ConsumeBaseKubeletConfiguration(j.cfg.NodeName); err != nil {
+		client, err := kubeletphase.GetLocalNodeTLSBootstrappedClient()
+		if err != nil {
+			return err
+		}
+
+		if err := kubeletphase.EnableDynamicConfigForNode(client, j.cfg.NodeName, kubeletVersion); err != nil {
 			return fmt.Errorf("error consuming base kubelet configuration: %v", err)
 		}
 	}

cmd/kubeadm/app/cmd/phases/kubelet.go

@@ -18,184 +18,172 @@ package phases
 
 import (
 	"fmt"
-	"io/ioutil"
+	"os"
 
 	"github.com/spf13/cobra"
 
-	"k8s.io/apimachinery/pkg/runtime"
-	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
 	kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
-	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
-	nodeutil "k8s.io/kubernetes/pkg/util/node"
 	"k8s.io/kubernetes/pkg/util/normalizer"
+	"k8s.io/kubernetes/pkg/util/version"
 )
 
 var (
-	kubeletWriteInitConfigLongDesc = normalizer.LongDesc(`
+	kubeletWriteConfigToDiskLongDesc = normalizer.LongDesc(`
-		Writes init kubelet configuration to disk for dynamic kubelet configuration feature.
+		Writes kubelet configuration to disk, either based on the kubelet-config-1.X ConfigMap in the cluster, or from the
-		Please note that the kubelet configuration can be passed to kubeadm as a value into the master configuration file.
+		configuration passed to the command via "--config".
 		` + cmdutil.AlphaDisclaimer)
 
-	kubeletWriteInitConfigExample = normalizer.Examples(`
+	kubeletWriteConfigToDiskExample = normalizer.Examples(`
-		# Writes init kubelet configuration to disk.
+		# Writes kubelet configuration for a node to disk. The information is fetched from the cluster ConfigMap
-		kubeadm alpha phase kubelet init
+		kubeadm alpha phase kubelet write-config-to-disk --kubelet-version v1.11.0 --kubeconfig /etc/kubernetes/kubelet.conf
+
+		# Writes kubelet configuration down to disk, based on the configuration flag passed to --config
+		kubeadm alpha phase kubelet write-config-to-disk --kubelet-version v1.11.0 --config kubeadm.yaml
 		`)
 
 	kubeletUploadDynamicConfigLongDesc = normalizer.LongDesc(`
-		Uploads dynamic kubelet configuration as ConfigMap and links it to the current node as ConfigMapRef.
+		Uploads kubelet configuration extracted from the kubeadm MasterConfiguration object to a ConfigMap
-		Please note that the kubelet configuration can be passed to kubeadm as a value into the master configuration file.
+		of the form kubelet-config-1.X in the cluster, where X is the minor version of the current Kubernetes version
 		` + cmdutil.AlphaDisclaimer)
 
 	kubeletUploadDynamicConfigExample = normalizer.Examples(`
-		# Uploads dynamic kubelet configuration as ConfigMap.
+		# Uploads the kubelet configuration from the kubeadm Config file to a ConfigMap in the cluster.
-		kubeadm alpha phase kubelet upload
+		kubeadm alpha phase kubelet upload-config --config kubeadm.yaml
 		`)
 
 	kubeletEnableDynamicConfigLongDesc = normalizer.LongDesc(`
-		Enables or updates dynamic kubelet configuration on node. This should be run on nodes.
+		Enables or updates dynamic kubelet configuration for a Node, against the kubelet-config-1.X ConfigMap in the cluster,
-		Please note that the kubelet configuration can be passed to kubeadm as a value into the master configuration file.
+		where X is the minor version of the desired kubelet version.
+
+		WARNING: This feature is still experimental, and disabled by default. Enable only if you know what you are doing, as it
+		may have surprising side-effects at this stage.
+
 		` + cmdutil.AlphaDisclaimer)
 
 	kubeletEnableDynamicConfigExample = normalizer.Examples(`
-		# Enables dynamic kubelet configuration on node.
+		# Enables dynamic kubelet configuration for a Node.
-		kubeadm alpha phase kubelet enable
+		kubeadm alpha phase kubelet enable-dynamic-config --node-name node-1 --kubelet-version v1.11.0
+
+		WARNING: This feature is still experimental, and disabled by default. Enable only if you know what you are doing, as it
+		may have surprising side-effects at this stage.
 		`)
 )
 
 // NewCmdKubelet returns main command for Kubelet phase
 func NewCmdKubelet() *cobra.Command {
+	var kubeConfigFile string
 	cmd := &cobra.Command{
 		Use:   "kubelet",
-		Short: "Adopts dynamic kubelet configuration.",
+		Short: "Handles kubelet configuration.",
 		Long:  cmdutil.MacroCommandLongDescription,
 	}
 
-	cmd.AddCommand(NewCmdKubeletWriteInitConfig())
+	cmd.PersistentFlags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use when talking to the cluster")
-	cmd.AddCommand(NewCmdKubeletUploadDynamicConfig())
-	cmd.AddCommand(NewCmdKubeletEnableDynamicConfig())
 
+	cmd.AddCommand(NewCmdKubeletWriteConfigToDisk(&kubeConfigFile))
+	cmd.AddCommand(NewCmdKubeletUploadConfig(&kubeConfigFile))
+	cmd.AddCommand(NewCmdKubeletEnableDynamicConfig(&kubeConfigFile))
 	return cmd
 }
 
-// NewCmdKubeletWriteInitConfig calls cobra.Command for writing init kubelet configuration
+// NewCmdKubeletUploadConfig calls cobra.Command for uploading dynamic kubelet configuration
-func NewCmdKubeletWriteInitConfig() *cobra.Command {
+func NewCmdKubeletUploadConfig(kubeConfigFile *string) *cobra.Command {
 	var cfgPath string
-	cmd := &cobra.Command{
-		Use:     "init",
-		Short:   "Writes init kubelet configuration to disk",
-		Long:    kubeletWriteInitConfigLongDesc,
-		Example: kubeletWriteInitConfigExample,
-		Run: func(cmd *cobra.Command, args []string) {
-			cfg := &kubeadmapiv1alpha2.MasterConfiguration{
-				// KubernetesVersion is not used by kubelet init, but we set this explicitly to avoid
-				// the lookup of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig
-				KubernetesVersion: "v1.9.0",
-			}
-			kubeadmscheme.Scheme.Default(cfg)
-
-			// This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags
-			internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg)
-			kubeadmutil.CheckErr(err)
-			if features.Enabled(internalcfg.FeatureGates, features.DynamicKubeletConfig) {
-				err = kubeletphase.WriteInitKubeletConfigToDiskOnMaster(internalcfg)
-				kubeadmutil.CheckErr(err)
-			} else {
-				fmt.Println("[kubelet] feature gate DynamicKubeletConfig is not enabled, do nothing.")
-			}
-		},
-	}
-
-	cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")
-
-	return cmd
-}
-
-// NewCmdKubeletUploadDynamicConfig calls cobra.Command for uploading dynamic kubelet configuration
-func NewCmdKubeletUploadDynamicConfig() *cobra.Command {
-	var cfgPath, kubeConfigFile string
 
 	cmd := &cobra.Command{
-		Use:     "upload",
+		Use:     "upload-config",
-		Short:   "Uploads dynamic kubelet configuration as ConfigMap",
+		Short:   "Uploads kubelet configuration to a ConfigMap",
 		Long:    kubeletUploadDynamicConfigLongDesc,
 		Example: kubeletUploadDynamicConfigExample,
 		Run: func(cmd *cobra.Command, args []string) {
-			cfg := &kubeadmapiv1alpha2.MasterConfiguration{
+			if len(cfgPath) == 0 {
-				// KubernetesVersion is not used by kubelet upload, but we set this explicitly to avoid
+				kubeadmutil.CheckErr(fmt.Errorf("The --config argument is required"))
-				// the lookup of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig
-				KubernetesVersion: "v1.9.0",
 			}
-			kubeadmscheme.Scheme.Default(cfg)
 
-			// This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags
+			// This call returns the ready-to-use configuration based on the configuration file
-			internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg)
+			internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, &kubeadmapiv1alpha2.MasterConfiguration{})
 			kubeadmutil.CheckErr(err)
-			if features.Enabled(internalcfg.FeatureGates, features.DynamicKubeletConfig) {
+
-				client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile)
+			client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile)
 			kubeadmutil.CheckErr(err)
-				err = kubeletphase.CreateBaseKubeletConfiguration(internalcfg, client)
+
+			err = kubeletphase.CreateConfigMap(internalcfg, client)
 			kubeadmutil.CheckErr(err)
-			} else {
-				fmt.Println("[kubelet] feature gate DynamicKubeletConfig is not enabled, do nothing.")
-			}
 		},
 	}
 
 	cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")
-	cmd.Flags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use when talking to the cluster")
+	return cmd
+}
 
+// NewCmdKubeletWriteConfigToDisk calls cobra.Command for writing init kubelet configuration
+func NewCmdKubeletWriteConfigToDisk(kubeConfigFile *string) *cobra.Command {
+	var cfgPath, kubeletVersionStr string
+	cmd := &cobra.Command{
+		Use:     "write-config-to-disk",
+		Short:   "Writes kubelet configuration to disk, either based on the --config argument or the kubeadm-config ConfigMap.",
+		Long:    kubeletWriteConfigToDiskLongDesc,
+		Example: kubeletWriteConfigToDiskExample,
+		Run: func(cmd *cobra.Command, args []string) {
+			if len(kubeletVersionStr) == 0 {
+				kubeadmutil.CheckErr(fmt.Errorf("The --kubelet-version argument is required"))
+			}
+
+			kubeletVersion, err := version.ParseSemantic(kubeletVersionStr)
+			kubeadmutil.CheckErr(err)
+
+			client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile)
+			kubeadmutil.CheckErr(err)
+
+			// This call returns the ready-to-use configuration based on the configuration file
+			internalcfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "kubelet", cfgPath)
+			kubeadmutil.CheckErr(err)
+
+			err = kubeletphase.WriteConfigToDisk(internalcfg.KubeletConfiguration.BaseConfig, kubeletVersion)
+			kubeadmutil.CheckErr(err)
+		},
+	}
+
+	cmd.Flags().StringVar(&kubeletVersionStr, "kubelet-version", kubeletVersionStr, "The desired version for the kubelet")
+	cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")
 	return cmd
 }
 
 // NewCmdKubeletEnableDynamicConfig calls cobra.Command for enabling dynamic kubelet configuration on node
-func NewCmdKubeletEnableDynamicConfig() *cobra.Command {
+// This feature is still in alpha and an experimental state
-	cfg := &kubeadmapiv1alpha2.NodeConfiguration{}
+func NewCmdKubeletEnableDynamicConfig(kubeConfigFile *string) *cobra.Command {
-	kubeadmscheme.Scheme.Default(cfg)
+	var nodeName, kubeletVersionStr string
 
-	var cfgPath string
 	cmd := &cobra.Command{
-		Use:     "enable",
+		Use:     "enable-dynamic-config",
-		Aliases: []string{"update"},
+		Short:   "EXPERIMENTAL: Enables or updates dynamic kubelet configuration for a Node",
-		Short:   "Enables or updates dynamic kubelet configuration on node",
 		Long:    kubeletEnableDynamicConfigLongDesc,
 		Example: kubeletEnableDynamicConfigExample,
 		Run: func(cmd *cobra.Command, args []string) {
-			nodeName, err := getNodeName(cfgPath, cfg)
+			if len(nodeName) == 0 {
-			kubeadmutil.CheckErr(err)
+				kubeadmutil.CheckErr(fmt.Errorf("The --node-name argument is required"))
-			if features.Enabled(cfg.FeatureGates, features.DynamicKubeletConfig) {
-				err = kubeletphase.ConsumeBaseKubeletConfiguration(nodeName)
-				kubeadmutil.CheckErr(err)
-			} else {
-				fmt.Println("[kubelet] feature gate DynamicKubeletConfig is not enabled, do nothing.")
 			}
+			if len(kubeletVersionStr) == 0 {
+				kubeadmutil.CheckErr(fmt.Errorf("The --kubelet-version argument is required"))
+			}
+
+			kubeletVersion, err := version.ParseSemantic(kubeletVersionStr)
+			kubeadmutil.CheckErr(err)
+
+			client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile)
+			kubeadmutil.CheckErr(err)
+
+			err = kubeletphase.EnableDynamicConfigForNode(client, nodeName, kubeletVersion)
+			kubeadmutil.CheckErr(err)
 		},
 	}
 
-	cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")
+	cmd.Flags().StringVar(&nodeName, "node-name", nodeName, "Name of the node that should enable the dynamic kubelet configuration")
-	cmd.Flags().StringVar(&cfg.NodeName, "node-name", cfg.NodeName, "Name of the node that should enable the dynamic kubelet configuration")
+	cmd.Flags().StringVar(&kubeletVersionStr, "kubelet-version", kubeletVersionStr, "The desired version for the kubelet")
-
 	return cmd
 }
-
-func getNodeName(cfgPath string, cfg *kubeadmapiv1alpha2.NodeConfiguration) (string, error) {
-	if cfgPath != "" {
-		b, err := ioutil.ReadFile(cfgPath)
-		if err != nil {
-			return "", fmt.Errorf("unable to read config from %q [%v]", cfgPath, err)
-		}
-		if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), b, cfg); err != nil {
-			return "", fmt.Errorf("unable to decode config from %q [%v]", cfgPath, err)
-		}
-	}
-
-	if cfg.NodeName == "" {
-		cfg.NodeName = nodeutil.GetHostname("")
-	}
-
-	return cfg.NodeName, nil
-}

cmd/kubeadm/app/cmd/phases/kubelet_test.go

@@ -25,33 +25,37 @@ import (
 )
 
 func TestKubeletSubCommandsHasFlags(t *testing.T) {
+	kubeConfigFile := "foo"
 	subCmds := []*cobra.Command{
-		NewCmdKubeletWriteInitConfig(),
+		NewCmdKubeletUploadConfig(&kubeConfigFile),
-		NewCmdKubeletUploadDynamicConfig(),
+		NewCmdKubeletWriteConfigToDisk(&kubeConfigFile),
-		NewCmdKubeletEnableDynamicConfig(),
+		NewCmdKubeletEnableDynamicConfig(&kubeConfigFile),
 	}
 
-	commonFlags := []string{
+	commonFlags := []string{}
-		"config",
-	}
 
 	var tests = []struct {
 		command         string
 		additionalFlags []string
 	}{
 		{
-			command: "init",
+			command: "upload-config",
-		},
-		{
-			command: "upload",
 			additionalFlags: []string{
-				"kubeconfig",
+				"config",
 			},
 		},
 		{
-			command: "enable",
+			command: "write-config-to-disk",
+			additionalFlags: []string{
+				"kubelet-version",
+				"config",
+			},
+		},
+		{
+			command: "enable-dynamic-config",
 			additionalFlags: []string{
 				"node-name",
+				"kubelet-version",
 			},
 		},
 	}

cmd/kubeadm/app/cmd/upgrade/common.go

@@ -24,17 +24,21 @@ import (
 	"os"
 	"strings"
 
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	fakediscovery "k8s.io/client-go/discovery/fake"
 	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
 	kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
+	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
+	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 )
@@ -61,8 +65,20 @@ func enforceRequirements(flags *cmdUpgradeFlags, dryRun bool, newK8sVersion stri
 	}
 
 	// Fetch the configuration from a file or ConfigMap and validate it
-	cfg, err := upgrade.FetchConfiguration(client, os.Stdout, flags.cfgPath)
+	fmt.Println("[upgrade/config] Making sure the configuration is correct:")
+	cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "upgrade/config", flags.cfgPath)
 	if err != nil {
+		if apierrors.IsNotFound(err) {
+			fmt.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %s namespace must exist.\n", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem)
+			fmt.Println("[upgrade/config] Without this information, 'kubeadm upgrade' won't know how to configure your upgraded cluster.")
+			fmt.Println("")
+			fmt.Println("[upgrade/config] Next steps:")
+			fmt.Printf("\t- OPTION 1: Run 'kubeadm config upload from-flags' and specify the same CLI arguments you passed to 'kubeadm init' when you created your master.\n")
+			fmt.Printf("\t- OPTION 2: Run 'kubeadm config upload from-file' and specify the same config file you passed to 'kubeadm init' when you created your master.\n")
+			fmt.Printf("\t- OPTION 3: Pass a config file to 'kubeadm upgrade' using the --config flag.\n")
+			fmt.Println("")
+			err = fmt.Errorf("the ConfigMap %q in the %s namespace used for getting configuration information was not found", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem)
+		}
 		return nil, fmt.Errorf("[upgrade/config] FATAL: %v", err)
 	}
 

cmd/kubeadm/app/constants/constants.go

@@ -161,9 +161,6 @@ const (
 	// system:nodes group subject is removed if present.
 	NodesClusterRoleBinding = "system:node"
 
-	// KubeletBaseConfigMapRoleName defines the base kubelet configuration ConfigMap.
-	KubeletBaseConfigMapRoleName = "kubeadm:kubelet-base-configmap"
-
 	// APICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation
 	APICallRetryInterval = 500 * time.Millisecond
 	// DiscoveryRetryInterval specifies how long kubeadm should wait before retrying to connect to the master when doing discovery
@@ -191,17 +188,17 @@ const (
 	// MasterConfigurationConfigMapKey specifies in what ConfigMap key the master configuration should be stored
 	MasterConfigurationConfigMapKey = "MasterConfiguration"
 
-	// KubeletBaseConfigurationConfigMap specifies in what ConfigMap in the kube-system namespace the initial remote configuration of kubelet should be stored
+	// KubeletBaseConfigurationConfigMapPrefix specifies in what ConfigMap in the kube-system namespace the initial remote configuration of kubelet should be stored
-	KubeletBaseConfigurationConfigMap = "kubelet-base-config-1.9"
+	KubeletBaseConfigurationConfigMapPrefix = "kubelet-config-"
 
 	// KubeletBaseConfigurationConfigMapKey specifies in what ConfigMap key the initial remote configuration of kubelet should be stored
 	KubeletBaseConfigurationConfigMapKey = "kubelet"
 
-	// KubeletBaseConfigurationDir specifies the directory on the node where stores the initial remote configuration of kubelet
+	// KubeletBaseConfigMapRolePrefix defines the base kubelet configuration ConfigMap.
-	KubeletBaseConfigurationDir = "/var/lib/kubelet/config/init"
+	KubeletBaseConfigMapRolePrefix = "kubeadm:kubelet-config-"
 
-	// KubeletBaseConfigurationFile specifies the file name on the node which stores initial remote configuration of kubelet
+	// KubeletConfigurationFile specifies the file name on the node which stores initial remote configuration of kubelet
-	KubeletBaseConfigurationFile = "kubelet"
+	KubeletConfigurationFile = "/var/lib/kubelet/config.yaml"
 
 	// MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports
 	MinExternalEtcdVersion = "3.2.17"
@@ -260,6 +257,14 @@ const (
 	// Copied from pkg/master/reconcilers to avoid pulling extra dependencies
 	// TODO: Import this constant from a consts only package, that does not pull any further dependencies.
 	LeaseEndpointReconcilerType = "lease"
+
+	// KubeletEnvFile is a file "kubeadm init" writes at runtime. Using that interface, kubeadm can customize certain
+	// kubelet flags conditionally based on the environment at runtime. Also, parameters given to the configuration file
+	// might be passed through this file. "kubeadm init" writes one variable, with the name ${KubeletEnvFileVariableName}.
+	KubeletEnvFile = "/var/lib/kubelet/kubeadm-flags.env"
+
+	// KubeletEnvFileVariableName specifies the shell script variable name "kubeadm init" should write a value to in KubeletEnvFile
+	KubeletEnvFileVariableName = "KUBELET_KUBEADM_ARGS"
 )
 
 var (
@@ -290,6 +295,9 @@ var (
 	// MinimumKubeletVersion specifies the minimum version of kubelet which kubeadm supports
 	MinimumKubeletVersion = version.MustParseSemantic("v1.10.0")
 
+	// MinimumKubeletConfigVersion specifies the minimum version of Kubernetes where kubeadm supports specifying --config to the kubelet
+	MinimumKubeletConfigVersion = version.MustParseSemantic("v1.11.0-alpha.1")
+
 	// SupportedEtcdVersion lists officially supported etcd versions with corresponding kubernetes releases
 	SupportedEtcdVersion = map[uint8]string{
 		10: "3.1.12",

cmd/kubeadm/app/phases/kubelet/config.go

@@ -0,0 +1,187 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubelet
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"k8s.io/api/core/v1"
+	rbac "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
+	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
+	rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
+	kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
+	kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
+	"k8s.io/kubernetes/pkg/util/version"
+)
+
+// WriteConfigToDisk writes the kubelet config object down to a file
+// Used at "kubeadm init" and "kubeadm upgrade" time
+func WriteConfigToDisk(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration, kubeletVersion *version.Version) error {
+
+	// If the kubelet version is v1.10.x, exit
+	if kubeletVersion.LessThan(kubeadmconstants.MinimumKubeletConfigVersion) {
+		return nil
+	}
+
+	kubeletBytes, err := getConfigBytes(kubeletConfig)
+	if err != nil {
+		return err
+	}
+	return writeConfigBytesToDisk(kubeletBytes)
+}
+
+// CreateConfigMap creates a ConfigMap with the generic kubelet configuration.
+// Used at "kubeadm init" and "kubeadm upgrade" time
+func CreateConfigMap(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
+
+	k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
+	if err != nil {
+		return err
+	}
+
+	// If Kubernetes version is v1.10.x, exit
+	if k8sVersion.LessThan(kubeadmconstants.MinimumKubeletConfigVersion) {
+		return nil
+	}
+
+	configMapName := configMapName(k8sVersion)
+	fmt.Printf("[kubelet] Creating a ConfigMap %q in namespace %s with the configuration for the kubelets in the cluster\n", configMapName, metav1.NamespaceSystem)
+
+	kubeletBytes, err := getConfigBytes(cfg.KubeletConfiguration.BaseConfig)
+	if err != nil {
+		return err
+	}
+
+	if err := apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      configMapName,
+			Namespace: metav1.NamespaceSystem,
+		},
+		Data: map[string]string{
+			kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(kubeletBytes),
+		},
+	}); err != nil {
+		return err
+	}
+
+	if err := createConfigMapRBACRules(client, k8sVersion); err != nil {
+		return fmt.Errorf("error creating kubelet configuration configmap RBAC rules: %v", err)
+	}
+	return nil
+}
+
+// createConfigMapRBACRules creates the RBAC rules for exposing the base kubelet ConfigMap in the kube-system namespace to unauthenticated users
+func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Version) error {
+	if err := apiclient.CreateOrUpdateRole(client, &rbac.Role{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      configMapRBACName(k8sVersion),
+			Namespace: metav1.NamespaceSystem,
+		},
+		Rules: []rbac.PolicyRule{
+			rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(configMapName(k8sVersion)).RuleOrDie(),
+		},
+	}); err != nil {
+		return err
+	}
+
+	return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      configMapRBACName(k8sVersion),
+			Namespace: metav1.NamespaceSystem,
+		},
+		RoleRef: rbac.RoleRef{
+			APIGroup: rbac.GroupName,
+			Kind:     "Role",
+			Name:     configMapRBACName(k8sVersion),
+		},
+		Subjects: []rbac.Subject{
+			{
+				Kind: rbac.GroupKind,
+				Name: kubeadmconstants.NodesGroup,
+			},
+			{
+				Kind: rbac.GroupKind,
+				Name: kubeadmconstants.NodeBootstrapTokenAuthGroup,
+			},
+		},
+	})
+}
+
+// DownloadConfig downloads the kubelet configuration from a ConfigMap and writes it to disk.
+// Used at "kubeadm join" time
+func DownloadConfig(kubeletKubeConfig string, kubeletVersion *version.Version) error {
+
+	// If the kubelet version is v1.10.x, exit
+	if kubeletVersion.LessThan(kubeadmconstants.MinimumKubeletConfigVersion) {
+		return nil
+	}
+
+	// Download the ConfigMap from the cluster based on what version the kubelet is
+	configMapName := configMapName(kubeletVersion)
+
+	fmt.Printf("[kubelet] Downloading configuration for the kubelet from the %q ConfigMap in the %s namespace\n",
+		configMapName, metav1.NamespaceSystem)
+
+	client, err := kubeconfigutil.ClientSetFromFile(kubeletKubeConfig)
+	if err != nil {
+		return fmt.Errorf("couldn't create client from kubeconfig file %q", kubeletKubeConfig)
+	}
+
+	kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	return writeConfigBytesToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey]))
+}
+
+// configMapName returns the right ConfigMap name for the right branch of k8s
+func configMapName(k8sVersion *version.Version) string {
+	return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigurationConfigMapPrefix, k8sVersion.Major(), k8sVersion.Minor())
+}
+
+// configMapRBACName returns the name for the Role/RoleBinding for the kubelet config configmap for the right branch of k8s
+func configMapRBACName(k8sVersion *version.Version) string {
+	return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigMapRolePrefix, k8sVersion.Major(), k8sVersion.Minor())
+}
+
+// getConfigBytes marshals a kubeletconfiguration object to bytes
+func getConfigBytes(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration) ([]byte, error) {
+	_, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	return kubeadmutil.MarshalToYamlForCodecs(kubeletConfig, kubeletconfigv1beta1.SchemeGroupVersion, *kubeletCodecs)
+}
+
+// writeConfigBytesToDisk writes a byte slice down to disk at the specific location of the kubelet config file
+func writeConfigBytesToDisk(b []byte) error {
+	fmt.Printf("[kubelet] Writing kubelet configuration to file %q\n", kubeadmconstants.KubeletConfigurationFile)
+
+	if err := ioutil.WriteFile(kubeadmconstants.KubeletConfigurationFile, b, 0644); err != nil {
+		return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", kubeadmconstants.KubeletConfigurationFile, err)
+	}
+	return nil
+}

cmd/kubeadm/app/phases/kubelet/config_test.go

@@ -0,0 +1,78 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubelet
+
+import (
+	"testing"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes/fake"
+	core "k8s.io/client-go/testing"
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
+	"k8s.io/kubernetes/pkg/util/version"
+)
+
+func TestCreateConfigMap(t *testing.T) {
+	nodeName := "fake-node"
+	client := fake.NewSimpleClientset()
+	cfg := &kubeadmapi.MasterConfiguration{
+		NodeName:          nodeName,
+		KubernetesVersion: "v1.11.0",
+		KubeletConfiguration: kubeadmapi.KubeletConfiguration{
+			BaseConfig: &kubeletconfigv1beta1.KubeletConfiguration{},
+		},
+	}
+
+	client.PrependReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) {
+		return true, &v1.Node{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: nodeName,
+			},
+			Spec: v1.NodeSpec{},
+		}, nil
+	})
+	client.PrependReactor("create", "roles", func(action core.Action) (bool, runtime.Object, error) {
+		return true, nil, nil
+	})
+	client.PrependReactor("create", "rolebindings", func(action core.Action) (bool, runtime.Object, error) {
+		return true, nil, nil
+	})
+	client.PrependReactor("create", "configmaps", func(action core.Action) (bool, runtime.Object, error) {
+		return true, nil, nil
+	})
+
+	if err := CreateConfigMap(cfg, client); err != nil {
+		t.Errorf("CreateConfigMap: unexpected error %v", err)
+	}
+}
+
+func TestCreateConfigMapRBACRules(t *testing.T) {
+	client := fake.NewSimpleClientset()
+	client.PrependReactor("create", "roles", func(action core.Action) (bool, runtime.Object, error) {
+		return true, nil, nil
+	})
+	client.PrependReactor("create", "rolebindings", func(action core.Action) (bool, runtime.Object, error) {
+		return true, nil, nil
+	})
+
+	if err := createConfigMapRBACRules(client, version.MustParseSemantic("v1.11.0")); err != nil {
+		t.Errorf("createConfigMapRBACRules: unexpected error %v", err)
+	}
+}

cmd/kubeadm/app/phases/kubelet/dynamic.go

@@ -0,0 +1,117 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubelet
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
+	"k8s.io/kubernetes/pkg/util/version"
+)
+
+// EnableDynamicConfigForNode updates the Node's ConfigSource to enable Dynamic Kubelet Configuration, depending on what version the kubelet is
+// Used at "kubeadm init", "kubeadm join" and "kubeadm upgrade" time
+// This func is ONLY run if the user enables the `DynamicKubeletConfig` feature gate, which is by default off
+func EnableDynamicConfigForNode(client clientset.Interface, nodeName string, kubeletVersion *version.Version) error {
+
+	// If the kubelet version is v1.10.x, exit
+	if kubeletVersion.LessThan(kubeadmconstants.MinimumKubeletConfigVersion) {
+		return nil
+	}
+
+	configMapName := configMapName(kubeletVersion)
+	fmt.Printf("[kubelet] Enabling Dynamic Kubelet Config for Node %q; config sourced from ConfigMap %q in namespace %s\n",
+		nodeName, configMapName, metav1.NamespaceSystem)
+	fmt.Println("[kubelet] WARNING: The Dynamic Kubelet Config feature is alpha and off by default. It hasn't been well-tested yet at this stage, use with caution.")
+
+	// Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned.
+	return wait.Poll(kubeadmconstants.APICallRetryInterval, kubeadmconstants.UpdateNodeTimeout, func() (bool, error) {
+		node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
+		if err != nil {
+			return false, nil
+		}
+
+		oldData, err := json.Marshal(node)
+		if err != nil {
+			return false, err
+		}
+
+		kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{})
+		if err != nil {
+			return false, nil
+		}
+
+		node.Spec.ConfigSource = &v1.NodeConfigSource{
+			ConfigMap: &v1.ConfigMapNodeConfigSource{
+				Name:             configMapName,
+				Namespace:        metav1.NamespaceSystem,
+				UID:              kubeletCfg.UID,
+				KubeletConfigKey: kubeadmconstants.KubeletBaseConfigurationConfigMapKey,
+			},
+		}
+
+		newData, err := json.Marshal(node)
+		if err != nil {
+			return false, err
+		}
+
+		patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{})
+		if err != nil {
+			return false, err
+		}
+
+		if _, err := client.CoreV1().Nodes().Patch(node.Name, types.StrategicMergePatchType, patchBytes); err != nil {
+			if apierrs.IsConflict(err) {
+				fmt.Println("Temporarily unable to update node metadata due to conflict (will retry)")
+				return false, nil
+			}
+			return false, err
+		}
+
+		return true, nil
+	})
+}
+
+// GetLocalNodeTLSBootstrappedClient waits for the kubelet to perform the TLS bootstrap
+// and then creates a client from config file /etc/kubernetes/kubelet.conf
+func GetLocalNodeTLSBootstrappedClient() (clientset.Interface, error) {
+	fmt.Println("[tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap...")
+
+	kubeletKubeConfig := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)
+
+	// Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned.
+	err := wait.PollImmediateInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
+		_, err := os.Stat(kubeletKubeConfig)
+		return (err == nil), nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return kubeconfigutil.ClientSetFromFile(kubeletKubeConfig)
+}

cmd/kubeadm/app/phases/kubelet/dynamic_test.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubelet
+
+import (
+	"testing"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes/fake"
+	core "k8s.io/client-go/testing"
+	"k8s.io/kubernetes/pkg/util/version"
+)
+
+func TestEnableDynamicConfigForNode(t *testing.T) {
+	nodeName := "fake-node"
+	client := fake.NewSimpleClientset()
+	client.PrependReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) {
+		return true, &v1.Node{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: nodeName,
+			},
+			Spec: v1.NodeSpec{
+				ConfigSource: &v1.NodeConfigSource{
+					ConfigMap: &v1.ConfigMapNodeConfigSource{
+						UID: "",
+					},
+				},
+			},
+		}, nil
+	})
+	client.PrependReactor("get", "configmaps", func(action core.Action) (bool, runtime.Object, error) {
+		return true, &v1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "kubelet-config-1.11",
+				Namespace: metav1.NamespaceSystem,
+				UID:       "fake-uid",
+			},
+		}, nil
+	})
+	client.PrependReactor("patch", "nodes", func(action core.Action) (bool, runtime.Object, error) {
+		return true, nil, nil
+	})
+
+	if err := EnableDynamicConfigForNode(client, nodeName, version.MustParseSemantic("v1.11.0")); err != nil {
+		t.Errorf("UpdateNodeWithConfigMap: unexpected error %v", err)
+	}
+}

cmd/kubeadm/app/phases/kubelet/flags.go

@@ -0,0 +1,78 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubelet
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
+	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+)
+
+// WriteKubeletDynamicEnvFile writes a environment file with dynamic flags to the kubelet.
+// Used at "kubeadm init" and "kubeadm join" time.
+func WriteKubeletDynamicEnvFile(cfg *kubeadmapi.MasterConfiguration) error {
+
+	// TODO: Pass through extra arguments from the config file here in the future
+	argList := kubeadmutil.BuildArgumentListFromMap(buildKubeletArgMap(cfg), map[string]string{})
+	envFileContent := fmt.Sprintf("%s=%s\n", constants.KubeletEnvFileVariableName, strings.Join(argList, " "))
+
+	return writeKubeletFlagBytesToDisk([]byte(envFileContent))
+}
+
+// buildKubeletArgMap takes a MasterConfiguration object and builds based on that a string-string map with flags
+// that should be given to the local kubelet daemon.
+func buildKubeletArgMap(cfg *kubeadmapi.MasterConfiguration) map[string]string {
+	kubeletFlags := map[string]string{}
+
+	if cfg.CRISocket == kubeadmapiv1alpha2.DefaultCRISocket {
+		// These flags should only be set when running docker
+		kubeletFlags["network-plugin"] = "cni"
+		kubeletFlags["cni-conf-dir"] = "/etc/cni/net.d"
+		kubeletFlags["cni-bin-dir"] = "/opt/cni/bin"
+	} else {
+		kubeletFlags["container-runtime"] = "remote"
+		kubeletFlags["container-runtime-endpoint"] = cfg.CRISocket
+	}
+	// TODO: Add support for registering custom Taints and Labels
+	// TODO: Add support for overriding flags with ExtraArgs
+	// TODO: Pass through --hostname-override if a custom name is used?
+	// TODO: Check if `systemd-resolved` is running, and set `--resolv-conf` based on that
+	// TODO: Conditionally set `--cgroup-driver` to either `systemd` or `cgroupfs`
+
+	return kubeletFlags
+}
+
+// writeKubeletFlagBytesToDisk writes a byte slice down to disk at the specific location of the kubelet flag overrides file
+func writeKubeletFlagBytesToDisk(b []byte) error {
+	fmt.Printf("[kubelet] Writing kubelet environment file with flags to file %q\n", constants.KubeletEnvFile)
+
+	// creates target folder if not already exists
+	if err := os.MkdirAll(filepath.Dir(constants.KubeletEnvFile), 0700); err != nil {
+		return fmt.Errorf("failed to create directory %q: %v", filepath.Dir(constants.KubeletEnvFile), err)
+	}
+	if err := ioutil.WriteFile(constants.KubeletEnvFile, b, 0644); err != nil {
+		return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", constants.KubeletEnvFile, err)
+	}
+	return nil
+}

cmd/kubeadm/app/phases/kubelet/kubelet.go

@@ -1,235 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package kubelet
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-
-	"k8s.io/api/core/v1"
-	rbac "k8s.io/api/rbac/v1"
-	apierrs "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/strategicpatch"
-	"k8s.io/apimachinery/pkg/util/wait"
-	clientset "k8s.io/client-go/kubernetes"
-	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
-	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
-	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
-	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
-	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
-	rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
-	kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
-	kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
-)
-
-// CreateBaseKubeletConfiguration creates base kubelet configuration for dynamic kubelet configuration feature.
-func CreateBaseKubeletConfiguration(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
-	fmt.Printf("[kubelet] Uploading a ConfigMap %q in namespace %s with base configuration for the kubelets in the cluster\n",
-		kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.NamespaceSystem)
-
-	_, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs()
-	if err != nil {
-		return err
-	}
-	kubeletBytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg.KubeletConfiguration.BaseConfig, kubeletconfigv1beta1.SchemeGroupVersion, *kubeletCodecs)
-	if err != nil {
-		return err
-	}
-
-	if err = apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      kubeadmconstants.KubeletBaseConfigurationConfigMap,
-			Namespace: metav1.NamespaceSystem,
-		},
-		Data: map[string]string{
-			kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(kubeletBytes),
-		},
-	}); err != nil {
-		return err
-	}
-
-	if err := createKubeletBaseConfigMapRBACRules(client); err != nil {
-		return fmt.Errorf("error creating base kubelet configmap RBAC rules: %v", err)
-	}
-
-	return updateNodeWithConfigMap(client, cfg.NodeName)
-}
-
-// ConsumeBaseKubeletConfiguration consumes base kubelet configuration for dynamic kubelet configuration feature.
-func ConsumeBaseKubeletConfiguration(nodeName string) error {
-	client, err := getLocalNodeTLSBootstrappedClient()
-	if err != nil {
-		return err
-	}
-
-	kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.GetOptions{})
-	if err != nil {
-		return err
-	}
-
-	if err := writeInitKubeletConfigToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey])); err != nil {
-		return fmt.Errorf("failed to write initial remote configuration of kubelet to disk for node %s: %v", nodeName, err)
-	}
-
-	return updateNodeWithConfigMap(client, nodeName)
-}
-
-// updateNodeWithConfigMap updates node ConfigSource with KubeletBaseConfigurationConfigMap
-func updateNodeWithConfigMap(client clientset.Interface, nodeName string) error {
-	fmt.Printf("[kubelet] Using Dynamic Kubelet Config for node %q; config sourced from ConfigMap %q in namespace %s\n",
-		nodeName, kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.NamespaceSystem)
-
-	// Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned.
-	return wait.Poll(kubeadmconstants.APICallRetryInterval, kubeadmconstants.UpdateNodeTimeout, func() (bool, error) {
-		node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
-		if err != nil {
-			return false, nil
-		}
-
-		oldData, err := json.Marshal(node)
-		if err != nil {
-			return false, err
-		}
-
-		kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.GetOptions{})
-		if err != nil {
-			return false, nil
-		}
-
-		node.Spec.ConfigSource = &v1.NodeConfigSource{
-			ConfigMap: &v1.ConfigMapNodeConfigSource{
-				Name:             kubeadmconstants.KubeletBaseConfigurationConfigMap,
-				Namespace:        metav1.NamespaceSystem,
-				UID:              kubeletCfg.UID,
-				KubeletConfigKey: kubeadmconstants.KubeletBaseConfigurationConfigMapKey,
-			},
-		}
-
-		newData, err := json.Marshal(node)
-		if err != nil {
-			return false, err
-		}
-
-		patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{})
-		if err != nil {
-			return false, err
-		}
-
-		if _, err := client.CoreV1().Nodes().Patch(node.Name, types.StrategicMergePatchType, patchBytes); err != nil {
-			if apierrs.IsConflict(err) {
-				fmt.Println("Temporarily unable to update node metadata due to conflict (will retry)")
-				return false, nil
-			}
-			return false, err
-		}
-
-		return true, nil
-	})
-}
-
-// createKubeletBaseConfigMapRBACRules creates the RBAC rules for exposing the base kubelet ConfigMap in the kube-system namespace to unauthenticated users
-func createKubeletBaseConfigMapRBACRules(client clientset.Interface) error {
-	if err := apiclient.CreateOrUpdateRole(client, &rbac.Role{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      kubeadmconstants.KubeletBaseConfigMapRoleName,
-			Namespace: metav1.NamespaceSystem,
-		},
-		Rules: []rbac.PolicyRule{
-			rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(kubeadmconstants.KubeletBaseConfigurationConfigMap).RuleOrDie(),
-		},
-	}); err != nil {
-		return err
-	}
-
-	return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      kubeadmconstants.KubeletBaseConfigMapRoleName,
-			Namespace: metav1.NamespaceSystem,
-		},
-		RoleRef: rbac.RoleRef{
-			APIGroup: rbac.GroupName,
-			Kind:     "Role",
-			Name:     kubeadmconstants.KubeletBaseConfigMapRoleName,
-		},
-		Subjects: []rbac.Subject{
-			{
-				Kind: rbac.GroupKind,
-				Name: kubeadmconstants.NodesGroup,
-			},
-			{
-				Kind: rbac.GroupKind,
-				Name: kubeadmconstants.NodeBootstrapTokenAuthGroup,
-			},
-		},
-	})
-}
-
-// getLocalNodeTLSBootstrappedClient waits for the kubelet to perform the TLS bootstrap
-// and then creates a client from config file /etc/kubernetes/kubelet.conf
-func getLocalNodeTLSBootstrappedClient() (clientset.Interface, error) {
-	fmt.Println("[tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap...")
-
-	kubeletKubeConfig := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)
-
-	// Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned.
-	err := wait.PollImmediateInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
-		_, err := os.Stat(kubeletKubeConfig)
-		return (err == nil), nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return kubeconfigutil.ClientSetFromFile(kubeletKubeConfig)
-}
-
-// WriteInitKubeletConfigToDiskOnMaster writes base kubelet configuration to disk on master.
-func WriteInitKubeletConfigToDiskOnMaster(cfg *kubeadmapi.MasterConfiguration) error {
-	fmt.Printf("[kubelet] Writing base configuration of kubelets to disk on master node %s\n", cfg.NodeName)
-
-	_, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs()
-	if err != nil {
-		return err
-	}
-
-	kubeletBytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg.KubeletConfiguration.BaseConfig, kubeletconfigv1beta1.SchemeGroupVersion, *kubeletCodecs)
-	if err != nil {
-		return err
-	}
-
-	if err := writeInitKubeletConfigToDisk(kubeletBytes); err != nil {
-		return fmt.Errorf("failed to write base configuration of kubelet to disk on master node %s: %v", cfg.NodeName, err)
-	}
-
-	return nil
-}
-
-func writeInitKubeletConfigToDisk(kubeletConfig []byte) error {
-	if err := os.MkdirAll(kubeadmconstants.KubeletBaseConfigurationDir, 0644); err != nil {
-		return fmt.Errorf("failed to create directory %q: %v", kubeadmconstants.KubeletBaseConfigurationDir, err)
-	}
-	baseConfigFile := filepath.Join(kubeadmconstants.KubeletBaseConfigurationDir, kubeadmconstants.KubeletBaseConfigurationFile)
-	if err := ioutil.WriteFile(baseConfigFile, kubeletConfig, 0644); err != nil {
-		return fmt.Errorf("failed to write initial remote configuration of kubelet into file %q: %v", baseConfigFile, err)
-	}
-	return nil
-}

cmd/kubeadm/app/phases/kubelet/kubelet_test.go

@@ -1,134 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package kubelet
-
-import (
-	"testing"
-
-	"k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/client-go/kubernetes/fake"
-	core "k8s.io/client-go/testing"
-	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
-	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
-	kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
-)
-
-func TestCreateBaseKubeletConfiguration(t *testing.T) {
-	nodeName := "fake-node"
-	client := fake.NewSimpleClientset()
-	cfg := &kubeadmapi.MasterConfiguration{
-		NodeName: nodeName,
-		KubeletConfiguration: kubeadmapi.KubeletConfiguration{
-			BaseConfig: &kubeletconfigv1beta1.KubeletConfiguration{
-				TypeMeta: metav1.TypeMeta{
-					Kind: "KubeletConfiguration",
-				},
-			},
-		},
-	}
-
-	client.PrependReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) {
-		return true, &v1.Node{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: nodeName,
-			},
-			Spec: v1.NodeSpec{
-				ConfigSource: &v1.NodeConfigSource{
-					ConfigMap: &v1.ConfigMapNodeConfigSource{
-						UID: "",
-					},
-				},
-			},
-		}, nil
-	})
-	client.PrependReactor("get", "configmaps", func(action core.Action) (bool, runtime.Object, error) {
-		return true, &v1.ConfigMap{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      kubeadmconstants.KubeletBaseConfigurationConfigMap,
-				Namespace: metav1.NamespaceSystem,
-				UID:       "fake-uid",
-			},
-		}, nil
-	})
-	client.PrependReactor("patch", "nodes", func(action core.Action) (bool, runtime.Object, error) {
-		return true, nil, nil
-	})
-	client.PrependReactor("create", "roles", func(action core.Action) (bool, runtime.Object, error) {
-		return true, nil, nil
-	})
-	client.PrependReactor("create", "rolebindings", func(action core.Action) (bool, runtime.Object, error) {
-		return true, nil, nil
-	})
-	client.PrependReactor("create", "configmaps", func(action core.Action) (bool, runtime.Object, error) {
-		return true, nil, nil
-	})
-
-	if err := CreateBaseKubeletConfiguration(cfg, client); err != nil {
-		t.Errorf("CreateBaseKubeletConfiguration: unexepected error %v", err)
-	}
-}
-
-func TestUpdateNodeWithConfigMap(t *testing.T) {
-	nodeName := "fake-node"
-	client := fake.NewSimpleClientset()
-	client.PrependReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) {
-		return true, &v1.Node{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: nodeName,
-			},
-			Spec: v1.NodeSpec{
-				ConfigSource: &v1.NodeConfigSource{
-					ConfigMap: &v1.ConfigMapNodeConfigSource{
-						UID: "",
-					},
-				},
-			},
-		}, nil
-	})
-	client.PrependReactor("get", "configmaps", func(action core.Action) (bool, runtime.Object, error) {
-		return true, &v1.ConfigMap{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      kubeadmconstants.KubeletBaseConfigurationConfigMap,
-				Namespace: metav1.NamespaceSystem,
-				UID:       "fake-uid",
-			},
-		}, nil
-	})
-	client.PrependReactor("patch", "nodes", func(action core.Action) (bool, runtime.Object, error) {
-		return true, nil, nil
-	})
-
-	if err := updateNodeWithConfigMap(client, nodeName); err != nil {
-		t.Errorf("UpdateNodeWithConfigMap: unexepected error %v", err)
-	}
-}
-
-func TestCreateKubeletBaseConfigMapRBACRules(t *testing.T) {
-	client := fake.NewSimpleClientset()
-	client.PrependReactor("create", "roles", func(action core.Action) (bool, runtime.Object, error) {
-		return true, nil, nil
-	})
-	client.PrependReactor("create", "rolebindings", func(action core.Action) (bool, runtime.Object, error) {
-		return true, nil, nil
-	})
-
-	if err := createKubeletBaseConfigMapRBACRules(client); err != nil {
-		t.Errorf("createKubeletBaseConfigMapRBACRules: unexepected error %v", err)
-	}
-}

cmd/kubeadm/app/phases/upgrade/configuration.go

@@ -1,77 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package upgrade
-
-import (
-	"fmt"
-	"io"
-	"io/ioutil"
-
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	clientset "k8s.io/client-go/kubernetes"
-	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
-	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
-	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
-)
-
-// FetchConfiguration fetches configuration required for upgrading your cluster from a file (which has precedence) or a ConfigMap in the cluster
-func FetchConfiguration(client clientset.Interface, w io.Writer, cfgPath string) (*kubeadmapi.MasterConfiguration, error) {
-	fmt.Println("[upgrade/config] Making sure the configuration is correct:")
-
-	// Load the configuration from a file or the cluster
-	configBytes, err := loadConfigurationBytes(client, w, cfgPath)
-	if err != nil {
-		return nil, err
-	}
-
-	// Take the versioned configuration populated from the file or configmap, convert it to internal, default and validate
-	versionedcfg, err := configutil.BytesToInternalConfig(configBytes)
-	if err != nil {
-		return nil, fmt.Errorf("could not decode configuration: %v", err)
-	}
-	return versionedcfg, nil
-}
-
-// loadConfigurationBytes loads the configuration byte slice from either a file or the cluster ConfigMap
-func loadConfigurationBytes(client clientset.Interface, w io.Writer, cfgPath string) ([]byte, error) {
-	// The config file has the highest priority
-	if cfgPath != "" {
-		fmt.Printf("[upgrade/config] Reading configuration options from a file: %s\n", cfgPath)
-		return ioutil.ReadFile(cfgPath)
-	}
-
-	fmt.Println("[upgrade/config] Reading configuration from the cluster...")
-
-	configMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(constants.MasterConfigurationConfigMap, metav1.GetOptions{})
-	if apierrors.IsNotFound(err) {
-		fmt.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %s namespace must exist.\n", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem)
-		fmt.Println("[upgrade/config] Without this information, 'kubeadm upgrade' won't know how to configure your upgraded cluster.")
-		fmt.Println("")
-		fmt.Println("[upgrade/config] Next steps:")
-		fmt.Printf("\t- OPTION 1: Run 'kubeadm config upload from-flags' and specify the same CLI arguments you passed to 'kubeadm init' when you created your master.\n")
-		fmt.Printf("\t- OPTION 2: Run 'kubeadm config upload from-file' and specify the same config file you passed to 'kubeadm init' when you created your master.\n")
-		fmt.Printf("\t- OPTION 3: Pass a config file to 'kubeadm upgrade' using the --config flag.\n")
-		fmt.Println("")
-		return []byte{}, fmt.Errorf("the ConfigMap %q in the %s namespace used for getting configuration information was not found", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem)
-	} else if err != nil {
-		return []byte{}, fmt.Errorf("an unexpected error happened when trying to get the ConfigMap %q in the %s namespace: %v", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem, err)
-	}
-
-	fmt.Printf("[upgrade/config] FYI: You can look at this config file with 'kubectl -n %s get cm %s -oyaml'\n", metav1.NamespaceSystem, constants.MasterConfigurationConfigMap)
-	return []byte(configMap.Data[constants.MasterConfigurationConfigMapKey]), nil
-}

cmd/kubeadm/app/phases/upgrade/postupgrade.go

@@ -36,6 +36,7 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
 	nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
 	certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
+	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
@@ -102,6 +103,11 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterC
 		}
 	}
 
+	// Create the new, version-branched kubelet ComponentConfig ConfigMap
+	if err := kubeletphase.CreateConfigMap(cfg, client); err != nil {
+		errs = append(errs, fmt.Errorf("error creating kubelet configuration ConfigMap: %v", err))
+	}
+
 	// Upgrade kube-dns/CoreDNS and kube-proxy
 	if err := dns.EnsureDNSAddon(cfg, client); err != nil {
 		errs = append(errs, err)

cmd/kubeadm/app/preflight/checks.go

@@ -1043,10 +1043,11 @@ func TryStartKubelet(ignorePreflightErrors sets.String) {
 	initSystem, err := initsystem.GetInitSystem()
 	if err != nil {
 		glog.Infoln("[preflight] no supported init system detected, won't ensure kubelet is running.")
-	} else if initSystem.ServiceExists("kubelet") && !initSystem.ServiceIsActive("kubelet") {
+	} else if initSystem.ServiceExists("kubelet") {
 
-		glog.Infoln("[preflight] starting the kubelet service")
+		glog.Infoln("[preflight] Activating the kubelet service")
-		if err := initSystem.ServiceStart("kubelet"); err != nil {
+		// This runs "systemctl daemon-reload && systemctl restart kubelet"
+		if err := initSystem.ServiceRestart("kubelet"); err != nil {
 			glog.Warningf("[preflight] unable to start the kubelet service: [%v]\n", err)
 			glog.Warningf("[preflight] please ensure kubelet is running manually.")
 		}

cmd/kubeadm/app/util/config/cluster.go

@@ -0,0 +1,65 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+)
+
+// TODO: Add unit tests for this file
+
+// FetchConfigFromFileOrCluster fetches configuration required for upgrading your cluster from a file (which has precedence) or a ConfigMap in the cluster
+func FetchConfigFromFileOrCluster(client clientset.Interface, w io.Writer, logPrefix, cfgPath string) (*kubeadmapi.MasterConfiguration, error) {
+	// Load the configuration from a file or the cluster
+	configBytes, err := loadConfigurationBytes(client, w, logPrefix, cfgPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// Take the versioned configuration populated from the file or ConfigMap, convert it to internal, default and validate
+	return BytesToInternalConfig(configBytes)
+}
+
+// loadConfigurationBytes loads the configuration byte slice from either a file or the cluster ConfigMap
+func loadConfigurationBytes(client clientset.Interface, w io.Writer, logPrefix, cfgPath string) ([]byte, error) {
+	// The config file has the highest priority
+	if cfgPath != "" {
+		fmt.Fprintf(w, "[%s] Reading configuration options from a file: %s\n", logPrefix, cfgPath)
+		return ioutil.ReadFile(cfgPath)
+	}
+
+	fmt.Fprintf(w, "[%s] Reading configuration from the cluster...\n", logPrefix)
+
+	configMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(constants.MasterConfigurationConfigMap, metav1.GetOptions{})
+	if apierrors.IsNotFound(err) {
+		// Return the apierror directly so the caller of this function can know what type of error occurred and act based on that
+		return []byte{}, err
+	} else if err != nil {
+		return []byte{}, fmt.Errorf("an unexpected error happened when trying to get the ConfigMap %q in the %s namespace: %v", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem, err)
+	}
+
+	fmt.Fprintf(w, "[%s] FYI: You can look at this config file with 'kubectl -n %s get cm %s -oyaml'\n", logPrefix, metav1.NamespaceSystem, constants.MasterConfigurationConfigMap)
+	return []byte(configMap.Data[constants.MasterConfigurationConfigMapKey]), nil
+}

pkg/util/initsystem/initsystem.go

@@ -29,6 +29,9 @@ type InitSystem interface {
 	// ServiceStop tries to stop a specific service
 	ServiceStop(service string) error
 
+	// ServiceRestart tries to reload the environment and restart the specific service
+	ServiceRestart(service string) error
+
 	// ServiceExists ensures the service is defined for this init system.
 	ServiceExists(service string) bool
 
@@ -47,6 +50,14 @@ func (sysd SystemdInitSystem) ServiceStart(service string) error {
 	return err
 }
 
+func (sysd SystemdInitSystem) ServiceRestart(service string) error {
+	if err := exec.Command("systemctl", "daemon-reload").Run(); err != nil {
+		return fmt.Errorf("failed to reload systemd: %v", err)
+	}
+	args := []string{"restart", service}
+	return exec.Command("systemctl", args...).Run()
+}
+
 func (sysd SystemdInitSystem) ServiceStop(service string) error {
 	args := []string{"stop", service}
 	err := exec.Command("systemctl", args...).Run()
@@ -95,6 +106,16 @@ func (sysd WindowsInitSystem) ServiceStart(service string) error {
 	return err
 }
 
+func (sysd WindowsInitSystem) ServiceRestart(service string) error {
+	if err := sysd.ServiceStop(service); err != nil {
+		return fmt.Errorf("couldn't stop service: %v", err)
+	}
+	if err := sysd.ServiceStart(service); err != nil {
+		return fmt.Errorf("couldn't start service: %v", err)
+	}
+	return nil
+}
+
 func (sysd WindowsInitSystem) ServiceStop(service string) error {
 	args := []string{"Stop-Service", service}
 	err := exec.Command("powershell", args...).Run()