Merge pull request #63799 from luxas/kubeadm_restructure_internal_config_usage
Automatic merge from submit-queue (batch tested with PRs 63314, 63884, 63799, 63521, 62242). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. kubeadm: Restructure internal config usage and fix bugs **What this PR does / why we need it**: - Moves the generic LoadYAML function from the versioned, external API package to a helper library so it can be consumed more easily - Makes the upgrading code use the internal version of the API (which always should be used anyway) - Moves all config-loading code to `configutil`, together with the migration code needed. This way we have everything in one centralized place, instead of duplicating that logic N times. - Makes `kubeadm init` use `configutil` for the reasons mentioned above. This PR is needed in order to support multiple external API groups (like v1alpha2) **Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*: Part of https://github.com/kubernetes/community/pull/2131 **Special notes for your reviewer**: This PR depends on: - https://github.com/kubernetes/kubernetes/pull/63782 - https://github.com/kubernetes/kubernetes/pull/63783 **Please review only the last (third) commit** **Release note**: ```release-note NONE ``` @kubernetes/sig-cluster-lifecycle-pr-reviews @liztio
This commit is contained in:
		| @@ -59,7 +59,6 @@ go_library( | ||||
|         "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library", | ||||
|         "//vendor/github.com/json-iterator/go:go_default_library", | ||||
|         "//vendor/github.com/ugorji/go/codec:go_default_library", | ||||
|         "//vendor/gopkg.in/yaml.v2:go_default_library", | ||||
|         "//vendor/k8s.io/api/core/v1:go_default_library", | ||||
|         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", | ||||
|         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", | ||||
| @@ -89,9 +88,5 @@ go_test( | ||||
|     srcs = ["upgrade_test.go"], | ||||
|     data = glob(["testdata/**"]), | ||||
|     embed = [":go_default_library"], | ||||
|     deps = [ | ||||
|         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", | ||||
|         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", | ||||
|         "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", | ||||
|     ], | ||||
|     deps = ["//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library"], | ||||
| ) | ||||
|   | ||||
| @@ -18,15 +18,13 @@ package v1alpha1 | ||||
|  | ||||
| import ( | ||||
| 	"bytes" | ||||
| 	"errors" | ||||
| 	"fmt" | ||||
| 	"reflect" | ||||
| 	"strconv" | ||||
| 	"strings" | ||||
|  | ||||
| 	"github.com/json-iterator/go" | ||||
| 	jsoniter "github.com/json-iterator/go" | ||||
| 	"github.com/ugorji/go/codec" | ||||
| 	yaml "gopkg.in/yaml.v2" | ||||
|  | ||||
| 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" | ||||
| 	"k8s.io/apimachinery/pkg/runtime" | ||||
| @@ -101,35 +99,3 @@ func proxyFeatureListToMap(m map[string]interface{}) error { | ||||
| 	unstructured.SetNestedMap(m, gateMap, featureGatePath...) | ||||
| 	return nil | ||||
| } | ||||
|  | ||||
| // LoadYAML is a small wrapper around go-yaml that ensures all nested structs are map[string]interface{} instead of map[interface{}]interface{}. | ||||
| func LoadYAML(bytes []byte) (map[string]interface{}, error) { | ||||
| 	var decoded map[interface{}]interface{} | ||||
| 	if err := yaml.Unmarshal(bytes, &decoded); err != nil { | ||||
| 		return map[string]interface{}{}, fmt.Errorf("couldn't unmarshal YAML: %v", err) | ||||
| 	} | ||||
|  | ||||
| 	converted, ok := convert(decoded).(map[string]interface{}) | ||||
| 	if !ok { | ||||
| 		return map[string]interface{}{}, errors.New("yaml is not a map") | ||||
| 	} | ||||
|  | ||||
| 	return converted, nil | ||||
| } | ||||
|  | ||||
| // https://stackoverflow.com/questions/40737122/convert-yaml-to-json-without-struct-golang | ||||
| func convert(i interface{}) interface{} { | ||||
| 	switch x := i.(type) { | ||||
| 	case map[interface{}]interface{}: | ||||
| 		m2 := map[string]interface{}{} | ||||
| 		for k, v := range x { | ||||
| 			m2[k.(string)] = convert(v) | ||||
| 		} | ||||
| 		return m2 | ||||
| 	case []interface{}: | ||||
| 		for i, v := range x { | ||||
| 			x[i] = convert(v) | ||||
| 		} | ||||
| 	} | ||||
| 	return i | ||||
| } | ||||
|   | ||||
| @@ -17,37 +17,11 @@ limitations under the License. | ||||
| package v1alpha1 | ||||
|  | ||||
| import ( | ||||
| 	"io/ioutil" | ||||
| 	"testing" | ||||
|  | ||||
| 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" | ||||
| 	"k8s.io/apimachinery/pkg/runtime" | ||||
| 	"k8s.io/apimachinery/pkg/runtime/serializer" | ||||
| ) | ||||
|  | ||||
| const test196 = "testdata/kubeadm196.yaml" | ||||
|  | ||||
| func TestUpgrade(t *testing.T) { | ||||
| 	testYAML, err := ioutil.ReadFile(test196) | ||||
| 	if err != nil { | ||||
| 		t.Fatalf("couldn't read test data: %v", err) | ||||
| 	} | ||||
|  | ||||
| 	decoded, err := LoadYAML(testYAML) | ||||
| 	if err != nil { | ||||
| 		t.Fatalf("couldn't unmarshal test yaml: %v", err) | ||||
| 	} | ||||
|  | ||||
| 	scheme := runtime.NewScheme() | ||||
| 	AddToScheme(scheme) | ||||
| 	codecs := serializer.NewCodecFactory(scheme) | ||||
|  | ||||
| 	obj := &MasterConfiguration{} | ||||
| 	if err := Migrate(decoded, obj, codecs); err != nil { | ||||
| 		t.Fatalf("couldn't decode migrated object: %v", err) | ||||
| 	} | ||||
| } | ||||
|  | ||||
| func TestProxyFeatureListToMap(t *testing.T) { | ||||
|  | ||||
| 	cases := []struct { | ||||
|   | ||||
| @@ -32,7 +32,6 @@ import ( | ||||
| 	flag "github.com/spf13/pflag" | ||||
|  | ||||
| 	"k8s.io/api/core/v1" | ||||
| 	"k8s.io/apimachinery/pkg/runtime" | ||||
| 	"k8s.io/apimachinery/pkg/util/sets" | ||||
| 	clientset "k8s.io/client-go/kubernetes" | ||||
| 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" | ||||
| @@ -115,8 +114,8 @@ var ( | ||||
|  | ||||
| // NewCmdInit returns "kubeadm init" command. | ||||
| func NewCmdInit(out io.Writer) *cobra.Command { | ||||
| 	cfg := &kubeadmapiv1alpha1.MasterConfiguration{} | ||||
| 	kubeadmscheme.Scheme.Default(cfg) | ||||
| 	externalcfg := &kubeadmapiv1alpha1.MasterConfiguration{} | ||||
| 	kubeadmscheme.Scheme.Default(externalcfg) | ||||
|  | ||||
| 	var cfgPath string | ||||
| 	var skipPreFlight bool | ||||
| @@ -130,26 +129,26 @@ func NewCmdInit(out io.Writer) *cobra.Command { | ||||
| 		Short: "Run this command in order to set up the Kubernetes master.", | ||||
| 		Run: func(cmd *cobra.Command, args []string) { | ||||
|  | ||||
| 			kubeadmscheme.Scheme.Default(externalcfg) | ||||
|  | ||||
| 			var err error | ||||
| 			if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil { | ||||
| 			if externalcfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil { | ||||
| 				kubeadmutil.CheckErr(err) | ||||
| 			} | ||||
|  | ||||
| 			kubeadmscheme.Scheme.Default(cfg) | ||||
| 			internalcfg := &kubeadmapi.MasterConfiguration{} | ||||
| 			kubeadmscheme.Scheme.Convert(cfg, internalcfg, nil) | ||||
|  | ||||
| 			ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(ignorePreflightErrors, skipPreFlight) | ||||
| 			kubeadmutil.CheckErr(err) | ||||
|  | ||||
| 			i, err := NewInit(cfgPath, internalcfg, ignorePreflightErrorsSet, skipTokenPrint, dryRun) | ||||
| 			err = validation.ValidateMixedArguments(cmd.Flags()) | ||||
| 			kubeadmutil.CheckErr(err) | ||||
|  | ||||
| 			i, err := NewInit(cfgPath, externalcfg, ignorePreflightErrorsSet, skipTokenPrint, dryRun) | ||||
| 			kubeadmutil.CheckErr(err) | ||||
| 			kubeadmutil.CheckErr(i.Validate(cmd)) | ||||
| 			kubeadmutil.CheckErr(i.Run(out)) | ||||
| 		}, | ||||
| 	} | ||||
|  | ||||
| 	AddInitConfigFlags(cmd.PersistentFlags(), cfg, &featureGatesString) | ||||
| 	AddInitConfigFlags(cmd.PersistentFlags(), externalcfg, &featureGatesString) | ||||
| 	AddInitOtherFlags(cmd.PersistentFlags(), &cfgPath, &skipPreFlight, &skipTokenPrint, &dryRun, &ignorePreflightErrors) | ||||
|  | ||||
| 	return cmd | ||||
| @@ -239,27 +238,15 @@ func AddInitOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipPreFlight, sk | ||||
| } | ||||
|  | ||||
| // NewInit validates given arguments and instantiates Init struct with provided information. | ||||
| func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, ignorePreflightErrors sets.String, skipTokenPrint, dryRun bool) (*Init, error) { | ||||
| func NewInit(cfgPath string, externalcfg *kubeadmapiv1alpha1.MasterConfiguration, ignorePreflightErrors sets.String, skipTokenPrint, dryRun bool) (*Init, error) { | ||||
|  | ||||
| 	if cfgPath != "" { | ||||
| 		glog.V(1).Infof("[init] reading config file from: " + cfgPath) | ||||
| 		b, err := ioutil.ReadFile(cfgPath) | ||||
| 		if err != nil { | ||||
| 			return nil, fmt.Errorf("unable to read config from %q [%v]", cfgPath, err) | ||||
| 		} | ||||
| 		if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), b, cfg); err != nil { | ||||
| 			return nil, fmt.Errorf("unable to decode config from %q [%v]", cfgPath, err) | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	// Set defaults dynamically that the API group defaulting can't (by fetching information from the internet, looking up network interfaces, etc.) | ||||
| 	glog.V(1).Infof("[init] setting dynamic defaults") | ||||
| 	err := configutil.SetInitDynamicDefaults(cfg) | ||||
| 	// Either use the config file if specified, or convert the defaults in the external to an internal cfg representation | ||||
| 	cfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, externalcfg) | ||||
| 	if err != nil { | ||||
| 		return nil, err | ||||
| 	} | ||||
|  | ||||
| 	glog.V(1).Infof("[init] validating Kubernetes version") | ||||
| 	glog.V(1).Infof("[init] validating feature gates") | ||||
| 	if err := features.ValidateVersion(features.InitFeatureGates, cfg.FeatureGates, cfg.KubernetesVersion); err != nil { | ||||
| 		return nil, err | ||||
| 	} | ||||
| @@ -293,14 +280,6 @@ type Init struct { | ||||
| 	dryRun         bool | ||||
| } | ||||
|  | ||||
| // Validate validates configuration passed to "kubeadm init" | ||||
| func (i *Init) Validate(cmd *cobra.Command) error { | ||||
| 	if err := validation.ValidateMixedArguments(cmd.Flags()); err != nil { | ||||
| 		return err | ||||
| 	} | ||||
| 	return validation.ValidateMasterConfiguration(i.cfg).ToAggregate() | ||||
| } | ||||
|  | ||||
| // Run executes master node provisioning, including certificates, needed static pod manifests, etc. | ||||
| func (i *Init) Run(out io.Writer) error { | ||||
| 	// Get directories to write files to; can be faked if we're dry-running | ||||
|   | ||||
| @@ -27,7 +27,6 @@ go_library( | ||||
|         "//cmd/kubeadm/app/util/dryrun:go_default_library", | ||||
|         "//cmd/kubeadm/app/util/etcd:go_default_library", | ||||
|         "//cmd/kubeadm/app/util/kubeconfig:go_default_library", | ||||
|         "//pkg/api/legacyscheme:go_default_library", | ||||
|         "//pkg/util/version:go_default_library", | ||||
|         "//vendor/github.com/golang/glog:go_default_library", | ||||
|         "//vendor/github.com/spf13/cobra:go_default_library", | ||||
| @@ -46,7 +45,7 @@ go_test( | ||||
|     ], | ||||
|     embed = [":go_default_library"], | ||||
|     deps = [ | ||||
|         "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library", | ||||
|         "//cmd/kubeadm/app/apis/kubeadm:go_default_library", | ||||
|         "//cmd/kubeadm/app/phases/upgrade:go_default_library", | ||||
|     ], | ||||
| ) | ||||
|   | ||||
| @@ -26,6 +26,7 @@ import ( | ||||
|  | ||||
| 	clientset "k8s.io/client-go/kubernetes" | ||||
| 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" | ||||
| 	kubeadmapiv1alpha1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" | ||||
| 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" | ||||
| 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" | ||||
| 	"k8s.io/kubernetes/cmd/kubeadm/app/constants" | ||||
| @@ -37,7 +38,6 @@ import ( | ||||
| 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" | ||||
| 	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" | ||||
| 	etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" | ||||
| 	"k8s.io/kubernetes/pkg/api/legacyscheme" | ||||
| 	"k8s.io/kubernetes/pkg/util/version" | ||||
| ) | ||||
|  | ||||
| @@ -87,7 +87,7 @@ func NewCmdApply(parentFlags *cmdUpgradeFlags) *cobra.Command { | ||||
| 			// If the version is specified in config file, pick up that value. | ||||
| 			if flags.parent.cfgPath != "" { | ||||
| 				glog.V(1).Infof("fetching configuration from file", flags.parent.cfgPath) | ||||
| 				cfg, err := upgrade.FetchConfigurationFromFile(flags.parent.cfgPath) | ||||
| 				cfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(parentFlags.cfgPath, &kubeadmapiv1alpha1.MasterConfiguration{}) | ||||
| 				kubeadmutil.CheckErr(err) | ||||
|  | ||||
| 				if cfg.KubernetesVersion != "" { | ||||
| @@ -147,26 +147,21 @@ func RunApply(flags *applyFlags) error { | ||||
| 		return err | ||||
| 	} | ||||
|  | ||||
| 	// Grab the external, versioned configuration and convert it to the internal type for usage here later | ||||
| 	glog.V(1).Infof("[upgrade/apply] converting configuration for internal use") | ||||
| 	internalcfg := &kubeadmapi.MasterConfiguration{} | ||||
| 	legacyscheme.Scheme.Convert(upgradeVars.cfg, internalcfg, nil) | ||||
|  | ||||
| 	// Validate requested and validate actual version | ||||
| 	glog.V(1).Infof("[upgrade/apply] validating requested and actual version") | ||||
| 	if err := configutil.NormalizeKubernetesVersion(internalcfg); err != nil { | ||||
| 	if err := configutil.NormalizeKubernetesVersion(upgradeVars.cfg); err != nil { | ||||
| 		return err | ||||
| 	} | ||||
|  | ||||
| 	// Use normalized version string in all following code. | ||||
| 	flags.newK8sVersionStr = internalcfg.KubernetesVersion | ||||
| 	flags.newK8sVersionStr = upgradeVars.cfg.KubernetesVersion | ||||
| 	k8sVer, err := version.ParseSemantic(flags.newK8sVersionStr) | ||||
| 	if err != nil { | ||||
| 		return fmt.Errorf("unable to parse normalized version %q as a semantic version", flags.newK8sVersionStr) | ||||
| 	} | ||||
| 	flags.newK8sVersion = k8sVer | ||||
|  | ||||
| 	if err := features.ValidateVersion(features.InitFeatureGates, internalcfg.FeatureGates, internalcfg.KubernetesVersion); err != nil { | ||||
| 	if err := features.ValidateVersion(features.InitFeatureGates, upgradeVars.cfg.FeatureGates, upgradeVars.cfg.KubernetesVersion); err != nil { | ||||
| 		return err | ||||
| 	} | ||||
|  | ||||
| @@ -186,18 +181,18 @@ func RunApply(flags *applyFlags) error { | ||||
| 	// Use a prepuller implementation based on creating DaemonSets | ||||
| 	// and block until all DaemonSets are ready; then we know for sure that all control plane images are cached locally | ||||
| 	glog.V(1).Infof("[upgrade/apply] creating prepuller") | ||||
| 	prepuller := upgrade.NewDaemonSetPrepuller(upgradeVars.client, upgradeVars.waiter, internalcfg) | ||||
| 	prepuller := upgrade.NewDaemonSetPrepuller(upgradeVars.client, upgradeVars.waiter, upgradeVars.cfg) | ||||
| 	upgrade.PrepullImagesInParallel(prepuller, flags.imagePullTimeout) | ||||
|  | ||||
| 	// Now; perform the upgrade procedure | ||||
| 	glog.V(1).Infof("[upgrade/apply] performing upgrade") | ||||
| 	if err := PerformControlPlaneUpgrade(flags, upgradeVars.client, upgradeVars.waiter, internalcfg); err != nil { | ||||
| 	if err := PerformControlPlaneUpgrade(flags, upgradeVars.client, upgradeVars.waiter, upgradeVars.cfg); err != nil { | ||||
| 		return fmt.Errorf("[upgrade/apply] FATAL: %v", err) | ||||
| 	} | ||||
|  | ||||
| 	// Upgrade RBAC rules and addons. | ||||
| 	glog.V(1).Infof("[upgrade/postupgrade] upgrading RBAC rules and addons") | ||||
| 	if err := upgrade.PerformPostUpgradeTasks(upgradeVars.client, internalcfg, flags.newK8sVersion, flags.dryRun); err != nil { | ||||
| 	if err := upgrade.PerformPostUpgradeTasks(upgradeVars.client, upgradeVars.cfg, flags.newK8sVersion, flags.dryRun); err != nil { | ||||
| 		return fmt.Errorf("[upgrade/postupgrade] FATAL post-upgrade error: %v", err) | ||||
| 	} | ||||
|  | ||||
|   | ||||
| @@ -27,6 +27,7 @@ import ( | ||||
| 	"k8s.io/apimachinery/pkg/util/sets" | ||||
| 	fakediscovery "k8s.io/client-go/discovery/fake" | ||||
| 	clientset "k8s.io/client-go/kubernetes" | ||||
| 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" | ||||
| 	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" | ||||
| 	kubeadmapiv1alpha1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" | ||||
| 	"k8s.io/kubernetes/cmd/kubeadm/app/features" | ||||
| @@ -42,7 +43,7 @@ import ( | ||||
| // TODO - Restructure or rename upgradeVariables | ||||
| type upgradeVariables struct { | ||||
| 	client        clientset.Interface | ||||
| 	cfg           *kubeadmapiv1alpha1.MasterConfiguration | ||||
| 	cfg           *kubeadmapi.MasterConfiguration | ||||
| 	versionGetter upgrade.VersionGetter | ||||
| 	waiter        apiclient.Waiter | ||||
| } | ||||
| @@ -94,13 +95,16 @@ func enforceRequirements(flags *cmdUpgradeFlags, dryRun bool, newK8sVersion stri | ||||
| } | ||||
|  | ||||
| // printConfiguration prints the external version of the API to yaml | ||||
| func printConfiguration(cfg *kubeadmapiv1alpha1.MasterConfiguration, w io.Writer) { | ||||
| func printConfiguration(cfg *kubeadmapi.MasterConfiguration, w io.Writer) { | ||||
| 	// Short-circuit if cfg is nil, so we can safely get the value of the pointer below | ||||
| 	if cfg == nil { | ||||
| 		return | ||||
| 	} | ||||
|  | ||||
| 	cfgYaml, err := kubeadmutil.MarshalToYamlForCodecs(cfg, kubeadmapiv1alpha1.SchemeGroupVersion, kubeadmscheme.Codecs) | ||||
| 	externalcfg := &kubeadmapiv1alpha1.MasterConfiguration{} | ||||
| 	kubeadmscheme.Scheme.Convert(cfg, externalcfg, nil) | ||||
|  | ||||
| 	cfgYaml, err := kubeadmutil.MarshalToYamlForCodecs(externalcfg, kubeadmapiv1alpha1.SchemeGroupVersion, kubeadmscheme.Codecs) | ||||
| 	if err == nil { | ||||
| 		fmt.Fprintln(w, "[upgrade/config] Configuration used:") | ||||
|  | ||||
|   | ||||
| @@ -20,12 +20,12 @@ import ( | ||||
| 	"bytes" | ||||
| 	"testing" | ||||
|  | ||||
| 	kubeadmapiv1alpha1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" | ||||
| 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" | ||||
| ) | ||||
|  | ||||
| func TestPrintConfiguration(t *testing.T) { | ||||
| 	var tests = []struct { | ||||
| 		cfg           *kubeadmapiv1alpha1.MasterConfiguration | ||||
| 		cfg           *kubeadmapi.MasterConfiguration | ||||
| 		buf           *bytes.Buffer | ||||
| 		expectedBytes []byte | ||||
| 	}{ | ||||
| @@ -34,7 +34,7 @@ func TestPrintConfiguration(t *testing.T) { | ||||
| 			expectedBytes: []byte(""), | ||||
| 		}, | ||||
| 		{ | ||||
| 			cfg: &kubeadmapiv1alpha1.MasterConfiguration{ | ||||
| 			cfg: &kubeadmapi.MasterConfiguration{ | ||||
| 				KubernetesVersion: "v1.7.1", | ||||
| 			}, | ||||
| 			expectedBytes: []byte(`[upgrade/config] Configuration used: | ||||
| @@ -71,9 +71,9 @@ func TestPrintConfiguration(t *testing.T) { | ||||
| `), | ||||
| 		}, | ||||
| 		{ | ||||
| 			cfg: &kubeadmapiv1alpha1.MasterConfiguration{ | ||||
| 			cfg: &kubeadmapi.MasterConfiguration{ | ||||
| 				KubernetesVersion: "v1.7.1", | ||||
| 				Networking: kubeadmapiv1alpha1.Networking{ | ||||
| 				Networking: kubeadmapi.Networking{ | ||||
| 					ServiceSubnet: "10.96.0.1/12", | ||||
| 				}, | ||||
| 			}, | ||||
| @@ -111,10 +111,10 @@ func TestPrintConfiguration(t *testing.T) { | ||||
| `), | ||||
| 		}, | ||||
| 		{ | ||||
| 			cfg: &kubeadmapiv1alpha1.MasterConfiguration{ | ||||
| 			cfg: &kubeadmapi.MasterConfiguration{ | ||||
| 				KubernetesVersion: "v1.7.1", | ||||
| 				Etcd: kubeadmapiv1alpha1.Etcd{ | ||||
| 					SelfHosted: &kubeadmapiv1alpha1.SelfHostedEtcd{ | ||||
| 				Etcd: kubeadmapi.Etcd{ | ||||
| 					SelfHosted: &kubeadmapi.SelfHostedEtcd{ | ||||
| 						CertificatesDir:    "/var/foo", | ||||
| 						ClusterServiceName: "foo", | ||||
| 						EtcdVersion:        "v0.1.0", | ||||
|   | ||||
| @@ -26,10 +26,12 @@ import ( | ||||
| 	"github.com/golang/glog" | ||||
| 	"github.com/spf13/cobra" | ||||
|  | ||||
| 	kubeadmapiv1alpha1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" | ||||
| 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" | ||||
| 	"k8s.io/kubernetes/cmd/kubeadm/app/constants" | ||||
| 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" | ||||
| 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" | ||||
| 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" | ||||
| 	etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" | ||||
| ) | ||||
|  | ||||
| @@ -58,7 +60,7 @@ func NewCmdPlan(parentFlags *cmdUpgradeFlags) *cobra.Command { | ||||
| 			// If the version is specified in config file, pick up that value. | ||||
| 			if parentFlags.cfgPath != "" { | ||||
| 				glog.V(1).Infof("fetching configuration from file", parentFlags.cfgPath) | ||||
| 				cfg, err := upgrade.FetchConfigurationFromFile(parentFlags.cfgPath) | ||||
| 				cfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(parentFlags.cfgPath, &kubeadmapiv1alpha1.MasterConfiguration{}) | ||||
| 				kubeadmutil.CheckErr(err) | ||||
|  | ||||
| 				if cfg.KubernetesVersion != "" { | ||||
|   | ||||
| @@ -17,9 +17,7 @@ go_library( | ||||
|     visibility = ["//visibility:public"], | ||||
|     deps = [ | ||||
|         "//cmd/kubeadm/app/apis/kubeadm:go_default_library", | ||||
|         "//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library", | ||||
|         "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library", | ||||
|         "//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library", | ||||
|         "//cmd/kubeadm/app/constants:go_default_library", | ||||
|         "//cmd/kubeadm/app/features:go_default_library", | ||||
|         "//cmd/kubeadm/app/images:go_default_library", | ||||
|   | ||||
| @@ -25,15 +25,12 @@ import ( | ||||
| 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||||
| 	clientset "k8s.io/client-go/kubernetes" | ||||
| 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" | ||||
| 	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" | ||||
| 	kubeadmapiv1alpha1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" | ||||
| 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" | ||||
| 	"k8s.io/kubernetes/cmd/kubeadm/app/constants" | ||||
| 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" | ||||
| ) | ||||
|  | ||||
| // FetchConfiguration fetches configuration required for upgrading your cluster from a file (which has precedence) or a ConfigMap in the cluster | ||||
| func FetchConfiguration(client clientset.Interface, w io.Writer, cfgPath string) (*kubeadmapiv1alpha1.MasterConfiguration, error) { | ||||
| func FetchConfiguration(client clientset.Interface, w io.Writer, cfgPath string) (*kubeadmapi.MasterConfiguration, error) { | ||||
| 	fmt.Println("[upgrade/config] Making sure the configuration is correct:") | ||||
|  | ||||
| 	// Load the configuration from a file or the cluster | ||||
| @@ -42,26 +39,8 @@ func FetchConfiguration(client clientset.Interface, w io.Writer, cfgPath string) | ||||
| 		return nil, err | ||||
| 	} | ||||
|  | ||||
| 	// Take the versioned configuration populated from the configmap, default it and validate | ||||
| 	// Return the internal version of the API object | ||||
| 	versionedcfg, err := bytesToValidatedMasterConfig(configBytes) | ||||
| 	if err != nil { | ||||
| 		return nil, fmt.Errorf("could not decode configuration: %v", err) | ||||
| 	} | ||||
| 	return versionedcfg, nil | ||||
| } | ||||
|  | ||||
| // FetchConfigurationFromFile fetch configuration from a file | ||||
| func FetchConfigurationFromFile(cfgPath string) (*kubeadmapiv1alpha1.MasterConfiguration, error) { | ||||
| 	// Load the configuration from a file or the cluster | ||||
| 	configBytes, err := ioutil.ReadFile(cfgPath) | ||||
| 	if err != nil { | ||||
| 		return nil, err | ||||
| 	} | ||||
|  | ||||
| 	// Take the versioned configuration populated from the configmap, default it and validate | ||||
| 	// Return the internal version of the API object | ||||
| 	versionedcfg, err := bytesToValidatedMasterConfig(configBytes) | ||||
| 	// Take the versioned configuration populated from the file or configmap, convert it to internal, default and validate | ||||
| 	versionedcfg, err := configutil.BytesToInternalConfig(configBytes) | ||||
| 	if err != nil { | ||||
| 		return nil, fmt.Errorf("could not decode configuration: %v", err) | ||||
| 	} | ||||
| @@ -70,6 +49,7 @@ func FetchConfigurationFromFile(cfgPath string) (*kubeadmapiv1alpha1.MasterConfi | ||||
|  | ||||
| // loadConfigurationBytes loads the configuration byte slice from either a file or the cluster ConfigMap | ||||
| func loadConfigurationBytes(client clientset.Interface, w io.Writer, cfgPath string) ([]byte, error) { | ||||
| 	// The config file has the highest priority | ||||
| 	if cfgPath != "" { | ||||
| 		fmt.Printf("[upgrade/config] Reading configuration options from a file: %s\n", cfgPath) | ||||
| 		return ioutil.ReadFile(cfgPath) | ||||
| @@ -95,34 +75,3 @@ func loadConfigurationBytes(client clientset.Interface, w io.Writer, cfgPath str | ||||
| 	fmt.Printf("[upgrade/config] FYI: You can look at this config file with 'kubectl -n %s get cm %s -oyaml'\n", metav1.NamespaceSystem, constants.MasterConfigurationConfigMap) | ||||
| 	return []byte(configMap.Data[constants.MasterConfigurationConfigMapKey]), nil | ||||
| } | ||||
|  | ||||
| // bytesToValidatedMasterConfig converts a byte array to an external, defaulted and validated configuration object | ||||
| func bytesToValidatedMasterConfig(b []byte) (*kubeadmapiv1alpha1.MasterConfiguration, error) { | ||||
| 	cfg := &kubeadmapiv1alpha1.MasterConfiguration{} | ||||
| 	finalCfg := &kubeadmapiv1alpha1.MasterConfiguration{} | ||||
| 	internalcfg := &kubeadmapi.MasterConfiguration{} | ||||
|  | ||||
| 	decoded, err := kubeadmapiv1alpha1.LoadYAML(b) | ||||
| 	if err != nil { | ||||
| 		return nil, fmt.Errorf("unable to decode config from bytes: %v", err) | ||||
| 	} | ||||
|  | ||||
| 	if err := kubeadmapiv1alpha1.Migrate(decoded, cfg, kubeadmscheme.Codecs); err != nil { | ||||
| 		return nil, fmt.Errorf("unable to migrate config from previous version: %v", err) | ||||
| 	} | ||||
| 	// Default and convert to the internal version | ||||
| 	kubeadmscheme.Scheme.Default(cfg) | ||||
| 	kubeadmscheme.Scheme.Convert(cfg, internalcfg, nil) | ||||
|  | ||||
| 	// Applies dynamic defaults to settings not provided with flags | ||||
| 	if err := configutil.SetInitDynamicDefaults(internalcfg); err != nil { | ||||
| 		return nil, err | ||||
| 	} | ||||
| 	// Validates cfg (flags/configs + defaults + dynamic defaults) | ||||
| 	if err := validation.ValidateMasterConfiguration(internalcfg).ToAggregate(); err != nil { | ||||
| 		return nil, err | ||||
| 	} | ||||
| 	// Finally converts back to the external version | ||||
| 	kubeadmscheme.Scheme.Convert(internalcfg, finalCfg, nil) | ||||
| 	return finalCfg, nil | ||||
| } | ||||
|   | ||||
| @@ -21,6 +21,7 @@ go_library( | ||||
|     deps = [ | ||||
|         "//cmd/kubeadm/app/apis/kubeadm:go_default_library", | ||||
|         "//cmd/kubeadm/app/preflight:go_default_library", | ||||
|         "//vendor/gopkg.in/yaml.v2:go_default_library", | ||||
|         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", | ||||
|         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", | ||||
|         "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", | ||||
|   | ||||
| @@ -21,7 +21,6 @@ go_library( | ||||
|         "//pkg/util/node:go_default_library", | ||||
|         "//pkg/util/version:go_default_library", | ||||
|         "//vendor/github.com/golang/glog:go_default_library", | ||||
|         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", | ||||
|         "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", | ||||
|     ], | ||||
| ) | ||||
|   | ||||
| @@ -23,7 +23,6 @@ import ( | ||||
|  | ||||
| 	"github.com/golang/glog" | ||||
|  | ||||
| 	"k8s.io/apimachinery/pkg/runtime" | ||||
| 	netutil "k8s.io/apimachinery/pkg/util/net" | ||||
| 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" | ||||
| 	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" | ||||
| @@ -76,52 +75,89 @@ func SetInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error { | ||||
| 	return nil | ||||
| } | ||||
|  | ||||
| // TryLoadMasterConfiguration tries to loads a Master configuration from the given file (if defined) | ||||
| func TryLoadMasterConfiguration(cfgPath string, cfg *kubeadmapiv1alpha1.MasterConfiguration) error { | ||||
|  | ||||
| 	if cfgPath != "" { | ||||
| 		b, err := ioutil.ReadFile(cfgPath) | ||||
| 		if err != nil { | ||||
| 			return fmt.Errorf("unable to read config from %q [%v]", cfgPath, err) | ||||
| 		} | ||||
| 		if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), b, cfg); err != nil { | ||||
| 			return fmt.Errorf("unable to decode config from %q [%v]", cfgPath, err) | ||||
| 		} | ||||
| 	} | ||||
|  | ||||
| 	return nil | ||||
| } | ||||
|  | ||||
| // ConfigFileAndDefaultsToInternalConfig takes a path to a config file and a versioned configuration that can serve as the default config | ||||
| // If cfgPath is specified, defaultversionedcfg will always get overridden. Otherwise, the default config (often populated by flags) will be used. | ||||
| // Then the external, versioned configuration is defaulted and converted to the internal type. | ||||
| // Right thereafter, the configuration is defaulted again with dynamic values (like IP addresses of a machine, etc) | ||||
| // Lastly, the internal config is validated and returned. | ||||
| func ConfigFileAndDefaultsToInternalConfig(cfgPath string, defaultversionedcfg *kubeadmapiv1alpha1.MasterConfiguration) (*kubeadmapi.MasterConfiguration, error) { | ||||
| 	glog.V(1).Infoln("configuring files and defaults to internal config") | ||||
| 	internalcfg := &kubeadmapi.MasterConfiguration{} | ||||
|  | ||||
| 	// Loads configuration from config file, if provided | ||||
| 	// Nb. --config overrides command line flags | ||||
| 	glog.V(1).Infoln("attempting to load configuration from config file") | ||||
| 	if err := TryLoadMasterConfiguration(cfgPath, defaultversionedcfg); err != nil { | ||||
| 		return nil, err | ||||
| 	if cfgPath != "" { | ||||
| 		// Loads configuration from config file, if provided | ||||
| 		// Nb. --config overrides command line flags | ||||
| 		glog.V(1).Infoln("loading configuration from the given file") | ||||
|  | ||||
| 		b, err := ioutil.ReadFile(cfgPath) | ||||
| 		if err != nil { | ||||
| 			return nil, fmt.Errorf("unable to read config from %q [%v]", cfgPath, err) | ||||
| 		} | ||||
| 		return BytesToInternalConfig(b) | ||||
| 	} | ||||
|  | ||||
| 	// Takes passed flags into account; the defaulting is executed once again enforcing assignement of | ||||
| 	// static default values to cfg only for values not provided with flags | ||||
| 	kubeadmscheme.Scheme.Default(defaultversionedcfg) | ||||
| 	kubeadmscheme.Scheme.Convert(defaultversionedcfg, internalcfg, nil) | ||||
| 	// Applies dynamic defaults to settings not provided with flags | ||||
| 	if err := SetInitDynamicDefaults(internalcfg); err != nil { | ||||
| 		return nil, err | ||||
|  | ||||
| 	return defaultAndValidate(internalcfg) | ||||
| } | ||||
|  | ||||
| // BytesToInternalConfig converts a byte array to an internal, defaulted and validated configuration object | ||||
| func BytesToInternalConfig(b []byte) (*kubeadmapi.MasterConfiguration, error) { | ||||
| 	internalcfg := &kubeadmapi.MasterConfiguration{} | ||||
|  | ||||
| 	decoded, err := kubeadmutil.LoadYAML(b) | ||||
| 	if err != nil { | ||||
| 		return nil, fmt.Errorf("unable to decode config from bytes: %v", err) | ||||
| 	} | ||||
|  | ||||
| 	// Validates cfg (flags/configs + defaults + dynamic defaults) | ||||
| 	if err := validation.ValidateMasterConfiguration(internalcfg).ToAggregate(); err != nil { | ||||
| 	// As there was a bug in kubeadm v1.10 and earlier that made the YAML uploaded to the cluster configmap NOT have metav1.TypeMeta information | ||||
| 	// we need to populate this here manually. If kind or apiVersion is empty, we know the apiVersion is v1alpha1, as by the time kubeadm had this bug, | ||||
| 	// it could only write | ||||
| 	// TODO: Remove this "hack" in v1.12 when we know the ConfigMap always contains v1alpha2 content written by kubeadm v1.11. Also, we will drop support for | ||||
| 	// v1alpha1 in v1.12 | ||||
| 	kind := decoded["kind"] | ||||
| 	apiVersion := decoded["apiVersion"] | ||||
| 	if kind == nil || len(kind.(string)) == 0 { | ||||
| 		decoded["kind"] = "MasterConfiguration" | ||||
| 	} | ||||
| 	if apiVersion == nil || len(apiVersion.(string)) == 0 { | ||||
| 		decoded["apiVersion"] = kubeadmapiv1alpha1.SchemeGroupVersion.String() | ||||
| 	} | ||||
|  | ||||
| 	// Between v1.9 and v1.10 the proxy componentconfig in the v1alpha1 MasterConfiguration changed unexpectedly, which broke unmarshalling out-of-the-box | ||||
| 	// Hence, we need to workaround this bug in the v1alpha1 API | ||||
| 	if decoded["apiVersion"] == kubeadmapiv1alpha1.SchemeGroupVersion.String() { | ||||
| 		v1alpha1cfg := &kubeadmapiv1alpha1.MasterConfiguration{} | ||||
| 		if err := kubeadmapiv1alpha1.Migrate(decoded, v1alpha1cfg, kubeadmscheme.Codecs); err != nil { | ||||
| 			return nil, fmt.Errorf("unable to migrate config from previous version: %v", err) | ||||
| 		} | ||||
|  | ||||
| 		// Default and convert to the internal version | ||||
| 		kubeadmscheme.Scheme.Default(v1alpha1cfg) | ||||
| 		kubeadmscheme.Scheme.Convert(v1alpha1cfg, internalcfg, nil) | ||||
| 	} else { | ||||
| 		// TODO: Add support for an upcoming v1alpha2 API | ||||
| 		// TODO: In the future, we can unmarshal any two or more external types into the internal object directly using the following syntax. | ||||
| 		// Long-term we don't need this if/else clause. In the future this will do | ||||
| 		// runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(kubeadmapiv1alpha2.SchemeGroupVersion, kubeadmapiv2alpha3.SchemeGroupVersion), b, internalcfg) | ||||
| 		return nil, fmt.Errorf("unknown API version for kubeadm configuration") | ||||
| 	} | ||||
|  | ||||
| 	return defaultAndValidate(internalcfg) | ||||
| } | ||||
|  | ||||
| func defaultAndValidate(cfg *kubeadmapi.MasterConfiguration) (*kubeadmapi.MasterConfiguration, error) { | ||||
| 	// Applies dynamic defaults to settings not provided with flags | ||||
| 	if err := SetInitDynamicDefaults(cfg); err != nil { | ||||
| 		return nil, err | ||||
| 	} | ||||
| 	return internalcfg, nil | ||||
| 	// Validates cfg (flags/configs + defaults + dynamic defaults) | ||||
| 	if err := validation.ValidateMasterConfiguration(cfg).ToAggregate(); err != nil { | ||||
| 		return nil, err | ||||
| 	} | ||||
| 	return cfg, nil | ||||
| } | ||||
|  | ||||
| // NormalizeKubernetesVersion resolves version labels, sets alternative | ||||
|   | ||||
| @@ -17,8 +17,11 @@ limitations under the License. | ||||
| package util | ||||
|  | ||||
| import ( | ||||
| 	"errors" | ||||
| 	"fmt" | ||||
|  | ||||
| 	yaml "gopkg.in/yaml.v2" | ||||
|  | ||||
| 	"k8s.io/apimachinery/pkg/runtime" | ||||
| 	"k8s.io/apimachinery/pkg/runtime/schema" | ||||
| 	"k8s.io/apimachinery/pkg/runtime/serializer" | ||||
| @@ -58,3 +61,35 @@ func UnmarshalFromYamlForCodecs(buffer []byte, gv schema.GroupVersion, codecs se | ||||
| 	decoder := codecs.DecoderToVersion(info.Serializer, gv) | ||||
| 	return runtime.Decode(decoder, buffer) | ||||
| } | ||||
|  | ||||
| // LoadYAML is a small wrapper around go-yaml that ensures all nested structs are map[string]interface{} instead of map[interface{}]interface{}. | ||||
| func LoadYAML(bytes []byte) (map[string]interface{}, error) { | ||||
| 	var decoded map[interface{}]interface{} | ||||
| 	if err := yaml.Unmarshal(bytes, &decoded); err != nil { | ||||
| 		return map[string]interface{}{}, fmt.Errorf("couldn't unmarshal YAML: %v", err) | ||||
| 	} | ||||
|  | ||||
| 	converted, ok := convert(decoded).(map[string]interface{}) | ||||
| 	if !ok { | ||||
| 		return map[string]interface{}{}, errors.New("yaml is not a map") | ||||
| 	} | ||||
|  | ||||
| 	return converted, nil | ||||
| } | ||||
|  | ||||
| // https://stackoverflow.com/questions/40737122/convert-yaml-to-json-without-struct-golang | ||||
| func convert(i interface{}) interface{} { | ||||
| 	switch x := i.(type) { | ||||
| 	case map[interface{}]interface{}: | ||||
| 		m2 := map[string]interface{}{} | ||||
| 		for k, v := range x { | ||||
| 			m2[k.(string)] = convert(v) | ||||
| 		} | ||||
| 		return m2 | ||||
| 	case []interface{}: | ||||
| 		for i, v := range x { | ||||
| 			x[i] = convert(v) | ||||
| 		} | ||||
| 	} | ||||
| 	return i | ||||
| } | ||||
|   | ||||
		Reference in New Issue
	
	Block a user
	 Kubernetes Submit Queue
					Kubernetes Submit Queue