diff --git a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go index 3bd46500aa1..13416b31e9e 100644 --- a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go +++ b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go @@ -19,7 +19,7 @@ package fuzzer import ( "time" - "github.com/google/gofuzz" + fuzz "github.com/google/gofuzz" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" @@ -41,15 +41,12 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { obj.Networking.DNSDomain = "foo" obj.CertificatesDir = "foo" obj.APIServerCertSANs = []string{"foo"} - obj.Etcd.ServerCertSANs = []string{"foo"} - obj.Etcd.PeerCertSANs = []string{"foo"} + obj.Token = "foo" obj.CRISocket = "foo" obj.TokenTTL = &metav1.Duration{Duration: 1 * time.Hour} obj.TokenUsages = []string{"foo"} obj.TokenGroups = []string{"foo"} - obj.Etcd.Image = "foo" - obj.Etcd.DataDir = "foo" obj.ImageRepository = "foo" obj.CIImageRepository = "" obj.UnifiedControlPlaneImage = "foo" @@ -62,7 +59,16 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { MountPath: "foo", Writable: false, }} - obj.Etcd.ExtraArgs = map[string]string{"foo": "foo"} + obj.Etcd.Local = &kubeadm.LocalEtcd{ + Image: "foo", + DataDir: "foo", + ServerCertSANs: []string{"foo"}, + PeerCertSANs: []string{"foo"}, + ExtraArgs: map[string]string{"foo": "foo"}, + } + // Note: We don't set values here for obj.Etcd.External, as these are mutually exlusive. + // And to make sure the fuzzer doesn't set a random value for obj.Etcd.External, we let + // kubeadmapi.Etcd implement fuzz.Interface (we handle that ourselves) obj.KubeletConfiguration = kubeadm.KubeletConfiguration{ BaseConfig: &kubeletconfigv1beta1.KubeletConfiguration{ StaticPodPath: "foo", diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index 17cbb15a67d..ea062c62184 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -17,6 +17,8 @@ limitations under the License. package kubeadm import ( + fuzz "github.com/google/gofuzz" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1" @@ -160,6 +162,49 @@ type Networking struct { // Etcd contains elements describing Etcd configuration. type Etcd struct { + + // Local provides configuration knobs for configuring the local etcd instance + // Local and External are mutually exclusive + Local *LocalEtcd + + // External describes how to connect to an external etcd cluster + // Local and External are mutually exclusive + External *ExternalEtcd +} + +// Fuzz is a dummy function here to get the roundtrip tests working in cmd/kubeadm/app/apis/kubeadm/fuzzer working. +// As we split the monolith-etcd struct into two smaller pieces with pointers and they are mutually exclusive, roundtrip +// tests that randomize all values in this struct isn't feasible. Instead, we override the fuzzing function for .Etcd with +// this func by letting Etcd implement the fuzz.Interface interface. As this func does nothing, we rely on the values given +// in fuzzer/fuzzer.go for the roundtrip tests, which is exactly what we want. +// TODO: Remove this function when we remove the v1alpha1 API +func (e Etcd) Fuzz(c fuzz.Continue) {} + +// LocalEtcd describes that kubeadm should run an etcd cluster locally +type LocalEtcd struct { + + // Image specifies which container image to use for running etcd. + // If empty, automatically populated by kubeadm using the image + // repository and default etcd version. + Image string + + // DataDir is the directory etcd will place its data. + // Defaults to "/var/lib/etcd". + DataDir string + + // ExtraArgs are extra arguments provided to the etcd binary + // when run inside a static pod. + ExtraArgs map[string]string + + // ServerCertSANs sets extra Subject Alternative Names for the etcd server signing cert. + ServerCertSANs []string + // PeerCertSANs sets extra Subject Alternative Names for the etcd peer signing cert. + PeerCertSANs []string +} + +// ExternalEtcd describes an external etcd cluster +type ExternalEtcd struct { + // Endpoints of etcd members. Useful for using external etcd. // If not provided, kubeadm will run etcd in a static pod. Endpoints []string @@ -169,22 +214,6 @@ type Etcd struct { CertFile string // KeyFile is an SSL key file used to secure etcd communication. KeyFile string - // DataDir is the directory etcd will place its data. - // Defaults to "/var/lib/etcd". - DataDir string - // ExtraArgs are extra arguments provided to the etcd binary - // when run inside a static pod. - ExtraArgs map[string]string - // Image specifies which container image to use for running etcd. - // If empty, automatically populated by kubeadm using the image - // repository and default etcd version. - Image string - // ServerCertSANs sets extra Subject Alternative Names for the etcd server - // signing cert. This is currently used for the etcd static-pod. - ServerCertSANs []string - // PeerCertSANs sets extra Subject Alternative Names for the etcd peer - // signing cert. This is currently used for the etcd static-pod. - PeerCertSANs []string } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/conversion.go index dcbb59a19d6..d5492c3333a 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/conversion.go @@ -30,6 +30,7 @@ func addConversionFuncs(scheme *runtime.Scheme) error { err := scheme.AddConversionFuncs( Convert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration, Convert_v1alpha1_Etcd_To_kubeadm_Etcd, + Convert_kubeadm_Etcd_To_v1alpha1_Etcd, ) if err != nil { return err @@ -56,10 +57,47 @@ func Convert_v1alpha1_Etcd_To_kubeadm_Etcd(in *Etcd, out *kubeadm.Etcd, s conver return err } + // The .Etcd schema changed between v1alpha1 and v1alpha2 API types. The change was to basically only split up the fields into two sub-structs, which can be seen here + if len(in.Endpoints) != 0 { + out.External = &kubeadm.ExternalEtcd{ + Endpoints: in.Endpoints, + CAFile: in.CAFile, + CertFile: in.CertFile, + KeyFile: in.KeyFile, + } + } else { + out.Local = &kubeadm.LocalEtcd{ + Image: in.Image, + DataDir: in.DataDir, + ExtraArgs: in.ExtraArgs, + ServerCertSANs: in.ServerCertSANs, + PeerCertSANs: in.PeerCertSANs, + } + } + // No need to transfer information about .Etcd.Selfhosted to v1alpha2 return nil } +// no-op, as we don't support converting from newer API to old alpha API +func Convert_kubeadm_Etcd_To_v1alpha1_Etcd(in *kubeadm.Etcd, out *Etcd, s conversion.Scope) error { + + if in.External != nil { + out.Endpoints = in.External.Endpoints + out.CAFile = in.External.CAFile + out.CertFile = in.External.CertFile + out.KeyFile = in.External.KeyFile + } else { + out.Image = in.Local.Image + out.DataDir = in.Local.DataDir + out.ExtraArgs = in.Local.ExtraArgs + out.ServerCertSANs = in.Local.ServerCertSANs + out.PeerCertSANs = in.Local.PeerCertSANs + } + + return nil +} + // UpgradeCloudProvider handles the removal of .CloudProvider as smoothly as possible func UpgradeCloudProvider(in *MasterConfiguration, out *kubeadm.MasterConfiguration) { if len(in.CloudProvider) != 0 { diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/defaults.go index 946d3f2e6e8..6ada8fc6146 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/defaults.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/defaults.go @@ -119,19 +119,28 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) { obj.ImageRepository = DefaultImageRepository } - if obj.Etcd.DataDir == "" { - obj.Etcd.DataDir = DefaultEtcdDataDir - } - if obj.ClusterName == "" { obj.ClusterName = DefaultClusterName } SetDefaults_KubeletConfiguration(obj) + SetDefaults_Etcd(obj) SetDefaults_ProxyConfiguration(obj) SetDefaults_AuditPolicyConfiguration(obj) } +// SetDefaults_Etcd assigns default values for the Proxy +func SetDefaults_Etcd(obj *MasterConfiguration) { + if obj.Etcd.External == nil && obj.Etcd.Local == nil { + obj.Etcd.Local = &LocalEtcd{} + } + if obj.Etcd.Local != nil { + if obj.Etcd.Local.DataDir == "" { + obj.Etcd.Local.DataDir = DefaultEtcdDataDir + } + } +} + // SetDefaults_ProxyConfiguration assigns default values for the Proxy func SetDefaults_ProxyConfiguration(obj *MasterConfiguration) { if obj.KubeProxy.Config == nil { diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go index 289f1a70afe..dd48f2b9277 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go @@ -153,6 +153,41 @@ type Networking struct { // Etcd contains elements describing Etcd configuration. type Etcd struct { + + // Local provides configuration knobs for configuring the local etcd instance + // Local and External are mutually exclusive + Local *LocalEtcd `json:"local,omitempty"` + + // External describes how to connect to an external etcd cluster + // Local and External are mutually exclusive + External *ExternalEtcd `json:"external,omitempty"` +} + +// LocalEtcd describes that kubeadm should run an etcd cluster locally +type LocalEtcd struct { + + // Image specifies which container image to use for running etcd. + // If empty, automatically populated by kubeadm using the image + // repository and default etcd version. + Image string `json:"image"` + + // DataDir is the directory etcd will place its data. + // Defaults to "/var/lib/etcd". + DataDir string `json:"dataDir"` + + // ExtraArgs are extra arguments provided to the etcd binary + // when run inside a static pod. + ExtraArgs map[string]string `json:"extraArgs,omitempty"` + + // ServerCertSANs sets extra Subject Alternative Names for the etcd server signing cert. + ServerCertSANs []string `json:"serverCertSANs,omitempty"` + // PeerCertSANs sets extra Subject Alternative Names for the etcd peer signing cert. + PeerCertSANs []string `json:"peerCertSANs,omitempty"` +} + +// ExternalEtcd describes an external etcd cluster +type ExternalEtcd struct { + // Endpoints of etcd members. Useful for using external etcd. // If not provided, kubeadm will run etcd in a static pod. Endpoints []string `json:"endpoints"` @@ -162,20 +197,6 @@ type Etcd struct { CertFile string `json:"certFile"` // KeyFile is an SSL key file used to secure etcd communication. KeyFile string `json:"keyFile"` - // DataDir is the directory etcd will place its data. - // Defaults to "/var/lib/etcd". - DataDir string `json:"dataDir"` - // ExtraArgs are extra arguments provided to the etcd binary - // when run inside a static pod. - ExtraArgs map[string]string `json:"extraArgs,omitempty"` - // Image specifies which container image to use for running etcd. - // If empty, automatically populated by kubeadm using the image - // repository and default etcd version. - Image string `json:"image"` - // ServerCertSANs sets extra Subject Alternative Names for the etcd server signing cert. - ServerCertSANs []string `json:"serverCertSANs,omitempty"` - // PeerCertSANs sets extra Subject Alternative Names for the etcd peer signing cert. - PeerCertSANs []string `json:"peerCertSANs,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go index a038a723591..116bfce6c8d 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go @@ -53,8 +53,6 @@ func ValidateMasterConfiguration(c *kubeadm.MasterConfiguration) field.ErrorList allErrs := field.ErrorList{} allErrs = append(allErrs, ValidateNetworking(&c.Networking, field.NewPath("networking"))...) allErrs = append(allErrs, ValidateCertSANs(c.APIServerCertSANs, field.NewPath("apiServerCertSANs"))...) - allErrs = append(allErrs, ValidateCertSANs(c.Etcd.ServerCertSANs, field.NewPath("etcd").Child("serverCertSANs"))...) - allErrs = append(allErrs, ValidateCertSANs(c.Etcd.PeerCertSANs, field.NewPath("etcd").Child("peerCertSANs"))...) allErrs = append(allErrs, ValidateAbsolutePath(c.CertificatesDir, field.NewPath("certificatesDir"))...) allErrs = append(allErrs, ValidateNodeName(c.NodeName, field.NewPath("nodeName"))...) allErrs = append(allErrs, ValidateToken(c.Token, field.NewPath("token"))...) @@ -63,6 +61,7 @@ func ValidateMasterConfiguration(c *kubeadm.MasterConfiguration) field.ErrorList allErrs = append(allErrs, ValidateFeatureGates(c.FeatureGates, field.NewPath("featureGates"))...) allErrs = append(allErrs, ValidateAPIEndpoint(&c.API, field.NewPath("api"))...) allErrs = append(allErrs, ValidateProxy(c.KubeProxy.Config, field.NewPath("kubeProxy").Child("config"))...) + allErrs = append(allErrs, ValidateEtcd(&c.Etcd, field.NewPath("etcd"))...) if features.Enabled(c.FeatureGates, features.DynamicKubeletConfig) { allErrs = append(allErrs, ValidateKubeletConfiguration(&c.KubeletConfiguration, field.NewPath("kubeletConfiguration"))...) } @@ -222,6 +221,54 @@ func ValidateTokenUsages(usages []string, fldPath *field.Path) field.ErrorList { return allErrs } +// ValidateEtcd validates the .Etcd sub-struct. +func ValidateEtcd(e *kubeadm.Etcd, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + localPath := fldPath.Child("local") + externalPath := fldPath.Child("external") + + if e.Local == nil && e.External == nil { + allErrs = append(allErrs, field.Invalid(fldPath, "", "either .Etcd.Local or .Etcd.External is required")) + return allErrs + } + if e.Local != nil && e.External != nil { + allErrs = append(allErrs, field.Invalid(fldPath, "", ".Etcd.Local and .Etcd.External are mutually exclusive")) + return allErrs + } + if e.Local != nil { + allErrs = append(allErrs, ValidateAbsolutePath(e.Local.DataDir, localPath.Child("dataDir"))...) + allErrs = append(allErrs, ValidateCertSANs(e.Local.ServerCertSANs, localPath.Child("serverCertSANs"))...) + allErrs = append(allErrs, ValidateCertSANs(e.Local.PeerCertSANs, localPath.Child("peerCertSANs"))...) + } + if e.External != nil { + requireHTTPS := true + // Only allow the http scheme if no certs/keys are passed + if e.External.CAFile == "" && e.External.CertFile == "" && e.External.KeyFile == "" { + requireHTTPS = false + } + // Require either none or both of the cert/key pair + if (e.External.CertFile == "" && e.External.KeyFile != "") || (e.External.CertFile != "" && e.External.KeyFile == "") { + allErrs = append(allErrs, field.Invalid(externalPath, "", "either both or none of .Etcd.External.CertFile and .Etcd.External.KeyFile must be set")) + } + // If the cert and key are specified, require the VA as well + if e.External.CertFile != "" && e.External.KeyFile != "" && e.External.CAFile == "" { + allErrs = append(allErrs, field.Invalid(externalPath, "", "setting .Etcd.External.CertFile and .Etcd.External.KeyFile requires .Etcd.External.CAFile")) + } + + allErrs = append(allErrs, ValidateURLs(e.External.Endpoints, requireHTTPS, externalPath.Child("endpoints"))...) + if e.External.CAFile != "" { + allErrs = append(allErrs, ValidateAbsolutePath(e.External.CAFile, externalPath.Child("caFile"))...) + } + if e.External.CertFile != "" { + allErrs = append(allErrs, ValidateAbsolutePath(e.External.CertFile, externalPath.Child("certFile"))...) + } + if e.External.KeyFile != "" { + allErrs = append(allErrs, ValidateAbsolutePath(e.External.KeyFile, externalPath.Child("keyFile"))...) + } + } + return allErrs +} + // ValidateCertSANs validates alternative names func ValidateCertSANs(altnames []string, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} @@ -233,6 +280,21 @@ func ValidateCertSANs(altnames []string, fldPath *field.Path) field.ErrorList { return allErrs } +// ValidateURLs validates the URLs given in the string slice, makes sure they are parseable. Optionally, it can enforcs HTTPS usage. +func ValidateURLs(urls []string, requireHTTPS bool, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + for _, urlstr := range urls { + u, err := url.Parse(urlstr) + if err != nil || u.Scheme == "" { + allErrs = append(allErrs, field.Invalid(fldPath, urlstr, "not a valid URL")) + } + if requireHTTPS && u.Scheme != "https" { + allErrs = append(allErrs, field.Invalid(fldPath, urlstr, "the URL must be using the HTTPS scheme")) + } + } + return allErrs +} + // ValidateIPFromString validates ip address func ValidateIPFromString(ipaddr string, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go index 8c51a354000..b0dd24a683b 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go @@ -457,6 +457,11 @@ func TestValidateMasterConfiguration(t *testing.T) { AdvertiseAddress: "1.2.3.4", BindPort: 6443, }, + Etcd: kubeadm.Etcd{ + Local: &kubeadm.LocalEtcd{ + DataDir: "/some/path", + }, + }, KubeProxy: kubeadm.KubeProxy{ Config: &kubeproxyconfigv1alpha1.KubeProxyConfiguration{ BindAddress: "192.168.59.103", @@ -498,6 +503,11 @@ func TestValidateMasterConfiguration(t *testing.T) { AdvertiseAddress: "1:2:3::4", BindPort: 3446, }, + Etcd: kubeadm.Etcd{ + Local: &kubeadm.LocalEtcd{ + DataDir: "/some/path", + }, + }, KubeProxy: kubeadm.KubeProxy{ Config: &kubeproxyconfigv1alpha1.KubeProxyConfiguration{ BindAddress: "192.168.59.103", diff --git a/cmd/kubeadm/app/cmd/config_test.go b/cmd/kubeadm/app/cmd/config_test.go index f68b504af92..622d6983e26 100644 --- a/cmd/kubeadm/app/cmd/config_test.go +++ b/cmd/kubeadm/app/cmd/config_test.go @@ -135,7 +135,9 @@ func TestConfigImagesListRunWithoutPath(t *testing.T) { name: "external etcd configuration", cfg: kubeadmapiv1alpha2.MasterConfiguration{ Etcd: kubeadmapiv1alpha2.Etcd{ - Endpoints: []string{"hi"}, + External: &kubeadmapiv1alpha2.ExternalEtcd{ + Endpoints: []string{"https://some.etcd.com:2379"}, + }, }, }, expectedImages: defaultNumberOfImages - 1, diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 7c03b5b4c49..e763d8fd7c0 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -95,7 +95,7 @@ var ( - {{ .APIServerImage }} - {{ .ControllerManagerImage }} - {{ .SchedulerImage }} - - {{ .EtcdImage }} (only if no external etcd endpoints are configured) +{{ .EtcdImage }} - You can check or miligate this in beforehand with "kubeadm config images pull" to make sure the images are downloaded locally and cached. @@ -338,7 +338,7 @@ func (i *Init) Run(out io.Writer) error { return fmt.Errorf("error creating init static pod manifest files: %v", err) } // Add etcd static pod spec only if external etcd is not configured - if len(i.cfg.Etcd.Endpoints) == 0 { + if i.cfg.Etcd.External == nil { glog.V(1).Infof("[init] no external etcd found. Creating manifest for local etcd static pod") if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(manifestDir, i.cfg); err != nil { return fmt.Errorf("error creating local etcd static pod manifest file: %v", err) @@ -380,7 +380,12 @@ func (i *Init) Run(out io.Writer) error { "APIServerImage": images.GetCoreImage(kubeadmconstants.KubeAPIServer, i.cfg.GetControlPlaneImageRepository(), i.cfg.KubernetesVersion, i.cfg.UnifiedControlPlaneImage), "ControllerManagerImage": images.GetCoreImage(kubeadmconstants.KubeControllerManager, i.cfg.GetControlPlaneImageRepository(), i.cfg.KubernetesVersion, i.cfg.UnifiedControlPlaneImage), "SchedulerImage": images.GetCoreImage(kubeadmconstants.KubeScheduler, i.cfg.GetControlPlaneImageRepository(), i.cfg.KubernetesVersion, i.cfg.UnifiedControlPlaneImage), - "EtcdImage": images.GetCoreImage(kubeadmconstants.Etcd, i.cfg.ImageRepository, i.cfg.KubernetesVersion, i.cfg.Etcd.Image), + } + // Set .EtcdImage conditionally + if i.cfg.Etcd.Local != nil { + ctx["EtcdImage"] = fmt.Sprintf(" - %s", images.GetCoreImage(kubeadmconstants.Etcd, i.cfg.ImageRepository, i.cfg.KubernetesVersion, i.cfg.Etcd.Local.Image)) + } else { + ctx["EtcdImage"] = "" } kubeletFailTempl.Execute(out, ctx) diff --git a/cmd/kubeadm/app/cmd/upgrade/common_test.go b/cmd/kubeadm/app/cmd/upgrade/common_test.go index e9864154cbe..ac170b04bb6 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common_test.go +++ b/cmd/kubeadm/app/cmd/upgrade/common_test.go @@ -36,6 +36,11 @@ func TestPrintConfiguration(t *testing.T) { { cfg: &kubeadmapi.MasterConfiguration{ KubernetesVersion: "v1.7.1", + Etcd: kubeadmapi.Etcd{ + Local: &kubeadmapi.LocalEtcd{ + DataDir: "/some/path", + }, + }, }, expectedBytes: []byte(`[upgrade/config] Configuration used: api: @@ -48,12 +53,9 @@ func TestPrintConfiguration(t *testing.T) { path: "" certificatesDir: "" etcd: - caFile: "" - certFile: "" - dataDir: "" - endpoints: null - image: "" - keyFile: "" + local: + dataDir: /some/path + image: "" imageRepository: "" kind: MasterConfiguration kubeProxy: {} @@ -74,6 +76,11 @@ func TestPrintConfiguration(t *testing.T) { Networking: kubeadmapi.Networking{ ServiceSubnet: "10.96.0.1/12", }, + Etcd: kubeadmapi.Etcd{ + External: &kubeadmapi.ExternalEtcd{ + Endpoints: []string{"https://one-etcd-instance:2379"}, + }, + }, }, expectedBytes: []byte(`[upgrade/config] Configuration used: api: @@ -86,12 +93,12 @@ func TestPrintConfiguration(t *testing.T) { path: "" certificatesDir: "" etcd: - caFile: "" - certFile: "" - dataDir: "" - endpoints: null - image: "" - keyFile: "" + external: + caFile: "" + certFile: "" + endpoints: + - https://one-etcd-instance:2379 + keyFile: "" imageRepository: "" kind: MasterConfiguration kubeProxy: {} diff --git a/cmd/kubeadm/app/cmd/upgrade/plan.go b/cmd/kubeadm/app/cmd/upgrade/plan.go index 0f21dae3671..b769709f370 100644 --- a/cmd/kubeadm/app/cmd/upgrade/plan.go +++ b/cmd/kubeadm/app/cmd/upgrade/plan.go @@ -94,13 +94,13 @@ func RunPlan(flags *planFlags) error { // Currently this is the only method we have for distinguishing // external etcd vs static pod etcd - isExternalEtcd := len(upgradeVars.cfg.Etcd.Endpoints) > 0 + isExternalEtcd := upgradeVars.cfg.Etcd.External != nil if isExternalEtcd { client, err := etcdutil.New( - upgradeVars.cfg.Etcd.Endpoints, - upgradeVars.cfg.Etcd.CAFile, - upgradeVars.cfg.Etcd.CertFile, - upgradeVars.cfg.Etcd.KeyFile) + upgradeVars.cfg.Etcd.External.Endpoints, + upgradeVars.cfg.Etcd.External.CAFile, + upgradeVars.cfg.Etcd.External.CertFile, + upgradeVars.cfg.Etcd.External.KeyFile) if err != nil { return err } diff --git a/cmd/kubeadm/app/images/images.go b/cmd/kubeadm/app/images/images.go index 3b25bd8bf25..999180828cd 100644 --- a/cmd/kubeadm/app/images/images.go +++ b/cmd/kubeadm/app/images/images.go @@ -56,8 +56,8 @@ func GetAllImages(cfg *kubeadmapi.MasterConfiguration) []string { imgs = append(imgs, fmt.Sprintf("%v/pause-%v:%v", cfg.ImageRepository, runtime.GOARCH, "3.1")) // if etcd is not external then add the image as it will be required - if len(cfg.Etcd.Endpoints) == 0 { - imgs = append(imgs, GetCoreImage(constants.Etcd, cfg.ImageRepository, cfg.KubernetesVersion, cfg.Etcd.Image)) + if cfg.Etcd.Local != nil { + imgs = append(imgs, GetCoreImage(constants.Etcd, cfg.ImageRepository, cfg.KubernetesVersion, cfg.Etcd.Local.Image)) } dnsImage := fmt.Sprintf("%v/k8s-dns-kube-dns-%v:%v", cfg.ImageRepository, runtime.GOARCH, dns.GetDNSVersion(nil, constants.KubeDNS)) diff --git a/cmd/kubeadm/app/phases/certs/certs.go b/cmd/kubeadm/app/phases/certs/certs.go index b67f4655c0e..b4d27a2e0db 100644 --- a/cmd/kubeadm/app/phases/certs/certs.go +++ b/cmd/kubeadm/app/phases/certs/certs.go @@ -51,8 +51,7 @@ func CreatePKIAssets(cfg *kubeadmapi.MasterConfiguration) error { CreateAPIServerEtcdClientCertAndKeyFiles, } - // Currently this is the only way we have to identify static pod etcd vs external etcd - if len(cfg.Etcd.Endpoints) == 0 { + if cfg.Etcd.Local != nil { certActions = append(certActions, etcdCertActions...) } diff --git a/cmd/kubeadm/app/phases/certs/certs_test.go b/cmd/kubeadm/app/phases/certs/certs_test.go index 502683d675b..c6fbdcca840 100644 --- a/cmd/kubeadm/app/phases/certs/certs_test.go +++ b/cmd/kubeadm/app/phases/certs/certs_test.go @@ -325,9 +325,11 @@ func TestNewEtcdServerCertAndKey(t *testing.T) { cfg := &kubeadmapi.MasterConfiguration{ Etcd: kubeadmapi.Etcd{ - ServerCertSANs: []string{ - proxy, - proxyIP, + Local: &kubeadmapi.LocalEtcd{ + ServerCertSANs: []string{ + proxy, + proxyIP, + }, }, }, } @@ -358,9 +360,11 @@ func TestNewEtcdPeerCertAndKey(t *testing.T) { API: kubeadmapi.API{AdvertiseAddress: addr}, NodeName: hostname, Etcd: kubeadmapi.Etcd{ - PeerCertSANs: []string{ - proxy, - proxyIP, + Local: &kubeadmapi.LocalEtcd{ + PeerCertSANs: []string{ + proxy, + proxyIP, + }, }, }, } @@ -693,13 +697,18 @@ func TestCreateCertificateFilesMethods(t *testing.T) { cfg := &kubeadmapi.MasterConfiguration{ API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4"}, + Etcd: kubeadmapi.Etcd{Local: &kubeadmapi.LocalEtcd{}}, Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, NodeName: "valid-hostname", CertificatesDir: tmpdir, } if test.externalEtcd { - cfg.Etcd.Endpoints = []string{"192.168.1.1:2379"} + if cfg.Etcd.External == nil { + cfg.Etcd.External = &kubeadmapi.ExternalEtcd{} + } + cfg.Etcd.Local = nil + cfg.Etcd.External.Endpoints = []string{"192.168.1.1:2379"} } // executes setup func (if necessary) diff --git a/cmd/kubeadm/app/phases/certs/doc.go b/cmd/kubeadm/app/phases/certs/doc.go index 0a60f992dcc..6f9801fd513 100644 --- a/cmd/kubeadm/app/phases/certs/doc.go +++ b/cmd/kubeadm/app/phases/certs/doc.go @@ -24,8 +24,8 @@ package certs From MasterConfiguration .API.AdvertiseAddress is an optional parameter that can be passed for an extra addition to the SAN IPs .APIServerCertSANs is an optional parameter for adding DNS names and IPs to the API Server serving cert SAN - .Etcd.ServerCertSANs is an optional parameter for adding DNS names and IPs to the etcd serving cert SAN - .Etcd.PeerCertSANs is an optional parameter for adding DNS names and IPs to the etcd peer cert SAN + .Etcd.Local.ServerCertSANs is an optional parameter for adding DNS names and IPs to the etcd serving cert SAN + .Etcd.Local.PeerCertSANs is an optional parameter for adding DNS names and IPs to the etcd peer cert SAN .Networking.DNSDomain is needed for knowing which DNS name the internal kubernetes service has .Networking.ServiceSubnet is needed for knowing which IP the internal kubernetes service is going to point to .CertificatesDir is required for knowing where all certificates should be stored diff --git a/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go b/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go index 6cebbe3a92e..39ca139cba7 100644 --- a/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go +++ b/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go @@ -316,7 +316,9 @@ func GetEtcdAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, e IPs: []net.IP{net.IPv4(127, 0, 0, 1)}, } - appendSANsToAltNames(altNames, cfg.Etcd.ServerCertSANs, kubeadmconstants.EtcdServerCertName) + if cfg.Etcd.Local != nil { + appendSANsToAltNames(altNames, cfg.Etcd.Local.ServerCertSANs, kubeadmconstants.EtcdServerCertName) + } return altNames, nil } @@ -338,7 +340,9 @@ func GetEtcdPeerAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltName IPs: []net.IP{advertiseAddress}, } - appendSANsToAltNames(altNames, cfg.Etcd.PeerCertSANs, kubeadmconstants.EtcdPeerCertName) + if cfg.Etcd.Local != nil { + appendSANsToAltNames(altNames, cfg.Etcd.Local.PeerCertSANs, kubeadmconstants.EtcdPeerCertName) + } return altNames, nil } diff --git a/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers_test.go b/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers_test.go index 2ff497c1550..9ff34b0b20a 100644 --- a/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers_test.go +++ b/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers_test.go @@ -508,11 +508,13 @@ func TestGetEtcdAltNames(t *testing.T) { proxyIP := "10.10.10.100" cfg := &kubeadmapi.MasterConfiguration{ Etcd: kubeadmapi.Etcd{ - ServerCertSANs: []string{ - proxy, - proxyIP, - "1.2.3.L", - "invalid,commas,in,DNS", + Local: &kubeadmapi.LocalEtcd{ + ServerCertSANs: []string{ + proxy, + proxyIP, + "1.2.3.L", + "invalid,commas,in,DNS", + }, }, }, } @@ -562,11 +564,13 @@ func TestGetEtcdPeerAltNames(t *testing.T) { API: kubeadmapi.API{AdvertiseAddress: advertiseIP}, NodeName: hostname, Etcd: kubeadmapi.Etcd{ - PeerCertSANs: []string{ - proxy, - proxyIP, - "1.2.3.L", - "invalid,commas,in,DNS", + Local: &kubeadmapi.LocalEtcd{ + PeerCertSANs: []string{ + proxy, + proxyIP, + "1.2.3.L", + "invalid,commas,in,DNS", + }, }, }, } diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 473472f50ee..17ff9de4390 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -169,16 +169,16 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration) []string { command := []string{"kube-apiserver"} // If the user set endpoints for an external etcd cluster - if len(cfg.Etcd.Endpoints) > 0 { - defaultArguments["etcd-servers"] = strings.Join(cfg.Etcd.Endpoints, ",") + if cfg.Etcd.External != nil { + defaultArguments["etcd-servers"] = strings.Join(cfg.Etcd.External.Endpoints, ",") // Use any user supplied etcd certificates - if cfg.Etcd.CAFile != "" { - defaultArguments["etcd-cafile"] = cfg.Etcd.CAFile + if cfg.Etcd.External.CAFile != "" { + defaultArguments["etcd-cafile"] = cfg.Etcd.External.CAFile } - if cfg.Etcd.CertFile != "" && cfg.Etcd.KeyFile != "" { - defaultArguments["etcd-certfile"] = cfg.Etcd.CertFile - defaultArguments["etcd-keyfile"] = cfg.Etcd.KeyFile + if cfg.Etcd.External.CertFile != "" && cfg.Etcd.External.KeyFile != "" { + defaultArguments["etcd-certfile"] = cfg.Etcd.External.CertFile + defaultArguments["etcd-keyfile"] = cfg.Etcd.External.KeyFile } } else { // Default to etcd static pod on localhost @@ -186,17 +186,6 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration) []string { defaultArguments["etcd-cafile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName) defaultArguments["etcd-certfile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientCertName) defaultArguments["etcd-keyfile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName) - - // Warn for unused user supplied variables - if cfg.Etcd.CAFile != "" { - glog.Warningf("[controlplane] configuration for %s CAFile, %s, is unused without providing Endpoints for external %s\n", kubeadmconstants.Etcd, cfg.Etcd.CAFile, kubeadmconstants.Etcd) - } - if cfg.Etcd.CertFile != "" { - glog.Warningf("[controlplane] configuration for %s CertFile, %s, is unused without providing Endpoints for external %s\n", kubeadmconstants.Etcd, cfg.Etcd.CertFile, kubeadmconstants.Etcd) - } - if cfg.Etcd.KeyFile != "" { - glog.Warningf("[controlplane] configuration for %s KeyFile, %s, is unused without providing Endpoints for external %s\n", kubeadmconstants.Etcd, cfg.Etcd.KeyFile, kubeadmconstants.Etcd) - } } if features.Enabled(cfg.FeatureGates, features.HighAvailability) { diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go index ea26780ff50..3e1a7325d33 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests_test.go @@ -181,50 +181,11 @@ func TestGetAPIServerCommand(t *testing.T) { "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", }, }, - { - name: "custom etcd cert and key files", - cfg: &kubeadmapi.MasterConfiguration{ - API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"}, - Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, - Etcd: kubeadmapi.Etcd{CertFile: "fiz", KeyFile: "faz"}, - CertificatesDir: testCertsDir, - }, - expected: []string{ - "kube-apiserver", - "--insecure-port=0", - "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota", - "--service-cluster-ip-range=bar", - "--service-account-key-file=" + testCertsDir + "/sa.pub", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--tls-cert-file=" + testCertsDir + "/apiserver.crt", - "--tls-private-key-file=" + testCertsDir + "/apiserver.key", - "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", - "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", - "--enable-bootstrap-token-auth=true", - "--secure-port=123", - "--allow-privileged=true", - "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", - "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", - "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", - "--requestheader-username-headers=X-Remote-User", - "--requestheader-group-headers=X-Remote-Group", - "--requestheader-extra-headers-prefix=X-Remote-Extra-", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - "--requestheader-allowed-names=front-proxy-client", - "--authorization-mode=Node,RBAC", - "--advertise-address=4.3.2.1", - "--etcd-servers=https://127.0.0.1:2379", - "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt", - "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", - "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", - }, - }, { name: "ignores the audit policy if the feature gate is not enabled", cfg: &kubeadmapi.MasterConfiguration{ API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"}, Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, - Etcd: kubeadmapi.Etcd{CertFile: "fiz", KeyFile: "faz"}, CertificatesDir: testCertsDir, AuditPolicyConfiguration: kubeadmapi.AuditPolicyConfiguration{ Path: "/foo/bar", @@ -267,7 +228,6 @@ func TestGetAPIServerCommand(t *testing.T) { cfg: &kubeadmapi.MasterConfiguration{ API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, - Etcd: kubeadmapi.Etcd{CertFile: "fiz", KeyFile: "faz"}, CertificatesDir: testCertsDir, }, expected: []string{ @@ -303,10 +263,17 @@ func TestGetAPIServerCommand(t *testing.T) { { name: "an external etcd with custom ca, certs and keys", cfg: &kubeadmapi.MasterConfiguration{ - API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, - Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, - FeatureGates: map[string]bool{features.HighAvailability: true}, - Etcd: kubeadmapi.Etcd{Endpoints: []string{"https://8.6.4.1:2379", "https://8.6.4.2:2379"}, CAFile: "fuz", CertFile: "fiz", KeyFile: "faz"}, + API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, + FeatureGates: map[string]bool{features.HighAvailability: true}, + Etcd: kubeadmapi.Etcd{ + External: &kubeadmapi.ExternalEtcd{ + Endpoints: []string{"https://8.6.4.1:2379", "https://8.6.4.2:2379"}, + CAFile: "fuz", + CertFile: "fiz", + KeyFile: "faz", + }, + }, CertificatesDir: testCertsDir, }, expected: []string{ @@ -343,9 +310,13 @@ func TestGetAPIServerCommand(t *testing.T) { { name: "an insecure etcd", cfg: &kubeadmapi.MasterConfiguration{ - API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, - Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, - Etcd: kubeadmapi.Etcd{Endpoints: []string{"http://127.0.0.1:2379", "http://127.0.0.1:2380"}}, + API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, + Etcd: kubeadmapi.Etcd{ + External: &kubeadmapi.ExternalEtcd{ + Endpoints: []string{"http://127.0.0.1:2379", "http://127.0.0.1:2380"}, + }, + }, CertificatesDir: testCertsDir, }, expected: []string{ diff --git a/cmd/kubeadm/app/phases/controlplane/volumes.go b/cmd/kubeadm/app/phases/controlplane/volumes.go index 5f8b967cdc0..9237d2c108a 100644 --- a/cmd/kubeadm/app/phases/controlplane/volumes.go +++ b/cmd/kubeadm/app/phases/controlplane/volumes.go @@ -62,8 +62,8 @@ func getHostPathVolumesForTheControlPlane(cfg *kubeadmapi.MasterConfiguration) c mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeAuditPolicyLogVolumeName, cfg.AuditPolicyConfiguration.LogDir, kubeadmconstants.StaticPodAuditPolicyLogDir, false, &hostPathDirectoryOrCreate) } // If external etcd is specified, mount the directories needed for accessing the CA/serving certs and the private key - if len(cfg.Etcd.Endpoints) != 0 { - etcdVols, etcdVolMounts := getEtcdCertVolumes(cfg.Etcd, cfg.CertificatesDir) + if cfg.Etcd.External != nil { + etcdVols, etcdVolMounts := getEtcdCertVolumes(cfg.Etcd.External, cfg.CertificatesDir) mounts.AddHostPathMounts(kubeadmconstants.KubeAPIServer, etcdVols, etcdVolMounts) } @@ -178,7 +178,7 @@ func (c *controlPlaneHostPathMounts) addComponentVolumeMount(component string, v } // getEtcdCertVolumes returns the volumes/volumemounts needed for talking to an external etcd cluster -func getEtcdCertVolumes(etcdCfg kubeadmapi.Etcd, k8sCertificatesDir string) ([]v1.Volume, []v1.VolumeMount) { +func getEtcdCertVolumes(etcdCfg *kubeadmapi.ExternalEtcd, k8sCertificatesDir string) ([]v1.Volume, []v1.VolumeMount) { certPaths := []string{etcdCfg.CAFile, etcdCfg.CertFile, etcdCfg.KeyFile} certDirs := sets.NewString() for _, certPath := range certPaths { diff --git a/cmd/kubeadm/app/phases/controlplane/volumes_test.go b/cmd/kubeadm/app/phases/controlplane/volumes_test.go index 68e943a0dba..492dc16ab02 100644 --- a/cmd/kubeadm/app/phases/controlplane/volumes_test.go +++ b/cmd/kubeadm/app/phases/controlplane/volumes_test.go @@ -234,7 +234,7 @@ func TestGetEtcdCertVolumes(t *testing.T) { } for _, rt := range tests { - actualVol, actualVolMount := getEtcdCertVolumes(kubeadmapi.Etcd{ + actualVol, actualVolMount := getEtcdCertVolumes(&kubeadmapi.ExternalEtcd{ CAFile: rt.ca, CertFile: rt.cert, KeyFile: rt.key, @@ -525,10 +525,12 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) { cfg: &kubeadmapi.MasterConfiguration{ CertificatesDir: testCertsDir, Etcd: kubeadmapi.Etcd{ - Endpoints: []string{"foo"}, - CAFile: "/etc/certs/etcd/my-etcd-ca.crt", - CertFile: testCertsDir + "/etcd/my-etcd.crt", - KeyFile: "/var/lib/etcd/certs/my-etcd.key", + External: &kubeadmapi.ExternalEtcd{ + Endpoints: []string{"foo"}, + CAFile: "/etc/certs/etcd/my-etcd-ca.crt", + CertFile: testCertsDir + "/etcd/my-etcd.crt", + KeyFile: "/var/lib/etcd/certs/my-etcd.key", + }, }, }, vol: volMap2, diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go index 8af1bd8db34..80000dabe1c 100644 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -54,17 +54,17 @@ func CreateLocalEtcdStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.Ma func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod { pathType := v1.HostPathDirectoryOrCreate etcdMounts := map[string]v1.Volume{ - etcdVolumeName: staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.DataDir, &pathType), + etcdVolumeName: staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.Local.DataDir, &pathType), certsVolumeName: staticpodutil.NewVolume(certsVolumeName, cfg.CertificatesDir+"/etcd", &pathType), } return staticpodutil.ComponentPod(v1.Container{ Name: kubeadmconstants.Etcd, Command: getEtcdCommand(cfg), - Image: images.GetCoreImage(kubeadmconstants.Etcd, cfg.ImageRepository, cfg.KubernetesVersion, cfg.Etcd.Image), + Image: images.GetCoreImage(kubeadmconstants.Etcd, cfg.ImageRepository, cfg.KubernetesVersion, cfg.Etcd.Local.Image), ImagePullPolicy: v1.PullIfNotPresent, // Mount the etcd datadir path read-write so etcd can store data in a more persistent manner VolumeMounts: []v1.VolumeMount{ - staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false), + staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.Local.DataDir, false), staticpodutil.NewVolumeMount(certsVolumeName, cfg.CertificatesDir+"/etcd", false), }, LivenessProbe: staticpodutil.EtcdProbe( @@ -79,7 +79,7 @@ func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string { defaultArguments := map[string]string{ "listen-client-urls": "https://127.0.0.1:2379", "advertise-client-urls": "https://127.0.0.1:2379", - "data-dir": cfg.Etcd.DataDir, + "data-dir": cfg.Etcd.Local.DataDir, "cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerCertName), "key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName), "trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName), @@ -92,6 +92,6 @@ func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string { } command := []string{"etcd"} - command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.Etcd.ExtraArgs)...) + command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.Etcd.Local.ExtraArgs)...) return command } diff --git a/cmd/kubeadm/app/phases/etcd/local_test.go b/cmd/kubeadm/app/phases/etcd/local_test.go index ff0a484ac67..7c884abc53b 100644 --- a/cmd/kubeadm/app/phases/etcd/local_test.go +++ b/cmd/kubeadm/app/phases/etcd/local_test.go @@ -34,6 +34,12 @@ func TestGetEtcdPodSpec(t *testing.T) { // Creates a Master Configuration cfg := &kubeadmapi.MasterConfiguration{ KubernetesVersion: "v1.7.0", + Etcd: kubeadmapi.Etcd{ + Local: &kubeadmapi.LocalEtcd{ + DataDir: "/var/lib/etcd", + Image: "", + }, + }, } // Executes GetEtcdPodSpec @@ -54,6 +60,12 @@ func TestCreateLocalEtcdStaticPodManifestFile(t *testing.T) { // Creates a Master Configuration cfg := &kubeadmapi.MasterConfiguration{ KubernetesVersion: "v1.7.0", + Etcd: kubeadmapi.Etcd{ + Local: &kubeadmapi.LocalEtcd{ + DataDir: "/var/lib/etcd", + Image: "k8s.gcr.io/etcd", + }, + }, } // Execute createStaticPodFunction @@ -75,7 +87,7 @@ func TestGetEtcdCommand(t *testing.T) { }{ { cfg: &kubeadmapi.MasterConfiguration{ - Etcd: kubeadmapi.Etcd{DataDir: "/var/lib/etcd"}, + Etcd: kubeadmapi.Etcd{Local: &kubeadmapi.LocalEtcd{DataDir: "/var/lib/etcd"}}, }, expected: []string{ "etcd", @@ -96,10 +108,12 @@ func TestGetEtcdCommand(t *testing.T) { { cfg: &kubeadmapi.MasterConfiguration{ Etcd: kubeadmapi.Etcd{ - DataDir: "/var/lib/etcd", - ExtraArgs: map[string]string{ - "listen-client-urls": "https://10.0.1.10:2379", - "advertise-client-urls": "https://10.0.1.10:2379", + Local: &kubeadmapi.LocalEtcd{ + DataDir: "/var/lib/etcd", + ExtraArgs: map[string]string{ + "listen-client-urls": "https://10.0.1.10:2379", + "advertise-client-urls": "https://10.0.1.10:2379", + }, }, }, }, @@ -121,7 +135,7 @@ func TestGetEtcdCommand(t *testing.T) { }, { cfg: &kubeadmapi.MasterConfiguration{ - Etcd: kubeadmapi.Etcd{DataDir: "/etc/foo"}, + Etcd: kubeadmapi.Etcd{Local: &kubeadmapi.LocalEtcd{DataDir: "/etc/foo"}}, }, expected: []string{ "etcd", diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods.go b/cmd/kubeadm/app/phases/upgrade/staticpods.go index 488a850636f..a041c9669cf 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods.go @@ -226,7 +226,7 @@ func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticP // performEtcdStaticPodUpgrade performs upgrade of etcd, it returns bool which indicates fatal error or not and the actual error. func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.MasterConfiguration, recoverManifests map[string]string, isTLSUpgrade bool, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) (bool, error) { // Add etcd static pod spec only if external etcd is not configured - if len(cfg.Etcd.Endpoints) != 0 { + if cfg.Etcd.External != nil { return false, fmt.Errorf("external etcd detected, won't try to change any etcd state") } @@ -238,7 +238,7 @@ func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathM // Backing up etcd data store backupEtcdDir := pathMgr.BackupEtcdDir() - runningEtcdDir := cfg.Etcd.DataDir + runningEtcdDir := cfg.Etcd.Local.DataDir if err := util.CopyDir(runningEtcdDir, backupEtcdDir); err != nil { return true, fmt.Errorf("failed to back up etcd data: %v", err) } @@ -382,14 +382,14 @@ func StaticPodControlPlane(waiter apiclient.Waiter, pathMgr StaticPodPathManager } if oldEtcdClient == nil { - if len(cfg.Etcd.Endpoints) > 0 { + if cfg.Etcd.External != nil { // External etcd isExternalEtcd = true client, err := etcdutil.New( - cfg.Etcd.Endpoints, - cfg.Etcd.CAFile, - cfg.Etcd.CertFile, - cfg.Etcd.KeyFile, + cfg.Etcd.External.Endpoints, + cfg.Etcd.External.CAFile, + cfg.Etcd.External.CertFile, + cfg.Etcd.External.KeyFile, ) if err != nil { return fmt.Errorf("failed to create etcd client for external etcd: %v", err) @@ -482,7 +482,7 @@ func rollbackOldManifests(oldManifests map[string]string, origErr error, pathMgr // When the folder contents are successfully rolled back, nil is returned, otherwise an error is returned. func rollbackEtcdData(cfg *kubeadmapi.MasterConfiguration, pathMgr StaticPodPathManager) error { backupEtcdDir := pathMgr.BackupEtcdDir() - runningEtcdDir := cfg.Etcd.DataDir + runningEtcdDir := cfg.Etcd.Local.DataDir if err := util.CopyDir(backupEtcdDir, runningEtcdDir); err != nil { // Let the user know there we're problems, but we tried to reçover diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go index 060f5e094be..44a299b5a3a 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go @@ -56,15 +56,9 @@ apiServerExtraArgs: null certificatesDir: %s controllerManagerExtraArgs: null etcd: - caFile: "" - certFile: "" - dataDir: %s - endpoints: null - extraArgs: null - image: "" - keyFile: "" - serverCertSANs: null - peerCertSANs: null + local: + dataDir: %s + image: "" featureFlags: null imageRepository: k8s.gcr.io kubernetesVersion: %s diff --git a/cmd/kubeadm/app/preflight/checks.go b/cmd/kubeadm/app/preflight/checks.go index d7cbbefb093..8b3c5cb6a98 100644 --- a/cmd/kubeadm/app/preflight/checks.go +++ b/cmd/kubeadm/app/preflight/checks.go @@ -689,8 +689,15 @@ func (ExternalEtcdVersionCheck) Name() string { } // Check validates external etcd version +// TODO: Use the official etcd Golang client for this instead? func (evc ExternalEtcdVersionCheck) Check() (warnings, errors []error) { glog.V(1).Infoln("validating the external etcd version") + + // Return quickly if the user isn't using external etcd + if evc.Etcd.External.Endpoints == nil { + return nil, nil + } + var config *tls.Config var err error if config, err = evc.configRootCAs(config); err != nil { @@ -703,7 +710,7 @@ func (evc ExternalEtcdVersionCheck) Check() (warnings, errors []error) { } client := evc.getHTTPClient(config) - for _, endpoint := range evc.Etcd.Endpoints { + for _, endpoint := range evc.Etcd.External.Endpoints { if _, err := url.Parse(endpoint); err != nil { errors = append(errors, fmt.Errorf("failed to parse external etcd endpoint %s : %v", endpoint, err)) continue @@ -739,10 +746,10 @@ func (evc ExternalEtcdVersionCheck) Check() (warnings, errors []error) { // configRootCAs configures and returns a reference to tls.Config instance if CAFile is provided func (evc ExternalEtcdVersionCheck) configRootCAs(config *tls.Config) (*tls.Config, error) { var CACertPool *x509.CertPool - if evc.Etcd.CAFile != "" { - CACert, err := ioutil.ReadFile(evc.Etcd.CAFile) + if evc.Etcd.External.CAFile != "" { + CACert, err := ioutil.ReadFile(evc.Etcd.External.CAFile) if err != nil { - return nil, fmt.Errorf("couldn't load external etcd's server certificate %s: %v", evc.Etcd.CAFile, err) + return nil, fmt.Errorf("couldn't load external etcd's server certificate %s: %v", evc.Etcd.External.CAFile, err) } CACertPool = x509.NewCertPool() CACertPool.AppendCertsFromPEM(CACert) @@ -759,11 +766,11 @@ func (evc ExternalEtcdVersionCheck) configRootCAs(config *tls.Config) (*tls.Conf // configCertAndKey configures and returns a reference to tls.Config instance if CertFile and KeyFile pair is provided func (evc ExternalEtcdVersionCheck) configCertAndKey(config *tls.Config) (*tls.Config, error) { var cert tls.Certificate - if evc.Etcd.CertFile != "" && evc.Etcd.KeyFile != "" { + if evc.Etcd.External.CertFile != "" && evc.Etcd.External.KeyFile != "" { var err error - cert, err = tls.LoadX509KeyPair(evc.Etcd.CertFile, evc.Etcd.KeyFile) + cert, err = tls.LoadX509KeyPair(evc.Etcd.External.CertFile, evc.Etcd.External.KeyFile) if err != nil { - return nil, fmt.Errorf("couldn't load external etcd's certificate and key pair %s, %s: %v", evc.Etcd.CertFile, evc.Etcd.KeyFile, err) + return nil, fmt.Errorf("couldn't load external etcd's certificate and key pair %s, %s: %v", evc.Etcd.External.CertFile, evc.Etcd.External.KeyFile, err) } if config == nil { config = &tls.Config{} @@ -874,26 +881,26 @@ func RunInitMasterChecks(execer utilsexec.Interface, cfg *kubeadmapi.MasterConfi ) } - if len(cfg.Etcd.Endpoints) == 0 { + if cfg.Etcd.Local != nil { // Only do etcd related checks when no external endpoints were specified checks = append(checks, PortOpenCheck{port: 2379}, - DirAvailableCheck{Path: cfg.Etcd.DataDir}, + DirAvailableCheck{Path: cfg.Etcd.Local.DataDir}, ) - } else { + } + + if cfg.Etcd.External != nil { // Only check etcd version when external endpoints are specified - if cfg.Etcd.CAFile != "" { - checks = append(checks, FileExistingCheck{Path: cfg.Etcd.CAFile}) + if cfg.Etcd.External.CAFile != "" { + checks = append(checks, FileExistingCheck{Path: cfg.Etcd.External.CAFile}) } - if cfg.Etcd.CertFile != "" { - checks = append(checks, FileExistingCheck{Path: cfg.Etcd.CertFile}) + if cfg.Etcd.External.CertFile != "" { + checks = append(checks, FileExistingCheck{Path: cfg.Etcd.External.CertFile}) } - if cfg.Etcd.KeyFile != "" { - checks = append(checks, FileExistingCheck{Path: cfg.Etcd.KeyFile}) + if cfg.Etcd.External.KeyFile != "" { + checks = append(checks, FileExistingCheck{Path: cfg.Etcd.External.KeyFile}) } - checks = append(checks, - ExternalEtcdVersionCheck{Etcd: cfg.Etcd}, - ) + checks = append(checks, ExternalEtcdVersionCheck{Etcd: cfg.Etcd}) } if ip := net.ParseIP(cfg.API.AdvertiseAddress); ip != nil { diff --git a/cmd/kubeadm/app/preflight/checks_test.go b/cmd/kubeadm/app/preflight/checks_test.go index 0c533a67f17..cfea0abc450 100644 --- a/cmd/kubeadm/app/preflight/checks_test.go +++ b/cmd/kubeadm/app/preflight/checks_test.go @@ -196,21 +196,21 @@ func TestRunInitMasterChecks(t *testing.T) { { name: "Test CA file exists if specfied", cfg: &kubeadmapi.MasterConfiguration{ - Etcd: kubeadmapi.Etcd{CAFile: "/foo"}, + Etcd: kubeadmapi.Etcd{External: &kubeadmapi.ExternalEtcd{CAFile: "/foo"}}, }, expected: false, }, { name: "Test Cert file exists if specfied", cfg: &kubeadmapi.MasterConfiguration{ - Etcd: kubeadmapi.Etcd{CertFile: "/foo"}, + Etcd: kubeadmapi.Etcd{External: &kubeadmapi.ExternalEtcd{CertFile: "/foo"}}, }, expected: false, }, { name: "Test Key file exists if specfied", cfg: &kubeadmapi.MasterConfiguration{ - Etcd: kubeadmapi.Etcd{CertFile: "/foo"}, + Etcd: kubeadmapi.Etcd{External: &kubeadmapi.ExternalEtcd{CertFile: "/foo"}}, }, expected: false, }, @@ -319,7 +319,7 @@ func TestConfigRootCAs(t *testing.T) { t.Errorf("failed configRootCAs:\n\texpected: succeed writing contents to temp CA file %s\n\tactual:%v", f.Name(), err) } - c := ExternalEtcdVersionCheck{Etcd: kubeadmapi.Etcd{CAFile: f.Name()}} + c := ExternalEtcdVersionCheck{Etcd: kubeadmapi.Etcd{External: &kubeadmapi.ExternalEtcd{CAFile: f.Name()}}} config, err := c.configRootCAs(nil) if err != nil { @@ -367,10 +367,14 @@ func TestConfigCertAndKey(t *testing.T) { err, ) } - c := ExternalEtcdVersionCheck{Etcd: kubeadmapi.Etcd{ - CertFile: certFile.Name(), - KeyFile: keyFile.Name(), - }} + c := ExternalEtcdVersionCheck{ + Etcd: kubeadmapi.Etcd{ + External: &kubeadmapi.ExternalEtcd{ + CertFile: certFile.Name(), + KeyFile: keyFile.Name(), + }, + }, + } config, err := c.configCertAndKey(nil) if err != nil { diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml index aa8c9942bbb..cf4d1d1c370 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml @@ -17,15 +17,13 @@ ClusterName: kubernetes ControllerManagerExtraArgs: null ControllerManagerExtraVolumes: null Etcd: - CAFile: "" - CertFile: "" - DataDir: /var/lib/etcd - Endpoints: null - ExtraArgs: null - Image: "" - KeyFile: "" - PeerCertSANs: null - ServerCertSANs: null + External: null + Local: + DataDir: /var/lib/etcd + ExtraArgs: null + Image: "" + PeerCertSANs: null + ServerCertSANs: null FeatureGates: null ImageRepository: k8s.gcr.io KubeProxy: diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha2.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha2.yaml index de6b2724910..959229b286a 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha2.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha2.yaml @@ -13,12 +13,9 @@ certificatesDir: /etc/kubernetes/pki clusterName: kubernetes criSocket: /var/run/dockershim.sock etcd: - caFile: "" - certFile: "" - dataDir: /var/lib/etcd - endpoints: null - image: "" - keyFile: "" + local: + dataDir: /var/lib/etcd + image: "" imageRepository: k8s.gcr.io kind: MasterConfiguration kubeProxy: diff --git a/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml b/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml index a852a56a357..7db1119f2f4 100644 --- a/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml +++ b/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml @@ -11,12 +11,9 @@ certificatesDir: /var/lib/kubernetes/pki clusterName: kubernetes criSocket: /var/run/criruntime.sock etcd: - caFile: "" - certFile: "" - dataDir: /var/lib/etcd - endpoints: null - image: "" - keyFile: "" + local: + dataDir: /var/lib/etcd + image: "" imageRepository: my-company.com kind: MasterConfiguration kubeProxy: diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go index e4ecb7cd16a..d09518e11ab 100644 --- a/cmd/kubeadm/app/util/staticpod/utils.go +++ b/cmd/kubeadm/app/util/staticpod/utils.go @@ -242,8 +242,8 @@ func GetProbeAddress(cfg *kubeadmapi.MasterConfiguration, componentName string) return addr } case componentName == kubeadmconstants.Etcd: - if cfg.Etcd.ExtraArgs != nil { - if arg, exists := cfg.Etcd.ExtraArgs[etcdListenClientURLsArg]; exists { + if cfg.Etcd.Local != nil && cfg.Etcd.Local.ExtraArgs != nil { + if arg, exists := cfg.Etcd.Local.ExtraArgs[etcdListenClientURLsArg]; exists { // Use the first url in the listen-client-urls if multiple url's are specified. if strings.ContainsAny(arg, ",") { arg = strings.Split(arg, ",")[0] diff --git a/cmd/kubeadm/app/util/staticpod/utils_test.go b/cmd/kubeadm/app/util/staticpod/utils_test.go index 834e1f5a0c6..96c3d8dc419 100644 --- a/cmd/kubeadm/app/util/staticpod/utils_test.go +++ b/cmd/kubeadm/app/util/staticpod/utils_test.go @@ -207,8 +207,10 @@ func TestEtcdProbe(t *testing.T) { name: "valid etcd probe using listen-client-urls IPv4 addresses", cfg: &kubeadmapi.MasterConfiguration{ Etcd: kubeadmapi.Etcd{ - ExtraArgs: map[string]string{ - "listen-client-urls": "http://1.2.3.4:2379,http://4.3.2.1:2379"}, + Local: &kubeadmapi.LocalEtcd{ + ExtraArgs: map[string]string{ + "listen-client-urls": "http://1.2.3.4:2379,http://4.3.2.1:2379"}, + }, }, }, component: kubeadmconstants.Etcd, @@ -223,8 +225,10 @@ func TestEtcdProbe(t *testing.T) { name: "valid etcd probe using listen-client-urls IPv6 addresses", cfg: &kubeadmapi.MasterConfiguration{ Etcd: kubeadmapi.Etcd{ - ExtraArgs: map[string]string{ - "listen-client-urls": "http://[2001:db8::1]:2379,http://[2001:db8::2]:2379"}, + Local: &kubeadmapi.LocalEtcd{ + ExtraArgs: map[string]string{ + "listen-client-urls": "http://[2001:db8::1]:2379,http://[2001:db8::2]:2379"}, + }, }, }, component: kubeadmconstants.Etcd, @@ -239,8 +243,10 @@ func TestEtcdProbe(t *testing.T) { name: "valid IPv4 etcd probe using hostname for listen-client-urls", cfg: &kubeadmapi.MasterConfiguration{ Etcd: kubeadmapi.Etcd{ - ExtraArgs: map[string]string{ - "listen-client-urls": "http://localhost:2379"}, + Local: &kubeadmapi.LocalEtcd{ + ExtraArgs: map[string]string{ + "listen-client-urls": "http://localhost:2379"}, + }, }, }, component: kubeadmconstants.Etcd,