diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go index f27f56f0f50..7b09327d2bb 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply.go @@ -167,7 +167,7 @@ func RunApply(flags *applyFlags) error { } // Upgrade RBAC rules and addons. - if err := upgrade.PerformPostUpgradeTasks(upgradeVars.client, internalcfg); err != nil { + if err := upgrade.PerformPostUpgradeTasks(upgradeVars.client, internalcfg, flags.newK8sVersion); err != nil { return fmt.Errorf("[upgrade/postupgrade] FATAL post-upgrade error: %v", err) } diff --git a/cmd/kubeadm/app/phases/upgrade/BUILD b/cmd/kubeadm/app/phases/upgrade/BUILD index de06c565e06..6a1a225bbe7 100644 --- a/cmd/kubeadm/app/phases/upgrade/BUILD +++ b/cmd/kubeadm/app/phases/upgrade/BUILD @@ -8,6 +8,7 @@ go_library( "health.go", "policy.go", "postupgrade.go", + "postupgrade_v18_19.go", "prepull.go", "selfhosted.go", "staticpods.go", @@ -25,6 +26,7 @@ go_library( "//cmd/kubeadm/app/phases/addons/proxy:go_default_library", "//cmd/kubeadm/app/phases/bootstraptoken/clusterinfo:go_default_library", "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library", + "//cmd/kubeadm/app/phases/certs:go_default_library", "//cmd/kubeadm/app/phases/controlplane:go_default_library", "//cmd/kubeadm/app/phases/etcd:go_default_library", "//cmd/kubeadm/app/phases/selfhosting:go_default_library", @@ -64,6 +66,7 @@ go_test( srcs = [ "compute_test.go", "policy_test.go", + "postupgrade_v18_19_test.go", "prepull_test.go", "staticpods_test.go", ], @@ -73,9 +76,12 @@ go_test( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/phases/certs:go_default_library", + "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library", "//cmd/kubeadm/app/phases/controlplane:go_default_library", "//cmd/kubeadm/app/phases/etcd:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", + "//cmd/kubeadm/test:go_default_library", "//pkg/api/legacyscheme:go_default_library", "//pkg/util/version:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go index 3452413c38b..37598e3b949 100644 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go @@ -17,19 +17,24 @@ limitations under the License. package upgrade import ( + "fmt" + "k8s.io/apimachinery/pkg/util/errors" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy" "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" + certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" + "k8s.io/kubernetes/pkg/util/version" ) // PerformPostUpgradeTasks runs nearly the same functions as 'kubeadm init' would do // Note that the markmaster phase is left out, not needed, and no token is created as that doesn't belong to the upgrade -func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterConfiguration) error { +func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterConfiguration, newK8sVer *version.Version) error { errs := []error{} // Upload currently used configuration to the cluster @@ -64,6 +69,21 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterC errs = append(errs, err) } + certAndKeyDir := kubeadmapiext.DefaultCertificatesDir + shouldBackup, err := shouldBackupAPIServerCertAndKey(certAndKeyDir, newK8sVer) + // Don't fail the upgrade phase if failing to determine to backup kube-apiserver cert and key. + if err != nil { + fmt.Printf("[postupgrade] WARNING: failed to determine to backup kube-apiserver cert and key: %v", err) + } else if shouldBackup { + // Don't fail the upgrade phase if failing to backup kube-apiserver cert and key. + if err := backupAPIServerCertAndKey(certAndKeyDir); err != nil { + fmt.Printf("[postupgrade] WARNING: failed to backup kube-apiserver cert and key: %v", err) + } + if err := certsphase.CreateAPIServerCertAndKeyFiles(cfg); err != nil { + errs = append(errs, err) + } + } + // Upgrade kube-dns and kube-proxy if err := dns.EnsureDNSAddon(cfg, client); err != nil { errs = append(errs, err) diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade_v18_19.go b/cmd/kubeadm/app/phases/upgrade/postupgrade_v18_19.go new file mode 100644 index 00000000000..8243a8100fa --- /dev/null +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade_v18_19.go @@ -0,0 +1,104 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "time" + + "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/pkg/util/version" +) + +var v190 = version.MustParseSemantic("v1.9.0") +var expiry = 180 * 24 * time.Hour + +// backupAPIServerCertAndKey backups the old cert and key of kube-apiserver to a specified directory. +func backupAPIServerCertAndKey(certAndKeyDir string) error { + subDir := filepath.Join(certAndKeyDir, "expired") + if err := os.Mkdir(subDir, 0766); err != nil { + return fmt.Errorf("failed to created backup directory %s: %v", subDir, err) + } + + filesToMove := map[string]string{ + filepath.Join(certAndKeyDir, constants.APIServerCertName): filepath.Join(subDir, constants.APIServerCertName), + filepath.Join(certAndKeyDir, constants.APIServerKeyName): filepath.Join(subDir, constants.APIServerKeyName), + } + return moveFiles(filesToMove) +} + +// moveFiles moves files from one directory to another. +func moveFiles(files map[string]string) error { + filesToRecover := map[string]string{} + for from, to := range files { + if err := os.Rename(from, to); err != nil { + return rollbackFiles(filesToRecover, err) + } + filesToRecover[to] = from + } + return nil +} + +// rollbackFiles moves the files back to the original directory. +func rollbackFiles(files map[string]string, originalErr error) error { + errs := []error{originalErr} + for from, to := range files { + if err := os.Rename(from, to); err != nil { + errs = append(errs, err) + } + } + return fmt.Errorf("couldn't move these files: %v. Got errors: %v", files, errors.NewAggregate(errs)) +} + +// shouldBackupAPIServerCertAndKey check if the new k8s version is at least 1.9.0 +// and kube-apiserver will be expired in 60 days. +func shouldBackupAPIServerCertAndKey(certAndKeyDir string, newK8sVer *version.Version) (bool, error) { + if newK8sVer.LessThan(v190) { + return false, nil + } + + apiServerCert := filepath.Join(certAndKeyDir, constants.APIServerCertName) + data, err := ioutil.ReadFile(apiServerCert) + if err != nil { + return false, fmt.Errorf("failed to read kube-apiserver certificate from disk: %v", err) + } + + block, _ := pem.Decode(data) + if block == nil { + return false, fmt.Errorf("expected the kube-apiserver certificate to be PEM encoded") + } + + certs, err := x509.ParseCertificates(block.Bytes) + if err != nil { + return false, fmt.Errorf("unable to parse certificate data: %v", err) + } + if len(certs) == 0 { + return false, fmt.Errorf("no certificate data found") + } + + if time.Now().Sub(certs[0].NotBefore) > expiry { + return true, nil + } + + return false, nil +} diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade_v18_19_test.go b/cmd/kubeadm/app/phases/upgrade/postupgrade_v18_19_test.go new file mode 100644 index 00000000000..d45e744f415 --- /dev/null +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade_v18_19_test.go @@ -0,0 +1,192 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" + certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" + testutil "k8s.io/kubernetes/cmd/kubeadm/test" + "k8s.io/kubernetes/pkg/util/version" +) + +func TestBackupAPIServerCertAndKey(t *testing.T) { + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + os.Chmod(tmpdir, 0766) + + certPath := filepath.Join(tmpdir, constants.APIServerCertName) + certFile, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + if err != nil { + t.Fatalf("Failed to create cert file %s: %v", certPath, err) + } + defer certFile.Close() + + keyPath := filepath.Join(tmpdir, constants.APIServerKeyName) + keyFile, err := os.OpenFile(keyPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + if err != nil { + t.Fatalf("Failed to create key file %s: %v", keyPath, err) + } + defer keyFile.Close() + + if err := backupAPIServerCertAndKey(tmpdir); err != nil { + t.Fatalf("Failed to backup cert and key in dir %s: %v", tmpdir, err) + } +} + +func TestMoveFiles(t *testing.T) { + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + os.Chmod(tmpdir, 0766) + + certPath := filepath.Join(tmpdir, constants.APIServerCertName) + certFile, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + if err != nil { + t.Fatalf("Failed to create cert file %s: %v", certPath, err) + } + defer certFile.Close() + + keyPath := filepath.Join(tmpdir, constants.APIServerKeyName) + keyFile, err := os.OpenFile(keyPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + if err != nil { + t.Fatalf("Failed to create key file %s: %v", keyPath, err) + } + defer keyFile.Close() + + subDir := filepath.Join(tmpdir, "expired") + if err := os.Mkdir(subDir, 0766); err != nil { + t.Fatalf("Failed to create backup directory %s: %v", subDir, err) + } + + filesToMove := map[string]string{ + filepath.Join(tmpdir, constants.APIServerCertName): filepath.Join(subDir, constants.APIServerCertName), + filepath.Join(tmpdir, constants.APIServerKeyName): filepath.Join(subDir, constants.APIServerKeyName), + } + + if err := moveFiles(filesToMove); err != nil { + t.Fatalf("Failed to move files %v: %v", filesToMove, err) + } +} + +func TestRollbackFiles(t *testing.T) { + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + os.Chmod(tmpdir, 0766) + + subDir := filepath.Join(tmpdir, "expired") + if err := os.Mkdir(subDir, 0766); err != nil { + t.Fatalf("Failed to create backup directory %s: %v", subDir, err) + } + + certPath := filepath.Join(subDir, constants.APIServerCertName) + certFile, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + if err != nil { + t.Fatalf("Failed to create cert file %s: %v", certPath, err) + } + defer certFile.Close() + + keyPath := filepath.Join(subDir, constants.APIServerKeyName) + keyFile, err := os.OpenFile(keyPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + if err != nil { + t.Fatalf("Failed to create key file %s: %v", keyPath, err) + } + defer keyFile.Close() + + filesToRollBack := map[string]string{ + filepath.Join(subDir, constants.APIServerCertName): filepath.Join(tmpdir, constants.APIServerCertName), + filepath.Join(subDir, constants.APIServerKeyName): filepath.Join(tmpdir, constants.APIServerKeyName), + } + + errString := "there are files need roll back" + originalErr := errors.New(errString) + err = rollbackFiles(filesToRollBack, originalErr) + if err == nil { + t.Fatalf("Expected error contains %q, got nil", errString) + } + if !strings.Contains(err.Error(), errString) { + t.Fatalf("Expected error contains %q, got %v", errString, err) + } +} + +func TestShouldBackupAPIServerCertAndKey(t *testing.T) { + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, + NodeName: "test-node", + } + caCert, caKey, err := certsphase.NewCACertAndKey() + if err != nil { + t.Fatalf("failed creation of ca cert and key: %v", err) + } + + for desc, test := range map[string]struct { + adjustedExpiry time.Duration + k8sVersion *version.Version + expected bool + }{ + "1.8 version doesn't need to backup": { + k8sVersion: version.MustParseSemantic("v1.8.0"), + expected: false, + }, + "1.9 version with cert not older than 180 days doesn't needs to backup": { + k8sVersion: version.MustParseSemantic("v1.9.0"), + expected: false, + }, + "1.9 version with cert older than 180 days need to backup": { + adjustedExpiry: expiry + 100*time.Hour, + k8sVersion: version.MustParseSemantic("v1.9.0"), + expected: true, + }, + } { + caCert.NotBefore = caCert.NotBefore.Add(-test.adjustedExpiry).UTC() + apiCert, apiKey, err := certsphase.NewAPIServerCertAndKey(cfg, caCert, caKey) + if err != nil { + t.Fatalf("Test %s: failed creation of cert and key: %v", desc, err) + } + + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + if err := pkiutil.WriteCertAndKey(tmpdir, constants.APIServerCertAndKeyBaseName, apiCert, apiKey); err != nil { + t.Fatalf("Test %s: failure while saving %s certificate and key: %v", desc, constants.APIServerCertAndKeyBaseName, err) + } + + certAndKey := []string{filepath.Join(tmpdir, constants.APIServerCertName), filepath.Join(tmpdir, constants.APIServerKeyName)} + for _, path := range certAndKey { + if _, err := os.Stat(path); os.IsNotExist(err) { + t.Fatalf("Test %s: %s not exist: %v", desc, path, err) + } + } + + shouldBackup, err := shouldBackupAPIServerCertAndKey(tmpdir, test.k8sVersion) + if err != nil { + t.Fatalf("Test %s: failed to check shouldBackupAPIServerCertAndKey: %v", desc, err) + } + + if shouldBackup != test.expected { + t.Fatalf("Test %s: shouldBackupAPIServerCertAndKey expected %v, got %v", desc, test.expected, shouldBackup) + } + } +}