Merge pull request #121305 from neolit123/1.29-super-admin-conf

kubeadm: add support for separate super-admin.conf kubeconfig file

cmd/kubeadm/app/cmd/certs_test.go

@@ -134,6 +134,7 @@ func TestRunRenewCommands(t *testing.T) {
 	// Generate all the kubeconfig files with embedded certs
 	for _, kubeConfig := range []string{
 		kubeadmconstants.AdminKubeConfigFileName,
+		kubeadmconstants.SuperAdminKubeConfigFileName,
 		kubeadmconstants.SchedulerKubeConfigFileName,
 		kubeadmconstants.ControllerManagerKubeConfigFileName,
 	} {
@@ -162,6 +163,7 @@ func TestRunRenewCommands(t *testing.T) {
 			},
 			KubeconfigFiles: []string{
 				kubeadmconstants.AdminKubeConfigFileName,
+				kubeadmconstants.SuperAdminKubeConfigFileName,
 				kubeadmconstants.SchedulerKubeConfigFileName,
 				kubeadmconstants.ControllerManagerKubeConfigFileName,
 			},
@@ -214,6 +216,12 @@ func TestRunRenewCommands(t *testing.T) {
 				kubeadmconstants.AdminKubeConfigFileName,
 			},
 		},
+		{
+			command: "super-admin.conf",
+			KubeconfigFiles: []string{
+				kubeadmconstants.SuperAdminKubeConfigFileName,
+			},
+		},
 		{
 			command: "scheduler.conf",
 			KubeconfigFiles: []string{

cmd/kubeadm/app/cmd/init.go

@@ -72,20 +72,21 @@ var _ phases.InitData = &initData{}
 // initData defines all the runtime information used when running the kubeadm init workflow;
 // this data is shared across all the phases that are included in the workflow.
 type initData struct {
-	cfg                     *kubeadmapi.InitConfiguration
-	skipTokenPrint          bool
-	dryRun                  bool
-	kubeconfigDir           string
-	kubeconfigPath          string
-	ignorePreflightErrors   sets.Set[string]
-	certificatesDir         string
-	dryRunDir               string
-	externalCA              bool
-	client                  clientset.Interface
-	outputWriter            io.Writer
-	uploadCerts             bool
-	skipCertificateKeyPrint bool
-	patchesDir              string
+	cfg                         *kubeadmapi.InitConfiguration
+	skipTokenPrint              bool
+	dryRun                      bool
+	kubeconfigDir               string
+	kubeconfigPath              string
+	ignorePreflightErrors       sets.Set[string]
+	certificatesDir             string
+	dryRunDir                   string
+	externalCA                  bool
+	client                      clientset.Interface
+	outputWriter                io.Writer
+	uploadCerts                 bool
+	skipCertificateKeyPrint     bool
+	patchesDir                  string
+	adminKubeConfigBootstrapped bool
 }
 
 // newCmdInit returns "kubeadm init" command.
@@ -495,12 +496,22 @@ func (d *initData) Client() (clientset.Interface, error) {
 			// If we're dry-running, we should create a faked client that answers some GETs in order to be able to do the full init flow and just logs the rest of requests
 			dryRunGetter := apiclient.NewInitDryRunGetter(d.cfg.NodeRegistration.Name, svcSubnetCIDR.String())
 			d.client = apiclient.NewDryRunClient(dryRunGetter, os.Stdout)
-		} else {
-			// If we're acting for real, we should create a connection to the API server and wait for it to come up
+		} else { // Use a real client
 			var err error
-			d.client, err = kubeconfigutil.ClientSetFromFile(d.KubeConfigPath())
-			if err != nil {
-				return nil, err
+			if !d.adminKubeConfigBootstrapped {
+				// Call EnsureAdminClusterRoleBinding() to obtain a working client from admin.conf.
+				d.client, err = kubeconfigphase.EnsureAdminClusterRoleBinding(kubeadmconstants.KubernetesDir, nil)
+				if err != nil {
+					return nil, errors.Wrapf(err, "could not bootstrap the admin user in file %s", kubeadmconstants.AdminKubeConfigFileName)
+				}
+				d.adminKubeConfigBootstrapped = true
+			} else {
+				// In case adminKubeConfigBootstrapped is already set just return a client from the default
+				// kubeconfig location.
+				d.client, err = kubeconfigutil.ClientSetFromFile(d.KubeConfigPath())
+				if err != nil {
+					return nil, err
+				}
 			}
 		}
 	}

cmd/kubeadm/app/cmd/phases/init/kubeconfig.go

@@ -41,6 +41,11 @@ var (
 			short: "Generate a kubeconfig file for the admin to use and for kubeadm itself",
 			long:  "Generate the kubeconfig file for the admin and for kubeadm itself, and save it to %s file.",
 		},
+		kubeadmconstants.SuperAdminKubeConfigFileName: {
+			name:  "super-admin",
+			short: "Generate a kubeconfig file for the super-admin",
+			long:  "Generate a kubeconfig file for the super-admin, and save it to %s file.",
+		},
 		kubeadmconstants.KubeletKubeConfigFileName: {
 			name:  "kubelet",
 			short: "Generate a kubeconfig file for the kubelet to use *only* for cluster bootstrapping purposes",
@@ -77,6 +82,7 @@ func NewKubeConfigPhase() workflow.Phase {
 				RunAllSiblings: true,
 			},
 			NewKubeConfigFilePhase(kubeadmconstants.AdminKubeConfigFileName),
+			NewKubeConfigFilePhase(kubeadmconstants.SuperAdminKubeConfigFileName),
 			NewKubeConfigFilePhase(kubeadmconstants.KubeletKubeConfigFileName),
 			NewKubeConfigFilePhase(kubeadmconstants.ControllerManagerKubeConfigFileName),
 			NewKubeConfigFilePhase(kubeadmconstants.SchedulerKubeConfigFileName),

cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go

@@ -169,6 +169,7 @@ func resetConfigDir(configPathDir string, dirsToClean []string, isDryRun bool) {
 
 	filesToClean := []string{
 		filepath.Join(configPathDir, kubeadmconstants.AdminKubeConfigFileName),
+		filepath.Join(configPathDir, kubeadmconstants.SuperAdminKubeConfigFileName),
 		filepath.Join(configPathDir, kubeadmconstants.KubeletKubeConfigFileName),
 		filepath.Join(configPathDir, kubeadmconstants.KubeletBootstrapKubeConfigFileName),
 		filepath.Join(configPathDir, kubeadmconstants.ControllerManagerKubeConfigFileName),

cmd/kubeadm/app/cmd/phases/reset/cleanupnode_test.go

@@ -68,6 +68,7 @@ func TestConfigDirCleaner(t *testing.T) {
 				"manifests/kube-apiserver.yaml",
 				"pki/ca.pem",
 				kubeadmconstants.AdminKubeConfigFileName,
+				kubeadmconstants.SuperAdminKubeConfigFileName,
 				kubeadmconstants.KubeletKubeConfigFileName,
 			},
 			verifyExists: []string{

cmd/kubeadm/app/constants/constants.go

@@ -146,8 +146,11 @@ const (
 	// FrontProxyClientCertCommonName defines front proxy certificate common name
 	FrontProxyClientCertCommonName = "front-proxy-client" //used as subject.commonname attribute (CN)
 
-	// AdminKubeConfigFileName defines name for the kubeconfig aimed to be used by the superuser/admin of the cluster
+	// AdminKubeConfigFileName defines name for the kubeconfig aimed to be used by the admin of the cluster
 	AdminKubeConfigFileName = "admin.conf"
+	// SuperAdminKubeConfigFileName defines name for the kubeconfig aimed to be used by the super-admin of the cluster
+	SuperAdminKubeConfigFileName = "super-admin.conf"
+
 	// KubeletBootstrapKubeConfigFileName defines the file name for the kubeconfig that the kubelet will use to do
 	// the TLS bootstrap to get itself an unique credential
 	KubeletBootstrapKubeConfigFileName = "bootstrap-kubelet.conf"
@@ -201,6 +204,10 @@ const (
 	NodeAutoApproveBootstrapClusterRoleBinding = "kubeadm:node-autoapprove-bootstrap"
 	// NodeAutoApproveCertificateRotationClusterRoleBinding defines name of the ClusterRoleBinding that makes the csrapprover approve node auto rotated CSRs
 	NodeAutoApproveCertificateRotationClusterRoleBinding = "kubeadm:node-autoapprove-certificate-rotation"
+	// ClusterAdminsGroupAndClusterRoleBinding is the name of the Group used for kubeadm generated cluster
+	// admin credentials and the name of the ClusterRoleBinding that binds the same Group to the "cluster-admin"
+	// built-in ClusterRole.
+	ClusterAdminsGroupAndClusterRoleBinding = "kubeadm:cluster-admins"
 
 	// APICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation
 	APICallRetryInterval = 500 * time.Millisecond
@@ -567,6 +574,11 @@ func GetAdminKubeConfigPath() string {
 	return filepath.Join(KubernetesDir, AdminKubeConfigFileName)
 }
 
+// GetSuperAdminKubeConfigPath returns the location on the disk where admin kubeconfig is located by default
+func GetSuperAdminKubeConfigPath() string {
+	return filepath.Join(KubernetesDir, SuperAdminKubeConfigFileName)
+}
+
 // GetBootstrapKubeletKubeConfigPath returns the location on the disk where bootstrap kubelet kubeconfig is located by default
 func GetBootstrapKubeletKubeConfigPath() string {
 	return filepath.Join(KubernetesDir, KubeletBootstrapKubeConfigFileName)

cmd/kubeadm/app/constants/constants_test.go

@@ -50,6 +50,19 @@ func TestGetAdminKubeConfigPath(t *testing.T) {
 	}
 }
 
+func TestGetSuperAdminKubeConfigPath(t *testing.T) {
+	expected := filepath.Join(KubernetesDir, SuperAdminKubeConfigFileName)
+	actual := GetSuperAdminKubeConfigPath()
+
+	if actual != expected {
+		t.Errorf(
+			"failed GetSuperAdminKubeConfigPath:\n\texpected: %s\n\t  actual: %s",
+			expected,
+			actual,
+		)
+	}
+}
+
 func TestGetBootstrapKubeletKubeConfigPath(t *testing.T) {
 	expected := filepath.FromSlash("/etc/kubernetes/bootstrap-kubelet.conf")
 	actual := GetBootstrapKubeletKubeConfigPath()

cmd/kubeadm/app/phases/certs/renewal/manager.go

@@ -141,6 +141,10 @@ func NewManager(cfg *kubeadmapi.ClusterConfiguration, kubernetesDir string) (*Ma
 			longName: "certificate embedded in the kubeconfig file for the admin to use and for kubeadm itself",
 			fileName: kubeadmconstants.AdminKubeConfigFileName,
 		},
+		{
+			longName: "certificate embedded in the kubeconfig file for the super-admin",
+			fileName: kubeadmconstants.SuperAdminKubeConfigFileName,
+		},
 		{
 			longName: "certificate embedded in the kubeconfig file for the controller manager to use",
 			fileName: kubeadmconstants.ControllerManagerKubeConfigFileName,

cmd/kubeadm/app/phases/certs/renewal/manager_test.go

@@ -64,7 +64,7 @@ func TestNewManager(t *testing.T) {
 		{
 			name:                 "cluster with local etcd",
 			cfg:                  &kubeadmapi.ClusterConfiguration{},
-			expectedCertificates: 10, //[admin apiserver apiserver-etcd-client apiserver-kubelet-client controller-manager etcd/healthcheck-client etcd/peer etcd/server front-proxy-client scheduler]
+			expectedCertificates: 11, // [admin super-admin apiserver apiserver-etcd-client apiserver-kubelet-client controller-manager etcd/healthcheck-client etcd/peer etcd/server front-proxy-client scheduler]
 		},
 		{
 			name: "cluster with external etcd",
@@ -73,7 +73,7 @@ func TestNewManager(t *testing.T) {
 					External: &kubeadmapi.ExternalEtcd{},
 				},
 			},
-			expectedCertificates: 6, // [admin apiserver apiserver-kubelet-client controller-manager front-proxy-client scheduler]
+			expectedCertificates: 7, // [admin super-admin apiserver apiserver-kubelet-client controller-manager front-proxy-client scheduler]
 		},
 	}
 

cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go

@@ -18,6 +18,7 @@ package kubeconfig
 
 import (
 	"bytes"
+	"context"
 	"crypto"
 	"crypto/x509"
 	"fmt"
@@ -28,6 +29,11 @@ import (
 
 	"github.com/pkg/errors"
 
+	rbac "k8s.io/api/rbac/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	certutil "k8s.io/client-go/util/cert"
@@ -97,6 +103,9 @@ func CreateJoinControlPlaneKubeConfigFiles(outDir string, cfg *kubeadmapi.InitCo
 	return nil
 }
 
+// CreateKubeConfigFileFunc defines a function type used for creating kubeconfig files.
+type CreateKubeConfigFileFunc func(string, string, *kubeadmapi.InitConfiguration) error
+
 // CreateKubeConfigFile creates a kubeconfig file.
 // If the kubeconfig file already exists, it is used only if evaluated equal; otherwise an error is returned.
 func CreateKubeConfigFile(kubeConfigFileName string, outDir string, cfg *kubeadmapi.InitConfiguration) error {
@@ -407,6 +416,7 @@ func ValidateKubeconfigsForExternalCA(outDir string, cfg *kubeadmapi.InitConfigu
 	validationConfigCPE := kubeconfigutil.CreateBasic(controlPlaneEndpoint, "dummy", "dummy", pkiutil.EncodeCertPEM(caCert))
 	kubeConfigFileNamesCPE := []string{
 		kubeadmconstants.AdminKubeConfigFileName,
+		kubeadmconstants.SuperAdminKubeConfigFileName,
 		kubeadmconstants.KubeletKubeConfigFileName,
 	}
 
@@ -433,6 +443,13 @@ func getKubeConfigSpecsBase(cfg *kubeadmapi.InitConfiguration) (map[string]*kube
 		kubeadmconstants.AdminKubeConfigFileName: {
 			APIServer:  controlPlaneEndpoint,
 			ClientName: "kubernetes-admin",
+			ClientCertAuth: &clientCertAuth{
+				Organizations: []string{kubeadmconstants.ClusterAdminsGroupAndClusterRoleBinding},
+			},
+		},
+		kubeadmconstants.SuperAdminKubeConfigFileName: {
+			APIServer:  controlPlaneEndpoint,
+			ClientName: "kubernetes-super-admin",
 			ClientCertAuth: &clientCertAuth{
 				Organizations: []string{kubeadmconstants.SystemPrivilegedGroup},
 			},
@@ -541,3 +558,140 @@ func CreateDefaultKubeConfigsAndCSRFiles(out io.Writer, kubeConfigDir string, ku
 	}
 	return nil
 }
+
+// EnsureRBACFunc defines a function type that can be passed to EnsureAdminClusterRoleBinding().
+type EnsureRBACFunc func(context.Context, clientset.Interface, clientset.Interface, time.Duration, time.Duration) (clientset.Interface, error)
+
+// EnsureAdminClusterRoleBinding constructs a client from admin.conf and optionally
+// constructs a client from super-admin.conf if the file exists. It then proceeds
+// to pass the clients to EnsureAdminClusterRoleBindingImpl. The function returns a
+// usable client from admin.conf with RBAC properly constructed or an error.
+func EnsureAdminClusterRoleBinding(outDir string, ensureRBACFunc EnsureRBACFunc) (clientset.Interface, error) {
+	var (
+		err                           error
+		adminClient, superAdminClient clientset.Interface
+	)
+
+	// Create a client from admin.conf.
+	adminClient, err = kubeconfigutil.ClientSetFromFile(filepath.Join(outDir, kubeadmconstants.AdminKubeConfigFileName))
+	if err != nil {
+		return nil, err
+	}
+
+	// Create a client from super-admin.conf.
+	superAdminPath := filepath.Join(outDir, kubeadmconstants.SuperAdminKubeConfigFileName)
+	if _, err := os.Stat(superAdminPath); err == nil {
+		superAdminClient, err = kubeconfigutil.ClientSetFromFile(superAdminPath)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if ensureRBACFunc == nil {
+		ensureRBACFunc = EnsureAdminClusterRoleBindingImpl
+	}
+
+	ctx := context.Background()
+	return ensureRBACFunc(
+		ctx, adminClient, superAdminClient, kubeadmconstants.APICallRetryInterval, kubeadmconstants.APICallWithWriteTimeout)
+}
+
+// EnsureAdminClusterRoleBindingImpl first attempts to see if the ClusterRoleBinding
+// kubeadm:cluster-admins exists by using adminClient. If it already exists,
+// it would mean the adminClient is usable. If it does not, attempt to create
+// the ClusterRoleBinding by using superAdminClient.
+func EnsureAdminClusterRoleBindingImpl(ctx context.Context, adminClient, superAdminClient clientset.Interface,
+	retryInterval, retryTimeout time.Duration) (clientset.Interface, error) {
+
+	klog.V(1).Infof("ensuring that the ClusterRoleBinding for the %s Group exists",
+		kubeadmconstants.ClusterAdminsGroupAndClusterRoleBinding)
+
+	var (
+		err, lastError     error
+		crbResult          *rbac.ClusterRoleBinding
+		clusterRoleBinding = &rbac.ClusterRoleBinding{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: kubeadmconstants.ClusterAdminsGroupAndClusterRoleBinding,
+			},
+			RoleRef: rbac.RoleRef{
+				APIGroup: rbac.GroupName,
+				Kind:     "ClusterRole",
+				Name:     "cluster-admin",
+			},
+			Subjects: []rbac.Subject{
+				{
+					Kind: rbac.GroupKind,
+					Name: kubeadmconstants.ClusterAdminsGroupAndClusterRoleBinding,
+				},
+			},
+		}
+	)
+
+	// First try to create the CRB with the admin.conf client. If the admin.conf contains a User bound
+	// to the built-in super-user group, this will pass. In all other cases an error will be returned.
+	// The poll here is required to ensure the API server is reachable during "kubeadm init" workflows.
+	err = wait.PollUntilContextTimeout(
+		ctx,
+		retryInterval,
+		retryTimeout,
+		true, func(ctx context.Context) (bool, error) {
+			if crbResult, err = adminClient.RbacV1().ClusterRoleBindings().Create(
+				ctx,
+				clusterRoleBinding,
+				metav1.CreateOptions{},
+			); err != nil {
+				if apierrors.IsForbidden(err) {
+					// If it encounters a forbidden error this means that the API server was reached
+					// but the CRB is missing - i.e. the admin.conf user does not have permissions
+					// to create its own permission RBAC yet.
+					//
+					// When a "create" call is made, but the resource is forbidden, a non-nil
+					// CRB will still be returned. Return true here, but update "crbResult" to nil,
+					// to ensure that the process continues with super-admin.conf.
+					crbResult = nil
+					return true, nil
+				} else if apierrors.IsAlreadyExists(err) {
+					// If the CRB exists it means the admin.conf already has the right
+					// permissions; return.
+					return true, nil
+				} else {
+					// Retry on any other error type.
+					lastError = errors.Wrap(err, "unable to create ClusterRoleBinding")
+					return false, nil
+				}
+			}
+			return true, nil
+		})
+	if err != nil {
+		return nil, lastError
+	}
+
+	// The CRB exists; return the admin.conf client.
+	if crbResult != nil {
+		return adminClient, nil
+	}
+
+	// If the superAdminClient is nil at this point we cannot proceed creating the CRB; return an error.
+	if superAdminClient == nil {
+		return nil, errors.Errorf("the ClusterRoleBinding for the %s Group is missing but there is no %s to create it",
+			kubeadmconstants.ClusterAdminsGroupAndClusterRoleBinding,
+			kubeadmconstants.SuperAdminKubeConfigFileName)
+	}
+
+	// Create the ClusterRoleBinding with the super-admin.conf client.
+	klog.V(1).Infof("creating the ClusterRoleBinding for the %s Group by using %s",
+		kubeadmconstants.ClusterAdminsGroupAndClusterRoleBinding,
+		kubeadmconstants.SuperAdminKubeConfigFileName)
+
+	if _, err := superAdminClient.RbacV1().ClusterRoleBindings().Create(
+		ctx,
+		clusterRoleBinding,
+		metav1.CreateOptions{},
+	); err != nil {
+		return nil, errors.Wrapf(err, "unable to create the %s ClusterRoleBinding",
+			kubeadmconstants.ClusterAdminsGroupAndClusterRoleBinding)
+	}
+
+	// Once the CRB is in place, start using the admin.conf client.
+	return adminClient, nil
+}

cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go

@@ -18,6 +18,7 @@ package kubeconfig
 
 import (
 	"bytes"
+	"context"
 	"crypto"
 	"crypto/x509"
 	"fmt"
@@ -26,14 +27,24 @@ import (
 	"path/filepath"
 	"reflect"
 	"testing"
+	"time"
 
 	"github.com/lithammer/dedent"
+	"github.com/pkg/errors"
 
+	rbac "k8s.io/api/rbac/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	clientset "k8s.io/client-go/kubernetes"
+	clientsetfake "k8s.io/client-go/kubernetes/fake"
+	clientgotesting "k8s.io/client-go/testing"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	certstestutil "k8s.io/kubernetes/cmd/kubeadm/app/util/certs"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
@@ -119,6 +130,11 @@ func TestGetKubeConfigSpecs(t *testing.T) {
 			{
 				kubeConfigFile: kubeadmconstants.AdminKubeConfigFileName,
 				clientName:     "kubernetes-admin",
+				organizations:  []string{kubeadmconstants.ClusterAdminsGroupAndClusterRoleBinding},
+			},
+			{
+				kubeConfigFile: kubeadmconstants.SuperAdminKubeConfigFileName,
+				clientName:     "kubernetes-super-admin",
 				organizations:  []string{kubeadmconstants.SystemPrivilegedGroup},
 			},
 			{
@@ -174,7 +190,7 @@ func TestGetKubeConfigSpecs(t *testing.T) {
 				}
 
 				switch assertion.kubeConfigFile {
-				case kubeadmconstants.AdminKubeConfigFileName, kubeadmconstants.KubeletKubeConfigFileName:
+				case kubeadmconstants.AdminKubeConfigFileName, kubeadmconstants.SuperAdminKubeConfigFileName, kubeadmconstants.KubeletKubeConfigFileName:
 					if spec.APIServer != controlPlaneEndpoint {
 						t.Errorf("expected getKubeConfigSpecs for %s to set cfg.APIServer to %s, got %s",
 							assertion.kubeConfigFile, controlPlaneEndpoint, spec.APIServer)
@@ -615,8 +631,9 @@ func TestValidateKubeconfigsForExternalCA(t *testing.T) {
 		},
 		"some files don't exist": {
 			filesToWrite: map[string]*clientcmdapi.Config{
-				kubeadmconstants.AdminKubeConfigFileName:   config,
-				kubeadmconstants.KubeletKubeConfigFileName: config,
+				kubeadmconstants.AdminKubeConfigFileName:      config,
+				kubeadmconstants.SuperAdminKubeConfigFileName: config,
+				kubeadmconstants.KubeletKubeConfigFileName:    config,
 			},
 			initConfig:    initConfig,
 			expectedError: true,
@@ -624,6 +641,7 @@ func TestValidateKubeconfigsForExternalCA(t *testing.T) {
 		"some files have invalid CA": {
 			filesToWrite: map[string]*clientcmdapi.Config{
 				kubeadmconstants.AdminKubeConfigFileName:             config,
+				kubeadmconstants.SuperAdminKubeConfigFileName:        config,
 				kubeadmconstants.KubeletKubeConfigFileName:           config,
 				kubeadmconstants.ControllerManagerKubeConfigFileName: configWithAnotherClusterCa,
 				kubeadmconstants.SchedulerKubeConfigFileName:         config,
@@ -634,6 +652,7 @@ func TestValidateKubeconfigsForExternalCA(t *testing.T) {
 		"some files have a different Server URL": {
 			filesToWrite: map[string]*clientcmdapi.Config{
 				kubeadmconstants.AdminKubeConfigFileName:             config,
+				kubeadmconstants.SuperAdminKubeConfigFileName:        config,
 				kubeadmconstants.KubeletKubeConfigFileName:           config,
 				kubeadmconstants.ControllerManagerKubeConfigFileName: config,
 				kubeadmconstants.SchedulerKubeConfigFileName:         configWithAnotherServerURL,
@@ -643,6 +662,7 @@ func TestValidateKubeconfigsForExternalCA(t *testing.T) {
 		"all files are valid": {
 			filesToWrite: map[string]*clientcmdapi.Config{
 				kubeadmconstants.AdminKubeConfigFileName:             config,
+				kubeadmconstants.SuperAdminKubeConfigFileName:        config,
 				kubeadmconstants.KubeletKubeConfigFileName:           config,
 				kubeadmconstants.ControllerManagerKubeConfigFileName: config,
 				kubeadmconstants.SchedulerKubeConfigFileName:         config,
@@ -715,3 +735,196 @@ func setupdKubeConfigWithTokenAuth(t *testing.T, caCert *x509.Certificate, APISe
 
 	return config
 }
+
+func TestEnsureAdminClusterRoleBinding(t *testing.T) {
+	dir := testutil.SetupTempDir(t)
+	defer os.RemoveAll(dir)
+
+	cfg := testutil.GetDefaultInternalConfig(t)
+	cfg.CertificatesDir = dir
+
+	ca := certsphase.KubeadmCertRootCA()
+	_, _, err := ca.CreateAsCA(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
+		name                  string
+		expectedRBACError     bool
+		expectedError         bool
+		missingAdminConf      bool
+		missingSuperAdminConf bool
+	}{
+		{
+			name: "no errors",
+		},
+		{
+			name:              "expect RBAC error",
+			expectedRBACError: true,
+			expectedError:     true,
+		},
+		{
+			name:             "admin.conf is missing",
+			missingAdminConf: true,
+			expectedError:    true,
+		},
+		{
+			name:                  "super-admin.conf is missing",
+			missingSuperAdminConf: true,
+			expectedError:         false, // The file is optional.
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			ensureRBACFunc := func(_ context.Context, adminClient clientset.Interface, superAdminClient clientset.Interface,
+				_ time.Duration, _ time.Duration) (clientset.Interface, error) {
+
+				if tc.expectedRBACError {
+					return nil, errors.New("ensureRBACFunc error")
+				}
+				return adminClient, nil
+			}
+
+			// Create the admin.conf and super-admin.conf so that EnsureAdminClusterRoleBinding
+			// can create clients from the files.
+			os.Remove(filepath.Join(dir, kubeadmconstants.AdminKubeConfigFileName))
+			if !tc.missingAdminConf {
+				if err := CreateKubeConfigFile(kubeadmconstants.AdminKubeConfigFileName, dir, cfg); err != nil {
+					t.Fatal(err)
+				}
+			}
+			os.Remove(filepath.Join(dir, kubeadmconstants.SuperAdminKubeConfigFileName))
+			if !tc.missingSuperAdminConf {
+				if err := CreateKubeConfigFile(kubeadmconstants.SuperAdminKubeConfigFileName, dir, cfg); err != nil {
+					t.Fatal(err)
+				}
+			}
+
+			client, err := EnsureAdminClusterRoleBinding(dir, ensureRBACFunc)
+			if (err != nil) != tc.expectedError {
+				t.Fatalf("expected error: %v, got: %v, error: %v", err != nil, tc.expectedError, err)
+			}
+
+			if err == nil && client == nil {
+				t.Fatal("got nil client")
+			}
+		})
+	}
+}
+
+func TestEnsureAdminClusterRoleBindingImpl(t *testing.T) {
+	tests := []struct {
+		name                  string
+		setupAdminClient      func(*clientsetfake.Clientset)
+		setupSuperAdminClient func(*clientsetfake.Clientset)
+		expectedError         bool
+	}{
+		{
+			name: "admin.conf: handle forbidden errors when the super-admin.conf client is nil",
+			setupAdminClient: func(client *clientsetfake.Clientset) {
+				client.PrependReactor("create", "clusterrolebindings", func(action clientgotesting.Action) (bool, runtime.Object, error) {
+					return true, nil, apierrors.NewForbidden(
+						schema.GroupResource{}, "name", errors.New(""))
+				})
+			},
+			expectedError: true,
+		},
+		{
+			// A "create" call against a real server can return a forbidden error and a non-nil CRB
+			name: "admin.conf: handle forbidden error and returned CRBs, when the super-admin.conf client is nil",
+			setupAdminClient: func(client *clientsetfake.Clientset) {
+				client.PrependReactor("create", "clusterrolebindings", func(action clientgotesting.Action) (bool, runtime.Object, error) {
+					return true, &rbac.ClusterRoleBinding{}, apierrors.NewForbidden(
+						schema.GroupResource{}, "name", errors.New(""))
+				})
+			},
+			expectedError: true,
+		},
+		{
+			name: "admin.conf: CRB already exists, use the admin.conf client",
+			setupAdminClient: func(client *clientsetfake.Clientset) {
+				client.PrependReactor("create", "clusterrolebindings", func(action clientgotesting.Action) (bool, runtime.Object, error) {
+					return true, nil, apierrors.NewAlreadyExists(
+						schema.GroupResource{}, "name")
+				})
+			},
+			expectedError: true,
+		},
+		{
+			name: "admin.conf: handle other errors, such as a server timeout",
+			setupAdminClient: func(client *clientsetfake.Clientset) {
+				client.PrependReactor("create", "clusterrolebindings", func(action clientgotesting.Action) (bool, runtime.Object, error) {
+					return true, nil, apierrors.NewServerTimeout(
+						schema.GroupResource{}, "create", 0)
+				})
+			},
+			expectedError: true,
+		},
+		{
+			name: "admin.conf: CRB exists, return a client from admin.conf",
+			setupAdminClient: func(client *clientsetfake.Clientset) {
+				client.PrependReactor("create", "clusterrolebindings", func(action clientgotesting.Action) (bool, runtime.Object, error) {
+					return true, &rbac.ClusterRoleBinding{}, nil
+				})
+			},
+			expectedError: false,
+		},
+		{
+			name: "super-admin.conf: error while creating CRB",
+			setupAdminClient: func(client *clientsetfake.Clientset) {
+				client.PrependReactor("create", "clusterrolebindings", func(action clientgotesting.Action) (bool, runtime.Object, error) {
+					return true, nil, apierrors.NewForbidden(
+						schema.GroupResource{}, "name", errors.New(""))
+				})
+			},
+			setupSuperAdminClient: func(client *clientsetfake.Clientset) {
+				client.PrependReactor("create", "clusterrolebindings", func(action clientgotesting.Action) (bool, runtime.Object, error) {
+					return true, nil, apierrors.NewServerTimeout(
+						schema.GroupResource{}, "create", 0)
+				})
+			},
+			expectedError: true,
+		},
+		{
+			name: "super-admin.conf: admin.conf cannot create CRB, create CRB with super-admin.conf, return client from admin.conf",
+			setupAdminClient: func(client *clientsetfake.Clientset) {
+				client.PrependReactor("create", "clusterrolebindings", func(action clientgotesting.Action) (bool, runtime.Object, error) {
+					return true, nil, apierrors.NewForbidden(
+						schema.GroupResource{}, "name", errors.New(""))
+				})
+			},
+			setupSuperAdminClient: func(client *clientsetfake.Clientset) {
+				client.PrependReactor("create", "clusterrolebindings", func(action clientgotesting.Action) (bool, runtime.Object, error) {
+					return true, &rbac.ClusterRoleBinding{}, nil
+				})
+			},
+			expectedError: false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			adminClient := clientsetfake.NewSimpleClientset()
+			tc.setupAdminClient(adminClient)
+
+			var superAdminClient clientset.Interface // ensure superAdminClient is nil by default
+			if tc.setupSuperAdminClient != nil {
+				fakeSuperAdminClient := clientsetfake.NewSimpleClientset()
+				tc.setupSuperAdminClient(fakeSuperAdminClient)
+				superAdminClient = fakeSuperAdminClient
+			}
+
+			client, err := EnsureAdminClusterRoleBindingImpl(
+				context.Background(), adminClient, superAdminClient, time.Millisecond*50, time.Millisecond*1000)
+			if (err != nil) != tc.expectedError {
+				t.Fatalf("expected error: %v, got %v, error: %v", tc.expectedError, err != nil, err)
+			}
+
+			if err == nil && client == nil {
+				t.Fatal("got nil client")
+			}
+		})
+	}
+}

cmd/kubeadm/app/phases/upgrade/postupgrade.go

@@ -40,6 +40,7 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
 	nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
+	kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
 	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
 	patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
@@ -69,6 +70,12 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitCon
 		errs = append(errs, err)
 	}
 
+	// TODO: remove this in the 1.30 release cycle:
+	// https://github.com/kubernetes/kubeadm/issues/2414
+	if err := createSuperAdminKubeConfig(cfg, kubeadmconstants.KubernetesDir, dryRun, nil, nil); err != nil {
+		errs = append(errs, err)
+	}
+
 	// Annotate the node with the crisocket information, sourced either from the InitConfiguration struct or
 	// --cri-socket.
 	// TODO: In the future we want to use something more official like NodeStatus or similar for detecting this properly
@@ -297,3 +304,64 @@ func GetKubeletDir(dryRun bool) (string, error) {
 	}
 	return kubeadmconstants.KubeletRunDirectory, nil
 }
+
+// createSuperAdminKubeConfig creates new admin.conf and super-admin.conf and then
+// ensures that the admin.conf client has RBAC permissions to be cluster-admin.
+// TODO: this code must not be present in the 1.30 release, remove it during the 1.30
+// release cycle:
+// https://github.com/kubernetes/kubeadm/issues/2414
+func createSuperAdminKubeConfig(cfg *kubeadmapi.InitConfiguration, outDir string, dryRun bool,
+	ensureRBACFunc kubeconfigphase.EnsureRBACFunc,
+	createKubeConfigFileFunc kubeconfigphase.CreateKubeConfigFileFunc) error {
+
+	if dryRun {
+		fmt.Printf("[dryrun] Would create a separate %s and RBAC for %s",
+			kubeadmconstants.SuperAdminKubeConfigFileName, kubeadmconstants.AdminKubeConfigFileName)
+		return nil
+	}
+
+	if ensureRBACFunc == nil {
+		ensureRBACFunc = kubeconfigphase.EnsureAdminClusterRoleBindingImpl
+	}
+	if createKubeConfigFileFunc == nil {
+		createKubeConfigFileFunc = kubeconfigphase.CreateKubeConfigFile
+	}
+
+	var (
+		err                  error
+		adminPath            = filepath.Join(outDir, kubeadmconstants.AdminKubeConfigFileName)
+		adminBackupPath      = adminPath + ".backup"
+		superAdminPath       = filepath.Join(outDir, kubeadmconstants.SuperAdminKubeConfigFileName)
+		superAdminBackupPath = superAdminPath + ".backup"
+	)
+
+	// Create new admin.conf and super-admin.conf.
+	// If something goes wrong, old existing files will be restored from backup as a best effort.
+
+	restoreBackup := func() {
+		_ = os.Rename(adminBackupPath, adminPath)
+		_ = os.Rename(superAdminBackupPath, superAdminPath)
+	}
+
+	_ = os.Rename(adminPath, adminBackupPath)
+	if err = createKubeConfigFileFunc(kubeadmconstants.AdminKubeConfigFileName, outDir, cfg); err != nil {
+		restoreBackup()
+		return err
+	}
+
+	_ = os.Rename(superAdminPath, superAdminBackupPath)
+	if err = createKubeConfigFileFunc(kubeadmconstants.SuperAdminKubeConfigFileName, outDir, cfg); err != nil {
+		restoreBackup()
+		return err
+	}
+
+	// Ensure the RBAC for admin.conf exists.
+	if _, err = kubeconfigphase.EnsureAdminClusterRoleBinding(outDir, ensureRBACFunc); err != nil {
+		restoreBackup()
+		return err
+	}
+
+	_ = os.Remove(adminBackupPath)
+	_ = os.Remove(superAdminBackupPath)
+	return nil
+}

cmd/kubeadm/app/phases/upgrade/postupgrade_test.go

@@ -17,18 +17,25 @@ limitations under the License.
 package upgrade
 
 import (
+	"context"
 	"os"
 	"path/filepath"
+	"reflect"
 	"regexp"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/pkg/errors"
 
 	errorsutil "k8s.io/apimachinery/pkg/util/errors"
+	clientset "k8s.io/client-go/kubernetes"
+
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
+	kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	testutil "k8s.io/kubernetes/cmd/kubeadm/test"
 )
@@ -230,3 +237,106 @@ func rollbackFiles(files map[string]string, originalErr error) error {
 	}
 	return errors.Errorf("couldn't move these files: %v. Got errors: %v", files, errorsutil.NewAggregate(errs))
 }
+
+// TODO: Remove this unit test during the 1.30 release cycle:
+// https://github.com/kubernetes/kubeadm/issues/2414
+func TestCreateSuperAdminKubeConfig(t *testing.T) {
+	dir := testutil.SetupTempDir(t)
+	defer os.RemoveAll(dir)
+
+	cfg := testutil.GetDefaultInternalConfig(t)
+	cfg.CertificatesDir = dir
+
+	ca := certsphase.KubeadmCertRootCA()
+	_, _, err := ca.CreateAsCA(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
+		name                  string
+		kubeConfigExist       bool
+		expectRBACError       bool
+		expectedError         bool
+		expectKubeConfigError bool
+	}{
+		{
+			name: "no error",
+		},
+		{
+			name:            "no error, kubeconfig files already exist",
+			kubeConfigExist: true,
+		},
+		{
+			name:            "return RBAC error",
+			expectRBACError: true,
+			expectedError:   true,
+		},
+		{
+			name:                  "return kubeconfig error",
+			expectKubeConfigError: true,
+			expectedError:         true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+
+			// Define a custom RBAC test function, so that there is no test coverage overlap.
+			ensureRBACFunc := func(context.Context, clientset.Interface, clientset.Interface,
+				time.Duration, time.Duration) (clientset.Interface, error) {
+
+				if tc.expectRBACError {
+					return nil, errors.New("ensureRBACFunc error")
+				}
+				return nil, nil
+			}
+
+			// Define a custom kubeconfig function so that we can fail at least one call.
+			kubeConfigFunc := func(a string, b string, cfg *kubeadmapi.InitConfiguration) error {
+				if tc.expectKubeConfigError {
+					return errors.New("kubeConfigFunc error")
+				}
+				return kubeconfigphase.CreateKubeConfigFile(a, b, cfg)
+			}
+
+			// If kubeConfigExist is true, pre-create the admin.conf and super-admin.conf files.
+			if tc.kubeConfigExist {
+				b := []byte("foo")
+				if err := os.WriteFile(filepath.Join(dir, constants.AdminKubeConfigFileName), b, 0644); err != nil {
+					t.Fatal(err)
+				}
+				if err := os.WriteFile(filepath.Join(dir, constants.SuperAdminKubeConfigFileName), b, 0644); err != nil {
+					t.Fatal(err)
+				}
+			}
+
+			// Call createSuperAdminKubeConfig() with a custom ensureRBACFunc().
+			err := createSuperAdminKubeConfig(cfg, dir, false, ensureRBACFunc, kubeConfigFunc)
+			if (err != nil) != tc.expectedError {
+				t.Fatalf("expected error: %v, got: %v, error: %v", err != nil, tc.expectedError, err)
+			}
+
+			// Obtain the list of files in the directory after createSuperAdminKubeConfig() is done.
+			var files []string
+			fileInfo, err := os.ReadDir(dir)
+			if err != nil {
+				t.Fatal(err)
+			}
+			for _, file := range fileInfo {
+				files = append(files, file.Name())
+			}
+
+			// Verify the expected files.
+			expectedFiles := []string{
+				constants.AdminKubeConfigFileName,
+				constants.CACertName,
+				constants.CAKeyName,
+				constants.SuperAdminKubeConfigFileName,
+			}
+			if !reflect.DeepEqual(expectedFiles, files) {
+				t.Fatalf("expected files: %v, got: %v", expectedFiles, files)
+			}
+		})
+	}
+}

cmd/kubeadm/app/phases/upgrade/staticpods.go

@@ -497,6 +497,20 @@ func StaticPodControlPlane(client clientset.Interface, waiter apiclient.Waiter,
 			// if not error, but not renewed because of external CA detected, inform the user
 			fmt.Printf("[upgrade/staticpods] External CA detected, %s certificate can't be renewed\n", constants.AdminKubeConfigFileName)
 		}
+
+		// Do the same for super-admin.conf, but only if it exists
+		if _, err := os.Stat(filepath.Join(pathMgr.KubernetesDir(), constants.SuperAdminKubeConfigFileName)); err == nil {
+			// renew the certificate embedded in the super-admin.conf file
+			renewed, err := certsRenewMgr.RenewUsingLocalCA(constants.SuperAdminKubeConfigFileName)
+			if err != nil {
+				return rollbackOldManifests(recoverManifests, errors.Wrapf(err, "failed to upgrade the %s certificates", constants.SuperAdminKubeConfigFileName), pathMgr, false)
+			}
+
+			if !renewed {
+				// if not error, but not renewed because of external CA detected, inform the user
+				fmt.Printf("[upgrade/staticpods] External CA detected, %s certificate can't be renewed\n", constants.SuperAdminKubeConfigFileName)
+			}
+		}
 	}
 
 	// Remove the temporary directories used on a best-effort (don't fail if the calls error out)

cmd/kubeadm/app/phases/upgrade/staticpods_test.go

@@ -329,9 +329,7 @@ func TestStaticPodControlPlane(t *testing.T) {
 				waitForHashChange:    nil,
 				waitForPodsWithLabel: nil,
 			},
-			moveFileFunc: func(oldPath, newPath string) error {
-				return os.Rename(oldPath, newPath)
-			},
+			moveFileFunc:         os.Rename,
 			expectedErr:          false,
 			manifestShouldChange: true,
 		},
@@ -342,9 +340,7 @@ func TestStaticPodControlPlane(t *testing.T) {
 				waitForHashChange:    nil,
 				waitForPodsWithLabel: nil,
 			},
-			moveFileFunc: func(oldPath, newPath string) error {
-				return os.Rename(oldPath, newPath)
-			},
+			moveFileFunc:         os.Rename,
 			expectedErr:          true,
 			manifestShouldChange: false,
 		},
@@ -355,9 +351,7 @@ func TestStaticPodControlPlane(t *testing.T) {
 				waitForHashChange:    errors.New("boo! failed"),
 				waitForPodsWithLabel: nil,
 			},
-			moveFileFunc: func(oldPath, newPath string) error {
-				return os.Rename(oldPath, newPath)
-			},
+			moveFileFunc:         os.Rename,
 			expectedErr:          true,
 			manifestShouldChange: false,
 		},
@@ -368,9 +362,7 @@ func TestStaticPodControlPlane(t *testing.T) {
 				waitForHashChange:    nil,
 				waitForPodsWithLabel: errors.New("boo! failed"),
 			},
-			moveFileFunc: func(oldPath, newPath string) error {
-				return os.Rename(oldPath, newPath)
-			},
+			moveFileFunc:         os.Rename,
 			expectedErr:          true,
 			manifestShouldChange: false,
 		},
@@ -432,9 +424,7 @@ func TestStaticPodControlPlane(t *testing.T) {
 				waitForHashChange:    nil,
 				waitForPodsWithLabel: nil,
 			},
-			moveFileFunc: func(oldPath, newPath string) error {
-				return os.Rename(oldPath, newPath)
-			},
+			moveFileFunc:         os.Rename,
 			skipKubeConfig:       constants.SchedulerKubeConfigFileName,
 			expectedErr:          true,
 			manifestShouldChange: false,
@@ -446,13 +436,34 @@ func TestStaticPodControlPlane(t *testing.T) {
 				waitForHashChange:    nil,
 				waitForPodsWithLabel: nil,
 			},
-			moveFileFunc: func(oldPath, newPath string) error {
-				return os.Rename(oldPath, newPath)
-			},
+			moveFileFunc:         os.Rename,
 			skipKubeConfig:       constants.AdminKubeConfigFileName,
 			expectedErr:          true,
 			manifestShouldChange: false,
 		},
+		{
+			description: "super-admin.conf is renewed if it exists",
+			waitErrsToReturn: map[string]error{
+				waitForHashes:        nil,
+				waitForHashChange:    nil,
+				waitForPodsWithLabel: nil,
+			},
+			moveFileFunc:         os.Rename,
+			expectedErr:          false,
+			manifestShouldChange: true,
+		},
+		{
+			description: "no error is thrown if super-admin.conf does not exist",
+			waitErrsToReturn: map[string]error{
+				waitForHashes:        nil,
+				waitForHashChange:    nil,
+				waitForPodsWithLabel: nil,
+			},
+			moveFileFunc:         os.Rename,
+			skipKubeConfig:       constants.SuperAdminKubeConfigFileName,
+			expectedErr:          false,
+			manifestShouldChange: true,
+		},
 	}
 
 	for i := range tests {
@@ -494,6 +505,7 @@ func TestStaticPodControlPlane(t *testing.T) {
 
 			for _, kubeConfig := range []string{
 				constants.AdminKubeConfigFileName,
+				constants.SuperAdminKubeConfigFileName,
 				constants.SchedulerKubeConfigFileName,
 				constants.ControllerManagerKubeConfigFileName,
 			} {