Add benchmark for ensureRBACPolicy

pkg/registry/rbac/rest/BUILD

@@ -3,6 +3,7 @@ package(default_visibility = ["//visibility:public"])
 load(
     "@io_bazel_rules_go//go:def.bzl",
     "go_library",
+    "go_test",
 )
 
 go_library(
@@ -39,7 +40,7 @@ go_library(
         "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library",
-        "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/typed/rbac/v1:go_default_library",
         "//staging/src/k8s.io/client-go/util/retry:go_default_library",
         "//staging/src/k8s.io/component-helpers/auth/rbac/reconciliation:go_default_library",
@@ -59,3 +60,13 @@ filegroup(
     srcs = [":package-srcs"],
     tags = ["automanaged"],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = ["storage_rbac_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//plugin/pkg/auth/authorizer/rbac/bootstrappolicy:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
+    ],
+)

pkg/registry/rbac/rest/storage_rbac.go

@@ -37,7 +37,7 @@ import (
 	"k8s.io/apiserver/pkg/registry/rest"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	serverstorage "k8s.io/apiserver/pkg/server/storage"
-	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	clientset "k8s.io/client-go/kubernetes"
 	rbacv1client "k8s.io/client-go/kubernetes/typed/rbac/v1"
 	"k8s.io/client-go/util/retry"
 	"k8s.io/component-helpers/auth/rbac/reconciliation"
@@ -174,174 +174,12 @@ func (p *PolicyData) EnsureRBACPolicy() genericapiserver.PostStartHookFunc {
 		// initializing roles is really important.  On some e2e runs, we've seen cases where etcd is down when the server
 		// starts, the roles don't initialize, and nothing works.
 		err := wait.Poll(1*time.Second, 30*time.Second, func() (done bool, err error) {
-			failedReconciliation := false
+			client, err := clientset.NewForConfig(hookContext.LoopbackClientConfig)
-
-			coreclientset, err := corev1client.NewForConfig(hookContext.LoopbackClientConfig)
 			if err != nil {
-				utilruntime.HandleError(fmt.Errorf("unable to initialize client: %v", err))
+				utilruntime.HandleError(fmt.Errorf("unable to initialize client set: %v", err))
 				return false, nil
 			}
-
+			return ensureRBACPolicy(p, client)
-			clientset, err := rbacv1client.NewForConfig(hookContext.LoopbackClientConfig)
-			if err != nil {
-				utilruntime.HandleError(fmt.Errorf("unable to initialize client: %v", err))
-				return false, nil
-			}
-			// Make sure etcd is responding before we start reconciling
-			if _, err := clientset.ClusterRoles().List(context.TODO(), metav1.ListOptions{}); err != nil {
-				utilruntime.HandleError(fmt.Errorf("unable to initialize clusterroles: %v", err))
-				return false, nil
-			}
-			if _, err := clientset.ClusterRoleBindings().List(context.TODO(), metav1.ListOptions{}); err != nil {
-				utilruntime.HandleError(fmt.Errorf("unable to initialize clusterrolebindings: %v", err))
-				return false, nil
-			}
-
-			// if the new cluster roles to aggregate do not yet exist, then we need to copy the old roles if they don't exist
-			// in new locations
-			if err := primeAggregatedClusterRoles(p.ClusterRolesToAggregate, clientset); err != nil {
-				utilruntime.HandleError(fmt.Errorf("unable to prime aggregated clusterroles: %v", err))
-				return false, nil
-			}
-
-			if err := primeSplitClusterRoleBindings(p.ClusterRoleBindingsToSplit, clientset); err != nil {
-				utilruntime.HandleError(fmt.Errorf("unable to prime split ClusterRoleBindings: %v", err))
-				return false, nil
-			}
-
-			// ensure bootstrap roles are created or reconciled
-			for _, clusterRole := range p.ClusterRoles {
-				opts := reconciliation.ReconcileRoleOptions{
-					Role:    reconciliation.ClusterRoleRuleOwner{ClusterRole: &clusterRole},
-					Client:  reconciliation.ClusterRoleModifier{Client: clientset.ClusterRoles()},
-					Confirm: true,
-				}
-				// ServiceUnavailble error is returned when the API server is blocked by storage version updates
-				err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
-					result, err := opts.Run()
-					if err != nil {
-						return err
-					}
-					switch {
-					case result.Protected && result.Operation != reconciliation.ReconcileNone:
-						klog.Warningf("skipped reconcile-protected clusterrole.%s/%s with missing permissions: %v", rbac.GroupName, clusterRole.Name, result.MissingRules)
-					case result.Operation == reconciliation.ReconcileUpdate:
-						klog.V(2).Infof("updated clusterrole.%s/%s with additional permissions: %v", rbac.GroupName, clusterRole.Name, result.MissingRules)
-					case result.Operation == reconciliation.ReconcileCreate:
-						klog.V(2).Infof("created clusterrole.%s/%s", rbac.GroupName, clusterRole.Name)
-					}
-					return nil
-				})
-				if err != nil {
-					// don't fail on failures, try to create as many as you can
-					utilruntime.HandleError(fmt.Errorf("unable to reconcile clusterrole.%s/%s: %v", rbac.GroupName, clusterRole.Name, err))
-					failedReconciliation = true
-				}
-			}
-
-			// ensure bootstrap rolebindings are created or reconciled
-			for _, clusterRoleBinding := range p.ClusterRoleBindings {
-				opts := reconciliation.ReconcileRoleBindingOptions{
-					RoleBinding: reconciliation.ClusterRoleBindingAdapter{ClusterRoleBinding: &clusterRoleBinding},
-					Client:      reconciliation.ClusterRoleBindingClientAdapter{Client: clientset.ClusterRoleBindings()},
-					Confirm:     true,
-				}
-				// ServiceUnavailble error is returned when the API server is blocked by storage version updates
-				err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
-					result, err := opts.Run()
-					if err != nil {
-						return err
-					}
-					switch {
-					case result.Protected && result.Operation != reconciliation.ReconcileNone:
-						klog.Warningf("skipped reconcile-protected clusterrolebinding.%s/%s with missing subjects: %v", rbac.GroupName, clusterRoleBinding.Name, result.MissingSubjects)
-					case result.Operation == reconciliation.ReconcileUpdate:
-						klog.V(2).Infof("updated clusterrolebinding.%s/%s with additional subjects: %v", rbac.GroupName, clusterRoleBinding.Name, result.MissingSubjects)
-					case result.Operation == reconciliation.ReconcileCreate:
-						klog.V(2).Infof("created clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name)
-					case result.Operation == reconciliation.ReconcileRecreate:
-						klog.V(2).Infof("recreated clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name)
-					}
-					return nil
-				})
-				if err != nil {
-					// don't fail on failures, try to create as many as you can
-					utilruntime.HandleError(fmt.Errorf("unable to reconcile clusterrolebinding.%s/%s: %v", rbac.GroupName, clusterRoleBinding.Name, err))
-					failedReconciliation = true
-				}
-			}
-
-			// ensure bootstrap namespaced roles are created or reconciled
-			for namespace, roles := range p.Roles {
-				for _, role := range roles {
-					opts := reconciliation.ReconcileRoleOptions{
-						Role:    reconciliation.RoleRuleOwner{Role: &role},
-						Client:  reconciliation.RoleModifier{Client: clientset, NamespaceClient: coreclientset.Namespaces()},
-						Confirm: true,
-					}
-					// ServiceUnavailble error is returned when the API server is blocked by storage version updates
-					err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
-						result, err := opts.Run()
-						if err != nil {
-							return err
-						}
-						switch {
-						case result.Protected && result.Operation != reconciliation.ReconcileNone:
-							klog.Warningf("skipped reconcile-protected role.%s/%s in %v with missing permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules)
-						case result.Operation == reconciliation.ReconcileUpdate:
-							klog.V(2).Infof("updated role.%s/%s in %v with additional permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules)
-						case result.Operation == reconciliation.ReconcileCreate:
-							klog.V(2).Infof("created role.%s/%s in %v", rbac.GroupName, role.Name, namespace)
-						}
-						return nil
-					})
-					if err != nil {
-						// don't fail on failures, try to create as many as you can
-						utilruntime.HandleError(fmt.Errorf("unable to reconcile role.%s/%s in %v: %v", rbac.GroupName, role.Name, namespace, err))
-						failedReconciliation = true
-					}
-				}
-			}
-
-			// ensure bootstrap namespaced rolebindings are created or reconciled
-			for namespace, roleBindings := range p.RoleBindings {
-				for _, roleBinding := range roleBindings {
-					opts := reconciliation.ReconcileRoleBindingOptions{
-						RoleBinding: reconciliation.RoleBindingAdapter{RoleBinding: &roleBinding},
-						Client:      reconciliation.RoleBindingClientAdapter{Client: clientset, NamespaceClient: coreclientset.Namespaces()},
-						Confirm:     true,
-					}
-					// ServiceUnavailble error is returned when the API server is blocked by storage version updates
-					err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
-						result, err := opts.Run()
-						if err != nil {
-							return err
-						}
-						switch {
-						case result.Protected && result.Operation != reconciliation.ReconcileNone:
-							klog.Warningf("skipped reconcile-protected rolebinding.%s/%s in %v with missing subjects: %v", rbac.GroupName, roleBinding.Name, namespace, result.MissingSubjects)
-						case result.Operation == reconciliation.ReconcileUpdate:
-							klog.V(2).Infof("updated rolebinding.%s/%s in %v with additional subjects: %v", rbac.GroupName, roleBinding.Name, namespace, result.MissingSubjects)
-						case result.Operation == reconciliation.ReconcileCreate:
-							klog.V(2).Infof("created rolebinding.%s/%s in %v", rbac.GroupName, roleBinding.Name, namespace)
-						case result.Operation == reconciliation.ReconcileRecreate:
-							klog.V(2).Infof("recreated rolebinding.%s/%s in %v", rbac.GroupName, roleBinding.Name, namespace)
-						}
-						return nil
-					})
-					if err != nil {
-						// don't fail on failures, try to create as many as you can
-						utilruntime.HandleError(fmt.Errorf("unable to reconcile rolebinding.%s/%s in %v: %v", rbac.GroupName, roleBinding.Name, namespace, err))
-						failedReconciliation = true
-					}
-				}
-			}
-			// failed to reconcile some objects, retry
-			if failedReconciliation {
-				return false, nil
-			}
-
-			return true, nil
 		})
 		// if we're never able to make it through initialization, kill the API server
 		if err != nil {
@@ -352,6 +190,165 @@ func (p *PolicyData) EnsureRBACPolicy() genericapiserver.PostStartHookFunc {
 	}
 }
 
+func ensureRBACPolicy(p *PolicyData, client clientset.Interface) (done bool, err error) {
+	failedReconciliation := false
+	// Make sure etcd is responding before we start reconciling
+	if _, err := client.RbacV1().ClusterRoles().List(context.TODO(), metav1.ListOptions{}); err != nil {
+		utilruntime.HandleError(fmt.Errorf("unable to initialize clusterroles: %v", err))
+		return false, nil
+	}
+	if _, err := client.RbacV1().ClusterRoleBindings().List(context.TODO(), metav1.ListOptions{}); err != nil {
+		utilruntime.HandleError(fmt.Errorf("unable to initialize clusterrolebindings: %v", err))
+		return false, nil
+	}
+
+	// if the new cluster roles to aggregate do not yet exist, then we need to copy the old roles if they don't exist
+	// in new locations
+	if err := primeAggregatedClusterRoles(p.ClusterRolesToAggregate, client.RbacV1()); err != nil {
+		utilruntime.HandleError(fmt.Errorf("unable to prime aggregated clusterroles: %v", err))
+		return false, nil
+	}
+
+	if err := primeSplitClusterRoleBindings(p.ClusterRoleBindingsToSplit, client.RbacV1()); err != nil {
+		utilruntime.HandleError(fmt.Errorf("unable to prime split ClusterRoleBindings: %v", err))
+		return false, nil
+	}
+
+	// ensure bootstrap roles are created or reconciled
+	for _, clusterRole := range p.ClusterRoles {
+		opts := reconciliation.ReconcileRoleOptions{
+			Role:    reconciliation.ClusterRoleRuleOwner{ClusterRole: &clusterRole},
+			Client:  reconciliation.ClusterRoleModifier{Client: client.RbacV1().ClusterRoles()},
+			Confirm: true,
+		}
+		// ServiceUnavailble error is returned when the API server is blocked by storage version updates
+		err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
+			result, err := opts.Run()
+			if err != nil {
+				return err
+			}
+			switch {
+			case result.Protected && result.Operation != reconciliation.ReconcileNone:
+				klog.Warningf("skipped reconcile-protected clusterrole.%s/%s with missing permissions: %v", rbac.GroupName, clusterRole.Name, result.MissingRules)
+			case result.Operation == reconciliation.ReconcileUpdate:
+				klog.V(2).Infof("updated clusterrole.%s/%s with additional permissions: %v", rbac.GroupName, clusterRole.Name, result.MissingRules)
+			case result.Operation == reconciliation.ReconcileCreate:
+				klog.V(2).Infof("created clusterrole.%s/%s", rbac.GroupName, clusterRole.Name)
+			}
+			return nil
+		})
+		if err != nil {
+			// don't fail on failures, try to create as many as you can
+			utilruntime.HandleError(fmt.Errorf("unable to reconcile clusterrole.%s/%s: %v", rbac.GroupName, clusterRole.Name, err))
+			failedReconciliation = true
+		}
+	}
+
+	// ensure bootstrap rolebindings are created or reconciled
+	for _, clusterRoleBinding := range p.ClusterRoleBindings {
+		opts := reconciliation.ReconcileRoleBindingOptions{
+			RoleBinding: reconciliation.ClusterRoleBindingAdapter{ClusterRoleBinding: &clusterRoleBinding},
+			Client:      reconciliation.ClusterRoleBindingClientAdapter{Client: client.RbacV1().ClusterRoleBindings()},
+			Confirm:     true,
+		}
+		// ServiceUnavailble error is returned when the API server is blocked by storage version updates
+		err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
+			result, err := opts.Run()
+			if err != nil {
+				return err
+			}
+			switch {
+			case result.Protected && result.Operation != reconciliation.ReconcileNone:
+				klog.Warningf("skipped reconcile-protected clusterrolebinding.%s/%s with missing subjects: %v", rbac.GroupName, clusterRoleBinding.Name, result.MissingSubjects)
+			case result.Operation == reconciliation.ReconcileUpdate:
+				klog.V(2).Infof("updated clusterrolebinding.%s/%s with additional subjects: %v", rbac.GroupName, clusterRoleBinding.Name, result.MissingSubjects)
+			case result.Operation == reconciliation.ReconcileCreate:
+				klog.V(2).Infof("created clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name)
+			case result.Operation == reconciliation.ReconcileRecreate:
+				klog.V(2).Infof("recreated clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name)
+			}
+			return nil
+		})
+		if err != nil {
+			// don't fail on failures, try to create as many as you can
+			utilruntime.HandleError(fmt.Errorf("unable to reconcile clusterrolebinding.%s/%s: %v", rbac.GroupName, clusterRoleBinding.Name, err))
+			failedReconciliation = true
+		}
+	}
+
+	// ensure bootstrap namespaced roles are created or reconciled
+	for namespace, roles := range p.Roles {
+		for _, role := range roles {
+			opts := reconciliation.ReconcileRoleOptions{
+				Role:    reconciliation.RoleRuleOwner{Role: &role},
+				Client:  reconciliation.RoleModifier{Client: client.RbacV1(), NamespaceClient: client.CoreV1().Namespaces()},
+				Confirm: true,
+			}
+			// ServiceUnavailble error is returned when the API server is blocked by storage version updates
+			err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
+				result, err := opts.Run()
+				if err != nil {
+					return err
+				}
+				switch {
+				case result.Protected && result.Operation != reconciliation.ReconcileNone:
+					klog.Warningf("skipped reconcile-protected role.%s/%s in %v with missing permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules)
+				case result.Operation == reconciliation.ReconcileUpdate:
+					klog.V(2).Infof("updated role.%s/%s in %v with additional permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules)
+				case result.Operation == reconciliation.ReconcileCreate:
+					klog.V(2).Infof("created role.%s/%s in %v", rbac.GroupName, role.Name, namespace)
+				}
+				return nil
+			})
+			if err != nil {
+				// don't fail on failures, try to create as many as you can
+				utilruntime.HandleError(fmt.Errorf("unable to reconcile role.%s/%s in %v: %v", rbac.GroupName, role.Name, namespace, err))
+				failedReconciliation = true
+			}
+		}
+	}
+
+	// ensure bootstrap namespaced rolebindings are created or reconciled
+	for namespace, roleBindings := range p.RoleBindings {
+		for _, roleBinding := range roleBindings {
+			opts := reconciliation.ReconcileRoleBindingOptions{
+				RoleBinding: reconciliation.RoleBindingAdapter{RoleBinding: &roleBinding},
+				Client:      reconciliation.RoleBindingClientAdapter{Client: client.RbacV1(), NamespaceClient: client.CoreV1().Namespaces()},
+				Confirm:     true,
+			}
+			// ServiceUnavailble error is returned when the API server is blocked by storage version updates
+			err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
+				result, err := opts.Run()
+				if err != nil {
+					return err
+				}
+				switch {
+				case result.Protected && result.Operation != reconciliation.ReconcileNone:
+					klog.Warningf("skipped reconcile-protected rolebinding.%s/%s in %v with missing subjects: %v", rbac.GroupName, roleBinding.Name, namespace, result.MissingSubjects)
+				case result.Operation == reconciliation.ReconcileUpdate:
+					klog.V(2).Infof("updated rolebinding.%s/%s in %v with additional subjects: %v", rbac.GroupName, roleBinding.Name, namespace, result.MissingSubjects)
+				case result.Operation == reconciliation.ReconcileCreate:
+					klog.V(2).Infof("created rolebinding.%s/%s in %v", rbac.GroupName, roleBinding.Name, namespace)
+				case result.Operation == reconciliation.ReconcileRecreate:
+					klog.V(2).Infof("recreated rolebinding.%s/%s in %v", rbac.GroupName, roleBinding.Name, namespace)
+				}
+				return nil
+			})
+			if err != nil {
+				// don't fail on failures, try to create as many as you can
+				utilruntime.HandleError(fmt.Errorf("unable to reconcile rolebinding.%s/%s in %v: %v", rbac.GroupName, roleBinding.Name, namespace, err))
+				failedReconciliation = true
+			}
+		}
+	}
+	// failed to reconcile some objects, retry
+	if failedReconciliation {
+		return false, nil
+	}
+
+	return true, nil
+}
+
 func (p RESTStorageProvider) GroupName() string {
 	return rbac.GroupName
 }

pkg/registry/rbac/rest/storage_rbac_test.go

@@ -0,0 +1,39 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rest
+
+import (
+	"testing"
+
+	"k8s.io/client-go/kubernetes/fake"
+	"k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac/bootstrappolicy"
+)
+
+func BenchmarkEnsureRBACPolicy(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		var policy = &PolicyData{
+			ClusterRoles:               append(bootstrappolicy.ClusterRoles(), bootstrappolicy.ControllerRoles()...),
+			ClusterRoleBindings:        append(bootstrappolicy.ClusterRoleBindings(), bootstrappolicy.ControllerRoleBindings()...),
+			Roles:                      bootstrappolicy.NamespaceRoles(),
+			RoleBindings:               bootstrappolicy.NamespaceRoleBindings(),
+			ClusterRolesToAggregate:    bootstrappolicy.ClusterRolesToAggregate(),
+			ClusterRoleBindingsToSplit: bootstrappolicy.ClusterRoleBindingsToSplit(),
+		}
+		coreClientSet := fake.NewSimpleClientset()
+		_, _ = ensureRBACPolicy(policy, coreClientSet)
+	}
+}