Merge pull request #123333 from liggitt/authz-metrics

Add allowed/denied metrics for authorizers

pkg/kubeapiserver/authorizer/reload.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
+	authorizationmetrics "k8s.io/apiserver/pkg/authorization/metrics"
 	"k8s.io/apiserver/pkg/authorization/union"
 	"k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics"
 	webhookutil "k8s.io/apiserver/pkg/util/webhook"
@@ -101,21 +102,21 @@ func (r *reloadableAuthorizerResolver) newForConfig(authzConfig *authzconfig.Aut
 			if r.nodeAuthorizer == nil {
 				return nil, nil, fmt.Errorf("authorizer type Node is not allowed if it was not enabled at initial server startup")
 			}
-			authorizers = append(authorizers, r.nodeAuthorizer)
+			authorizers = append(authorizers, authorizationmetrics.InstrumentedAuthorizer(string(configuredAuthorizer.Type), configuredAuthorizer.Name, r.nodeAuthorizer))
 			ruleResolvers = append(ruleResolvers, r.nodeAuthorizer)
 		case authzconfig.AuthorizerType(modes.ModeAlwaysAllow):
 			alwaysAllowAuthorizer := authorizerfactory.NewAlwaysAllowAuthorizer()
-			authorizers = append(authorizers, alwaysAllowAuthorizer)
+			authorizers = append(authorizers, authorizationmetrics.InstrumentedAuthorizer(string(configuredAuthorizer.Type), configuredAuthorizer.Name, alwaysAllowAuthorizer))
 			ruleResolvers = append(ruleResolvers, alwaysAllowAuthorizer)
 		case authzconfig.AuthorizerType(modes.ModeAlwaysDeny):
 			alwaysDenyAuthorizer := authorizerfactory.NewAlwaysDenyAuthorizer()
-			authorizers = append(authorizers, alwaysDenyAuthorizer)
+			authorizers = append(authorizers, authorizationmetrics.InstrumentedAuthorizer(string(configuredAuthorizer.Type), configuredAuthorizer.Name, alwaysDenyAuthorizer))
 			ruleResolvers = append(ruleResolvers, alwaysDenyAuthorizer)
 		case authzconfig.AuthorizerType(modes.ModeABAC):
 			if r.abacAuthorizer == nil {
 				return nil, nil, fmt.Errorf("authorizer type ABAC is not allowed if it was not enabled at initial server startup")
 			}
-			authorizers = append(authorizers, r.abacAuthorizer)
+			authorizers = append(authorizers, authorizationmetrics.InstrumentedAuthorizer(string(configuredAuthorizer.Type), configuredAuthorizer.Name, r.abacAuthorizer))
 			ruleResolvers = append(ruleResolvers, r.abacAuthorizer)
 		case authzconfig.AuthorizerType(modes.ModeWebhook):
 			if r.initialConfig.WebhookRetryBackoff == nil {
@@ -145,13 +146,13 @@ func (r *reloadableAuthorizerResolver) newForConfig(authzConfig *authzconfig.Aut
 			if err != nil {
 				return nil, nil, err
 			}
-			authorizers = append(authorizers, webhookAuthorizer)
+			authorizers = append(authorizers, authorizationmetrics.InstrumentedAuthorizer(string(configuredAuthorizer.Type), configuredAuthorizer.Name, webhookAuthorizer))
 			ruleResolvers = append(ruleResolvers, webhookAuthorizer)
 		case authzconfig.AuthorizerType(modes.ModeRBAC):
 			if r.rbacAuthorizer == nil {
 				return nil, nil, fmt.Errorf("authorizer type RBAC is not allowed if it was not enabled at initial server startup")
 			}
-			authorizers = append(authorizers, r.rbacAuthorizer)
+			authorizers = append(authorizers, authorizationmetrics.InstrumentedAuthorizer(string(configuredAuthorizer.Type), configuredAuthorizer.Name, r.rbacAuthorizer))
 			ruleResolvers = append(ruleResolvers, r.rbacAuthorizer)
 		default:
 			return nil, nil, fmt.Errorf("unknown authorization mode %s specified", configuredAuthorizer.Type)

staging/src/k8s.io/apiserver/pkg/authorization/metrics/metrics.go

@@ -0,0 +1,92 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"context"
+	"sync"
+
+	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+const (
+	namespace = "apiserver"
+	subsystem = "authorization"
+)
+
+var (
+	authorizationDecisionsTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "decisions_total",
+			Help:           "Total number of terminal decisions made by an authorizer split by authorizer type, name, and decision.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"type", "name", "decision"},
+	)
+)
+
+var registerMetrics sync.Once
+
+func RegisterMetrics() {
+	registerMetrics.Do(func() {
+		legacyregistry.MustRegister(authorizationDecisionsTotal)
+	})
+}
+
+func ResetMetricsForTest() {
+	authorizationDecisionsTotal.Reset()
+}
+
+func RecordAuthorizationDecision(authorizerType, authorizerName, decision string) {
+	authorizationDecisionsTotal.WithLabelValues(authorizerType, authorizerName, decision).Inc()
+}
+
+func InstrumentedAuthorizer(authorizerType string, authorizerName string, delegate authorizer.Authorizer) authorizer.Authorizer {
+	RegisterMetrics()
+	return &instrumentedAuthorizer{
+		authorizerType: string(authorizerType),
+		authorizerName: authorizerName,
+		delegate:       delegate,
+	}
+}
+
+type instrumentedAuthorizer struct {
+	authorizerType string
+	authorizerName string
+	delegate       authorizer.Authorizer
+}
+
+func (a *instrumentedAuthorizer) Authorize(ctx context.Context, attributes authorizer.Attributes) (authorizer.Decision, string, error) {
+	decision, reason, err := a.delegate.Authorize(ctx, attributes)
+	switch decision {
+	case authorizer.DecisionNoOpinion:
+		// non-terminal, not reported
+	case authorizer.DecisionAllow:
+		// matches SubjectAccessReview status.allowed field name
+		RecordAuthorizationDecision(a.authorizerType, a.authorizerName, "allowed")
+	case authorizer.DecisionDeny:
+		// matches SubjectAccessReview status.denied field name
+		RecordAuthorizationDecision(a.authorizerType, a.authorizerName, "denied")
+	default:
+		RecordAuthorizationDecision(a.authorizerType, a.authorizerName, "unknown")
+	}
+	return decision, reason, err
+}

staging/src/k8s.io/apiserver/pkg/authorization/metrics/metrics_test.go

@@ -0,0 +1,105 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/component-base/metrics/testutil"
+)
+
+func TestRecordAuthorizationDecisionsTotal(t *testing.T) {
+	prefix := `
+    # HELP apiserver_authorization_decisions_total [ALPHA] Total number of terminal decisions made by an authorizer split by authorizer type, name, and decision.
+    # TYPE apiserver_authorization_decisions_total counter`
+	metrics := []string{
+		namespace + "_" + subsystem + "_decisions_total",
+	}
+
+	authorizationDecisionsTotal.Reset()
+	RegisterMetrics()
+
+	dummyAuthorizer := &dummyAuthorizer{}
+	a := InstrumentedAuthorizer("mytype", "myname", dummyAuthorizer)
+
+	// allow
+	{
+		dummyAuthorizer.decision = authorizer.DecisionAllow
+		_, _, _ = a.Authorize(context.Background(), nil)
+		expectedValue := prefix + `
+			apiserver_authorization_decisions_total{decision="allowed",name="myname",type="mytype"} 1
+		`
+		if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
+			t.Fatal(err)
+		}
+		authorizationDecisionsTotal.Reset()
+	}
+
+	// deny
+	{
+		dummyAuthorizer.decision = authorizer.DecisionDeny
+		_, _, _ = a.Authorize(context.Background(), nil)
+		_, _, _ = a.Authorize(context.Background(), nil)
+		expectedValue := prefix + `
+			apiserver_authorization_decisions_total{decision="denied",name="myname",type="mytype"} 2
+		`
+		if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
+			t.Fatal(err)
+		}
+		authorizationDecisionsTotal.Reset()
+	}
+
+	// no-opinion emits no metric
+	{
+		dummyAuthorizer.decision = authorizer.DecisionNoOpinion
+		_, _, _ = a.Authorize(context.Background(), nil)
+		_, _, _ = a.Authorize(context.Background(), nil)
+		expectedValue := prefix + `
+		`
+		if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
+			t.Fatal(err)
+		}
+		authorizationDecisionsTotal.Reset()
+	}
+
+	// unknown decision emits a metric
+	{
+		dummyAuthorizer.decision = authorizer.DecisionDeny + 10
+		_, _, _ = a.Authorize(context.Background(), nil)
+		expectedValue := prefix + `
+			apiserver_authorization_decisions_total{decision="unknown",name="myname",type="mytype"} 1
+		`
+		if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
+			t.Fatal(err)
+		}
+		authorizationDecisionsTotal.Reset()
+	}
+
+}
+
+type dummyAuthorizer struct {
+	decision authorizer.Decision
+	err      error
+}
+
+func (d *dummyAuthorizer) Authorize(ctx context.Context, attrs authorizer.Attributes) (authorizer.Decision, string, error) {
+	return d.decision, "", d.err
+}

staging/src/k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics/metrics.go

@@ -73,7 +73,6 @@ func RegisterMetrics() {
 func ResetMetricsForTest() {
 	authorizationConfigAutomaticReloadsTotal.Reset()
 	authorizationConfigAutomaticReloadLastTimestampSeconds.Reset()
-	legacyregistry.Reset()
 }
 
 func RecordAuthorizationConfigAutomaticReloadFailure(apiServerID string) {

test/integration/auth/authz_config_test.go

@@ -25,6 +25,7 @@ import (
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 	"sync/atomic"
@@ -35,6 +36,7 @@ import (
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	authorizationmetrics "k8s.io/apiserver/pkg/authorization/metrics"
 	"k8s.io/apiserver/pkg/features"
 	authzmetrics "k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
@@ -178,6 +180,7 @@ users:
 	}
 
 	// returns a deny response when called
+	denyName := "deny.example.com"
 	serverDenyCalled := atomic.Int32{}
 	serverDeny := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		serverDenyCalled.Add(1)
@@ -221,6 +224,7 @@ users:
 	}
 
 	// returns an allow response when called
+	allowName := "allow.example.com"
 	serverAllowCalled := atomic.Int32{}
 	serverAllow := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		serverAllowCalled.Add(1)
@@ -242,6 +246,7 @@ users:
 	}
 
 	// returns an allow response when called
+	allowReloadedName := "allowreloaded.example.com"
 	serverAllowReloadedCalled := atomic.Int32{}
 	serverAllowReloaded := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		serverAllowReloadedCalled.Add(1)
@@ -269,9 +274,15 @@ users:
 		serverNoOpinionCalled.Store(0)
 		serverAllowCalled.Store(0)
 		serverAllowReloadedCalled.Store(0)
+		authorizationmetrics.ResetMetricsForTest()
 	}
+	var adminClient *clientset.Clientset
 	assertCounts := func(errorCount, timeoutCount, denyCount, noOpinionCount, allowCount, allowReloadedCount int32) {
 		t.Helper()
+		metrics, err := getMetrics(t, adminClient)
+		if err != nil {
+			t.Errorf("error getting metrics: %v", err)
+		}
 		if e, a := errorCount, serverErrorCalled.Load(); e != a {
 			t.Errorf("expected fail webhook calls: %d, got %d", e, a)
 		}
@@ -281,15 +292,24 @@ users:
 		if e, a := denyCount, serverDenyCalled.Load(); e != a {
 			t.Errorf("expected deny webhook calls: %d, got %d", e, a)
 		}
+		if e, a := denyCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: denyName}]["denied"]; e != int32(a) {
+			t.Errorf("expected deny webhook denied metrics calls: %d, got %d", e, a)
+		}
 		if e, a := noOpinionCount, serverNoOpinionCalled.Load(); e != a {
 			t.Errorf("expected noOpinion webhook calls: %d, got %d", e, a)
 		}
 		if e, a := allowCount, serverAllowCalled.Load(); e != a {
 			t.Errorf("expected allow webhook calls: %d, got %d", e, a)
 		}
+		if e, a := allowCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: allowName}]["allowed"]; e != int32(a) {
+			t.Errorf("expected allow webhook allowed metrics calls: %d, got %d", e, a)
+		}
 		if e, a := allowReloadedCount, serverAllowReloadedCalled.Load(); e != a {
 			t.Errorf("expected allowReloaded webhook calls: %d, got %d", e, a)
 		}
+		if e, a := allowReloadedCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: allowReloadedName}]["allowed"]; e != int32(a) {
+			t.Errorf("expected allowReloaded webhook allowed metrics calls: %d, got %d", e, a)
+		}
 		resetCounts()
 	}
 
@@ -333,7 +353,7 @@ authorizers:
     - expression: 'request.resourceAttributes.name == "timeout"'
 
 - type: Webhook
-  name: deny.example.com
+  name: `+denyName+`
   webhook:
     timeout: 5s
     failurePolicy: NoOpinion
@@ -361,7 +381,7 @@ authorizers:
       kubeConfigFile: `+serverNoOpinionKubeconfigName+`
 
 - type: Webhook
-  name: allow.example.com
+  name: `+allowName+`
   webhook:
     timeout: 5s
     failurePolicy: Deny
@@ -383,7 +403,7 @@ authorizers:
 	)
 	t.Cleanup(server.TearDownFn)
 
-	adminClient := clientset.NewForConfigOrDie(server.ClientConfig)
+	adminClient = clientset.NewForConfigOrDie(server.ClientConfig)
 
 	// malformed webhook short circuits
 	t.Log("checking error")
@@ -470,14 +490,14 @@ authorizers:
 	}
 
 	// check last loaded success/failure metric timestamps, ensure success is present, failure is not
-	initialReloadSuccess, initialReloadFailure, err := getReloadTimes(t, adminClient)
+	initialMetrics, err := getMetrics(t, adminClient)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if initialReloadSuccess == nil {
+	if initialMetrics.reloadSuccess == nil {
 		t.Fatal("expected success timestamp, got none")
 	}
-	if initialReloadFailure != nil {
+	if initialMetrics.reloadFailure != nil {
 		t.Fatal("expected no failure timestamp, got one")
 	}
 
@@ -487,24 +507,24 @@ authorizers:
 	}
 
 	// wait for failure timestamp > success timestamp
-	var reload1Success, reload1Failure *time.Time
+	var reload1Metrics *metrics
 	err = wait.PollUntilContextTimeout(context.TODO(), time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
-		reload1Success, reload1Failure, err = getReloadTimes(t, adminClient)
+		reload1Metrics, err = getMetrics(t, adminClient)
 		if err != nil {
 			t.Fatal(err)
 		}
-		if reload1Success == nil {
+		if reload1Metrics.reloadSuccess == nil {
 			t.Fatal("expected success timestamp, got none")
 		}
-		if !reload1Success.Equal(*initialReloadSuccess) {
-			t.Fatalf("success timestamp changed from initial success %s to %s unexpectedly", initialReloadSuccess.String(), reload1Success.String())
+		if !reload1Metrics.reloadSuccess.Equal(*initialMetrics.reloadSuccess) {
+			t.Fatalf("success timestamp changed from initial success %s to %s unexpectedly", initialMetrics.reloadSuccess.String(), reload1Metrics.reloadSuccess.String())
 		}
-		if reload1Failure == nil {
+		if reload1Metrics.reloadFailure == nil {
 			t.Log("expected failure timestamp, got nil, retrying")
 			return false, nil
 		}
-		if !reload1Failure.After(*reload1Success) {
-			t.Fatalf("expected failure timestamp to be more recent than success timestamp, got %s <= %s", reload1Failure.String(), reload1Success.String())
+		if !reload1Metrics.reloadFailure.After(*reload1Metrics.reloadSuccess) {
+			t.Fatalf("expected failure timestamp to be more recent than success timestamp, got %s <= %s", reload1Metrics.reloadFailure.String(), reload1Metrics.reloadSuccess.String())
 		}
 		return true, nil
 	})
@@ -539,7 +559,7 @@ apiVersion: apiserver.config.k8s.io/v1alpha1
 kind: AuthorizationConfiguration
 authorizers:
 - type: Webhook
-  name: allowreloaded.example.com
+  name: `+allowReloadedName+`
   webhook:
     timeout: 5s
     failurePolicy: Deny
@@ -553,29 +573,29 @@ authorizers:
 		t.Fatal(err)
 	}
 
-	// wait for success timestamp > reload1Failure timestamp
-	var reload2Success, reload2Failure *time.Time
+	// wait for success timestamp > reload1Metrics.reloadFailure timestamp
+	var reload2Metrics *metrics
 	err = wait.PollUntilContextTimeout(context.TODO(), time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
-		reload2Success, reload2Failure, err = getReloadTimes(t, adminClient)
+		reload2Metrics, err = getMetrics(t, adminClient)
 		if err != nil {
 			t.Fatal(err)
 		}
-		if reload2Failure == nil {
+		if reload2Metrics.reloadFailure == nil {
 			t.Log("expected failure timestamp, got nil, retrying")
 			return false, nil
 		}
-		if !reload2Failure.Equal(*reload1Failure) {
-			t.Fatalf("failure timestamp changed from reload1Failure %s to %s unexpectedly", reload1Failure.String(), reload2Failure.String())
+		if !reload2Metrics.reloadFailure.Equal(*reload1Metrics.reloadFailure) {
+			t.Fatalf("failure timestamp changed from reload1Metrics.reloadFailure %s to %s unexpectedly", reload1Metrics.reloadFailure.String(), reload2Metrics.reloadFailure.String())
 		}
-		if reload2Success == nil {
+		if reload2Metrics.reloadSuccess == nil {
 			t.Fatal("expected success timestamp, got none")
 		}
-		if reload2Success.Equal(*initialReloadSuccess) {
+		if reload2Metrics.reloadSuccess.Equal(*initialMetrics.reloadSuccess) {
 			t.Log("success timestamp hasn't updated from initial success, retrying")
 			return false, nil
 		}
-		if !reload2Success.After(*reload2Failure) {
-			t.Fatalf("expected success timestamp to be more recent than failure, got %s <= %s", reload2Success.String(), reload2Failure.String())
+		if !reload2Metrics.reloadSuccess.After(*reload2Metrics.reloadFailure) {
+			t.Fatalf("expected success timestamp to be more recent than failure, got %s <= %s", reload2Metrics.reloadSuccess.String(), reload2Metrics.reloadFailure.String())
 		}
 		return true, nil
 	})
@@ -610,28 +630,28 @@ authorizers:
 	}
 
 	// wait for failure timestamp > success timestamp
-	var reload3Success, reload3Failure *time.Time
+	var reload3Metrics *metrics
 	err = wait.PollUntilContextTimeout(context.TODO(), time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
-		reload3Success, reload3Failure, err = getReloadTimes(t, adminClient)
+		reload3Metrics, err = getMetrics(t, adminClient)
 		if err != nil {
 			t.Fatal(err)
 		}
-		if reload3Success == nil {
+		if reload3Metrics.reloadSuccess == nil {
 			t.Fatal("expected success timestamp, got none")
 		}
-		if !reload3Success.Equal(*reload2Success) {
-			t.Fatalf("success timestamp changed from %s to %s unexpectedly", reload2Success.String(), reload3Success.String())
+		if !reload3Metrics.reloadSuccess.Equal(*reload2Metrics.reloadSuccess) {
+			t.Fatalf("success timestamp changed from %s to %s unexpectedly", reload2Metrics.reloadSuccess.String(), reload3Metrics.reloadSuccess.String())
 		}
-		if reload3Failure == nil {
+		if reload3Metrics.reloadFailure == nil {
 			t.Log("expected failure timestamp, got nil, retrying")
 			return false, nil
 		}
-		if reload3Failure.Equal(*reload2Failure) {
+		if reload3Metrics.reloadFailure.Equal(*reload2Metrics.reloadFailure) {
 			t.Log("failure timestamp hasn't updated, retrying")
 			return false, nil
 		}
-		if !reload3Failure.After(*reload3Success) {
-			t.Fatalf("expected failure timestamp to be more recent than success, got %s <= %s", reload3Failure.String(), reload3Success.String())
+		if !reload3Metrics.reloadFailure.After(*reload3Metrics.reloadSuccess) {
+			t.Fatalf("expected failure timestamp to be more recent than success, got %s <= %s", reload3Metrics.reloadFailure.String(), reload3Metrics.reloadSuccess.String())
 		}
 		return true, nil
 	})
@@ -661,36 +681,69 @@ authorizers:
 	}
 }
 
-func getReloadTimes(t *testing.T, client *clientset.Clientset) (*time.Time, *time.Time, error) {
+type metrics struct {
+	reloadSuccess *time.Time
+	reloadFailure *time.Time
+	decisions     map[authorizerKey]map[string]int
+}
+type authorizerKey struct {
+	authorizerType string
+	authorizerName string
+}
+
+var decisionMetric = regexp.MustCompile(`apiserver_authorization_decisions_total\{decision="(.*?)",name="(.*?)",type="(.*?)"\} (\d+)`)
+
+func getMetrics(t *testing.T, client *clientset.Clientset) (*metrics, error) {
 	data, err := client.RESTClient().Get().AbsPath("/metrics").DoRaw(context.TODO())
 
 	//  apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:4b86cfa719a83dd63a4dc6a9831edb2b59240d0f59cf215b2d51aacb3f5c395e",status="success"} 1.7002567356895502e+09
 	//  apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:4b86cfa719a83dd63a4dc6a9831edb2b59240d0f59cf215b2d51aacb3f5c395e",status="failure"} 1.7002567356895502e+09
+	//  apiserver_authorization_decisions_total{decision="allowed",name="allow.example.com",type="Webhook"} 2
+	//  apiserver_authorization_decisions_total{decision="allowed",name="allowreloaded.example.com",type="Webhook"} 1
+	//  apiserver_authorization_decisions_total{decision="denied",name="deny.example.com",type="Webhook"} 1
+	//  apiserver_authorization_decisions_total{decision="denied",name="error.example.com",type="Webhook"} 1
+	//  apiserver_authorization_decisions_total{decision="denied",name="timeout.example.com",type="Webhook"} 1
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	var success, failure *time.Time
+	var m metrics
 	for _, line := range strings.Split(string(data), "\n") {
+		if matches := decisionMetric.FindStringSubmatch(line); matches != nil {
+			t.Log(line)
+			if m.decisions == nil {
+				m.decisions = map[authorizerKey]map[string]int{}
+			}
+			key := authorizerKey{authorizerType: matches[3], authorizerName: matches[2]}
+			if m.decisions[key] == nil {
+				m.decisions[key] = map[string]int{}
+			}
+			count, err := strconv.Atoi(matches[4])
+			if err != nil {
+				return nil, err
+			}
+			m.decisions[key][matches[1]] = count
+
+		}
 		if strings.HasPrefix(line, "apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds") {
 			t.Log(line)
 			values := strings.Split(line, " ")
 			value, err := strconv.ParseFloat(values[len(values)-1], 64)
 			if err != nil {
-				return nil, nil, err
+				return nil, err
 			}
 			seconds := int64(value)
 			nanoseconds := int64((value - float64(seconds)) * 1000000000)
 			tm := time.Unix(seconds, nanoseconds)
 			if strings.Contains(line, `"success"`) {
-				success = &tm
-				t.Log("success", success.String())
+				m.reloadSuccess = &tm
+				t.Log("success", m.reloadSuccess.String())
 			}
 			if strings.Contains(line, `"failure"`) {
-				failure = &tm
-				t.Log("failure", failure.String())
+				m.reloadFailure = &tm
+				t.Log("failure", m.reloadFailure.String())
 			}
 		}
 	}
-	return success, failure, nil
+	return &m, nil
 }

vendor/modules.txt

@@ -1428,6 +1428,7 @@ k8s.io/apiserver/pkg/authentication/user
 k8s.io/apiserver/pkg/authorization/authorizer
 k8s.io/apiserver/pkg/authorization/authorizerfactory
 k8s.io/apiserver/pkg/authorization/cel
+k8s.io/apiserver/pkg/authorization/metrics
 k8s.io/apiserver/pkg/authorization/path
 k8s.io/apiserver/pkg/authorization/union
 k8s.io/apiserver/pkg/cel