Merge pull request #119008 from nilekhc/hotreload-update-metrics

[KMSv2] feat: implements metrics for encryption config hot reload

staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/apiserver/pkg/server/options/encryptionconfig"
+	"k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog/v2"
 )
@@ -163,16 +164,19 @@ func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem(serverCtx contex
 	ctx, closeTransformers := context.WithCancel(serverCtx)
 
 	defer func() {
-		// TODO: increment success metric when updatedEffectiveConfig=true
-
 		// TODO can work queue metrics help here?
 
 		if !updatedEffectiveConfig {
 			// avoid leaking if we're not using the newly constructed transformers (due to an error or them not being changed)
 			closeTransformers()
 		}
+
+		if updatedEffectiveConfig && err == nil {
+			metrics.RecordEncryptionConfigAutomaticReloadSuccess()
+		}
+
 		if err != nil {
-			// TODO: increment failure metric
+			metrics.RecordEncryptionConfigAutomaticReloadFailure()
 			utilruntime.HandleError(fmt.Errorf("error processing encryption config file %s: %v", d.filePath, err))
 			// add dummy item back to the queue to trigger file content processing.
 			d.queue.AddRateLimited(key)

staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"sync"
+
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+const (
+	namespace = "apiserver"
+	subsystem = "encryption_config_controller"
+)
+
+var (
+	encryptionConfigAutomaticReloadFailureTotal = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "automatic_reload_failures_total",
+			Help:           "Total number of failed automatic reloads of encryption configuration.",
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
+
+	encryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "automatic_reload_success_total",
+			Help:           "Total number of successful automatic reloads of encryption configuration.",
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
+
+	encryptionConfigAutomaticReloadLastTimestampSeconds = metrics.NewGaugeVec(
+		&metrics.GaugeOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "automatic_reload_last_timestamp_seconds",
+			Help:           "Timestamp of the last successful or failed automatic reload of encryption configuration.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"status"},
+	)
+)
+
+var registerMetrics sync.Once
+
+func RegisterMetrics() {
+	registerMetrics.Do(func() {
+		legacyregistry.MustRegister(encryptionConfigAutomaticReloadFailureTotal)
+		legacyregistry.MustRegister(encryptionConfigAutomaticReloadSuccessTotal)
+		legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds)
+	})
+}
+
+func RecordEncryptionConfigAutomaticReloadFailure() {
+	encryptionConfigAutomaticReloadFailureTotal.Inc()
+	recordEncryptionConfigAutomaticReloadTimestamp("failure")
+}
+
+func RecordEncryptionConfigAutomaticReloadSuccess() {
+	encryptionConfigAutomaticReloadSuccessTotal.Inc()
+	recordEncryptionConfigAutomaticReloadTimestamp("success")
+}
+
+func recordEncryptionConfigAutomaticReloadTimestamp(result string) {
+	encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result).SetToCurrentTime()
+}

staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics_test.go

@@ -0,0 +1,104 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"strings"
+	"testing"
+
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/component-base/metrics/testutil"
+)
+
+func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) {
+	expectedValue := `
+	# HELP apiserver_encryption_config_controller_automatic_reload_failures_total [ALPHA] Total number of failed automatic reloads of encryption configuration.
+    # TYPE apiserver_encryption_config_controller_automatic_reload_failures_total counter
+    apiserver_encryption_config_controller_automatic_reload_failures_total 1
+	`
+	metrics := []string{
+		namespace + "_" + subsystem + "_automatic_reload_failures_total",
+	}
+
+	encryptionConfigAutomaticReloadFailureTotal.Reset()
+	RegisterMetrics()
+
+	RecordEncryptionConfigAutomaticReloadFailure()
+	if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestRecordEncryptionConfigAutomaticReloadSuccess(t *testing.T) {
+	expectedValue := `
+	# HELP apiserver_encryption_config_controller_automatic_reload_success_total [ALPHA] Total number of successful automatic reloads of encryption configuration.
+    # TYPE apiserver_encryption_config_controller_automatic_reload_success_total counter
+    apiserver_encryption_config_controller_automatic_reload_success_total 1
+	`
+	metrics := []string{
+		namespace + "_" + subsystem + "_automatic_reload_success_total",
+	}
+
+	encryptionConfigAutomaticReloadSuccessTotal.Reset()
+	RegisterMetrics()
+
+	RecordEncryptionConfigAutomaticReloadSuccess()
+	if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestEncryptionConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
+	testCases := []struct {
+		expectedValue string
+		resultLabel   string
+		timestamp     int64
+	}{
+		{
+			expectedValue: `
+                # HELP apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last successful or failed automatic reload of encryption configuration.
+                # TYPE apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds gauge
+                apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{status="failure"} 1.689101941e+09
+            `,
+			resultLabel: "failure",
+			timestamp:   1689101941,
+		},
+		{
+			expectedValue: `
+                # HELP apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last successful or failed automatic reload of encryption configuration.
+                # TYPE apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds gauge
+                apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{status="success"} 1.689101941e+09
+            `,
+			resultLabel: "success",
+			timestamp:   1689101941,
+		},
+	}
+
+	metrics := []string{
+		namespace + "_" + subsystem + "_automatic_reload_last_timestamp_seconds",
+	}
+	RegisterMetrics()
+
+	for _, tc := range testCases {
+		encryptionConfigAutomaticReloadLastTimestampSeconds.Reset()
+		encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(tc.resultLabel).Set(float64(tc.timestamp))
+
+		if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.expectedValue), metrics...); err != nil {
+			t.Fatal(err)
+		}
+	}
+}

vendor/modules.txt

@@ -1530,6 +1530,7 @@ k8s.io/apiserver/pkg/server/mux
 k8s.io/apiserver/pkg/server/options
 k8s.io/apiserver/pkg/server/options/encryptionconfig
 k8s.io/apiserver/pkg/server/options/encryptionconfig/controller
+k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics
 k8s.io/apiserver/pkg/server/resourceconfig
 k8s.io/apiserver/pkg/server/routes
 k8s.io/apiserver/pkg/server/storage