add "kubelet_volume_stat_cal_duration_seconds_bucket" VolumeStatCalDuration metrics for fsquato monitoring benchmark

pkg/kubelet/server/metrics/metrics.go

@@ -64,6 +64,19 @@ var (
 		},
 		[]string{"method", "path", "server_type", "long_running"},
 	)
+	// VolumeStatCalDuration tracks the duration in seconds to calculate volume stats.
+	// this metric is mainly for comparison between fsquota monitoring and `du` for disk usage.
+	VolumeStatCalDuration = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Subsystem: kubeletSubsystem,
+			Name:      "volume_stat_cal_duration_seconds",
+			Help:      "Duration in seconds to calculate volume stats",
+			// Use DefBuckets for now, will customize the buckets if necessary.
+			Buckets:        metrics.DefBuckets,
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{},
+	)
 )
 
 var registerMetrics sync.Once
@@ -74,6 +87,7 @@ func Register() {
 		legacyregistry.MustRegister(HTTPRequests)
 		legacyregistry.MustRegister(HTTPRequestsDuration)
 		legacyregistry.MustRegister(HTTPInflightRequests)
+		legacyregistry.MustRegister(VolumeStatCalDuration)
 	})
 }
 

pkg/kubelet/server/stats/volume_stat_calculator.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/klog/v2"
 	stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
 	"k8s.io/kubernetes/pkg/features"
+	servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics"
 	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/util"
 )
@@ -133,7 +134,13 @@ func (s *volumeStatCalculator) calcAndStoreStats() {
 	var ephemeralStats []stats.VolumeStats
 	var persistentStats []stats.VolumeStats
 	for name, v := range metricVolumes {
-		metric, err := v.GetMetrics()
+		metric, err := func() (*volume.Metrics, error) {
+			startTime := time.Now()
+			defer func() {
+				servermetrics.VolumeStatCalDuration.WithLabelValues().Observe(servermetrics.SinceInSeconds(startTime))
+			}()
+			return v.GetMetrics()
+		}()
 		if err != nil {
 			// Expected for Volumes that don't support Metrics
 			if !volume.IsNotSupported(err) {