add volume type and seperated histogram for volume stat collection

pkg/kubelet/server/metrics/metrics.go

@@ -68,14 +68,13 @@ var (
 	// this metric is mainly for comparison between fsquota monitoring and `du` for disk usage.
 	VolumeStatCalDuration = metrics.NewHistogramVec(
 		&metrics.HistogramOpts{
-			Subsystem: kubeletSubsystem,
+			Subsystem:      kubeletSubsystem,
-			Name:      "volume_stat_cal_duration_seconds",
+			Name:           "volume_metric_collection_duration_seconds",
-			Help:      "Duration in seconds to calculate volume stats",
+			Help:           "Duration in seconds to calculate volume stats",
-			// Use DefBuckets for now, will customize the buckets if necessary.
 			Buckets:        metrics.DefBuckets,
 			StabilityLevel: metrics.ALPHA,
 		},
-		[]string{},
+		[]string{"metric_source"},
 	)
 )
 
@@ -95,3 +94,8 @@ func Register() {
 func SinceInSeconds(start time.Time) float64 {
 	return time.Since(start).Seconds()
 }
+
+// CollectVolumeStatCalDuration collects the duration in seconds to calculate volume stats.
+func CollectVolumeStatCalDuration(metricSource string, start time.Time) {
+	VolumeStatCalDuration.WithLabelValues(metricSource).Observe(SinceInSeconds(start))
+}

pkg/kubelet/server/stats/volume_stat_calculator.go

@@ -30,7 +30,6 @@ import (
 	"k8s.io/klog/v2"
 	stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
 	"k8s.io/kubernetes/pkg/features"
-	servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics"
 	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/util"
 )
@@ -134,13 +133,7 @@ func (s *volumeStatCalculator) calcAndStoreStats() {
 	var ephemeralStats []stats.VolumeStats
 	var persistentStats []stats.VolumeStats
 	for name, v := range metricVolumes {
-		metric, err := func() (*volume.Metrics, error) {
+		metric, err := v.GetMetrics()
-			startTime := time.Now()
-			defer func() {
-				servermetrics.VolumeStatCalDuration.WithLabelValues().Observe(servermetrics.SinceInSeconds(startTime))
-			}()
-			return v.GetMetrics()
-		}()
 		if err != nil {
 			// Expected for Volumes that don't support Metrics
 			if !volume.IsNotSupported(err) {

pkg/volume/csi/csi_metrics.go

@@ -23,6 +23,7 @@ import (
 
 	"google.golang.org/grpc"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics"
 	"k8s.io/kubernetes/pkg/volume"
 	volumeutil "k8s.io/kubernetes/pkg/volume/util"
 )
@@ -51,6 +52,8 @@ func NewMetricsCsi(volumeID string, targetPath string, driverName csiDriverName)
 }
 
 func (mc *metricsCsi) GetMetrics() (*volume.Metrics, error) {
+	startTime := time.Now()
+	defer servermetrics.CollectVolumeStatCalDuration(string(mc.csiClientGetter.driverName), startTime)
 	currentTime := metav1.Now()
 	ctx, cancel := context.WithTimeout(context.Background(), csiTimeout)
 	defer cancel()

pkg/volume/metrics_block.go

@@ -21,9 +21,11 @@ import (
 	"io"
 	"os"
 	"runtime"
+	"time"
 
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics"
 )
 
 var _ MetricsProvider = &metricsBlock{}
@@ -49,6 +51,9 @@ func NewMetricsBlock(device string) MetricsProvider {
 // tools. Storage systems may have more information that they can provide by
 // going through specialized APIs.
 func (mb *metricsBlock) GetMetrics() (*Metrics, error) {
+	startTime := time.Now()
+	defer servermetrics.CollectVolumeStatCalDuration("block", startTime)
+
 	// TODO: Windows does not yet support VolumeMode=Block
 	if runtime.GOOS == "windows" {
 		return nil, NewNotImplementedError("Windows does not support Block volumes")

pkg/volume/metrics_du.go

@@ -17,8 +17,11 @@ limitations under the License.
 package volume
 
 import (
+	"time"
+
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics"
 	"k8s.io/kubernetes/pkg/volume/util/fs"
 )
 
@@ -41,6 +44,9 @@ func NewMetricsDu(path string) MetricsProvider {
 // and gathering filesystem info for the Volume path.
 // See MetricsProvider.GetMetrics
 func (md *metricsDu) GetMetrics() (*Metrics, error) {
+	startTime := time.Now()
+	defer servermetrics.CollectVolumeStatCalDuration("du", startTime)
+
 	metrics := &Metrics{Time: metav1.Now()}
 	if md.path == "" {
 		return metrics, NewNoPathDefinedError()

pkg/volume/metrics_statfs.go

@@ -17,8 +17,11 @@ limitations under the License.
 package volume
 
 import (
+	"time"
+
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics"
 	"k8s.io/kubernetes/pkg/volume/util/fs"
 )
 
@@ -40,6 +43,9 @@ func NewMetricsStatFS(path string) MetricsProvider {
 // GetMetrics calculates the volume usage and device free space by executing "du"
 // and gathering filesystem info for the Volume path.
 func (md *metricsStatFS) GetMetrics() (*Metrics, error) {
+	startTime := time.Now()
+	defer servermetrics.CollectVolumeStatCalDuration("statfs", startTime)
+
 	metrics := &Metrics{Time: metav1.Now()}
 	if md.path == "" {
 		return metrics, NewNoPathDefinedError()