add new metric to record the latency to allocate an IP address

pkg/registry/core/service/ipallocator/ipallocator.go

@@ -206,7 +206,13 @@ func (a *Allocator) allocateService(svc *api.Service, ip net.IP, dryRun bool) er
 	if dryRun {
 		return nil
 	}
-	return a.createIPAddress(ip.String(), svc, "static")
+	start := time.Now()
+	err = a.createIPAddress(ip.String(), svc, "static")
+	if err != nil {
+		return err
+	}
+	a.metrics.setLatency(a.metricLabel, time.Since(start))
+	return nil
 }
 
 // AllocateNext return an IP address that wasn't allocated yet.
@@ -239,6 +245,7 @@ func (a *Allocator) allocateNextService(svc *api.Service, dryRun bool) (net.IP,
 
 	trace := utiltrace.New("allocate dynamic ClusterIP address")
 	defer trace.LogIfLong(500 * time.Millisecond)
+	start := time.Now()
 
 	// rand.Int63n panics for n <= 0 so we need to avoid problems when
 	// converting from uint64 to int64
@@ -255,6 +262,7 @@ func (a *Allocator) allocateNextService(svc *api.Service, dryRun bool) (net.IP,
 	iterator := ipIterator(a.offsetAddress, a.lastAddress, offset)
 	ip, err := a.allocateFromRange(iterator, svc)
 	if err == nil {
+		a.metrics.setLatency(a.metricLabel, time.Since(start))
 		return ip, nil
 	}
 	// check the lower range
@@ -263,6 +271,7 @@ func (a *Allocator) allocateNextService(svc *api.Service, dryRun bool) (net.IP,
 		iterator = ipIterator(a.firstAddress, a.offsetAddress.Prev(), offset)
 		ip, err = a.allocateFromRange(iterator, svc)
 		if err == nil {
+			a.metrics.setLatency(a.metricLabel, time.Since(start))
 			return ip, nil
 		}
 	}

pkg/registry/core/service/ipallocator/metrics.go

@@ -18,6 +18,7 @@ package ipallocator
 
 import (
 	"sync"
+	"time"
 
 	"k8s.io/component-base/metrics"
 	"k8s.io/component-base/metrics/legacyregistry"
@@ -73,6 +74,17 @@ var (
 		},
 		[]string{"cidr", "scope"},
 	)
+	clusterIPAllocationLatency = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "allocation_duration_seconds",
+			Help:           "Duration in seconds to allocate a Cluster IP by ServiceCIDR",
+			Buckets:        metrics.DefBuckets,
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"cidr"},
+	)
 )
 
 var registerMetricsOnce sync.Once
@@ -83,6 +95,7 @@ func registerMetrics() {
 		legacyregistry.MustRegister(clusterIPAvailable)
 		legacyregistry.MustRegister(clusterIPAllocations)
 		legacyregistry.MustRegister(clusterIPAllocationErrors)
+		legacyregistry.MustRegister(clusterIPAllocationLatency)
 	})
 }
 
@@ -90,6 +103,7 @@ func registerMetrics() {
 type metricsRecorderInterface interface {
 	setAllocated(cidr string, allocated int)
 	setAvailable(cidr string, available int)
+	setLatency(cidr string, latency time.Duration)
 	incrementAllocations(cidr, scope string)
 	incrementAllocationErrors(cidr, scope string)
 }
@@ -105,6 +119,10 @@ func (m *metricsRecorder) setAvailable(cidr string, available int) {
 	clusterIPAvailable.WithLabelValues(cidr).Set(float64(available))
 }
 
+func (m *metricsRecorder) setLatency(cidr string, latency time.Duration) {
+	clusterIPAllocationLatency.WithLabelValues(cidr).Observe(latency.Seconds())
+}
+
 func (m *metricsRecorder) incrementAllocations(cidr, scope string) {
 	clusterIPAllocations.WithLabelValues(cidr, scope).Inc()
 }
@@ -118,5 +136,6 @@ type emptyMetricsRecorder struct{}
 
 func (*emptyMetricsRecorder) setAllocated(cidr string, allocated int)       {}
 func (*emptyMetricsRecorder) setAvailable(cidr string, available int)       {}
+func (*emptyMetricsRecorder) setLatency(cidr string, latency time.Duration) {}
 func (*emptyMetricsRecorder) incrementAllocations(cidr, scope string)       {}
 func (*emptyMetricsRecorder) incrementAllocationErrors(cidr, scope string)  {}