pkg/proxy: add last-queued-timestamp metric

This adds a metric, kubeproxy_sync_proxy_rules_last_queued_timestamp,
that captures the last time a change was queued to be applied to the
proxy. This matches the healthz logic, which fails if a pending change
is stale.

This allows us to write alerts that mirror healthz.

Signed-off-by: Casey Callendrello <cdc@redhat.com>

pkg/proxy/iptables/proxier.go

@@ -504,6 +504,7 @@ func (proxier *Proxier) probability(n int) string {
 func (proxier *Proxier) Sync() {
 	if proxier.healthzServer != nil {
 		proxier.healthzServer.QueuedUpdate()
+		metrics.SyncProxyRulesLastQueuedTimestamp.SetToCurrentTime()
 	}
 	proxier.syncRunner.Run()
 }

pkg/proxy/ipvs/proxier.go

@@ -810,6 +810,7 @@ func CleanupLeftovers(ipvs utilipvs.Interface, ipt utiliptables.Interface, ipset
 func (proxier *Proxier) Sync() {
 	if proxier.healthzServer != nil {
 		proxier.healthzServer.QueuedUpdate()
+		metrics.SyncProxyRulesLastQueuedTimestamp.SetToCurrentTime()
 	}
 	proxier.syncRunner.Run()
 }

pkg/proxy/metrics/metrics.go

@@ -125,6 +125,18 @@ var (
 			StabilityLevel: metrics.ALPHA,
 		},
 	)
+
+	// SyncProxyRulesLastQueuedTimestamp is the last time a proxy sync was
+	// requested. If this is much larger than
+	// kubeproxy_sync_proxy_rules_last_timestamp_seconds, then something is hung.
+	SyncProxyRulesLastQueuedTimestamp = metrics.NewGauge(
+		&metrics.GaugeOpts{
+			Subsystem:      kubeProxySubsystem,
+			Name:           "sync_proxy_rules_last_queued_timestamp_seconds",
+			Help:           "The last time a sync of proxy rules was queued",
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
 )
 
 var registerMetricsOnce sync.Once
@@ -140,6 +152,7 @@ func RegisterMetrics() {
 		legacyregistry.MustRegister(ServiceChangesPending)
 		legacyregistry.MustRegister(ServiceChangesTotal)
 		legacyregistry.MustRegister(IptablesRestoreFailuresTotal)
+		legacyregistry.MustRegister(SyncProxyRulesLastQueuedTimestamp)
 	})
 }
 

pkg/proxy/winkernel/proxier.go

@@ -736,6 +736,7 @@ func getHnsNetworkInfo(hnsNetworkName string) (*hnsNetworkInfo, error) {
 func (proxier *Proxier) Sync() {
 	if proxier.healthzServer != nil {
 		proxier.healthzServer.QueuedUpdate()
+		metrics.SyncProxyRulesLastQueuedTimestamp.SetToCurrentTime()
 	}
 	proxier.syncRunner.Run()
 }