Add nftables cleanup failure metric, fix cleanup bug

If the sync fails, don't try to cleanup, since it's guaranteed to fail
too.

pkg/proxy/metrics/metrics.go

@@ -196,6 +196,17 @@ var (
 		},
 	)
 
+	// NFTablesCleanupFailuresTotal is the number of nftables stale chain cleanup
+	// failures that the proxy has seen.
+	NFTablesCleanupFailuresTotal = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Subsystem:      kubeProxySubsystem,
+			Name:           "sync_proxy_rules_nftables_cleanup_failures_total",
+			Help:           "Cumulative proxy nftables cleanup failures",
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
+
 	// ProxyHealthzTotal is the number of returned HTTP Status for each
 	// healthz probe.
 	ProxyHealthzTotal = metrics.NewCounterVec(
@@ -280,6 +291,7 @@ func RegisterMetrics(mode kubeproxyconfig.ProxyMode) {
 
 		case kubeproxyconfig.ProxyModeNFTables:
 			legacyregistry.MustRegister(NFTablesSyncFailuresTotal)
+			legacyregistry.MustRegister(NFTablesCleanupFailuresTotal)
 
 		case kubeproxyconfig.ProxyModeKernelspace:
 			// currently no winkernel-specific metrics

pkg/proxy/nftables/proxier.go

@@ -1033,7 +1033,7 @@ func (proxier *Proxier) syncProxyRules() {
 				// the chains still exist, they'll just get added back
 				// (with a later timestamp) at the end of the sync.
 				proxier.logger.Error(err, "Unable to delete stale chains; will retry later")
-				// FIXME: metric
+				metrics.NFTablesCleanupFailuresTotal.Inc()
 			}
 		}
 	}
@@ -1621,6 +1621,10 @@ func (proxier *Proxier) syncProxyRules() {
 	if err != nil {
 		proxier.logger.Error(err, "nftables sync failed")
 		metrics.NFTablesSyncFailuresTotal.Inc()
+
+		// staleChains is now incorrect since we didn't actually flush the
+		// chains in it. We can recompute it next time.
+		clear(proxier.staleChains)
 		return
 	}
 	success = true