Merge pull request #44968 from MrHohn/kube-proxy-healthcheck

Automatic merge from submit-queue (batch tested with PRs 44727, 45409, 44968, 45122, 45493)

Separate healthz server from metrics server in kube-proxy

From #14661, proposal is on kubernetes/community#552.

Couple bullet points as in commit:
- /healthz will be served on 0.0.0.0:10256 by default.
- /metrics and /proxyMode will be served on port 10249 as before.
- Healthz handler will verify timestamp in iptables mode.

/assign @nicksardo @bowei @thockin 

**Release note**:

```release-note
NONE
```

cmd/kube-proxy/BUILD

@@ -31,7 +31,6 @@ go_library(
         "//pkg/version/prometheus:go_default_library",
         "//pkg/version/verflag:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/logs:go_default_library",
     ],

cmd/kube-proxy/app/BUILD

@@ -25,6 +25,7 @@ go_library(
         "//pkg/kubelet/qos:go_default_library",
         "//pkg/proxy:go_default_library",
         "//pkg/proxy/config:go_default_library",
+        "//pkg/proxy/healthcheck:go_default_library",
         "//pkg/proxy/iptables:go_default_library",
         "//pkg/proxy/userspace:go_default_library",
         "//pkg/proxy/winuserspace:go_default_library",

cmd/kube-proxy/app/server.go

@@ -51,6 +51,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/qos"
 	"k8s.io/kubernetes/pkg/proxy"
 	proxyconfig "k8s.io/kubernetes/pkg/proxy/config"
+	"k8s.io/kubernetes/pkg/proxy/healthcheck"
 	"k8s.io/kubernetes/pkg/proxy/iptables"
 	"k8s.io/kubernetes/pkg/proxy/userspace"
 	"k8s.io/kubernetes/pkg/proxy/winuserspace"
@@ -249,7 +250,7 @@ func applyDefaults(in *componentconfig.KubeProxyConfiguration) (*componentconfig
 func NewProxyCommand() *cobra.Command {
 	opts := Options{
 		config:      new(componentconfig.KubeProxyConfiguration),
-		healthzPort: 10249,
+		healthzPort: 10256,
 	}
 
 	cmd := &cobra.Command{
@@ -296,7 +297,7 @@ type ProxyServer struct {
 	ProxyMode              string
 	NodeRef                *clientv1.ObjectReference
 	CleanupAndExit         bool
-	HealthzBindAddress     string
+	MetricsBindAddress     string
 	OOMScoreAdj            *int32
 	ResourceContainer      string
 	ConfigSyncPeriod       time.Duration
@@ -305,6 +306,7 @@ type ProxyServer struct {
 	// get rid of this one.
 	ServiceHandler        proxyconfig.ServiceConfigHandler
 	EndpointsEventHandler proxyconfig.EndpointsHandler
+	HealthzServer         *healthcheck.HealthzServer
 }
 
 // createClients creates a kube client and an event client from the given config and masterOverride.
@@ -388,6 +390,11 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx
 	eventBroadcaster := record.NewBroadcaster()
 	recorder := eventBroadcaster.NewRecorder(api.Scheme, clientv1.EventSource{Component: "kube-proxy", Host: hostname})
 
+	var healthzServer *healthcheck.HealthzServer
+	if len(config.HealthzBindAddress) > 0 {
+		healthzServer = healthcheck.NewDefaultHealthzServer(config.HealthzBindAddress, 2*config.IPTables.SyncPeriod.Duration)
+	}
+
 	var proxier proxy.ProxyProvider
 	var serviceEventHandler proxyconfig.ServiceHandler
 	// TODO: Migrate all handlers to ServiceHandler types and
@@ -416,6 +423,7 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx
 			hostname,
 			getNodeIP(client, hostname),
 			recorder,
+			healthzServer,
 		)
 		if err != nil {
 			return nil, fmt.Errorf("unable to create proxier: %v", err)
@@ -504,13 +512,14 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx
 		Conntracker:            &realConntracker{},
 		ProxyMode:              proxyMode,
 		NodeRef:                nodeRef,
-		HealthzBindAddress:     config.HealthzBindAddress,
+		MetricsBindAddress:     config.MetricsBindAddress,
 		OOMScoreAdj:            config.OOMScoreAdj,
 		ResourceContainer:      config.ResourceContainer,
 		ConfigSyncPeriod:       config.ConfigSyncPeriod.Duration,
 		ServiceEventHandler:    serviceEventHandler,
 		ServiceHandler:         serviceHandler,
 		EndpointsEventHandler:  endpointsEventHandler,
+		HealthzServer:          healthzServer,
 	}, nil
 }
 
@@ -546,17 +555,22 @@ func (s *ProxyServer) Run() error {
 
 	s.Broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: s.EventClient.Events("")})
 
-	// Start up a webserver if requested
-	if len(s.HealthzBindAddress) > 0 {
+	// Start up a healthz server if requested
+	if s.HealthzServer != nil {
+		s.HealthzServer.Run()
+	}
+
+	// Start up a metrics server if requested
+	if len(s.MetricsBindAddress) > 0 {
 		http.HandleFunc("/proxyMode", func(w http.ResponseWriter, r *http.Request) {
 			fmt.Fprintf(w, "%s", s.ProxyMode)
 		})
 		http.Handle("/metrics", prometheus.Handler())
 		configz.InstallHandler(http.DefaultServeMux)
 		go wait.Until(func() {
-			err := http.ListenAndServe(s.HealthzBindAddress, nil)
+			err := http.ListenAndServe(s.MetricsBindAddress, nil)
 			if err != nil {
-				utilruntime.HandleError(fmt.Errorf("starting health server failed: %v", err))
+				utilruntime.HandleError(fmt.Errorf("starting metrics server failed: %v", err))
 			}
 		}, 5*time.Second, wait.NeverStop)
 	}

cmd/kube-proxy/proxy.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/spf13/pflag"
 
-	"k8s.io/apiserver/pkg/server/healthz"
 	utilflag "k8s.io/apiserver/pkg/util/flag"
 	"k8s.io/apiserver/pkg/util/logs"
 	"k8s.io/kubernetes/cmd/kube-proxy/app"
@@ -32,8 +31,6 @@ import (
 )
 
 func main() {
-	healthz.DefaultHealthz()
-
 	command := app.NewProxyCommand()
 
 	// TODO: once we switch everything over to Cobra commands, we can go back to calling