Augmented kubelet healthz with syncLoop check.

Monit uses read-only port for health monitoring.

pkg/kubelet/server.go

@@ -80,14 +80,12 @@ func ListenAndServeKubeletServer(host HostInterface, address net.IP, port uint,
 // ListenAndServeKubeletReadOnlyServer initializes a server to respond to HTTP network requests on the Kubelet.
 func ListenAndServeKubeletReadOnlyServer(host HostInterface, address net.IP, port uint) {
 	glog.V(1).Infof("Starting to listen read-only on %s:%d", address, port)
-	s := &Server{host, http.NewServeMux()}
-	healthz.InstallHandler(s.mux)
-	s.mux.HandleFunc("/stats/", s.handleStats)
+	s := NewServer(host, false)
 	s.mux.Handle("/metrics", prometheus.Handler())
 
 	server := &http.Server{
 		Addr:           net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)),
-		Handler:        s,
+		Handler:        &s,
 		ReadTimeout:    5 * time.Minute,
 		WriteTimeout:   5 * time.Minute,
 		MaxHeaderBytes: 1 << 20,
@@ -110,7 +108,9 @@ type HostInterface interface {
 	ServeLogs(w http.ResponseWriter, req *http.Request)
 	PortForward(name string, uid types.UID, port uint16, stream io.ReadWriteCloser) error
 	StreamingConnectionIdleTimeout() time.Duration
+	ResyncInterval() time.Duration
 	GetHostname() string
+	LatestLoopEntryTime() time.Time
 }
 
 // NewServer initializes and configures a kubelet.Server object to handle HTTP requests.
@@ -132,6 +132,7 @@ func (s *Server) InstallDefaultHandlers() {
 		healthz.PingHealthz,
 		healthz.NamedCheck("docker", s.dockerHealthCheck),
 		healthz.NamedCheck("hostname", s.hostnameHealthCheck),
+		healthz.NamedCheck("syncloop", s.syncLoopHealthCheck),
 	)
 	s.mux.HandleFunc("/pods", s.handlePods)
 	s.mux.HandleFunc("/stats/", s.handleStats)
@@ -195,6 +196,20 @@ func (s *Server) hostnameHealthCheck(req *http.Request) error {
 	return nil
 }
 
+// Checks if kubelet's sync loop  that updates containers is working.
+func (s *Server) syncLoopHealthCheck(req *http.Request) error {
+	duration := s.host.ResyncInterval() * 2
+	minDuration := time.Minute * 5
+	if duration < minDuration {
+		duration = minDuration
+	}
+	enterLoopTime := s.host.LatestLoopEntryTime()
+	if !enterLoopTime.IsZero() && time.Now().After(enterLoopTime.Add(duration)) {
+		return fmt.Errorf("Sync Loop took longer than expected.")
+	}
+	return nil
+}
+
 // handleContainerLogs handles containerLogs request against the Kubelet
 func (s *Server) handleContainerLogs(w http.ResponseWriter, req *http.Request) {
 	defer req.Body.Close()