Adds TCPCloseWaitTimeout option to kube-proxy for sysctl nf_conntrack_tcp_timeout_time_wait

Fixes issue-32551

cmd/kube-proxy/app/conntrack.go

@@ -27,24 +27,33 @@ import (
 	"k8s.io/kubernetes/pkg/util/sysctl"
 )
 
+// Conntracker is an interface to the global sysctl. Descriptions of the various
+// sysctl fields can be found here:
+//
+// https://www.kernel.org/doc/Documentation/networking/nf_conntrack-sysctl.txt
 type Conntracker interface {
+	// SetMax adjusts nf_conntrack_max.
 	SetMax(max int) error
+	// SetTCPEstablishedTimeout adjusts nf_conntrack_tcp_timeout_established.
 	SetTCPEstablishedTimeout(seconds int) error
+	// SetTCPCloseWaitTimeout nf_conntrack_tcp_timeout_close_wait.
+	SetTCPCloseWaitTimeout(seconds int) error
 }
 
 type realConntracker struct{}
 
 var readOnlySysFSError = errors.New("readOnlySysFS")
 
-func (realConntracker) SetMax(max int) error {
-	glog.Infof("Setting nf_conntrack_max to %d", max)
-	if err := sysctl.New().SetSysctl("net/netfilter/nf_conntrack_max", max); err != nil {
+func (rct realConntracker) SetMax(max int) error {
+	if err := rct.setIntSysCtl("nf_conntrack_max", max); err != nil {
 		return err
 	}
-	// sysfs is expected to be mounted as 'rw'. However, it may be unexpectedly mounted as
-	// 'ro' by docker because of a known docker issue (https://github.com/docker/docker/issues/24000).
-	// Setting conntrack will fail when sysfs is readonly. When that happens, we don't set conntrack
-	// hashsize and return a special error readOnlySysFSError here. The caller should deal with
+	// sysfs is expected to be mounted as 'rw'. However, it may be
+	// unexpectedly mounted as 'ro' by docker because of a known docker
+	// issue (https://github.com/docker/docker/issues/24000). Setting
+	// conntrack will fail when sysfs is readonly. When that happens, we
+	// don't set conntrack hashsize and return a special error
+	// readOnlySysFSError here. The caller should deal with
 	// readOnlySysFSError differently.
 	writable, err := isSysFSWritable()
 	if err != nil {
@@ -58,9 +67,22 @@ func (realConntracker) SetMax(max int) error {
 	return writeIntStringFile("/sys/module/nf_conntrack/parameters/hashsize", max/4)
 }
 
-func (realConntracker) SetTCPEstablishedTimeout(seconds int) error {
-	glog.Infof("Setting nf_conntrack_tcp_timeout_established to %d", seconds)
-	return sysctl.New().SetSysctl("net/netfilter/nf_conntrack_tcp_timeout_established", seconds)
+func (rct realConntracker) SetTCPEstablishedTimeout(seconds int) error {
+	return rct.setIntSysCtl("nf_conntrack_tcp_timeout_established", seconds)
+}
+
+func (rct realConntracker) SetTCPCloseWaitTimeout(seconds int) error {
+	return rct.setIntSysCtl("nf_conntrack_tcp_timeout_close_wait", seconds)
+}
+
+func (realConntracker) setIntSysCtl(name string, value int) error {
+	entry := "net/netfilter/" + name
+
+	glog.Infof("Set sysctl '%v' to %v", entry, value)
+	if err := sysctl.New().SetSysctl(entry, value); err != nil {
+		return err
+	}
+	return nil
 }
 
 // isSysFSWritable checks /proc/mounts to see whether sysfs is 'rw' or not.
@@ -73,18 +95,23 @@ func isSysFSWritable() (bool, error) {
 		glog.Errorf("failed to list mount points: %v", err)
 		return false, err
 	}
+
 	for _, mountPoint := range mountPoints {
+		const sysfsDevice = "sysfs"
 		if mountPoint.Device != sysfsDevice {
 			continue
 		}
 		// Check whether sysfs is 'rw'
+		const permWritable = "rw"
 		if len(mountPoint.Opts) > 0 && mountPoint.Opts[0] == permWritable {
 			return true, nil
 		}
-		glog.Errorf("sysfs is not writable: %+v", mountPoint)
-		break
+		glog.Errorf("sysfs is not writable: %+v (mount options are %v)",
+			mountPoint, mountPoint.Opts)
+		return false, readOnlySysFSError
 	}
-	return false, nil
+
+	return false, errors.New("No sysfs mounted")
 }
 
 func writeIntStringFile(filename string, value int) error {

cmd/kube-proxy/app/options/options.go

@@ -94,5 +94,10 @@ func (s *ProxyServerConfig) AddFlags(fs *pflag.FlagSet) {
 	fs.Int32Var(&s.ConntrackMin, "conntrack-min", s.ConntrackMin,
 		"Minimum number of conntrack entries to allocate, regardless of conntrack-max-per-core (set conntrack-max-per-core=0 to leave the limit as-is).")
 	fs.DurationVar(&s.ConntrackTCPEstablishedTimeout.Duration, "conntrack-tcp-timeout-established", s.ConntrackTCPEstablishedTimeout.Duration, "Idle timeout for established TCP connections (0 to leave as-is)")
+	fs.DurationVar(
+		&s.ConntrackTCPCloseWaitTimeout.Duration, "conntrack-tcp-timeout-close-wait",
+		s.ConntrackTCPCloseWaitTimeout.Duration,
+		"NAT timeout for TCP connections in the CLOSE_WAIT state")
+
 	config.DefaultFeatureGate.AddFlag(fs)
 }

cmd/kube-proxy/app/server.go

@@ -316,12 +316,22 @@ func (s *ProxyServer) Run() error {
 				// administrator should decide whether and how to handle this issue,
 				// whether to drain the node and restart docker.
 				// TODO(random-liu): Remove this when the docker bug is fixed.
-				const message = "DOCKER RESTART NEEDED (docker issue #24000): /sys is read-only: can't raise conntrack limits, problems may arise later."
+				const message = "DOCKER RESTART NEEDED (docker issue #24000): /sys is read-only: " +
+					"cannot modify conntrack limits, problems may arise later."
 				s.Recorder.Eventf(s.Config.NodeRef, api.EventTypeWarning, err.Error(), message)
 			}
 		}
+
 		if s.Config.ConntrackTCPEstablishedTimeout.Duration > 0 {
-			if err := s.Conntracker.SetTCPEstablishedTimeout(int(s.Config.ConntrackTCPEstablishedTimeout.Duration / time.Second)); err != nil {
+			timeout := int(s.Config.ConntrackTCPEstablishedTimeout.Duration / time.Second)
+			if err := s.Conntracker.SetTCPEstablishedTimeout(timeout); err != nil {
+				return err
+			}
+		}
+
+		if s.Config.ConntrackTCPCloseWaitTimeout.Duration > 0 {
+			timeout := int(s.Config.ConntrackTCPCloseWaitTimeout.Duration / time.Second)
+			if err := s.Conntracker.SetTCPCloseWaitTimeout(timeout); err != nil {
 				return err
 			}
 		}