From ac2228705118ee5ba83ce3c3ca980397425bdfb1 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Mon, 14 Feb 2022 17:32:24 +0100 Subject: [PATCH 1/2] kubelet apiserver: be gentle closing connections on heartbeat failures --- cmd/kubelet/app/server.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 7500510e3b6..db114fbe37e 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -845,6 +845,17 @@ func buildKubeletClientConfig(ctx context.Context, s *options.KubeletServer, nod if err != nil { return nil, nil, err } + // Kubelet needs to be able to recover from stale http connections. + // HTTP2 has a mechanism to detect broken connections by sending periodical pings. + // HTTP1 only can have one persistent connection, and it will close all Idle connections + // once the Kubelet heartbeat fails. However, since there are many edge cases that we can't + // control, users can still opt-in to the previous behavior for closing the connections by + // setting the environment variable DISABLE_HTTP2. + if s := os.Getenv("DISABLE_HTTP2"); len(s) > 0 { + klog.InfoS("HTTP2 has been explicitly disabled, Kubelet will forcefully close active connections on heartbeat failures") + } else { + closeAllConns = func() { utilnet.CloseIdleConnectionsFor(transportConfig.Transport) } + } klog.V(2).InfoS("Starting client certificate rotation") clientCertificateManager.Start() From 39e9b19b88276f50fa860007d5cd6f0194679c2c Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Fri, 18 Feb 2022 11:00:51 +0100 Subject: [PATCH 2/2] kubelet: rename closeAllConns to onHeartbeatFailure Since the behavior of the kubelet OnHeartbeatFailure hook has changed to close only the idle connections, rename this function to match the semantics. --- cmd/kubelet/app/server.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index db114fbe37e..bcf88f9dd1a 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -554,14 +554,14 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend klog.InfoS("Standalone mode, no API client") case kubeDeps.KubeClient == nil, kubeDeps.EventClient == nil, kubeDeps.HeartbeatClient == nil: - clientConfig, closeAllConns, err := buildKubeletClientConfig(ctx, s, nodeName) + clientConfig, onHeartbeatFailure, err := buildKubeletClientConfig(ctx, s, nodeName) if err != nil { return err } - if closeAllConns == nil { - return errors.New("closeAllConns must be a valid function other than nil") + if onHeartbeatFailure == nil { + return errors.New("onHeartbeatFailure must be a valid function other than nil") } - kubeDeps.OnHeartbeatFailure = closeAllConns + kubeDeps.OnHeartbeatFailure = onHeartbeatFailure kubeDeps.KubeClient, err = clientset.NewForConfig(clientConfig) if err != nil { @@ -845,6 +845,7 @@ func buildKubeletClientConfig(ctx context.Context, s *options.KubeletServer, nod if err != nil { return nil, nil, err } + var onHeartbeatFailure func() // Kubelet needs to be able to recover from stale http connections. // HTTP2 has a mechanism to detect broken connections by sending periodical pings. // HTTP1 only can have one persistent connection, and it will close all Idle connections @@ -853,14 +854,15 @@ func buildKubeletClientConfig(ctx context.Context, s *options.KubeletServer, nod // setting the environment variable DISABLE_HTTP2. if s := os.Getenv("DISABLE_HTTP2"); len(s) > 0 { klog.InfoS("HTTP2 has been explicitly disabled, Kubelet will forcefully close active connections on heartbeat failures") + onHeartbeatFailure = closeAllConns } else { - closeAllConns = func() { utilnet.CloseIdleConnectionsFor(transportConfig.Transport) } + onHeartbeatFailure = func() { utilnet.CloseIdleConnectionsFor(transportConfig.Transport) } } klog.V(2).InfoS("Starting client certificate rotation") clientCertificateManager.Start() - return transportConfig, closeAllConns, nil + return transportConfig, onHeartbeatFailure, nil } if len(s.BootstrapKubeconfig) > 0 { @@ -884,19 +886,19 @@ func buildKubeletClientConfig(ctx context.Context, s *options.KubeletServer, nod // once the Kubelet heartbeat fails. However, since there are many edge cases that we can't // control, users can still opt-in to the previous behavior for closing the connections by // setting the environment variable DISABLE_HTTP2. - var closeAllConns func() + var onHeartbeatFailure func() if s := os.Getenv("DISABLE_HTTP2"); len(s) > 0 { klog.InfoS("HTTP2 has been explicitly disabled, updating Kubelet client Dialer to forcefully close active connections on heartbeat failures") - closeAllConns, err = updateDialer(clientConfig) + onHeartbeatFailure, err = updateDialer(clientConfig) if err != nil { return nil, nil, err } } else { - closeAllConns = func() { + onHeartbeatFailure = func() { utilnet.CloseIdleConnectionsFor(clientConfig.Transport) } } - return clientConfig, closeAllConns, nil + return clientConfig, onHeartbeatFailure, nil } // updateDialer instruments a restconfig with a dial. the returned function allows forcefully closing all active connections.