From 49209b3394d1002adeae5041e14d06eee855365f Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Fri, 28 Apr 2017 11:57:19 -0400 Subject: [PATCH] Make timeouts in the Kubelet slightly offset to aid debugging Several of these loops overlap, and when they are the reason a failure is happening it is difficult to sort them out. Slighly misalign these loops to make their impact obvious. --- pkg/kubelet/dockertools/kube_docker_client.go | 4 +++- pkg/kubelet/rkt/rkt.go | 4 +++- pkg/kubelet/volumemanager/volume_manager.go | 4 +++- .../goroutinemap/exponentialbackoff/exponential_backoff.go | 4 +++- pkg/util/goroutinemap/goroutinemap.go | 4 +++- 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pkg/kubelet/dockertools/kube_docker_client.go b/pkg/kubelet/dockertools/kube_docker_client.go index 20e2acaab94..c0dca47d58a 100644 --- a/pkg/kubelet/dockertools/kube_docker_client.go +++ b/pkg/kubelet/dockertools/kube_docker_client.go @@ -70,7 +70,9 @@ var _ DockerInterface = &kubeDockerClient{} // kubeDockerClient only applies timeout on non-long running operations. const ( // defaultTimeout is the default timeout of short running docker operations. - defaultTimeout = 2 * time.Minute + // Value is slightly offset from 2 minutes to make timeouts due to this + // constant recognizable. + defaultTimeout = 2*time.Minute - 1*time.Second // defaultShmSize is the default ShmSize to use (in bytes) if not specified. defaultShmSize = int64(1024 * 1024 * 64) diff --git a/pkg/kubelet/rkt/rkt.go b/pkg/kubelet/rkt/rkt.go index c765c1a046c..e60c2f86109 100644 --- a/pkg/kubelet/rkt/rkt.go +++ b/pkg/kubelet/rkt/rkt.go @@ -135,7 +135,9 @@ const ( defaultNetworkName = "rkt.kubernetes.io" // defaultRequestTimeout is the default timeout of rkt requests. - defaultRequestTimeout = 2 * time.Minute + // Value is slightly offset from 2 minutes to make timeouts due to this + // constant recognizable. + defaultRequestTimeout = 2*time.Minute - 1*time.Second etcHostsPath = "/etc/hosts" etcResolvConfPath = "/etc/resolv.conf" diff --git a/pkg/kubelet/volumemanager/volume_manager.go b/pkg/kubelet/volumemanager/volume_manager.go index 47e7c4b5824..da2f4a5f478 100644 --- a/pkg/kubelet/volumemanager/volume_manager.go +++ b/pkg/kubelet/volumemanager/volume_manager.go @@ -71,7 +71,9 @@ const ( // will retry in the next sync iteration. This frees the associated // goroutine of the pod to process newer updates if needed (e.g., a delete // request to the pod). - podAttachAndMountTimeout time.Duration = 2 * time.Minute + // Value is slightly offset from 2 minutes to make timeouts due to this + // constant recognizable. + podAttachAndMountTimeout time.Duration = 2*time.Minute + 3*time.Second // podAttachAndMountRetryInterval is the amount of time the GetVolumesForPod // call waits before retrying diff --git a/pkg/util/goroutinemap/exponentialbackoff/exponential_backoff.go b/pkg/util/goroutinemap/exponentialbackoff/exponential_backoff.go index c00fe46823b..9dbad16a203 100644 --- a/pkg/util/goroutinemap/exponentialbackoff/exponential_backoff.go +++ b/pkg/util/goroutinemap/exponentialbackoff/exponential_backoff.go @@ -32,7 +32,9 @@ const ( // maxDurationBeforeRetry is the maximum amount of time that // durationBeforeRetry will grow to due to exponential backoff. - maxDurationBeforeRetry time.Duration = 2 * time.Minute + // Value is slightly offset from 2 minutes to make timeouts due to this + // constant recognizable. + maxDurationBeforeRetry time.Duration = 2*time.Minute + 2*time.Second ) // ExponentialBackoff contains the last occurrence of an error and the duration diff --git a/pkg/util/goroutinemap/goroutinemap.go b/pkg/util/goroutinemap/goroutinemap.go index 2c6c3aca61d..b28ca30a615 100644 --- a/pkg/util/goroutinemap/goroutinemap.go +++ b/pkg/util/goroutinemap/goroutinemap.go @@ -40,7 +40,9 @@ const ( // maxDurationBeforeRetry is the maximum amount of time that // durationBeforeRetry will grow to due to exponential backoff. - maxDurationBeforeRetry = 2 * time.Minute + // Value is slightly offset from 2 minutes to make timeouts due to this + // constant recognizable. + maxDurationBeforeRetry = 2*time.Minute + 1*time.Second ) // GoRoutineMap defines a type that can run named goroutines and track their