kubelet: add feature gate check for exec probe timeouts

This change also involves adding a custom error type for probe timeouts
so that the kubelet exec prober can distinguish between failed probes
that have exited or probes that have timed out.

Signed-off-by: Andrew Sy Kim <kim.andrewsy@gmail.com>

pkg/probe/exec/BUILD

@@ -8,11 +8,16 @@ load(
 
 go_library(
     name = "go_default_library",
-    srcs = ["exec.go"],
+    srcs = [
+        "errors.go",
+        "exec.go",
+    ],
     importpath = "k8s.io/kubernetes/pkg/probe/exec",
     deps = [
+        "//pkg/features:go_default_library",
         "//pkg/kubelet/util/ioutils:go_default_library",
         "//pkg/probe:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
         "//vendor/k8s.io/utils/exec:go_default_library",
     ],

pkg/probe/exec/errors.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package exec
+
+import (
+	"time"
+)
+
+// NewTimeoutError returns a new TimeoutError.
+func NewTimeoutError(err error, timeout time.Duration) *TimeoutError {
+	return &TimeoutError{
+		err:     err,
+		timeout: timeout,
+	}
+}
+
+// TimeoutError is an error returned on exec probe timeouts. It should be returned by CRI implementations
+// in order for the exec prober to interpret exec timeouts as failed probes.
+// TODO: this error type can likely be removed when we support CRI errors.
+type TimeoutError struct {
+	err     error
+	timeout time.Duration
+}
+
+// Error returns the error string.
+func (t *TimeoutError) Error() string {
+	return t.err.Error()
+}
+
+// Timeout returns the timeout duration of the exec probe.
+func (t *TimeoutError) Timeout() time.Duration {
+	return t.timeout
+}

pkg/probe/exec/exec.go

@@ -19,6 +19,8 @@ package exec
 import (
 	"bytes"
 
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet/util/ioutils"
 	"k8s.io/kubernetes/pkg/probe"
 
@@ -66,6 +68,16 @@ func (pr execProber) Probe(e exec.Cmd) (probe.Result, string, error) {
 			}
 			return probe.Failure, string(data), nil
 		}
+
+		timeoutErr, ok := err.(*TimeoutError)
+		if ok {
+			if utilfeature.DefaultFeatureGate.Enabled(features.ExecProbeTimeout) {
+				return probe.Failure, string(data), nil
+			}
+
+			klog.Warningf("Exec probe timed out after %s but ExecProbeTimeout feature gate was disabled", timeoutErr.Timeout())
+		}
+
 		return probe.Unknown, "", err
 	}
 	return probe.Success, string(data), nil