Add a 5x exponential backoff on 429s & 5xxs to the webhook Authenticator/Authorizer.

plugin/pkg/webhook/webhook.go

@@ -19,6 +19,7 @@ package webhook
 
 import (
 	"fmt"
+	"time"
 
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/unversioned"
@@ -27,16 +28,18 @@ import (
 	"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
 	"k8s.io/kubernetes/pkg/runtime"
 	runtimeserializer "k8s.io/kubernetes/pkg/runtime/serializer"
+	"k8s.io/kubernetes/pkg/util/wait"
 
 	_ "k8s.io/kubernetes/pkg/apis/authorization/install"
 )
 
 type GenericWebhook struct {
-	RestClient *restclient.RESTClient
+	RestClient     *restclient.RESTClient
+	initialBackoff time.Duration
 }
 
 // New creates a new GenericWebhook from the provided kubeconfig file.
-func NewGenericWebhook(kubeConfigFile string, groupVersions []unversioned.GroupVersion) (*GenericWebhook, error) {
+func NewGenericWebhook(kubeConfigFile string, groupVersions []unversioned.GroupVersion, initialBackoff time.Duration) (*GenericWebhook, error) {
 	for _, groupVersion := range groupVersions {
 		if !registered.IsEnabledVersion(groupVersion) {
 			return nil, fmt.Errorf("webhook plugin requires enabling extension resource: %s", groupVersion)
@@ -64,5 +67,31 @@ func NewGenericWebhook(kubeConfigFile string, groupVersions []unversioned.GroupV
 
 	// TODO(ericchiang): Can we ensure remote service is reachable?
 
-	return &GenericWebhook{restClient}, nil
+	return &GenericWebhook{restClient, initialBackoff}, nil
+}
+
+// WithExponentialBackoff will retry webhookFn 5 times w/ exponentially
+// increasing backoff when a 429 or a 5xx response code is returned.
+func (g *GenericWebhook) WithExponentialBackoff(webhookFn func() restclient.Result) restclient.Result {
+	backoff := wait.Backoff{
+		Duration: g.initialBackoff,
+		Factor:   1.5,
+		Jitter:   0.2,
+		Steps:    5,
+	}
+	var result restclient.Result
+	wait.ExponentialBackoff(backoff, func() (bool, error) {
+		result = webhookFn()
+		// Return from Request.Do() errors immediately.
+		if err := result.Error(); err != nil {
+			return false, err
+		}
+		// Retry 429s, and 5xxs.
+		var statusCode int
+		if result.StatusCode(&statusCode); statusCode == 429 || statusCode >= 500 {
+			return false, nil
+		}
+		return true, nil
+	})
+	return result
 }