kms: add wiring to support automatic encryption config reload

This change adds a flag --encryption-provider-config-automatic-reload
which will be used to drive automatic reloading of the encryption
config at runtime.  While this flag is set to true, or when KMS v2
plugins are used without KMS v1 plugins, the /healthz endpoints
associated with said plugins are collapsed into a single endpoint at
/healthz/kms-providers - in this state, it is not possible to
configure exclusions for specific KMS providers while including the
remaining ones - ex: using /readyz?exclude=kms-provider-1 to exclude
a particular KMS is not possible.  This single healthz check handles
checking  all configured KMS providers.  When reloading is enabled
but no KMS providers are configured, it is a no-op.

k8s.io/apiserver does not support dynamic addition and removal of
healthz checks at runtime.  Reloading will instead have a single
static healthz check and swap the underlying implementation at
runtime when a config change occurs.

Signed-off-by: Monis Khan <mok@microsoft.com>

test/integration/controlplane/transformation/all_transformation_test.go

@@ -95,7 +95,7 @@ resources:
         secret: c2VjcmV0IGlzIHNlY3VyZQ==
 `
 
-	test, err := newTransformTest(t, encryptionConfig)
+	test, err := newTransformTest(t, encryptionConfig, false)
 	if err != nil {
 		t.Fatalf("failed to start Kube API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
 	}

test/integration/controlplane/transformation/kms_transformation_test.go

@@ -26,19 +26,16 @@ import (
 	"encoding/base64"
 	"encoding/binary"
 	"fmt"
-	"net/http"
 	"strings"
 	"testing"
 	"time"
 
 	"golang.org/x/crypto/cryptobyte"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/storage/value"
 	aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
 	mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/v1beta1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
 	kmsapi "k8s.io/kms/apis/v1beta1"
 )
 
@@ -131,7 +128,7 @@ resources:
 	}
 	defer pluginMock.CleanUp()
 
-	test, err := newTransformTest(t, encryptionConfig)
+	test, err := newTransformTest(t, encryptionConfig, false)
 	if err != nil {
 		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
 	}
@@ -320,7 +317,7 @@ resources:
 		t.Fatalf("Failed to start KMS Plugin #2: err: %v", err)
 	}
 
-	test, err := newTransformTest(t, encryptionConfig)
+	test, err := newTransformTest(t, encryptionConfig, false)
 	if err != nil {
 		t.Fatalf("Failed to start kube-apiserver, error: %v", err)
 	}
@@ -331,66 +328,103 @@ resources:
 
 	// Stage 1 - Since all kms-plugins are guaranteed to be up, healthz checks for:
 	// healthz/kms-provider-0 and /healthz/kms-provider-1 should be OK.
-	mustBeHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig)
-	mustBeHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig)
+	mustBeHealthy(t, "/kms-provider-0", "ok", test.kubeAPIServer.ClientConfig)
+	mustBeHealthy(t, "/kms-provider-1", "ok", test.kubeAPIServer.ClientConfig)
 
 	// Stage 2 - kms-plugin for provider-1 is down. Therefore, expect the health check for provider-1
 	// to fail, but provider-2 should still be OK
 	pluginMock1.EnterFailedState()
-	mustBeUnHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig)
-	mustBeHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig)
+	mustBeUnHealthy(t, "/kms-provider-0",
+		"internal server error: rpc error: code = FailedPrecondition desc = failed precondition - key disabled",
+		test.kubeAPIServer.ClientConfig)
+	mustBeHealthy(t, "/kms-provider-1", "ok", test.kubeAPIServer.ClientConfig)
 	pluginMock1.ExitFailedState()
 
 	// Stage 3 - kms-plugin for provider-1 is now up. Therefore, expect the health check for provider-1
 	// to succeed now, but provider-2 is now down.
-	// Need to sleep since health check chases responses for 3 seconds.
 	pluginMock2.EnterFailedState()
-	mustBeHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig)
-	mustBeUnHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig)
+	mustBeHealthy(t, "/kms-provider-0", "ok", test.kubeAPIServer.ClientConfig)
+	mustBeUnHealthy(t, "/kms-provider-1",
+		"internal server error: rpc error: code = FailedPrecondition desc = failed precondition - key disabled",
+		test.kubeAPIServer.ClientConfig)
+	pluginMock2.ExitFailedState()
+
+	// Stage 4 - All kms-plugins are once again up,
+	// the healthz check should be OK.
+	mustBeHealthy(t, "/kms-provider-0", "ok", test.kubeAPIServer.ClientConfig)
+	mustBeHealthy(t, "/kms-provider-1", "ok", test.kubeAPIServer.ClientConfig)
 }
 
-func mustBeHealthy(t *testing.T, checkName string, clientConfig *rest.Config) {
-	t.Helper()
-	var restErr error
-	pollErr := wait.PollImmediate(2*time.Second, wait.ForeverTestTimeout, func() (bool, error) {
-		status, err := getHealthz(checkName, clientConfig)
-		restErr = err
-		if err != nil {
-			return false, err
-		}
-		return status == http.StatusOK, nil
-	})
+func TestKMSHealthzWithReload(t *testing.T) {
+	encryptionConfig := `
+kind: EncryptionConfiguration
+apiVersion: apiserver.config.k8s.io/v1
+resources:
+  - resources:
+    - secrets
+    providers:
+    - kms:
+       name: provider-1
+       endpoint: unix:///@kms-provider-1.sock
+    - kms:
+       name: provider-2
+       endpoint: unix:///@kms-provider-2.sock
+`
 
-	if pollErr == wait.ErrWaitTimeout {
-		t.Fatalf("failed to get the expected healthz status of OK for check: %s, error: %v, debug inner error: %v", checkName, pollErr, restErr)
-	}
-}
-
-func mustBeUnHealthy(t *testing.T, checkName string, clientConfig *rest.Config) {
-	t.Helper()
-	var restErr error
-	pollErr := wait.PollImmediate(2*time.Second, wait.ForeverTestTimeout, func() (bool, error) {
-		status, err := getHealthz(checkName, clientConfig)
-		restErr = err
-		if err != nil {
-			return false, err
-		}
-		return status != http.StatusOK, nil
-	})
-
-	if pollErr == wait.ErrWaitTimeout {
-		t.Fatalf("failed to get the expected healthz status of !OK for check: %s, error: %v, debug inner error: %v", checkName, pollErr, restErr)
-	}
-}
-
-func getHealthz(checkName string, clientConfig *rest.Config) (int, error) {
-	client, err := kubernetes.NewForConfig(clientConfig)
+	pluginMock1, err := mock.NewBase64Plugin("@kms-provider-1.sock")
 	if err != nil {
-		return 0, fmt.Errorf("failed to create a client: %v", err)
+		t.Fatalf("failed to create mock of KMS Plugin #1: %v", err)
 	}
 
-	result := client.CoreV1().RESTClient().Get().AbsPath(fmt.Sprintf("/healthz/%v", checkName)).Do(context.TODO())
-	status := 0
-	result.StatusCode(&status)
-	return status, nil
+	if err := pluginMock1.Start(); err != nil {
+		t.Fatalf("Failed to start kms-plugin, err: %v", err)
+	}
+	defer pluginMock1.CleanUp()
+	if err := mock.WaitForBase64PluginToBeUp(pluginMock1); err != nil {
+		t.Fatalf("Failed to start plugin #1, err: %v", err)
+	}
+
+	pluginMock2, err := mock.NewBase64Plugin("@kms-provider-2.sock")
+	if err != nil {
+		t.Fatalf("Failed to create mock of KMS Plugin #2: err: %v", err)
+	}
+	if err := pluginMock2.Start(); err != nil {
+		t.Fatalf("Failed to start kms-plugin, err: %v", err)
+	}
+	defer pluginMock2.CleanUp()
+	if err := mock.WaitForBase64PluginToBeUp(pluginMock2); err != nil {
+		t.Fatalf("Failed to start KMS Plugin #2: err: %v", err)
+	}
+
+	test, err := newTransformTest(t, encryptionConfig, true)
+	if err != nil {
+		t.Fatalf("Failed to start kube-apiserver, error: %v", err)
+	}
+	defer test.cleanUp()
+
+	// Name of the healthz check is always "kms-provider-0" and it covers all kms plugins.
+
+	// Stage 1 - Since all kms-plugins are guaranteed to be up,
+	// the healthz check should be OK.
+	mustBeHealthy(t, "/kms-providers", "ok", test.kubeAPIServer.ClientConfig)
+
+	// Stage 2 - kms-plugin for provider-1 is down. Therefore, expect the healthz check
+	// to fail and report that provider-1 is down
+	pluginMock1.EnterFailedState()
+	mustBeUnHealthy(t, "/kms-providers",
+		"internal server error: kms-provider-0: rpc error: code = FailedPrecondition desc = failed precondition - key disabled",
+		test.kubeAPIServer.ClientConfig)
+	pluginMock1.ExitFailedState()
+
+	// Stage 3 - kms-plugin for provider-1 is now up. Therefore, expect the health check for provider-1
+	// to succeed now, but provider-2 is now down.
+	pluginMock2.EnterFailedState()
+	mustBeUnHealthy(t, "/kms-providers",
+		"internal server error: kms-provider-1: rpc error: code = FailedPrecondition desc = failed precondition - key disabled",
+		test.kubeAPIServer.ClientConfig)
+	pluginMock2.ExitFailedState()
+
+	// Stage 4 - All kms-plugins are once again up,
+	// the healthz check should be OK.
+	mustBeHealthy(t, "/kms-providers", "ok", test.kubeAPIServer.ClientConfig)
 }

test/integration/controlplane/transformation/kmsv2_transformation_test.go

@@ -140,7 +140,7 @@ resources:
 	}
 	defer pluginMock.CleanUp()
 
-	test, err := newTransformTest(t, encryptionConfig)
+	test, err := newTransformTest(t, encryptionConfig, false)
 	if err != nil {
 		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
 	}
@@ -253,33 +253,46 @@ resources:
 		t.Fatalf("Failed to start KMS Plugin #2: err: %v", err)
 	}
 
-	test, err := newTransformTest(t, encryptionConfig)
+	test, err := newTransformTest(t, encryptionConfig, false)
 	if err != nil {
 		t.Fatalf("Failed to start kube-apiserver, error: %v", err)
 	}
 	defer test.cleanUp()
 
-	// Name of the healthz check is calculated based on a constant "kms-provider-" + position of the
-	// provider in the config.
+	// Name of the healthz check is always "kms-provider-0" and it covers all kms plugins.
 
-	// Stage 1 - Since all kms-plugins are guaranteed to be up, healthz checks for:
-	// healthz/kms-provider-0 and /healthz/kms-provider-1 should be OK.
-	mustBeHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig)
-	mustBeHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig)
+	// Stage 1 - Since all kms-plugins are guaranteed to be up,
+	// the healthz check should be OK.
+	mustBeHealthy(t, "/kms-providers", "ok", test.kubeAPIServer.ClientConfig)
 
-	// Stage 2 - kms-plugin for provider-1 is down. Therefore, expect the health check for provider-1
-	// to fail, but provider-2 should still be OK
+	// Stage 2 - kms-plugin for provider-1 is down. Therefore, expect the healthz check
+	// to fail and report that provider-1 is down
 	pluginMock1.EnterFailedState()
-	mustBeUnHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig)
-	mustBeHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig)
+	mustBeUnHealthy(t, "/kms-providers",
+		"internal server error: kms-provider-0: rpc error: code = FailedPrecondition desc = failed precondition - key disabled",
+		test.kubeAPIServer.ClientConfig)
 	pluginMock1.ExitFailedState()
 
 	// Stage 3 - kms-plugin for provider-1 is now up. Therefore, expect the health check for provider-1
 	// to succeed now, but provider-2 is now down.
-	// Need to sleep since health check chases responses for 3 seconds.
 	pluginMock2.EnterFailedState()
-	mustBeHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig)
-	mustBeUnHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig)
+	mustBeUnHealthy(t, "/kms-providers",
+		"internal server error: kms-provider-1: rpc error: code = FailedPrecondition desc = failed precondition - key disabled",
+		test.kubeAPIServer.ClientConfig)
+	pluginMock2.ExitFailedState()
+
+	// Stage 4 - All kms-plugins are once again up,
+	// the healthz check should be OK.
+	mustBeHealthy(t, "/kms-providers", "ok", test.kubeAPIServer.ClientConfig)
+
+	// Stage 5 - All kms-plugins are unhealthy at the same time and we can observe both failures.
+	pluginMock1.EnterFailedState()
+	pluginMock2.EnterFailedState()
+	mustBeUnHealthy(t, "/kms-providers",
+		"internal server error: "+
+			"[kms-provider-0: failed to perform status section of the healthz check for KMS Provider provider-1, error: rpc error: code = FailedPrecondition desc = failed precondition - key disabled,"+
+			" kms-provider-1: failed to perform status section of the healthz check for KMS Provider provider-2, error: rpc error: code = FailedPrecondition desc = failed precondition - key disabled]",
+		test.kubeAPIServer.ClientConfig)
 }
 
 func TestKMSv2SingleService(t *testing.T) {
@@ -328,7 +341,7 @@ resources:
 	}
 	t.Cleanup(pluginMock.CleanUp)
 
-	test, err := newTransformTest(t, encryptionConfig)
+	test, err := newTransformTest(t, encryptionConfig, false)
 	if err != nil {
 		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
 	}

test/integration/controlplane/transformation/secrets_transformation_test.go

@@ -85,7 +85,7 @@ func TestSecretsShouldBeTransformed(t *testing.T) {
 		// TODO: add secretbox
 	}
 	for _, tt := range testCases {
-		test, err := newTransformTest(t, tt.transformerConfigContent)
+		test, err := newTransformTest(t, tt.transformerConfigContent, false)
 		if err != nil {
 			test.cleanUp()
 			t.Errorf("failed to setup test for envelop %s, error was %v", tt.transformerPrefix, err)
@@ -120,7 +120,7 @@ func BenchmarkAESCBCEnvelopeWrite(b *testing.B) {
 
 func runBenchmark(b *testing.B, transformerConfig string) {
 	b.StopTimer()
-	test, err := newTransformTest(b, transformerConfig)
+	test, err := newTransformTest(b, transformerConfig, false)
 	defer test.cleanUp()
 	if err != nil {
 		b.Fatalf("failed to setup benchmark for config %s, error was %v", transformerConfig, err)

test/integration/controlplane/transformation/transformation_test.go

@@ -26,26 +26,28 @@ import (
 	"strconv"
 	"strings"
 	"testing"
-
-	"k8s.io/klog/v2"
+	"time"
 
 	clientv3 "go.etcd.io/etcd/client/v3"
-	"k8s.io/component-base/metrics/legacyregistry"
-	"sigs.k8s.io/yaml"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
 	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 	"k8s.io/apiserver/pkg/storage/value"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/klog/v2"
 	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
 	"k8s.io/kubernetes/test/integration"
 	"k8s.io/kubernetes/test/integration/etcd"
 	"k8s.io/kubernetes/test/integration/framework"
+	"sigs.k8s.io/yaml"
 )
 
 const (
@@ -78,7 +80,7 @@ type transformTest struct {
 	secret            *corev1.Secret
 }
 
-func newTransformTest(l kubeapiservertesting.Logger, transformerConfigYAML string) (*transformTest, error) {
+func newTransformTest(l kubeapiservertesting.Logger, transformerConfigYAML string, reload bool) (*transformTest, error) {
 	e := transformTest{
 		logger:            l,
 		transformerConfig: transformerConfigYAML,
@@ -92,7 +94,7 @@ func newTransformTest(l kubeapiservertesting.Logger, transformerConfigYAML strin
 		}
 	}
 
-	if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(l, nil, e.getEncryptionOptions(), e.storageConfig); err != nil {
+	if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(l, nil, e.getEncryptionOptions(reload), e.storageConfig); err != nil {
 		return nil, fmt.Errorf("failed to start KubeAPI server: %v", err)
 	}
 	klog.Infof("Started kube-apiserver %v", e.kubeAPIServer.ClientConfig.Host)
@@ -105,6 +107,15 @@ func newTransformTest(l kubeapiservertesting.Logger, transformerConfigYAML strin
 		return nil, err
 	}
 
+	if transformerConfigYAML != "" && reload {
+		// when reloading is enabled, this healthz endpoint is always present
+		mustBeHealthy(l, "/kms-providers", "ok", e.kubeAPIServer.ClientConfig)
+
+		// excluding healthz endpoints even if they do not exist should work
+		mustBeHealthy(l, "", `warn: some health checks cannot be excluded: no matches for "kms-provider-0","kms-provider-1","kms-provider-2","kms-provider-3"`,
+			e.kubeAPIServer.ClientConfig, "kms-provider-0", "kms-provider-1", "kms-provider-2", "kms-provider-3")
+	}
+
 	return &e, nil
 }
 
@@ -228,10 +239,11 @@ func (e *transformTest) getRawSecretFromETCD() ([]byte, error) {
 	return etcdResponse.Kvs[0].Value, nil
 }
 
-func (e *transformTest) getEncryptionOptions() []string {
+func (e *transformTest) getEncryptionOptions(reload bool) []string {
 	if e.transformerConfig != "" {
 		return []string{
 			"--encryption-provider-config", path.Join(e.configDir, encryptionConfigFileName),
+			fmt.Sprintf("--encryption-provider-config-automatic-reload=%v", reload),
 			"--disable-admission-plugins", "ServiceAccount"}
 	}
 
@@ -401,3 +413,59 @@ func (e *transformTest) printMetrics() error {
 
 	return nil
 }
+
+func mustBeHealthy(t kubeapiservertesting.Logger, checkName, wantBodyContains string, clientConfig *rest.Config, excludes ...string) {
+	t.Helper()
+	var restErr error
+	pollErr := wait.PollImmediate(2*time.Second, wait.ForeverTestTimeout, func() (bool, error) {
+		body, ok, err := getHealthz(checkName, clientConfig, excludes...)
+		restErr = err
+		if err != nil {
+			return false, err
+		}
+		done := ok && strings.Contains(body, wantBodyContains)
+		if !done {
+			t.Logf("expected server check %q to be healthy with message %q but it is not: %s", checkName, wantBodyContains, body)
+		}
+		return done, nil
+	})
+
+	if pollErr == wait.ErrWaitTimeout {
+		t.Fatalf("failed to get the expected healthz status of OK for check: %s, error: %v, debug inner error: %v", checkName, pollErr, restErr)
+	}
+}
+
+func mustBeUnHealthy(t kubeapiservertesting.Logger, checkName, wantBodyContains string, clientConfig *rest.Config, excludes ...string) {
+	t.Helper()
+	var restErr error
+	pollErr := wait.PollImmediate(2*time.Second, wait.ForeverTestTimeout, func() (bool, error) {
+		body, ok, err := getHealthz(checkName, clientConfig, excludes...)
+		restErr = err
+		if err != nil {
+			return false, err
+		}
+		done := !ok && strings.Contains(body, wantBodyContains)
+		if !done {
+			t.Logf("expected server check %q to be unhealthy with message %q but it is not: %s", checkName, wantBodyContains, body)
+		}
+		return done, nil
+	})
+
+	if pollErr == wait.ErrWaitTimeout {
+		t.Fatalf("failed to get the expected healthz status of !OK for check: %s, error: %v, debug inner error: %v", checkName, pollErr, restErr)
+	}
+}
+
+func getHealthz(checkName string, clientConfig *rest.Config, excludes ...string) (string, bool, error) {
+	client, err := kubernetes.NewForConfig(clientConfig)
+	if err != nil {
+		return "", false, fmt.Errorf("failed to create a client: %v", err)
+	}
+
+	req := client.CoreV1().RESTClient().Get().AbsPath(fmt.Sprintf("/healthz%v", checkName)).Param("verbose", "true")
+	for _, exclude := range excludes {
+		req.Param("exclude", exclude)
+	}
+	body, err := req.DoRaw(context.TODO()) // we can still have a response body during an error case
+	return string(body), err == nil, nil
+}