Allow kube-apiserver to test the status of kms-plugin.

2019-05-30 11:15:35 -07:00
parent 49c34bb597
commit 05fdbb201f
15 changed files with 565 additions and 105 deletions
--- a/test/integration/master/kms_transformation_test.go
+++ b/test/integration/master/kms_transformation_test.go
@@ -24,52 +24,70 @@ import (
 	"crypto/aes"
 	"encoding/binary"
 	"fmt"
+
+	"net/http"
 	"strings"
 	"testing"
+	"time"

 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/storage/value"
 	aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
-
 	kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 )

 const (
-	kmsPrefix     = "k8s:enc:kms:v1:grpc-kms-provider:"
 	dekKeySizeLen = 2
-
-	kmsConfigYAML = `
-kind: EncryptionConfiguration
-apiVersion: apiserver.config.k8s.io/v1
-resources:
-  - resources:
-    - secrets
-    providers:
-    - kms:
-       name: grpc-kms-provider
-       cachesize: 1000
-       endpoint: unix:///@kms-provider.sock
-`
 )

-// rawDEKKEKSecret provides operations for working with secrets transformed with Data Encryption Key(DEK) Key Encryption Kye(KEK) envelop.
-type rawDEKKEKSecret []byte
+type envelope struct {
+	providerName string
+	rawEnvelope  []byte
+	plainTextDEK []byte
+}

-func (r rawDEKKEKSecret) getDEKLen() int {
+func (r envelope) prefix() string {
+	return fmt.Sprintf("k8s:enc:kms:v1:%s:", r.providerName)
+}
+
+func (r envelope) prefixLen() int {
+	return len(r.prefix())
+}
+
+func (r envelope) dekLen() int {
 	// DEK's length is stored in the two bytes that follow the prefix.
-	return int(binary.BigEndian.Uint16(r[len(kmsPrefix) : len(kmsPrefix)+dekKeySizeLen]))
+	return int(binary.BigEndian.Uint16(r.rawEnvelope[r.prefixLen() : r.prefixLen()+dekKeySizeLen]))
 }

-func (r rawDEKKEKSecret) getDEK() []byte {
-	return r[len(kmsPrefix)+dekKeySizeLen : len(kmsPrefix)+dekKeySizeLen+r.getDEKLen()]
+func (r envelope) cipherTextDEK() []byte {
+	return r.rawEnvelope[r.prefixLen()+dekKeySizeLen : r.prefixLen()+dekKeySizeLen+r.dekLen()]
 }

-func (r rawDEKKEKSecret) getStartOfPayload() int {
-	return len(kmsPrefix) + dekKeySizeLen + r.getDEKLen()
+func (r envelope) startOfPayload(providerName string) int {
+	return r.prefixLen() + dekKeySizeLen + r.dekLen()
 }

-func (r rawDEKKEKSecret) getPayload() []byte {
-	return r[r.getStartOfPayload():]
+func (r envelope) cipherTextPayload() []byte {
+	return r.rawEnvelope[r.startOfPayload(r.providerName):]
+}
+
+func (r envelope) plainTextPayload(secretETCDPath string) ([]byte, error) {
+	block, err := aes.NewCipher(r.plainTextDEK)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize AES Cipher: %v", err)
+	}
+	// etcd path of the key is used as the authenticated context - need to pass it to decrypt
+	ctx := value.DefaultContext([]byte(secretETCDPath))
+	aescbcTransformer := aestransformer.NewCBCTransformer(block)
+	plainSecret, _, err := aescbcTransformer.TransformFromStorage(r.cipherTextPayload(), ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to transform from storage via AESCBC, err: %v", err)
+	}
+
+	return plainSecret, nil
 }

 // TestKMSProvider is an integration test between KubeAPI, ETCD and KMS Plugin
@@ -77,60 +95,77 @@ func (r rawDEKKEKSecret) getPayload() []byte {
 // 1. Raw records in ETCD that were processed by KMS Provider should be prefixed with k8s:enc:kms:v1:grpc-kms-provider-name:
 // 2. Data Encryption Key (DEK) should be generated by envelopeTransformer and passed to KMS gRPC Plugin
 // 3. KMS gRPC Plugin should encrypt the DEK with a Key Encryption Key (KEK) and pass it back to envelopeTransformer
-// 4. The payload (ex. Secret) should be encrypted via AES CBC transform
+// 4. The cipherTextPayload (ex. Secret) should be encrypted via AES CBC transform
 // 5. Prefix-EncryptedDEK-EncryptedPayload structure should be deposited to ETCD
 func TestKMSProvider(t *testing.T) {
-	pluginMock, err := newBase64Plugin()
+	encryptionConfig := `
+kind: EncryptionConfiguration
+apiVersion: apiserver.config.k8s.io/v1
+resources:
+  - resources:
+    - secrets
+    providers:
+    - kms:
+       name: kms-provider
+       cachesize: 1000
+       endpoint: unix:///@kms-provider.sock
+`
+
+	providerName := "kms-provider"
+	pluginMock, err := newBase64Plugin("@kms-provider.sock")
 	if err != nil {
 		t.Fatalf("failed to create mock of KMS Plugin: %v", err)
 	}
-	defer pluginMock.cleanUp()
-	serveErr := make(chan error, 1)
-	go func() {
-		serveErr <- pluginMock.grpcServer.Serve(pluginMock.listener)
-	}()

-	test, err := newTransformTest(t, kmsConfigYAML)
+	go pluginMock.grpcServer.Serve(pluginMock.listener)
+	defer pluginMock.cleanUp()
+	kmsPluginMustBeUp(t, pluginMock)
+
+	test, err := newTransformTest(t, encryptionConfig)
 	if err != nil {
-		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s", kmsConfigYAML)
+		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
 	}
 	defer test.cleanUp()

-	// As part of newTransformTest a new secret was created, so KMS Mock should have been exercised by this point.
-	if len(serveErr) != 0 {
-		t.Fatalf("KMSPlugin failed while serving requests: %v", <-serveErr)
-	}
-
-	secretETCDPath := test.getETCDPath()
-	var rawSecretAsSeenByETCD rawDEKKEKSecret
-	rawSecretAsSeenByETCD, err = test.getRawSecretFromETCD()
+	test.secret, err = test.createSecret(testSecret, testNamespace)
 	if err != nil {
-		t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err)
-	}
-
-	if !bytes.HasPrefix(rawSecretAsSeenByETCD, []byte(kmsPrefix)) {
-		t.Fatalf("expected secret to be prefixed with %s, but got %s", kmsPrefix, rawSecretAsSeenByETCD)
+		t.Fatalf("Failed to create test secret, error: %v", err)
 	}

 	// Since Data Encryption Key (DEK) is randomly generated (per encryption operation), we need to ask KMS Mock for it.
-	dekPlainAsSeenByKMS, err := getDEKFromKMSPlugin(pluginMock)
+	plainTextDEK := pluginMock.lastEncryptRequest.Plain
 	if err != nil {
 		t.Fatalf("failed to get DEK from KMS: %v", err)
 	}

-	decryptResponse, err := pluginMock.Decrypt(context.Background(),
-		&kmsapi.DecryptRequest{Version: kmsAPIVersion, Cipher: rawSecretAsSeenByETCD.getDEK()})
+	secretETCDPath := test.getETCDPath()
+	rawEnvelope, err := test.getRawSecretFromETCD()
+	if err != nil {
+		t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err)
+	}
+	envelope := envelope{
+		providerName: providerName,
+		rawEnvelope:  rawEnvelope,
+		plainTextDEK: plainTextDEK,
+	}
+
+	wantPrefix := "k8s:enc:kms:v1:kms-provider:"
+	if !bytes.HasPrefix(rawEnvelope, []byte(wantPrefix)) {
+		t.Fatalf("expected secret to be prefixed with %s, but got %s", wantPrefix, rawEnvelope)
+	}
+
+	decryptResponse, err := pluginMock.Decrypt(context.Background(), &kmsapi.DecryptRequest{Version: kmsAPIVersion, Cipher: envelope.cipherTextDEK()})
 	if err != nil {
 		t.Fatalf("failed to decrypt DEK, %v", err)
 	}
 	dekPlainAsWouldBeSeenByETCD := decryptResponse.Plain

-	if !bytes.Equal(dekPlainAsSeenByKMS, dekPlainAsWouldBeSeenByETCD) {
-		t.Fatalf("expected dekPlainAsSeenByKMS %v to be passed to KMS Plugin, but got %s",
-			dekPlainAsSeenByKMS, dekPlainAsWouldBeSeenByETCD)
+	if !bytes.Equal(plainTextDEK, dekPlainAsWouldBeSeenByETCD) {
+		t.Fatalf("expected plainTextDEK %v to be passed to KMS Plugin, but got %s",
+			plainTextDEK, dekPlainAsWouldBeSeenByETCD)
 	}

-	plainSecret, err := decryptPayload(dekPlainAsWouldBeSeenByETCD, rawSecretAsSeenByETCD, secretETCDPath)
+	plainSecret, err := envelope.plainTextPayload(secretETCDPath)
 	if err != nil {
 		t.Fatalf("failed to transform from storage via AESCBC, err: %v", err)
 	}
@@ -144,32 +179,124 @@ func TestKMSProvider(t *testing.T) {
 	if secretVal != string(s.Data[secretKey]) {
 		t.Fatalf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey]))
 	}
-	test.printMetrics()
 }

-func getDEKFromKMSPlugin(pluginMock *base64Plugin) ([]byte, error) {
-	// We expect KMS to already have seen an encryptRequest. Hence non-blocking call.
-	e, ok := <-pluginMock.encryptRequest
+func TestKMSHealthz(t *testing.T) {
+	encryptionConfig := `
+kind: EncryptionConfiguration
+apiVersion: apiserver.config.k8s.io/v1
+resources:
+  - resources:
+    - secrets
+    providers:
+    - kms:
+       name: provider-1
+       endpoint: unix:///@kms-provider-1.sock
+    - kms:
+       name: provider-2
+       endpoint: unix:///@kms-provider-2.sock
+`

-	if !ok {
-		return nil, fmt.Errorf("failed to sense encryptRequest from KMS Plugin Mock")
-	}
-
-	return e.Plain, nil
-}
-
-func decryptPayload(key []byte, secret rawDEKKEKSecret, secretETCDPath string) ([]byte, error) {
-	block, err := aes.NewCipher(key)
+	pluginMock1, err := newBase64Plugin("@kms-provider-1.sock")
 	if err != nil {
-		return nil, fmt.Errorf("failed to initialize AES Cipher: %v", err)
-	}
-	// etcd path of the key is used as the authenticated context - need to pass it to decrypt
-	ctx := value.DefaultContext([]byte(secretETCDPath))
-	aescbcTransformer := aestransformer.NewCBCTransformer(block)
-	plainSecret, _, err := aescbcTransformer.TransformFromStorage(secret.getPayload(), ctx)
-	if err != nil {
-		return nil, fmt.Errorf("failed to transform from storage via AESCBC, err: %v", err)
+		t.Fatalf("failed to create mock of KMS Plugin #1: %v", err)
 	}

-	return plainSecret, nil
+	go pluginMock1.grpcServer.Serve(pluginMock1.listener)
+	defer pluginMock1.cleanUp()
+	kmsPluginMustBeUp(t, pluginMock1)
+
+	pluginMock2, err := newBase64Plugin("@kms-provider-2.sock")
+	if err != nil {
+		t.Fatalf("failed to create mock of KMS Plugin #2: %v", err)
+	}
+
+	go pluginMock2.grpcServer.Serve(pluginMock2.listener)
+	defer pluginMock2.cleanUp()
+	kmsPluginMustBeUp(t, pluginMock2)
+
+	test, err := newTransformTest(t, encryptionConfig)
+	if err != nil {
+		t.Fatalf("Failed to start kube-apiserver, error: %v", err)
+	}
+	defer test.cleanUp()
+
+	// Name of the healthz check is calculated based on a constant "kms-provider-" + position of the
+	// provider in the config.
+
+	// Stage 1 - Since all kms-plugins are guaranteed to be up, healthz checks for:
+	// healthz/kms-provider-0 and /healthz/kms-provider-1 should be OK.
+	mustBeHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig)
+	mustBeHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig)
+
+	// Stage 2 - kms-plugin for provider-1 is down. Therefore, expect the health check for provider-1
+	// to fail, but provider-2 should still be OK
+	pluginMock1.enterFailedState()
+	mustBeUnHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig)
+	mustBeHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig)
+	pluginMock1.exitFailedState()
+
+	// Stage 3 - kms-plugin for provider-1 is now up. Therefore, expect the health check for provider-1
+	// to succeed now, but provider-2 is now down.
+	// Need to sleep since health check chases responses for 3 seconds.
+	pluginMock2.enterFailedState()
+	mustBeHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig)
+	mustBeUnHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig)
+}
+
+func kmsPluginMustBeUp(t *testing.T, plugin *base64Plugin) {
+	t.Helper()
+	var gRPCErr error
+	pollErr := wait.PollImmediate(1*time.Second, wait.ForeverTestTimeout, func() (bool, error) {
+		_, gRPCErr = plugin.Encrypt(context.Background(), &kmsapi.EncryptRequest{Plain: []byte("foo")})
+		return gRPCErr == nil, nil
+	})
+
+	if pollErr == wait.ErrWaitTimeout {
+		t.Fatalf("failed to start kms-plugin, error: %v", gRPCErr)
+	}
+}
+
+func mustBeHealthy(t *testing.T, checkName string, clientConfig *rest.Config) {
+	t.Helper()
+	var restErr error
+	pollErr := wait.PollImmediate(2*time.Second, wait.ForeverTestTimeout, func() (bool, error) {
+		status, err := getHealthz(checkName, clientConfig)
+		if err != nil {
+			return false, err
+		}
+		return status == http.StatusOK, nil
+	})
+
+	if pollErr == wait.ErrWaitTimeout {
+		t.Fatalf("failed to get the expected healthz status of OK for check: %s, error: %v", restErr, checkName)
+	}
+}
+
+func mustBeUnHealthy(t *testing.T, checkName string, clientConfig *rest.Config) {
+	t.Helper()
+	var restErr error
+	pollErr := wait.PollImmediate(2*time.Second, wait.ForeverTestTimeout, func() (bool, error) {
+		status, err := getHealthz(checkName, clientConfig)
+		if err != nil {
+			return false, err
+		}
+		return status != http.StatusOK, nil
+	})
+
+	if pollErr == wait.ErrWaitTimeout {
+		t.Fatalf("failed to get the expected healthz status of !OK for check: %s, error: %v", restErr, checkName)
+	}
+}
+
+func getHealthz(checkName string, clientConfig *rest.Config) (int, error) {
+	client, err := kubernetes.NewForConfig(clientConfig)
+	if err != nil {
+		return 0, fmt.Errorf("failed to create a client: %v", err)
+	}
+
+	result := client.CoreV1().RESTClient().Get().AbsPath(fmt.Sprintf("/healthz/%v", checkName)).Do()
+	status := 0
+	result.StatusCode(&status)
+	return status, nil
 }