Allow kube-apiserver to test the status of kms-plugin.

2019-05-30 11:15:35 -07:00
parent 49c34bb597
commit 05fdbb201f
15 changed files with 565 additions and 105 deletions
--- a/test/integration/master/BUILD
+++ b/test/integration/master/BUILD
@@ -135,57 +135,68 @@ go_library(
        "//test/integration/framework:go_default_library",
        "//vendor/github.com/coreos/etcd/clientv3:go_default_library",
        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+        "//vendor/k8s.io/klog:go_default_library",
        "//vendor/sigs.k8s.io/yaml:go_default_library",
    ] + select({
        "@io_bazel_rules_go//go/platform:android": [
            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/k8s.io/klog:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
        ],
        "@io_bazel_rules_go//go/platform:darwin": [
            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/k8s.io/klog:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
        ],
        "@io_bazel_rules_go//go/platform:dragonfly": [
            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/k8s.io/klog:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
        ],
        "@io_bazel_rules_go//go/platform:freebsd": [
            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/k8s.io/klog:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
        ],
        "@io_bazel_rules_go//go/platform:linux": [
            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/k8s.io/klog:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
        ],
        "@io_bazel_rules_go//go/platform:nacl": [
            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/k8s.io/klog:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
        ],
        "@io_bazel_rules_go//go/platform:netbsd": [
            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/k8s.io/klog:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
        ],
        "@io_bazel_rules_go//go/platform:openbsd": [
            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/k8s.io/klog:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
        ],
        "@io_bazel_rules_go//go/platform:plan9": [
            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/k8s.io/klog:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
        ],
        "@io_bazel_rules_go//go/platform:solaris": [
            "//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
            "//vendor/google.golang.org/grpc:go_default_library",
-            "//vendor/k8s.io/klog:go_default_library",
+            "//vendor/google.golang.org/grpc/codes:go_default_library",
+            "//vendor/google.golang.org/grpc/status:go_default_library",
        ],
        "//conditions:default": [],
    }),
--- a/test/integration/master/kms_plugin_mock.go
+++ b/test/integration/master/kms_plugin_mock.go
@@ -23,8 +23,11 @@ import (
 	"encoding/base64"
 	"fmt"
 	"net"
+	"sync"

 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"

 	kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1"
 	"k8s.io/klog"
@@ -32,33 +35,32 @@ import (

 const (
 	kmsAPIVersion = "v1beta1"
-	sockFile      = "@kms-provider.sock"
 	unixProtocol  = "unix"
 )

 // base64Plugin gRPC sever for a mock KMS provider.
 // Uses base64 to simulate encrypt and decrypt.
 type base64Plugin struct {
-	grpcServer *grpc.Server
-	listener   net.Listener
-
-	// Allow users of the plugin to sense requests that were passed to KMS.
-	encryptRequest chan *kmsapi.EncryptRequest
+	grpcServer         *grpc.Server
+	listener           net.Listener
+	mu                 *sync.Mutex
+	lastEncryptRequest *kmsapi.EncryptRequest
+	inFailedState      bool
 }

-func newBase64Plugin() (*base64Plugin, error) {
-	listener, err := net.Listen(unixProtocol, sockFile)
+func newBase64Plugin(socketPath string) (*base64Plugin, error) {
+	listener, err := net.Listen(unixProtocol, socketPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to listen on the unix socket, error: %v", err)
 	}
-	klog.Infof("Listening on %s", sockFile)
+	klog.Infof("Listening on %s", socketPath)

 	server := grpc.NewServer()

 	result := &base64Plugin{
-		grpcServer:     server,
-		listener:       listener,
-		encryptRequest: make(chan *kmsapi.EncryptRequest, 1),
+		grpcServer: server,
+		listener:   listener,
+		mu:         &sync.Mutex{},
 	}

 	kmsapi.RegisterKeyManagementServiceServer(server, result)
@@ -73,6 +75,18 @@ func (s *base64Plugin) cleanUp() {

 var testProviderAPIVersion = kmsAPIVersion

+func (s *base64Plugin) enterFailedState() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.inFailedState = true
+}
+
+func (s *base64Plugin) exitFailedState() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.inFailedState = false
+}
+
 func (s *base64Plugin) Version(ctx context.Context, request *kmsapi.VersionRequest) (*kmsapi.VersionResponse, error) {
 	return &kmsapi.VersionResponse{Version: testProviderAPIVersion, RuntimeName: "testKMS", RuntimeVersion: "0.0.1"}, nil
 }
@@ -80,6 +94,12 @@ func (s *base64Plugin) Version(ctx context.Context, request *kmsapi.VersionReque
 func (s *base64Plugin) Decrypt(ctx context.Context, request *kmsapi.DecryptRequest) (*kmsapi.DecryptResponse, error) {
 	klog.Infof("Received Decrypt Request for DEK: %s", string(request.Cipher))

+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.inFailedState {
+		return nil, status.Error(codes.FailedPrecondition, "failed precondition - key disabled")
+	}
+
 	buf := make([]byte, base64.StdEncoding.DecodedLen(len(request.Cipher)))
 	n, err := base64.StdEncoding.Decode(buf, request.Cipher)
 	if err != nil {
@@ -91,7 +111,13 @@ func (s *base64Plugin) Decrypt(ctx context.Context, request *kmsapi.DecryptReque

 func (s *base64Plugin) Encrypt(ctx context.Context, request *kmsapi.EncryptRequest) (*kmsapi.EncryptResponse, error) {
 	klog.Infof("Received Encrypt Request for DEK: %x", request.Plain)
-	s.encryptRequest <- request
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.lastEncryptRequest = request
+
+	if s.inFailedState {
+		return nil, status.Error(codes.FailedPrecondition, "failed precondition - key disabled")
+	}

 	buf := make([]byte, base64.StdEncoding.EncodedLen(len(request.Plain)))
 	base64.StdEncoding.Encode(buf, request.Plain)
--- a/test/integration/master/kms_transformation_test.go
+++ b/test/integration/master/kms_transformation_test.go
@@ -24,52 +24,70 @@ import (
 	"crypto/aes"
 	"encoding/binary"
 	"fmt"
+
+	"net/http"
 	"strings"
 	"testing"
+	"time"

 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/storage/value"
 	aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
-
 	kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 )

 const (
-	kmsPrefix     = "k8s:enc:kms:v1:grpc-kms-provider:"
 	dekKeySizeLen = 2
-
-	kmsConfigYAML = `
-kind: EncryptionConfiguration
-apiVersion: apiserver.config.k8s.io/v1
-resources:
-  - resources:
-    - secrets
-    providers:
-    - kms:
-       name: grpc-kms-provider
-       cachesize: 1000
-       endpoint: unix:///@kms-provider.sock
-`
 )

-// rawDEKKEKSecret provides operations for working with secrets transformed with Data Encryption Key(DEK) Key Encryption Kye(KEK) envelop.
-type rawDEKKEKSecret []byte
+type envelope struct {
+	providerName string
+	rawEnvelope  []byte
+	plainTextDEK []byte
+}

-func (r rawDEKKEKSecret) getDEKLen() int {
+func (r envelope) prefix() string {
+	return fmt.Sprintf("k8s:enc:kms:v1:%s:", r.providerName)
+}
+
+func (r envelope) prefixLen() int {
+	return len(r.prefix())
+}
+
+func (r envelope) dekLen() int {
 	// DEK's length is stored in the two bytes that follow the prefix.
-	return int(binary.BigEndian.Uint16(r[len(kmsPrefix) : len(kmsPrefix)+dekKeySizeLen]))
+	return int(binary.BigEndian.Uint16(r.rawEnvelope[r.prefixLen() : r.prefixLen()+dekKeySizeLen]))
 }

-func (r rawDEKKEKSecret) getDEK() []byte {
-	return r[len(kmsPrefix)+dekKeySizeLen : len(kmsPrefix)+dekKeySizeLen+r.getDEKLen()]
+func (r envelope) cipherTextDEK() []byte {
+	return r.rawEnvelope[r.prefixLen()+dekKeySizeLen : r.prefixLen()+dekKeySizeLen+r.dekLen()]
 }

-func (r rawDEKKEKSecret) getStartOfPayload() int {
-	return len(kmsPrefix) + dekKeySizeLen + r.getDEKLen()
+func (r envelope) startOfPayload(providerName string) int {
+	return r.prefixLen() + dekKeySizeLen + r.dekLen()
 }

-func (r rawDEKKEKSecret) getPayload() []byte {
-	return r[r.getStartOfPayload():]
+func (r envelope) cipherTextPayload() []byte {
+	return r.rawEnvelope[r.startOfPayload(r.providerName):]
+}
+
+func (r envelope) plainTextPayload(secretETCDPath string) ([]byte, error) {
+	block, err := aes.NewCipher(r.plainTextDEK)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize AES Cipher: %v", err)
+	}
+	// etcd path of the key is used as the authenticated context - need to pass it to decrypt
+	ctx := value.DefaultContext([]byte(secretETCDPath))
+	aescbcTransformer := aestransformer.NewCBCTransformer(block)
+	plainSecret, _, err := aescbcTransformer.TransformFromStorage(r.cipherTextPayload(), ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to transform from storage via AESCBC, err: %v", err)
+	}
+
+	return plainSecret, nil
 }

 // TestKMSProvider is an integration test between KubeAPI, ETCD and KMS Plugin
@@ -77,60 +95,77 @@ func (r rawDEKKEKSecret) getPayload() []byte {
 // 1. Raw records in ETCD that were processed by KMS Provider should be prefixed with k8s:enc:kms:v1:grpc-kms-provider-name:
 // 2. Data Encryption Key (DEK) should be generated by envelopeTransformer and passed to KMS gRPC Plugin
 // 3. KMS gRPC Plugin should encrypt the DEK with a Key Encryption Key (KEK) and pass it back to envelopeTransformer
-// 4. The payload (ex. Secret) should be encrypted via AES CBC transform
+// 4. The cipherTextPayload (ex. Secret) should be encrypted via AES CBC transform
 // 5. Prefix-EncryptedDEK-EncryptedPayload structure should be deposited to ETCD
 func TestKMSProvider(t *testing.T) {
-	pluginMock, err := newBase64Plugin()
+	encryptionConfig := `
+kind: EncryptionConfiguration
+apiVersion: apiserver.config.k8s.io/v1
+resources:
+  - resources:
+    - secrets
+    providers:
+    - kms:
+       name: kms-provider
+       cachesize: 1000
+       endpoint: unix:///@kms-provider.sock
+`
+
+	providerName := "kms-provider"
+	pluginMock, err := newBase64Plugin("@kms-provider.sock")
 	if err != nil {
 		t.Fatalf("failed to create mock of KMS Plugin: %v", err)
 	}
-	defer pluginMock.cleanUp()
-	serveErr := make(chan error, 1)
-	go func() {
-		serveErr <- pluginMock.grpcServer.Serve(pluginMock.listener)
-	}()

-	test, err := newTransformTest(t, kmsConfigYAML)
+	go pluginMock.grpcServer.Serve(pluginMock.listener)
+	defer pluginMock.cleanUp()
+	kmsPluginMustBeUp(t, pluginMock)
+
+	test, err := newTransformTest(t, encryptionConfig)
 	if err != nil {
-		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s", kmsConfigYAML)
+		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
 	}
 	defer test.cleanUp()

-	// As part of newTransformTest a new secret was created, so KMS Mock should have been exercised by this point.
-	if len(serveErr) != 0 {
-		t.Fatalf("KMSPlugin failed while serving requests: %v", <-serveErr)
-	}
-
-	secretETCDPath := test.getETCDPath()
-	var rawSecretAsSeenByETCD rawDEKKEKSecret
-	rawSecretAsSeenByETCD, err = test.getRawSecretFromETCD()
+	test.secret, err = test.createSecret(testSecret, testNamespace)
 	if err != nil {
-		t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err)
-	}
-
-	if !bytes.HasPrefix(rawSecretAsSeenByETCD, []byte(kmsPrefix)) {
-		t.Fatalf("expected secret to be prefixed with %s, but got %s", kmsPrefix, rawSecretAsSeenByETCD)
+		t.Fatalf("Failed to create test secret, error: %v", err)
 	}

 	// Since Data Encryption Key (DEK) is randomly generated (per encryption operation), we need to ask KMS Mock for it.
-	dekPlainAsSeenByKMS, err := getDEKFromKMSPlugin(pluginMock)
+	plainTextDEK := pluginMock.lastEncryptRequest.Plain
 	if err != nil {
 		t.Fatalf("failed to get DEK from KMS: %v", err)
 	}

-	decryptResponse, err := pluginMock.Decrypt(context.Background(),
-		&kmsapi.DecryptRequest{Version: kmsAPIVersion, Cipher: rawSecretAsSeenByETCD.getDEK()})
+	secretETCDPath := test.getETCDPath()
+	rawEnvelope, err := test.getRawSecretFromETCD()
+	if err != nil {
+		t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err)
+	}
+	envelope := envelope{
+		providerName: providerName,
+		rawEnvelope:  rawEnvelope,
+		plainTextDEK: plainTextDEK,
+	}
+
+	wantPrefix := "k8s:enc:kms:v1:kms-provider:"
+	if !bytes.HasPrefix(rawEnvelope, []byte(wantPrefix)) {
+		t.Fatalf("expected secret to be prefixed with %s, but got %s", wantPrefix, rawEnvelope)
+	}
+
+	decryptResponse, err := pluginMock.Decrypt(context.Background(), &kmsapi.DecryptRequest{Version: kmsAPIVersion, Cipher: envelope.cipherTextDEK()})
 	if err != nil {
 		t.Fatalf("failed to decrypt DEK, %v", err)
 	}
 	dekPlainAsWouldBeSeenByETCD := decryptResponse.Plain

-	if !bytes.Equal(dekPlainAsSeenByKMS, dekPlainAsWouldBeSeenByETCD) {
-		t.Fatalf("expected dekPlainAsSeenByKMS %v to be passed to KMS Plugin, but got %s",
-			dekPlainAsSeenByKMS, dekPlainAsWouldBeSeenByETCD)
+	if !bytes.Equal(plainTextDEK, dekPlainAsWouldBeSeenByETCD) {
+		t.Fatalf("expected plainTextDEK %v to be passed to KMS Plugin, but got %s",
+			plainTextDEK, dekPlainAsWouldBeSeenByETCD)
 	}

-	plainSecret, err := decryptPayload(dekPlainAsWouldBeSeenByETCD, rawSecretAsSeenByETCD, secretETCDPath)
+	plainSecret, err := envelope.plainTextPayload(secretETCDPath)
 	if err != nil {
 		t.Fatalf("failed to transform from storage via AESCBC, err: %v", err)
 	}
@@ -144,32 +179,124 @@ func TestKMSProvider(t *testing.T) {
 	if secretVal != string(s.Data[secretKey]) {
 		t.Fatalf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey]))
 	}
-	test.printMetrics()
 }

-func getDEKFromKMSPlugin(pluginMock *base64Plugin) ([]byte, error) {
-	// We expect KMS to already have seen an encryptRequest. Hence non-blocking call.
-	e, ok := <-pluginMock.encryptRequest
+func TestKMSHealthz(t *testing.T) {
+	encryptionConfig := `
+kind: EncryptionConfiguration
+apiVersion: apiserver.config.k8s.io/v1
+resources:
+  - resources:
+    - secrets
+    providers:
+    - kms:
+       name: provider-1
+       endpoint: unix:///@kms-provider-1.sock
+    - kms:
+       name: provider-2
+       endpoint: unix:///@kms-provider-2.sock
+`

-	if !ok {
-		return nil, fmt.Errorf("failed to sense encryptRequest from KMS Plugin Mock")
-	}
-
-	return e.Plain, nil
-}
-
-func decryptPayload(key []byte, secret rawDEKKEKSecret, secretETCDPath string) ([]byte, error) {
-	block, err := aes.NewCipher(key)
+	pluginMock1, err := newBase64Plugin("@kms-provider-1.sock")
 	if err != nil {
-		return nil, fmt.Errorf("failed to initialize AES Cipher: %v", err)
-	}
-	// etcd path of the key is used as the authenticated context - need to pass it to decrypt
-	ctx := value.DefaultContext([]byte(secretETCDPath))
-	aescbcTransformer := aestransformer.NewCBCTransformer(block)
-	plainSecret, _, err := aescbcTransformer.TransformFromStorage(secret.getPayload(), ctx)
-	if err != nil {
-		return nil, fmt.Errorf("failed to transform from storage via AESCBC, err: %v", err)
+		t.Fatalf("failed to create mock of KMS Plugin #1: %v", err)
 	}

-	return plainSecret, nil
+	go pluginMock1.grpcServer.Serve(pluginMock1.listener)
+	defer pluginMock1.cleanUp()
+	kmsPluginMustBeUp(t, pluginMock1)
+
+	pluginMock2, err := newBase64Plugin("@kms-provider-2.sock")
+	if err != nil {
+		t.Fatalf("failed to create mock of KMS Plugin #2: %v", err)
+	}
+
+	go pluginMock2.grpcServer.Serve(pluginMock2.listener)
+	defer pluginMock2.cleanUp()
+	kmsPluginMustBeUp(t, pluginMock2)
+
+	test, err := newTransformTest(t, encryptionConfig)
+	if err != nil {
+		t.Fatalf("Failed to start kube-apiserver, error: %v", err)
+	}
+	defer test.cleanUp()
+
+	// Name of the healthz check is calculated based on a constant "kms-provider-" + position of the
+	// provider in the config.
+
+	// Stage 1 - Since all kms-plugins are guaranteed to be up, healthz checks for:
+	// healthz/kms-provider-0 and /healthz/kms-provider-1 should be OK.
+	mustBeHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig)
+	mustBeHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig)
+
+	// Stage 2 - kms-plugin for provider-1 is down. Therefore, expect the health check for provider-1
+	// to fail, but provider-2 should still be OK
+	pluginMock1.enterFailedState()
+	mustBeUnHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig)
+	mustBeHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig)
+	pluginMock1.exitFailedState()
+
+	// Stage 3 - kms-plugin for provider-1 is now up. Therefore, expect the health check for provider-1
+	// to succeed now, but provider-2 is now down.
+	// Need to sleep since health check chases responses for 3 seconds.
+	pluginMock2.enterFailedState()
+	mustBeHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig)
+	mustBeUnHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig)
+}
+
+func kmsPluginMustBeUp(t *testing.T, plugin *base64Plugin) {
+	t.Helper()
+	var gRPCErr error
+	pollErr := wait.PollImmediate(1*time.Second, wait.ForeverTestTimeout, func() (bool, error) {
+		_, gRPCErr = plugin.Encrypt(context.Background(), &kmsapi.EncryptRequest{Plain: []byte("foo")})
+		return gRPCErr == nil, nil
+	})
+
+	if pollErr == wait.ErrWaitTimeout {
+		t.Fatalf("failed to start kms-plugin, error: %v", gRPCErr)
+	}
+}
+
+func mustBeHealthy(t *testing.T, checkName string, clientConfig *rest.Config) {
+	t.Helper()
+	var restErr error
+	pollErr := wait.PollImmediate(2*time.Second, wait.ForeverTestTimeout, func() (bool, error) {
+		status, err := getHealthz(checkName, clientConfig)
+		if err != nil {
+			return false, err
+		}
+		return status == http.StatusOK, nil
+	})
+
+	if pollErr == wait.ErrWaitTimeout {
+		t.Fatalf("failed to get the expected healthz status of OK for check: %s, error: %v", restErr, checkName)
+	}
+}
+
+func mustBeUnHealthy(t *testing.T, checkName string, clientConfig *rest.Config) {
+	t.Helper()
+	var restErr error
+	pollErr := wait.PollImmediate(2*time.Second, wait.ForeverTestTimeout, func() (bool, error) {
+		status, err := getHealthz(checkName, clientConfig)
+		if err != nil {
+			return false, err
+		}
+		return status != http.StatusOK, nil
+	})
+
+	if pollErr == wait.ErrWaitTimeout {
+		t.Fatalf("failed to get the expected healthz status of !OK for check: %s, error: %v", restErr, checkName)
+	}
+}
+
+func getHealthz(checkName string, clientConfig *rest.Config) (int, error) {
+	client, err := kubernetes.NewForConfig(clientConfig)
+	if err != nil {
+		return 0, fmt.Errorf("failed to create a client: %v", err)
+	}
+
+	result := client.CoreV1().RESTClient().Get().AbsPath(fmt.Sprintf("/healthz/%v", checkName)).Do()
+	status := 0
+	result.StatusCode(&status)
+	return status, nil
 }
--- a/test/integration/master/secrets_transformation_test.go
+++ b/test/integration/master/secrets_transformation_test.go
@@ -90,6 +90,10 @@ func TestSecretsShouldBeTransformed(t *testing.T) {
 			t.Errorf("failed to setup test for envelop %s, error was %v", tt.transformerPrefix, err)
 			continue
 		}
+		test.secret, err = test.createSecret(testSecret, testNamespace)
+		if err != nil {
+			t.Fatalf("Failed to create test secret, error: %v", err)
+		}
 		test.run(tt.unSealFunc, tt.transformerPrefix)
 		test.cleanUp()
 	}
--- a/test/integration/master/transformation_testcase.go
+++ b/test/integration/master/transformation_testcase.go
@@ -27,6 +27,8 @@ import (
 	"strings"
 	"testing"

+	"k8s.io/klog"
+
 	"github.com/coreos/etcd/clientv3"
 	"github.com/prometheus/client_golang/prometheus"
 	"sigs.k8s.io/yaml"
@@ -81,6 +83,7 @@ func newTransformTest(l kubeapiservertesting.Logger, transformerConfigYAML strin
 	if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(l, nil, e.getEncryptionOptions(), e.storageConfig); err != nil {
 		return nil, fmt.Errorf("failed to start KubeAPI server: %v", err)
 	}
+	klog.Infof("Started kube-apiserver %v", e.kubeAPIServer.ClientConfig.Host)

 	if e.restClient, err = kubernetes.NewForConfig(e.kubeAPIServer.ClientConfig); err != nil {
 		return nil, fmt.Errorf("error while creating rest client: %v", err)
@@ -90,10 +93,6 @@ func newTransformTest(l kubeapiservertesting.Logger, transformerConfigYAML strin
 		return nil, err
 	}

-	if e.secret, err = e.createSecret(testSecret, e.ns.Name); err != nil {
-		return nil, err
-	}
-
 	return &e, nil
 }

@@ -118,7 +117,7 @@ func (e *transformTest) run(unSealSecretFunc unSealSecret, expectedEnvelopePrefi

 	// etcd path of the key is used as the authenticated context - need to pass it to decrypt
 	ctx := value.DefaultContext([]byte(e.getETCDPath()))
-	// Envelope header precedes the payload
+	// Envelope header precedes the cipherTextPayload
 	sealedData := response.Kvs[0].Value[len(expectedEnvelopePrefix):]
 	transformerConfig, err := e.getEncryptionConfig()
 	if err != nil {