Enable encryption for custom resources

Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com>

test/integration/controlplane/transformation/all_transformation_test.go

@@ -0,0 +1,129 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package transformation
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/kubernetes/test/integration/etcd"
+)
+
+func createResources(t *testing.T, test *transformTest,
+	group,
+	version,
+	kind,
+	resource,
+	name,
+	namespace string,
+) {
+	switch resource {
+	case "pods":
+		_, err := test.createPod(namespace, dynamic.NewForConfigOrDie(test.kubeAPIServer.ClientConfig))
+		if err != nil {
+			t.Fatalf("Failed to create test pod, error: %v, name: %s, ns: %s", err, name, namespace)
+		}
+	case "configmaps":
+		_, err := test.createConfigMap(name, namespace)
+		if err != nil {
+			t.Fatalf("Failed to create test configmap, error: %v, name: %s, ns: %s", err, name, namespace)
+		}
+	default:
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		t.Cleanup(cancel)
+
+		gvr := schema.GroupVersionResource{Group: group, Version: version, Resource: resource}
+		data := etcd.GetEtcdStorageData()[gvr]
+		stub := data.Stub
+		dynamicClient, obj, err := etcd.JSONToUnstructured(stub, namespace, &meta.RESTMapping{
+			Resource:         gvr,
+			GroupVersionKind: gvr.GroupVersion().WithKind(kind),
+			Scope:            meta.RESTScopeRoot,
+		}, dynamic.NewForConfigOrDie(test.kubeAPIServer.ClientConfig))
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, err = dynamicClient.Create(ctx, obj, metav1.CreateOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, err := dynamicClient.Get(ctx, obj.GetName(), metav1.GetOptions{}); err != nil {
+			t.Fatalf("object should exist: %v", err)
+		}
+	}
+}
+
+func TestEncryptSupportedForAllResourceTypes(t *testing.T) {
+	// check resources provided by the three servers that we have wired together
+	// - pods and configmaps from KAS
+	// - CRDs and CRs from API extensions
+	// - API services from aggregator
+	encryptionConfig := `
+kind: EncryptionConfiguration
+apiVersion: apiserver.config.k8s.io/v1
+resources:
+- resources:
+  - pods
+  - configmaps
+  - customresourcedefinitions.apiextensions.k8s.io
+  - pandas.awesome.bears.com
+  - apiservices.apiregistration.k8s.io
+  providers:
+  - aescbc:
+      keys:
+      - name: key1
+        secret: c2VjcmV0IGlzIHNlY3VyZQ==
+`
+
+	test, err := newTransformTest(t, encryptionConfig)
+	if err != nil {
+		t.Fatalf("failed to start Kube API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
+	}
+	t.Cleanup(test.cleanUp)
+
+	// the storage registry for CRs is dynamic so create one to exercise the wiring
+	etcd.CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(test.kubeAPIServer.ClientConfig), false, etcd.GetCustomResourceDefinitionData()...)
+
+	for _, tt := range []struct {
+		group     string
+		version   string
+		kind      string
+		resource  string
+		name      string
+		namespace string
+	}{
+		{"", "v1", "ConfigMap", "configmaps", "cm1", testNamespace},
+		{"apiextensions.k8s.io", "v1", "CustomResourceDefinition", "customresourcedefinitions", "pandas.awesome.bears.com", ""},
+		{"awesome.bears.com", "v1", "Panda", "pandas", "cr3panda", ""},
+		{"apiregistration.k8s.io", "v1", "APIService", "apiservices", "as2.foo.com", ""},
+		{"", "v1", "Pod", "pods", "pod1", testNamespace},
+	} {
+		tt := tt
+		t.Run(tt.resource, func(t *testing.T) {
+			t.Parallel()
+
+			createResources(t, test, tt.group, tt.version, tt.kind, tt.resource, tt.name, tt.namespace)
+			test.runResource(t, unSealWithCBCTransformer, aesCBCPrefix, tt.group, tt.version, tt.resource, tt.name, tt.namespace)
+		})
+	}
+}

test/integration/controlplane/transformation/kms_transformation_test.go

@@ -145,7 +145,7 @@ resources:
 	// Since Data Encryption Key (DEK) is randomly generated (per encryption operation), we need to ask KMS Mock for it.
 	plainTextDEK := pluginMock.LastEncryptRequest()
 
-	secretETCDPath := test.getETCDPath()
+	secretETCDPath := test.getETCDPathForResource(test.storageConfig.Prefix, "", "secrets", test.secret.Name, test.secret.Namespace)
 	rawEnvelope, err := test.getRawSecretFromETCD()
 	if err != nil {
 		t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err)

test/integration/controlplane/transformation/kmsv2_transformation_test.go

@@ -154,7 +154,7 @@ resources:
 	// Since Data Encryption Key (DEK) is randomly generated (per encryption operation), we need to ask KMS Mock for it.
 	plainTextDEK := pluginMock.LastEncryptRequest()
 
-	secretETCDPath := test.getETCDPath()
+	secretETCDPath := test.getETCDPathForResource(test.storageConfig.Prefix, "", "secrets", test.secret.Name, test.secret.Namespace)
 	rawEnvelope, err := test.getRawSecretFromETCD()
 	if err != nil {
 		t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err)

test/integration/controlplane/transformation/secrets_transformation_test.go

@@ -95,7 +95,7 @@ func TestSecretsShouldBeTransformed(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Failed to create test secret, error: %v", err)
 		}
-		test.run(tt.unSealFunc, tt.transformerPrefix)
+		test.runResource(test.logger, tt.unSealFunc, tt.transformerPrefix, "", "v1", "secrets", test.secret.Name, test.secret.Namespace)
 		test.cleanUp()
 	}
 }

test/integration/controlplane/transformation/transformation_testcase.go

@@ -19,6 +19,7 @@ package transformation
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path"
@@ -34,12 +35,16 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 	"k8s.io/apiserver/pkg/storage/value"
+	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
 	"k8s.io/kubernetes/test/integration"
+	"k8s.io/kubernetes/test/integration/etcd"
 	"k8s.io/kubernetes/test/integration/framework"
 )
 
@@ -50,6 +55,8 @@ const (
 	testNamespace            = "secret-encryption-test"
 	testSecret               = "test-secret"
 	metricsPrefix            = "apiserver_storage_"
+	configMapKey             = "foo"
+	configMapVal             = "bar"
 
 	// precomputed key and secret for use with AES CBC
 	// this looks exactly the same as the AES GCM secret but with a different value
@@ -107,44 +114,88 @@ func (e *transformTest) cleanUp() {
 	e.kubeAPIServer.TearDownFn()
 }
 
-func (e *transformTest) run(unSealSecretFunc unSealSecret, expectedEnvelopePrefix string) {
-	response, err := e.readRawRecordFromETCD(e.getETCDPath())
+func (e *transformTest) runResource(l kubeapiservertesting.Logger, unSealSecretFunc unSealSecret, expectedEnvelopePrefix,
+	group,
+	version,
+	resource,
+	name,
+	namespaceName string,
+) {
+	response, err := e.readRawRecordFromETCD(e.getETCDPathForResource(e.storageConfig.Prefix, group, resource, name, namespaceName))
 	if err != nil {
-		e.logger.Errorf("failed to read from etcd: %v", err)
+		l.Errorf("failed to read from etcd: %v", err)
 		return
 	}
 
 	if !bytes.HasPrefix(response.Kvs[0].Value, []byte(expectedEnvelopePrefix)) {
-		e.logger.Errorf("expected secret to be prefixed with %s, but got %s",
+		l.Errorf("expected data to be prefixed with %s, but got %s",
 			expectedEnvelopePrefix, response.Kvs[0].Value)
 		return
 	}
 
 	// etcd path of the key is used as the authenticated context - need to pass it to decrypt
 	ctx := context.Background()
-	dataCtx := value.DefaultContext([]byte(e.getETCDPath()))
+	dataCtx := value.DefaultContext(e.getETCDPathForResource(e.storageConfig.Prefix, group, resource, name, namespaceName))
 	// Envelope header precedes the cipherTextPayload
 	sealedData := response.Kvs[0].Value[len(expectedEnvelopePrefix):]
 	transformerConfig, err := e.getEncryptionConfig()
 	if err != nil {
-		e.logger.Errorf("failed to parse transformer config: %v", err)
+		l.Errorf("failed to parse transformer config: %v", err)
 	}
 	v, err := unSealSecretFunc(ctx, sealedData, dataCtx, *transformerConfig)
 	if err != nil {
-		e.logger.Errorf("failed to unseal secret: %v", err)
+		l.Errorf("failed to unseal secret: %v", err)
 		return
 	}
-	if !strings.Contains(string(v), secretVal) {
-		e.logger.Errorf("expected %q after decryption, but got %q", secretVal, string(v))
+	if resource == "secrets" {
+		if !strings.Contains(string(v), secretVal) {
+			l.Errorf("expected %q after decryption, but got %q", secretVal, string(v))
+		}
+	} else if resource == "configmaps" {
+		if !strings.Contains(string(v), configMapVal) {
+			l.Errorf("expected %q after decryption, but got %q", configMapVal, string(v))
+		}
+	} else {
+		if !strings.Contains(string(v), name) {
+			l.Errorf("expected %q after decryption, but got %q", name, string(v))
+		}
 	}
 
-	// Secrets should be un-enveloped on direct reads from Kube API Server.
-	s, err := e.restClient.CoreV1().Secrets(testNamespace).Get(context.TODO(), testSecret, metav1.GetOptions{})
-	if err != nil {
-		e.logger.Errorf("failed to get Secret from %s, err: %v", testNamespace, err)
-	}
-	if secretVal != string(s.Data[secretKey]) {
-		e.logger.Errorf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey]))
+	// Data should be un-enveloped on direct reads from Kube API Server.
+	if resource == "secrets" {
+		s, err := e.restClient.CoreV1().Secrets(testNamespace).Get(context.TODO(), testSecret, metav1.GetOptions{})
+		if err != nil {
+			l.Fatalf("failed to get Secret from %s, err: %v", testNamespace, err)
+		}
+		if secretVal != string(s.Data[secretKey]) {
+			l.Errorf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey]))
+		}
+	} else if resource == "configmaps" {
+		s, err := e.restClient.CoreV1().ConfigMaps(namespaceName).Get(context.TODO(), name, metav1.GetOptions{})
+		if err != nil {
+			l.Fatalf("failed to get ConfigMap from %s, err: %v", namespaceName, err)
+		}
+		if configMapVal != string(s.Data[configMapKey]) {
+			l.Errorf("expected %s from KubeAPI, but got %s", configMapVal, string(s.Data[configMapKey]))
+		}
+	} else if resource == "pods" {
+		p, err := e.restClient.CoreV1().Pods(namespaceName).Get(context.TODO(), name, metav1.GetOptions{})
+		if err != nil {
+			l.Fatalf("failed to get Pod from %s, err: %v", namespaceName, err)
+		}
+		if p.Name != name {
+			l.Errorf("expected %s from KubeAPI, but got %s", name, p.Name)
+		}
+	} else {
+		l.Logf("Get object with dynamic client")
+		fooResource := schema.GroupVersionResource{Group: group, Version: version, Resource: resource}
+		obj, err := dynamic.NewForConfigOrDie(e.kubeAPIServer.ClientConfig).Resource(fooResource).Namespace(namespaceName).Get(context.TODO(), name, metav1.GetOptions{})
+		if err != nil {
+			l.Fatalf("Failed to get test instance: %v, name: %s", err, name)
+		}
+		if obj.GetObjectKind().GroupVersionKind().Group == group && obj.GroupVersionKind().Version == version && obj.GetKind() == resource && obj.GetNamespace() == namespaceName && obj.GetName() != name {
+			l.Errorf("expected %s from KubeAPI, but got %s", name, obj.GetName())
+		}
 	}
 }
 
@@ -157,12 +208,19 @@ func (e *transformTest) benchmark(b *testing.B) {
 	}
 }
 
-func (e *transformTest) getETCDPath() string {
-	return fmt.Sprintf("/%s/secrets/%s/%s", e.storageConfig.Prefix, e.ns.Name, e.secret.Name)
+func (e *transformTest) getETCDPathForResource(storagePrefix, group, resource, name, namespaceName string) string {
+	groupResource := resource
+	if group != "" {
+		groupResource = fmt.Sprintf("%s/%s", group, resource)
+	}
+	if namespaceName == "" {
+		return fmt.Sprintf("/%s/%s/%s", storagePrefix, groupResource, name)
+	}
+	return fmt.Sprintf("/%s/%s/%s/%s", storagePrefix, groupResource, namespaceName, name)
 }
 
 func (e *transformTest) getRawSecretFromETCD() ([]byte, error) {
-	secretETCDPath := e.getETCDPath()
+	secretETCDPath := e.getETCDPathForResource(e.storageConfig.Prefix, "", "secrets", e.secret.Name, e.secret.Namespace)
 	etcdResponse, err := e.readRawRecordFromETCD(secretETCDPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read %s from etcd: %v", secretETCDPath, err)
@@ -172,7 +230,9 @@ func (e *transformTest) getRawSecretFromETCD() ([]byte, error) {
 
 func (e *transformTest) getEncryptionOptions() []string {
 	if e.transformerConfig != "" {
-		return []string{"--encryption-provider-config", path.Join(e.configDir, encryptionConfigFileName)}
+		return []string{
+			"--encryption-provider-config", path.Join(e.configDir, encryptionConfigFileName),
+			"--disable-admission-plugins", "ServiceAccount"}
 	}
 
 	return nil
@@ -235,6 +295,60 @@ func (e *transformTest) createSecret(name, namespace string) (*corev1.Secret, er
 	return secret, nil
 }
 
+func (e *transformTest) createConfigMap(name, namespace string) (*corev1.ConfigMap, error) {
+	cm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Data: map[string]string{
+			configMapKey: configMapVal,
+		},
+	}
+	if _, err := e.restClient.CoreV1().ConfigMaps(cm.Namespace).Create(context.TODO(), cm, metav1.CreateOptions{}); err != nil {
+		return nil, fmt.Errorf("error while writing configmap: %v", err)
+	}
+
+	return cm, nil
+}
+
+func gvr(group, version, resource string) schema.GroupVersionResource {
+	return schema.GroupVersionResource{Group: group, Version: version, Resource: resource}
+}
+
+func createResource(client dynamic.Interface, gvr schema.GroupVersionResource, ns string) (*unstructured.Unstructured, error) {
+	stubObj, err := getStubObj(gvr)
+	if err != nil {
+		return nil, err
+	}
+	return client.Resource(gvr).Namespace(ns).Create(context.TODO(), stubObj, metav1.CreateOptions{})
+}
+
+func getStubObj(gvr schema.GroupVersionResource) (*unstructured.Unstructured, error) {
+	stub := ""
+	if data, ok := etcd.GetEtcdStorageDataForNamespace(testNamespace)[gvr]; ok {
+		stub = data.Stub
+	}
+	if len(stub) == 0 {
+		return nil, fmt.Errorf("no stub data for %#v", gvr)
+	}
+
+	stubObj := &unstructured.Unstructured{Object: map[string]interface{}{}}
+	if err := json.Unmarshal([]byte(stub), &stubObj.Object); err != nil {
+		return nil, fmt.Errorf("error unmarshaling stub for %#v: %v", gvr, err)
+	}
+	return stubObj, nil
+}
+
+func (e *transformTest) createPod(namespace string, dynamicInterface dynamic.Interface) (*unstructured.Unstructured, error) {
+	podGVR := gvr("", "v1", "pods")
+	pod, err := createResource(dynamicInterface, podGVR, namespace)
+	if err != nil {
+		return nil, fmt.Errorf("error while writing pod: %v", err)
+	}
+	return pod, nil
+}
+
 func (e *transformTest) readRawRecordFromETCD(path string) (*clientv3.GetResponse, error) {
 	rawClient, etcdClient, err := integration.GetEtcdClients(e.kubeAPIServer.ServerOpts.Etcd.StorageConfig.Transport)
 	if err != nil {