Load encryption config once

This change updates the API server code to load the encryption
config once at start up instead of multiple times.  Previously the
code would set up the storage transformers and the etcd healthz
checks in separate parse steps.  This is problematic for KMS v2 key
ID based staleness checks which need to be able to assert that the
API server has a single view into the KMS plugin's current key ID.

Signed-off-by: Monis Khan <mok@microsoft.com>

test/integration/controlplane/transformation/kmsv2_transformation_test.go

@@ -29,17 +29,25 @@ import (
 	"time"
 
 	"github.com/gogo/protobuf/proto"
+
+	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apiserver/pkg/features"
+	"k8s.io/apiserver/pkg/server/options/encryptionconfig"
 	"k8s.io/apiserver/pkg/storage/value"
 	aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
+	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2"
 	kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2alpha1"
 	kmsv2mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/v2alpha1"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/client-go/dynamic"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	kmsv2api "k8s.io/kms/apis/v2alpha1"
+	"k8s.io/kubernetes/test/integration/etcd"
 )
 
 type envelopekmsv2 struct {
@@ -273,3 +281,81 @@ resources:
 	mustBeHealthy(t, "kms-provider-0", test.kubeAPIServer.ClientConfig)
 	mustBeUnHealthy(t, "kms-provider-1", test.kubeAPIServer.ClientConfig)
 }
+
+func TestKMSv2SingleService(t *testing.T) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
+
+	var kmsv2Calls int
+	origEnvelopeKMSv2ServiceFactory := encryptionconfig.EnvelopeKMSv2ServiceFactory
+	encryptionconfig.EnvelopeKMSv2ServiceFactory = func(ctx context.Context, endpoint string, callTimeout time.Duration) (kmsv2.Service, error) {
+		kmsv2Calls++
+		return origEnvelopeKMSv2ServiceFactory(ctx, endpoint, callTimeout)
+	}
+	t.Cleanup(func() {
+		encryptionconfig.EnvelopeKMSv2ServiceFactory = origEnvelopeKMSv2ServiceFactory
+	})
+
+	// check resources provided by the three servers that we have wired together
+	// - pods and config maps from KAS
+	// - CRDs and CRs from API extensions
+	// - API services from aggregator
+	encryptionConfig := `
+kind: EncryptionConfiguration
+apiVersion: apiserver.config.k8s.io/v1
+resources:
+  - resources:
+    - pods
+    - configmaps
+    - customresourcedefinitions.apiextensions.k8s.io
+    - pandas.awesome.bears.com
+    - apiservices.apiregistration.k8s.io
+    providers:
+    - kms:
+       apiVersion: v2
+       name: kms-provider
+       cachesize: 1000
+       endpoint: unix:///@kms-provider.sock
+`
+
+	pluginMock, err := kmsv2mock.NewBase64Plugin("@kms-provider.sock")
+	if err != nil {
+		t.Fatalf("failed to create mock of KMSv2 Plugin: %v", err)
+	}
+
+	go pluginMock.Start()
+	if err := kmsv2mock.WaitForBase64PluginToBeUp(pluginMock); err != nil {
+		t.Fatalf("Failed start plugin, err: %v", err)
+	}
+	t.Cleanup(pluginMock.CleanUp)
+
+	test, err := newTransformTest(t, encryptionConfig)
+	if err != nil {
+		t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
+	}
+	t.Cleanup(test.cleanUp)
+
+	// the storage registry for CRs is dynamic so create one to exercise the wiring
+	etcd.CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(test.kubeAPIServer.ClientConfig), false, etcd.GetCustomResourceDefinitionData()...)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	t.Cleanup(cancel)
+
+	gvr := schema.GroupVersionResource{Group: "awesome.bears.com", Version: "v1", Resource: "pandas"}
+	stub := etcd.GetEtcdStorageData()[gvr].Stub
+	dynamicClient, obj, err := etcd.JSONToUnstructured(stub, "", &meta.RESTMapping{
+		Resource:         gvr,
+		GroupVersionKind: gvr.GroupVersion().WithKind("Panda"),
+		Scope:            meta.RESTScopeRoot,
+	}, dynamic.NewForConfigOrDie(test.kubeAPIServer.ClientConfig))
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = dynamicClient.Create(ctx, obj, metav1.CreateOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if kmsv2Calls != 1 {
+		t.Fatalf("expected a single call to KMS v2 service factory: %v", kmsv2Calls)
+	}
+}