E2E Test

test/e2e/apimachinery/crd_conversion_webhook.go

@@ -0,0 +1,396 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apimachinery
+
+import (
+	"k8s.io/kubernetes/staging/src/k8s.io/apiextensions-apiserver/test/integration"
+	"time"
+
+	apps "k8s.io/api/apps/v1"
+	"k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	utilversion "k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/client-go/dynamic"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	_ "github.com/stretchr/testify/assert"
+)
+
+const (
+	secretCRDName      = "sample-custom-resource-conversion-webhook-secret"
+	deploymentCRDName  = "sample-crd-conversion-webhook-deployment"
+	serviceCRDName     = "e2e-test-crd-conversion-webhook"
+	roleBindingCRDName = "crd-conversion-webhook-auth-reader"
+)
+
+var serverCRDConversionWebhookVersion = utilversion.MustParseSemantic("v1.13.0-alpha")
+
+var apiVersions = []v1beta1.CustomResourceDefinitionVersion{
+	{
+		Name:    "v1",
+		Served:  true,
+		Storage: true,
+	},
+	{
+		Name:    "v2",
+		Served:  true,
+		Storage: false,
+	},
+}
+
+var alternativeApiVersions = []v1beta1.CustomResourceDefinitionVersion{
+	{
+		Name:    "v1",
+		Served:  true,
+		Storage: false,
+	},
+	{
+		Name:    "v2",
+		Served:  true,
+		Storage: true,
+	},
+}
+
+var _ = SIGDescribe("CustomResourceConversionWebhook [Feature:CustomResourceWebhookConversion]", func() {
+	var context *certContext
+	f := framework.NewDefaultFramework("crd-webhook")
+
+	var client clientset.Interface
+	var namespaceName string
+
+	BeforeEach(func() {
+		client = f.ClientSet
+		namespaceName = f.Namespace.Name
+
+		// Make sure the relevant provider supports conversion webhook
+		framework.SkipUnlessServerVersionGTE(serverCRDConversionWebhookVersion, f.ClientSet.Discovery())
+
+		By("Setting up server cert")
+		context = setupServerCert(f.Namespace.Name, serviceCRDName)
+		createAuthReaderRoleBindingForCRDConversion(f, f.Namespace.Name)
+
+		deployCustomResourceWebhookAndService(f, imageutils.GetE2EImage(imageutils.CRDConversionWebhook), context)
+	})
+
+	AfterEach(func() {
+		cleanCRDWebhookTest(client, namespaceName)
+	})
+
+	It("Should be able to convert from CR v1 to CR v2", func() {
+		testcrd, err := framework.CreateMultiVersionTestCRD(f, "stable.example.com", apiVersions,
+			&v1beta1.WebhookClientConfig{
+				CABundle: context.signingCert,
+				Service: &v1beta1.ServiceReference{
+					Namespace: f.Namespace.Name,
+					Name:      serviceCRDName,
+					Path:      strPtr("/crdconvert"),
+				}})
+		if err != nil {
+			return
+		}
+		defer testcrd.CleanUp()
+		testCustomResourceConversionWebhook(f, testcrd.Crd, testcrd.DynamicClients)
+	})
+
+	It("Should be able to convert a non homogeneous list of CRs", func() {
+		testcrd, err := framework.CreateMultiVersionTestCRD(f, "stable.example.com", apiVersions,
+			&v1beta1.WebhookClientConfig{
+				CABundle: context.signingCert,
+				Service: &v1beta1.ServiceReference{
+					Namespace: f.Namespace.Name,
+					Name:      serviceCRDName,
+					Path:      strPtr("/crdconvert"),
+				}})
+		if err != nil {
+			return
+		}
+		defer testcrd.CleanUp()
+		testCRListConversion(f, testcrd)
+	})
+})
+
+func cleanCRDWebhookTest(client clientset.Interface, namespaceName string) {
+	_ = client.CoreV1().Services(namespaceName).Delete(serviceCRDName, nil)
+	_ = client.AppsV1().Deployments(namespaceName).Delete(deploymentCRDName, nil)
+	_ = client.CoreV1().Secrets(namespaceName).Delete(secretCRDName, nil)
+	_ = client.RbacV1().RoleBindings("kube-system").Delete(roleBindingCRDName, nil)
+}
+
+func createAuthReaderRoleBindingForCRDConversion(f *framework.Framework, namespace string) {
+	By("Create role binding to let cr conversion webhook read extension-apiserver-authentication")
+	client := f.ClientSet
+	// Create the role binding to allow the webhook read the extension-apiserver-authentication configmap
+	_, err := client.RbacV1().RoleBindings("kube-system").Create(&rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: roleBindingCRDName,
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: "",
+			Kind:     "Role",
+			Name:     "extension-apiserver-authentication-reader",
+		},
+		// Webhook uses the default service account.
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      "default",
+				Namespace: namespace,
+			},
+		},
+	})
+	if err != nil && errors.IsAlreadyExists(err) {
+		framework.Logf("role binding %s already exists", roleBindingCRDName)
+	} else {
+		framework.ExpectNoError(err, "creating role binding %s:webhook to access configMap", namespace)
+	}
+}
+
+func deployCustomResourceWebhookAndService(f *framework.Framework, image string, context *certContext) {
+	By("Deploying the custom resource conversion webhook pod")
+	client := f.ClientSet
+
+	// Creating the secret that contains the webhook's cert.
+	secret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: secretCRDName,
+		},
+		Type: v1.SecretTypeOpaque,
+		Data: map[string][]byte{
+			"tls.crt": context.cert,
+			"tls.key": context.key,
+		},
+	}
+	namespace := f.Namespace.Name
+	_, err := client.CoreV1().Secrets(namespace).Create(secret)
+	framework.ExpectNoError(err, "creating secret %q in namespace %q", secretName, namespace)
+
+	// Create the deployment of the webhook
+	podLabels := map[string]string{"app": "sample-crd-conversion-webhook", "crd-webhook": "true"}
+	replicas := int32(1)
+	zero := int64(0)
+	mounts := []v1.VolumeMount{
+		{
+			Name:      "crd-conversion-webhook-certs",
+			ReadOnly:  true,
+			MountPath: "/webhook.local.config/certificates",
+		},
+	}
+	volumes := []v1.Volume{
+		{
+			Name: "crd-conversion-webhook-certs",
+			VolumeSource: v1.VolumeSource{
+				Secret: &v1.SecretVolumeSource{SecretName: secretCRDName},
+			},
+		},
+	}
+	containers := []v1.Container{
+		{
+			Name:         "sample-crd-conversion-webhook",
+			VolumeMounts: mounts,
+			Args: []string{
+				"--tls-cert-file=/webhook.local.config/certificates/tls.crt",
+				"--tls-private-key-file=/webhook.local.config/certificates/tls.key",
+				"--alsologtostderr",
+				"-v=4",
+				"2>&1",
+			},
+			Image: image,
+		},
+	}
+	d := &apps.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   deploymentCRDName,
+			Labels: podLabels,
+		},
+		Spec: apps.DeploymentSpec{
+			Replicas: &replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: podLabels,
+			},
+			Strategy: apps.DeploymentStrategy{
+				Type: apps.RollingUpdateDeploymentStrategyType,
+			},
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: podLabels,
+				},
+				Spec: v1.PodSpec{
+					TerminationGracePeriodSeconds: &zero,
+					Containers:                    containers,
+					Volumes:                       volumes,
+				},
+			},
+		},
+	}
+	deployment, err := client.AppsV1().Deployments(namespace).Create(d)
+	framework.ExpectNoError(err, "creating deployment %s in namespace %s", deploymentCRDName, namespace)
+	By("Wait for the deployment to be ready")
+	err = framework.WaitForDeploymentRevisionAndImage(client, namespace, deploymentCRDName, "1", image)
+	framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", image, deploymentName, namespace)
+	err = framework.WaitForDeploymentComplete(client, deployment)
+	framework.ExpectNoError(err, "waiting for the deployment status valid", image, deploymentCRDName, namespace)
+
+	By("Deploying the webhook service")
+
+	serviceLabels := map[string]string{"crd-webhook": "true"}
+	service := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      serviceCRDName,
+			Labels:    map[string]string{"test": "crd-webhook"},
+		},
+		Spec: v1.ServiceSpec{
+			Selector: serviceLabels,
+			Ports: []v1.ServicePort{
+				{
+					Protocol:   "TCP",
+					Port:       443,
+					TargetPort: intstr.FromInt(443),
+				},
+			},
+		},
+	}
+	_, err = client.CoreV1().Services(namespace).Create(service)
+	framework.ExpectNoError(err, "creating service %s in namespace %s", serviceCRDName, namespace)
+
+	By("Verifying the service has paired with the endpoint")
+	err = framework.WaitForServiceEndpointsNum(client, namespace, serviceCRDName, 1, 1*time.Second, 30*time.Second)
+	framework.ExpectNoError(err, "waiting for service %s/%s have %d endpoint", namespace, serviceCRDName, 1)
+}
+
+func verifyV1Object(f *framework.Framework, crd *v1beta1.CustomResourceDefinition, obj *unstructured.Unstructured) {
+	Expect(obj.GetAPIVersion()).To(BeEquivalentTo(crd.Spec.Group + "/v1"))
+	hostPort, exists := obj.Object["hostPort"]
+	Expect(exists).To(BeTrue())
+	Expect(hostPort).To(BeEquivalentTo("localhost:8080"))
+	_, hostExists := obj.Object["host"]
+	Expect(hostExists).To(BeFalse())
+	_, portExists := obj.Object["port"]
+	Expect(portExists).To(BeFalse())
+}
+
+func verifyV2Object(f *framework.Framework, crd *v1beta1.CustomResourceDefinition, obj *unstructured.Unstructured) {
+	Expect(obj.GetAPIVersion()).To(BeEquivalentTo(crd.Spec.Group + "/v2"))
+	_, hostPortExists := obj.Object["hostPort"]
+	Expect(hostPortExists).To(BeFalse())
+	host, hostExists := obj.Object["host"]
+	Expect(hostExists).To(BeTrue())
+	Expect(host).To(BeEquivalentTo("localhost"))
+	port, portExists := obj.Object["port"]
+	Expect(portExists).To(BeTrue())
+	Expect(port).To(BeEquivalentTo("8080"))
+}
+
+func testCustomResourceConversionWebhook(f *framework.Framework, crd *v1beta1.CustomResourceDefinition, customResourceClients map[string]dynamic.ResourceInterface) {
+	name := "cr-instance-1"
+	By("Creating a v1 custom resource")
+	crInstance := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"kind":       crd.Spec.Names.Kind,
+			"apiVersion": crd.Spec.Group + "/v1",
+			"metadata": map[string]interface{}{
+				"name":      name,
+				"namespace": f.Namespace.Name,
+			},
+			"hostPort": "localhost:8080",
+		},
+	}
+	_, err := customResourceClients["v1"].Create(crInstance, metav1.CreateOptions{})
+	Expect(err).To(BeNil())
+	By("v2 custom resource should be converted")
+	v2crd, err := customResourceClients["v2"].Get(name, metav1.GetOptions{})
+	verifyV2Object(f, crd, v2crd)
+}
+
+func testCRListConversion(f *framework.Framework, testCrd *framework.TestCrd) {
+	crd := testCrd.Crd
+	customResourceClients := testCrd.DynamicClients
+	name1 := "cr-instance-1"
+	name2 := "cr-instance-2"
+	By("Creating a v1 custom resource")
+	crInstance := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"kind":       crd.Spec.Names.Kind,
+			"apiVersion": crd.Spec.Group + "/v1",
+			"metadata": map[string]interface{}{
+				"name":      name1,
+				"namespace": f.Namespace.Name,
+			},
+			"hostPort": "localhost:8080",
+		},
+	}
+	_, err := customResourceClients["v1"].Create(crInstance, metav1.CreateOptions{})
+	Expect(err).To(BeNil())
+
+	// Now cr-instance-1 is stored as v1. lets change storage version
+	crd, err = integration.UpdateCustomResourceDefinitionWithRetry(testCrd.ApiExtensionClient, crd.Name, func(c *v1beta1.CustomResourceDefinition) {
+		c.Spec.Versions = alternativeApiVersions
+	})
+	Expect(err).To(BeNil())
+	By("Create a v2 custom resource")
+	crInstance = &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"kind":       crd.Spec.Names.Kind,
+			"apiVersion": crd.Spec.Group + "/v1",
+			"metadata": map[string]interface{}{
+				"name":      name2,
+				"namespace": f.Namespace.Name,
+			},
+			"hostPort": "localhost:8080",
+		},
+	}
+
+	// After changing a CRD, the resources for versions will be re-created that can be result in
+	// cancelled connection (e.g. "grpc connection closed" or "context canceled").
+	// Just retrying fixes that.
+	for i := 0; i < 5; i++ {
+		_, err = customResourceClients["v1"].Create(crInstance, metav1.CreateOptions{})
+		if err == nil {
+			break
+		}
+	}
+	Expect(err).To(BeNil())
+
+	// Now that we have a v1 and v2 object, both list operation in v1 and v2 should work as expected.
+
+	By("List CRs in v1")
+	list, err := customResourceClients["v1"].List(metav1.ListOptions{})
+	Expect(err).To(BeNil())
+	Expect(len(list.Items)).To(BeIdenticalTo(2))
+	Expect((list.Items[0].GetName() == name1 && list.Items[1].GetName() == name2) ||
+		(list.Items[0].GetName() == name2 && list.Items[1].GetName() == name1)).To(BeTrue())
+	verifyV1Object(f, crd, &list.Items[0])
+	verifyV1Object(f, crd, &list.Items[1])
+
+	By("List CRs in v2")
+	list, err = customResourceClients["v2"].List(metav1.ListOptions{})
+	Expect(err).To(BeNil())
+	Expect(len(list.Items)).To(BeIdenticalTo(2))
+	Expect((list.Items[0].GetName() == name1 && list.Items[1].GetName() == name2) ||
+		(list.Items[0].GetName() == name2 && list.Items[1].GetName() == name1)).To(BeTrue())
+	verifyV2Object(f, crd, &list.Items[0])
+	verifyV2Object(f, crd, &list.Items[1])
+}

test/e2e/apimachinery/webhook.go

@@ -136,7 +136,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() {
 		defer testcrd.CleanUp()
 		webhookCleanup := registerWebhookForCustomResource(f, context, testcrd)
 		defer webhookCleanup()
-		testCustomResourceWebhook(f, testcrd.Crd, testcrd.DynamicClient)
+		testCustomResourceWebhook(f, testcrd.Crd, testcrd.GetV1DynamicClient())
 	})
 
 	It("Should unconditionally reject operations on fail closed webhook", func() {
@@ -173,7 +173,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() {
 		defer testcrd.CleanUp()
 		webhookCleanup := registerMutatingWebhookForCustomResource(f, context, testcrd)
 		defer webhookCleanup()
-		testMutatingCustomResourceWebhook(f, testcrd.Crd, testcrd.DynamicClient)
+		testMutatingCustomResourceWebhook(f, testcrd.Crd, testcrd.GetV1DynamicClient())
 	})
 
 	It("Should deny crd creation", func() {
@@ -1157,7 +1157,7 @@ func registerWebhookForCustomResource(f *framework.Framework, context *certConte
 					Operations: []v1beta1.OperationType{v1beta1.Create},
 					Rule: v1beta1.Rule{
 						APIGroups:   []string{testcrd.ApiGroup},
-						APIVersions: []string{testcrd.ApiVersion},
+						APIVersions: testcrd.GetAPIVersions(),
 						Resources:   []string{testcrd.GetPluralName()},
 					},
 				}},
@@ -1198,7 +1198,7 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, context *c
 					Operations: []v1beta1.OperationType{v1beta1.Create},
 					Rule: v1beta1.Rule{
 						APIGroups:   []string{testcrd.ApiGroup},
-						APIVersions: []string{testcrd.ApiVersion},
+						APIVersions: testcrd.GetAPIVersions(),
 						Resources:   []string{testcrd.GetPluralName()},
 					},
 				}},
@@ -1217,7 +1217,7 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, context *c
 					Operations: []v1beta1.OperationType{v1beta1.Create},
 					Rule: v1beta1.Rule{
 						APIGroups:   []string{testcrd.ApiGroup},
-						APIVersions: []string{testcrd.ApiVersion},
+						APIVersions: testcrd.GetAPIVersions(),
 						Resources:   []string{testcrd.GetPluralName()},
 					},
 				}},
@@ -1343,12 +1343,18 @@ func testCRDDenyWebhook(f *framework.Framework) {
 	name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, "deny")
 	kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, "deny")
 	group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName)
-	apiVersion := "v1"
+	apiVersions := []apiextensionsv1beta1.CustomResourceDefinitionVersion{
+		{
+			Name:    "v1",
+			Served:  true,
+			Storage: true,
+		},
+	}
 	testcrd := &framework.TestCrd{
-		Name:       name,
+		Name:     name,
-		Kind:       kind,
+		Kind:     kind,
-		ApiGroup:   group,
+		ApiGroup: group,
-		ApiVersion: apiVersion,
+		Versions: apiVersions,
 	}
 
 	// Creating a custom resource definition for use by assorted tests.
@@ -1370,8 +1376,8 @@ func testCRDDenyWebhook(f *framework.Framework) {
 			},
 		},
 		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
-			Group:   testcrd.ApiGroup,
+			Group:    testcrd.ApiGroup,
-			Version: testcrd.ApiVersion,
+			Versions: testcrd.Versions,
 			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
 				Plural:   testcrd.GetPluralName(),
 				Singular: testcrd.Name,

test/e2e/framework/crd_util.go

@@ -35,25 +35,23 @@ type TestCrd struct {
 	Name               string
 	Kind               string
 	ApiGroup           string
-	ApiVersion         string
+	Versions           []apiextensionsv1beta1.CustomResourceDefinitionVersion
 	ApiExtensionClient *crdclientset.Clientset
 	Crd                *apiextensionsv1beta1.CustomResourceDefinition
-	DynamicClient      dynamic.ResourceInterface
+	DynamicClients     map[string]dynamic.ResourceInterface
 	CleanUp            CleanCrdFn
 }
 
 // CreateTestCRD creates a new CRD specifically for the calling test.
-func CreateTestCRD(f *Framework) (*TestCrd, error) {
+func CreateMultiVersionTestCRD(f *Framework, group string, apiVersions []apiextensionsv1beta1.CustomResourceDefinitionVersion, conversionWebhook *apiextensionsv1beta1.WebhookClientConfig) (*TestCrd, error) {
 	suffix := randomSuffix()
 	name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, suffix)
 	kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, suffix)
-	group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName)
-	apiVersion := "v1"
 	testcrd := &TestCrd{
-		Name:       name,
+		Name:     name,
-		Kind:       kind,
+		Kind:     kind,
-		ApiGroup:   group,
+		ApiGroup: group,
-		ApiVersion: apiVersion,
+		Versions: apiVersions,
 	}
 
 	// Creating a custom resource definition for use by assorted tests.
@@ -75,6 +73,13 @@ func CreateTestCRD(f *Framework) (*TestCrd, error) {
 
 	crd := newCRDForTest(testcrd)
 
+	if conversionWebhook != nil {
+		crd.Spec.Conversion = &apiextensionsv1beta1.CustomResourceConversion{
+			Strategy:            "Webhook",
+			WebhookClientConfig: conversionWebhook,
+		}
+	}
+
 	//create CRD and waits for the resource to be recognized and available.
 	crd, err = fixtures.CreateNewCustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient)
 	if err != nil {
@@ -82,12 +87,17 @@ func CreateTestCRD(f *Framework) (*TestCrd, error) {
 		return nil, err
 	}
 
-	gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: crd.Spec.Version, Resource: crd.Spec.Names.Plural}
+	resourceClients := map[string]dynamic.ResourceInterface{}
-	resourceClient := dynamicClient.Resource(gvr).Namespace(f.Namespace.Name)
+	for _, v := range crd.Spec.Versions {
+		if v.Served {
+			gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: v.Name, Resource: crd.Spec.Names.Plural}
+			resourceClients[v.Name] = dynamicClient.Resource(gvr).Namespace(f.Namespace.Name)
+		}
+	}
 
 	testcrd.ApiExtensionClient = apiExtensionClient
 	testcrd.Crd = crd
-	testcrd.DynamicClient = resourceClient
+	testcrd.DynamicClients = resourceClients
 	testcrd.CleanUp = func() error {
 		err := fixtures.DeleteCustomResourceDefinition(crd, apiExtensionClient)
 		if err != nil {
@@ -98,13 +108,26 @@ func CreateTestCRD(f *Framework) (*TestCrd, error) {
 	return testcrd, nil
 }
 
+// CreateTestCRD creates a new CRD specifically for the calling test.
+func CreateTestCRD(f *Framework) (*TestCrd, error) {
+	group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName)
+	apiVersions := []apiextensionsv1beta1.CustomResourceDefinitionVersion{
+		{
+			Name:    "v1",
+			Served:  true,
+			Storage: true,
+		},
+	}
+	return CreateMultiVersionTestCRD(f, group, apiVersions, nil)
+}
+
 // newCRDForTest generates a CRD definition for the test
 func newCRDForTest(testcrd *TestCrd) *apiextensionsv1beta1.CustomResourceDefinition {
 	return &apiextensionsv1beta1.CustomResourceDefinition{
 		ObjectMeta: metav1.ObjectMeta{Name: testcrd.GetMetaName()},
 		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
-			Group:   testcrd.ApiGroup,
+			Group:    testcrd.ApiGroup,
-			Version: testcrd.ApiVersion,
+			Versions: testcrd.Versions,
 			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
 				Plural:   testcrd.GetPluralName(),
 				Singular: testcrd.Name,
@@ -130,3 +153,17 @@ func (c *TestCrd) GetPluralName() string {
 func (c *TestCrd) GetListName() string {
 	return c.Name + "List"
 }
+
+func (c *TestCrd) GetAPIVersions() []string {
+	ret := []string{}
+	for _, v := range c.Versions {
+		if v.Served {
+			ret = append(ret, v.Name)
+		}
+	}
+	return ret
+}
+
+func (c *TestCrd) GetV1DynamicClient() dynamic.ResourceInterface {
+	return c.DynamicClients["v1"]
+}

test/utils/image/manifest.go

@@ -92,6 +92,7 @@ var (
 
 // Preconfigured image configs
 var (
+	CRDConversionWebhook     = Config{e2eRegistry, "crd-conversion-webhook", "1.13rev2"}
 	AdmissionWebhook         = Config{e2eRegistry, "webhook", "1.13v1"}
 	APIServer                = Config{e2eRegistry, "sample-apiserver", "1.10"}
 	AppArmorLoader           = Config{e2eRegistry, "apparmor-loader", "1.0"}