Add version testing

test/e2e/apimachinery/crd_selectable_fields.go

@@ -18,12 +18,11 @@ package apimachinery
 
 import (
 	"context"
-	"encoding/json"
+	"fmt"
 	"github.com/onsi/gomega"
 	"time"
 
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
-	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features"
 	"k8s.io/apiextensions-apiserver/test/integration/fixtures"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -35,7 +34,10 @@ import (
 	"k8s.io/apiserver/pkg/storage/names"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/utils/crd"
+	imageutils "k8s.io/kubernetes/test/utils/image"
 	admissionapi "k8s.io/pod-security-admission/api"
+	"k8s.io/utils/ptr"
 
 	"github.com/onsi/ginkgo/v2"
 )
@@ -46,34 +48,67 @@ var _ = SIGDescribe("CustomResourceFieldSelectors [Privileged:ClusterAdmin]", fr
 	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
 
 	ginkgo.Context("CustomResourceFieldSelectors", func() {
-		var apiExtensionClient *clientset.Clientset
+		customResourceClient := func(crd *apiextensionsv1.CustomResourceDefinition, version string) (dynamic.NamespaceableResourceInterface, schema.GroupVersionResource) {
-		ginkgo.BeforeEach(func() {
-			var err error
-			apiExtensionClient, err = clientset.NewForConfig(f.ClientConfig())
-			framework.ExpectNoError(err, "initializing apiExtensionClient")
-		})
-
-		customResourceClient := func(crd *apiextensionsv1.CustomResourceDefinition) (dynamic.NamespaceableResourceInterface, schema.GroupVersionResource) {
 			gvrs := fixtures.GetGroupVersionResourcesOfCustomResource(crd)
-			if len(gvrs) != 1 {
+			for _, gvr := range gvrs {
-				ginkgo.Fail("Expected one version in custom resource definition")
+				if gvr.Version == version {
-			}
+					return f.DynamicClient.Resource(gvr), gvr
-			gvr := gvrs[0]
-			return f.DynamicClient.Resource(gvr), gvr
-		}
-
-		var schemaWithValidationExpression = unmarshalSchema([]byte(`{
-			"type":"object",
-			"properties":{
-				"spec":{
-					"type":"object",
-					"properties":{
-						"color":{ "type":"string" },
-						"quantity":{ "type":"integer" }
-					}
 				}
 			}
-		}`))
+			ginkgo.Fail(fmt.Sprintf("Expected version '%s' in custom resource definition", version))
+			return nil, schema.GroupVersionResource{}
+		}
+
+		var apiVersions = []apiextensionsv1.CustomResourceDefinitionVersion{
+			{
+				Name:    "v1",
+				Served:  true,
+				Storage: true,
+				Schema: &apiextensionsv1.CustomResourceValidation{
+					OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
+						Type: "object",
+						Properties: map[string]apiextensionsv1.JSONSchemaProps{
+							"hostPort": {Type: "string"},
+						},
+					},
+				},
+				SelectableFields: []apiextensionsv1.SelectableField{
+					{JSONPath: ".hostPort"},
+				},
+			},
+			{
+				Name:    "v2",
+				Served:  true,
+				Storage: false,
+				Schema: &apiextensionsv1.CustomResourceValidation{
+					OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
+						Type: "object",
+						Properties: map[string]apiextensionsv1.JSONSchemaProps{
+							"host": {Type: "string"},
+							"port": {Type: "string"},
+						},
+					},
+				},
+				SelectableFields: []apiextensionsv1.SelectableField{
+					{JSONPath: ".host"},
+					{JSONPath: ".port"},
+				},
+			},
+		}
+
+		var certCtx *certContext
+		servicePort := int32(9443)
+		containerPort := int32(9444)
+
+		ginkgo.BeforeEach(func(ctx context.Context) {
+			ginkgo.DeferCleanup(cleanCRDWebhookTest, f.ClientSet, f.Namespace.Name)
+
+			ginkgo.By("Setting up server cert")
+			certCtx = setupServerCert(f.Namespace.Name, serviceCRDName)
+			createAuthReaderRoleBindingForCRDConversion(ctx, f, f.Namespace.Name)
+
+			deployCustomResourceWebhookAndService(ctx, f, imageutils.GetE2EImage(imageutils.Agnhost), certCtx, servicePort, containerPort)
+		})
 
 		/*
 			Release: v1.31
@@ -86,46 +121,58 @@ var _ = SIGDescribe("CustomResourceFieldSelectors [Privileged:ClusterAdmin]", fr
 		*/
 		framework.It("MUST list and watch custom resources matching the field selector", func(ctx context.Context) {
 			ginkgo.By("Creating a custom resource definition with selectable fields")
-			crd := fixtures.NewRandomNameV1CustomResourceDefinitionWithSchema(apiextensionsv1.NamespaceScoped, schemaWithValidationExpression, false)
+			testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *apiextensionsv1.CustomResourceDefinition) {
-			for i := range crd.Spec.Versions {
+				crd.Spec.Versions = apiVersions
-				crd.Spec.Versions[i].SelectableFields = []apiextensionsv1.SelectableField{
+				crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{
-					{JSONPath: ".spec.color"},
+					Strategy: apiextensionsv1.WebhookConverter,
-					{JSONPath: ".spec.quantity"},
+					Webhook: &apiextensionsv1.WebhookConversion{
+						ClientConfig: &apiextensionsv1.WebhookClientConfig{
+							CABundle: certCtx.signingCert,
+							Service: &apiextensionsv1.ServiceReference{
+								Namespace: f.Namespace.Name,
+								Name:      serviceCRDName,
+								Path:      ptr.To("/crdconvert"),
+								Port:      ptr.To(servicePort),
+							},
+						},
+						ConversionReviewVersions: []string{"v1", "v1beta1"},
+					},
 				}
+				crd.Spec.PreserveUnknownFields = false
+			})
+			if err != nil {
+				return
 			}
-			crd, err := fixtures.CreateNewV1CustomResourceDefinition(crd, apiExtensionClient, f.DynamicClient)
+			ginkgo.DeferCleanup(testcrd.CleanUp)
-			framework.ExpectNoError(err, "creating CustomResourceDefinition")
+
-			defer func() {
+			ginkgo.By("Creating a custom resource conversion webhook")
-				err = fixtures.DeleteV1CustomResourceDefinition(crd, apiExtensionClient)
+			waitWebhookConversionReady(ctx, f, testcrd.Crd, testcrd.DynamicClients, "v2")
-				framework.ExpectNoError(err, "deleting CustomResourceDefinition")
+			crd := testcrd.Crd
-			}()
 
 			ginkgo.By("Watching with field selectors")
-			crClient, gvr := customResourceClient(crd)
 
-			watchSimpleSelector, err := crClient.Namespace(f.Namespace.Name).Watch(ctx, metav1.ListOptions{FieldSelector: "spec.color=blue"})
+			v2Client, gvr := customResourceClient(crd, "v2")
+			hostWatch, err := v2Client.Namespace(f.Namespace.Name).Watch(ctx, metav1.ListOptions{FieldSelector: "host=host1"})
 			framework.ExpectNoError(err, "watching custom resources with field selector")
-			defer func() {
+			v2hostPortWatch, err := v2Client.Namespace(f.Namespace.Name).Watch(ctx, metav1.ListOptions{FieldSelector: "host=host1,port=80"})
-				watchSimpleSelector.Stop()
+			framework.ExpectNoError(err, "watching custom resources with field selector")
-			}()
+
-			watchCompoundSelector, err := crClient.Namespace(f.Namespace.Name).Watch(ctx, metav1.ListOptions{FieldSelector: "spec.color=blue,spec.quantity=2"})
+			v1Client, _ := customResourceClient(crd, "v1")
+			v1hostPortWatch, err := v1Client.Namespace(f.Namespace.Name).Watch(ctx, metav1.ListOptions{FieldSelector: "hostPort=host1:80"})
 			framework.ExpectNoError(err, "watching custom resources with field selector")
-			defer func() {
-				watchCompoundSelector.Stop()
-			}()
 
 			ginkgo.By("Creating custom resources")
 			toCreate := []map[string]any{
 				{
-					"color":    "blue",
+					"host": "host1",
-					"quantity": int64(2),
+					"port": "80",
 				},
 				{
-					"color":    "blue",
+					"host": "host1",
-					"quantity": int64(3),
+					"port": "8080",
 				},
 				{
-					"color": "green",
+					"host": "host2",
 				},
 			}
 
@@ -133,76 +180,91 @@ var _ = SIGDescribe("CustomResourceFieldSelectors [Privileged:ClusterAdmin]", fr
 			for i, spec := range toCreate {
 				name := names.SimpleNameGenerator.GenerateName("selectable-field-cr")
 				crNames[i] = name
-				_, err = crClient.Namespace(f.Namespace.Name).Create(ctx, &unstructured.Unstructured{Object: map[string]interface{}{
+
+				obj := map[string]interface{}{
 					"apiVersion": gvr.Group + "/" + gvr.Version,
 					"kind":       crd.Spec.Names.Kind,
 					"metadata": map[string]interface{}{
 						"name":      name,
 						"namespace": f.Namespace.Name,
 					},
-					"spec": spec,
+				}
-				}}, metav1.CreateOptions{})
+				for k, v := range spec {
+					obj[k] = v
+				}
+				_, err = v2Client.Namespace(f.Namespace.Name).Create(ctx, &unstructured.Unstructured{Object: obj}, metav1.CreateOptions{})
 				framework.ExpectNoError(err, "creating custom resource")
 			}
-
+			ginkgo.By("Listing v2 custom resources with field selector host=host1")
-			ginkgo.By("Listing custom resources with field selector spec.color=blue")
+			list, err := v2Client.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "host=host1"})
-			list, err := crClient.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "spec.color=blue"})
 			framework.ExpectNoError(err, "listing custom resources with field selector")
 			gomega.Expect(listResultToNames(list)).To(gomega.Equal(sets.New(crNames[0], crNames[1])))
 
-			ginkgo.By("Listing custom resources with field selector spec.color=blue,spec.quantity=2")
+			ginkgo.By("Listing v2 custom resources with field selector host=host1,port=80")
-			list, err = crClient.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "spec.color=blue,spec.quantity=2"})
+			list, err = v2Client.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "host=host1,port=80"})
 			framework.ExpectNoError(err, "listing custom resources with field selector")
 			gomega.Expect(listResultToNames(list)).To(gomega.Equal(sets.New(crNames[0])))
 
-			ginkgo.By("Waiting for watch events to contain custom resources for field selector spec.color=blue")
+			ginkgo.By("Listing v1 custom resources with field selector hostPort=host1:80")
-			gomega.Eventually(ctx, watchAccumulator(watchSimpleSelector)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second).
+			list, err = v1Client.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "hostPort=host1:80"})
+			framework.ExpectNoError(err, "listing custom resources with field selector")
+			gomega.Expect(listResultToNames(list)).To(gomega.Equal(sets.New(crNames[0])))
+
+			ginkgo.By("Listing v1 custom resources with field selector hostPort=host1:8080")
+			list, err = v1Client.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "hostPort=host1:8080"})
+			framework.ExpectNoError(err, "listing custom resources with field selector")
+			gomega.Expect(listResultToNames(list)).To(gomega.Equal(sets.New(crNames[1])))
+
+			ginkgo.By("Waiting for watch events to contain v2 custom resources for field selector host=host1")
+			gomega.Eventually(ctx, watchAccumulator(hostWatch)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second).
 				Should(gomega.Equal(addedEvents(sets.New(crNames[0], crNames[1]))))
 
-			ginkgo.By("Waiting for watch events to contain custom resources for field selector spec.color=blue,spec.quantity=2")
+			ginkgo.By("Waiting for watch events to contain v2 custom resources for field selector host=host1,port=80")
-			gomega.Eventually(ctx, watchAccumulator(watchCompoundSelector)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second).
+			gomega.Eventually(ctx, watchAccumulator(v2hostPortWatch)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second).
+				Should(gomega.Equal(addedEvents(sets.New(crNames[0]))))
+
+			ginkgo.By("Waiting for watch events to contain v1 custom resources for field selector hostPort=host1:80")
+			gomega.Eventually(ctx, watchAccumulator(v1hostPortWatch)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second).
 				Should(gomega.Equal(addedEvents(sets.New(crNames[0]))))
 
 			ginkgo.By("Deleting one custom resources to ensure that deletions are observed")
 			var gracePeriod int64 = 0
-			err = crClient.Namespace(f.Namespace.Name).Delete(ctx, crNames[0], metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod})
+			err = v2Client.Namespace(f.Namespace.Name).DeleteCollection(ctx, metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod}, metav1.ListOptions{FieldSelector: "host=host1,port=80"})
 			framework.ExpectNoError(err, "deleting custom resource")
 
 			ginkgo.By("Updating one custom resources to ensure that deletions are observed")
-			u, err := crClient.Namespace(f.Namespace.Name).Get(ctx, crNames[1], metav1.GetOptions{})
+			u, err := v2Client.Namespace(f.Namespace.Name).Get(ctx, crNames[1], metav1.GetOptions{})
 			framework.ExpectNoError(err, "getting custom resource")
-			u.Object["spec"].(map[string]any)["color"] = "green"
+			u.Object["host"] = "host2"
-			_, err = crClient.Namespace(f.Namespace.Name).Update(ctx, u, metav1.UpdateOptions{})
+			_, err = v2Client.Namespace(f.Namespace.Name).Update(ctx, u, metav1.UpdateOptions{})
 			framework.ExpectNoError(err, "updating custom resource")
 
-			ginkgo.By("Listing custom resources after updates and deletes for field selector spec.color=blue")
+			ginkgo.By("Listing v2 custom resources after updates and deletes for field selector host=host1")
-			list, err = crClient.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "spec.color=blue"})
+			list, err = v2Client.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "host=host1"})
 			framework.ExpectNoError(err, "listing custom resources with field selector")
 			gomega.Expect(listResultToNames(list)).To(gomega.Equal(sets.New[string]()))
 
-			ginkgo.By("Listing custom resources after updates and deletes for field selector spec.color=blue,spec.quantity=2")
+			ginkgo.By("Listing v2 custom resources after updates and deletes for field selector host=host1,port=80")
-			list, err = crClient.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "spec.color=blue,spec.quantity=2"})
+			list, err = v2Client.Namespace(f.Namespace.Name).List(ctx, metav1.ListOptions{FieldSelector: "host=host1,port=80"})
 			framework.ExpectNoError(err, "listing custom resources with field selector")
 			gomega.Expect(listResultToNames(list)).To(gomega.Equal(sets.New[string]()))
 
-			ginkgo.By("Waiting for watch events after updates and deletes for field selector spec.color=blue")
+			ginkgo.By("Waiting for v2 watch events after updates and deletes for field selector host=host1")
-			gomega.Eventually(ctx, watchAccumulator(watchSimpleSelector)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second).
+			gomega.Eventually(ctx, watchAccumulator(hostWatch)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second).
 				Should(gomega.Equal(deletedEvents(sets.New(crNames[0], crNames[1]))))
 
-			ginkgo.By("Waiting for watch events after updates and deletes for field selector spec.color=blue,spec.quantity=2")
+			ginkgo.By("Waiting for v2 watch events after updates and deletes for field selector host=host1,port=80")
-			gomega.Eventually(ctx, watchAccumulator(watchCompoundSelector)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second).
+			gomega.Eventually(ctx, watchAccumulator(v2hostPortWatch)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second).
+				Should(gomega.Equal(deletedEvents(sets.New(crNames[0]))))
+
+			ginkgo.By("Waiting for v1 watch events after updates and deletes for field selector hostPort=host1:80")
+			gomega.Eventually(ctx, watchAccumulator(v1hostPortWatch)).WithPolling(5 * time.Millisecond).WithTimeout(30 * time.Second).
 				Should(gomega.Equal(deletedEvents(sets.New(crNames[0]))))
 		})
+
 	})
 })
 
-func unmarshalSchema(schemaJSON []byte) *apiextensionsv1.JSONSchemaProps {
-	var c apiextensionsv1.JSONSchemaProps
-	err := json.Unmarshal(schemaJSON, &c)
-	framework.ExpectNoError(err, "unmarshalling OpenAPIv3 schema")
-	return &c
-}
-
 type accumulatedEvents struct {
 	added, deleted sets.Set[string]
 }