Integration test (please review with -w to ignore

whitespace changes)

staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/resources.go

@@ -370,13 +370,13 @@ func DeleteCustomResourceDefinition(crd *apiextensionsv1beta1.CustomResourceDefi
 	return nil
 }
 
-// CreateNewScaleClient returns a scale client.
-func CreateNewScaleClient(crd *apiextensionsv1beta1.CustomResourceDefinition, config *rest.Config) (scale.ScalesGetter, error) {
+// CreateNewVersionedScaleClient returns a scale client.
+func CreateNewVersionedScaleClient(crd *apiextensionsv1beta1.CustomResourceDefinition, config *rest.Config, version string) (scale.ScalesGetter, error) {
 	discoveryClient, err := discovery.NewDiscoveryClientForConfig(config)
 	if err != nil {
 		return nil, err
 	}
-	groupResource, err := discoveryClient.ServerResourcesForGroupVersion(crd.Spec.Group + "/" + crd.Spec.Version)
+	groupResource, err := discoveryClient.ServerResourcesForGroupVersion(crd.Spec.Group + "/" + version)
 	if err != nil {
 		return nil, err
 	}
@@ -386,12 +386,12 @@ func CreateNewScaleClient(crd *apiextensionsv1beta1.CustomResourceDefinition, co
 			Group: metav1.APIGroup{
 				Name: crd.Spec.Group,
 				Versions: []metav1.GroupVersionForDiscovery{
-					{Version: crd.Spec.Version},
+					{Version: version},
 				},
-				PreferredVersion: metav1.GroupVersionForDiscovery{Version: crd.Spec.Version},
+				PreferredVersion: metav1.GroupVersionForDiscovery{Version: version},
 			},
 			VersionedResources: map[string][]metav1.APIResource{
-				crd.Spec.Version: groupResource.APIResources,
+				version: groupResource.APIResources,
 			},
 		},
 	}

staging/src/k8s.io/apiextensions-apiserver/test/integration/helpers.go

@@ -30,6 +30,8 @@ import (
 	"k8s.io/client-go/dynamic"
 )
 
+var swaggerMetadataDescriptions = metav1.ObjectMeta{}.SwaggerDoc()
+
 func instantiateCustomResource(t *testing.T, instanceToCreate *unstructured.Unstructured, client dynamic.ResourceInterface, definition *apiextensionsv1beta1.CustomResourceDefinition) (*unstructured.Unstructured, error) {
 	return instantiateVersionedCustomResource(t, instanceToCreate, client, definition, definition.Spec.Versions[0].Name)
 }
@@ -92,3 +94,97 @@ func updateCustomResourceDefinitionWithRetry(client clientset.Interface, name st
 	}
 	return nil, fmt.Errorf("too many retries after conflicts updating CustomResourceDefinition %q", name)
 }
+
+// getSchemaForVersion returns the validation schema for given version in given CRD.
+func getSchemaForVersion(crd *apiextensionsv1beta1.CustomResourceDefinition, version string) (*apiextensionsv1beta1.CustomResourceValidation, error) {
+	if !hasPerVersionSchema(crd.Spec.Versions) {
+		return crd.Spec.Validation, nil
+	}
+	if crd.Spec.Validation != nil {
+		return nil, fmt.Errorf("malformed CustomResourceDefinition %s version %s: top-level and per-version schemas must be mutual exclusive", crd.Name, version)
+	}
+	for _, v := range crd.Spec.Versions {
+		if version == v.Name {
+			return v.Schema, nil
+		}
+	}
+	return nil, fmt.Errorf("version %s not found in CustomResourceDefinition: %v", version, crd.Name)
+}
+
+// getSubresourcesForVersion returns the subresources for given version in given CRD.
+func getSubresourcesForVersion(crd *apiextensionsv1beta1.CustomResourceDefinition, version string) (*apiextensionsv1beta1.CustomResourceSubresources, error) {
+	if !hasPerVersionSubresources(crd.Spec.Versions) {
+		return crd.Spec.Subresources, nil
+	}
+	if crd.Spec.Subresources != nil {
+		return nil, fmt.Errorf("malformed CustomResourceDefinition %s version %s: top-level and per-version subresources must be mutual exclusive", crd.Name, version)
+	}
+	for _, v := range crd.Spec.Versions {
+		if version == v.Name {
+			return v.Subresources, nil
+		}
+	}
+	return nil, fmt.Errorf("version %s not found in CustomResourceDefinition: %v", version, crd.Name)
+}
+
+// getColumnsForVersion returns the columns for given version in given CRD.
+// NOTE: the newly logically-defaulted columns is not pointing to the original CRD object.
+// One cannot mutate the original CRD columns using the logically-defaulted columns. Please iterate through
+// the original CRD object instead.
+func getColumnsForVersion(crd *apiextensionsv1beta1.CustomResourceDefinition, version string) ([]apiextensionsv1beta1.CustomResourceColumnDefinition, error) {
+	if !hasPerVersionColumns(crd.Spec.Versions) {
+		return serveDefaultColumnsIfEmpty(crd.Spec.AdditionalPrinterColumns), nil
+	}
+	if len(crd.Spec.AdditionalPrinterColumns) > 0 {
+		return nil, fmt.Errorf("malformed CustomResourceDefinition %s version %s: top-level and per-version additionalPrinterColumns must be mutual exclusive", crd.Name, version)
+	}
+	for _, v := range crd.Spec.Versions {
+		if version == v.Name {
+			return serveDefaultColumnsIfEmpty(v.AdditionalPrinterColumns), nil
+		}
+	}
+	return nil, fmt.Errorf("version %s not found in CustomResourceDefinition: %v", version, crd.Name)
+}
+
+// serveDefaultColumnsIfEmpty applies logically defaulting to columns, if the input columns is empty.
+// NOTE: in this way, the newly logically-defaulted columns is not pointing to the original CRD object.
+// One cannot mutate the original CRD columns using the logically-defaulted columns. Please iterate through
+// the original CRD object instead.
+func serveDefaultColumnsIfEmpty(columns []apiextensionsv1beta1.CustomResourceColumnDefinition) []apiextensionsv1beta1.CustomResourceColumnDefinition {
+	if len(columns) > 0 {
+		return columns
+	}
+	return []apiextensionsv1beta1.CustomResourceColumnDefinition{
+		{Name: "Age", Type: "date", Description: swaggerMetadataDescriptions["creationTimestamp"], JSONPath: ".metadata.creationTimestamp"},
+	}
+}
+
+// hasPerVersionSchema returns true if a CRD uses per-version schema.
+func hasPerVersionSchema(versions []apiextensionsv1beta1.CustomResourceDefinitionVersion) bool {
+	for _, v := range versions {
+		if v.Schema != nil {
+			return true
+		}
+	}
+	return false
+}
+
+// hasPerVersionSubresources returns true if a CRD uses per-version subresources.
+func hasPerVersionSubresources(versions []apiextensionsv1beta1.CustomResourceDefinitionVersion) bool {
+	for _, v := range versions {
+		if v.Subresources != nil {
+			return true
+		}
+	}
+	return false
+}
+
+// hasPerVersionColumns returns true if a CRD uses per-version columns.
+func hasPerVersionColumns(versions []apiextensionsv1beta1.CustomResourceDefinitionVersion) bool {
+	for _, v := range versions {
+		if len(v.AdditionalPrinterColumns) > 0 {
+			return true
+		}
+	}
+	return false
+}

staging/src/k8s.io/apiextensions-apiserver/test/integration/subresources_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package integration
 
 import (
+	"fmt"
 	"math"
 	"reflect"
 	"sort"
@@ -29,17 +30,23 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
 	"k8s.io/client-go/dynamic"
 
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features"
 	"k8s.io/apiextensions-apiserver/test/integration/fixtures"
 )
 
 var labelSelectorPath = ".status.labelSelector"
+var anotherLabelSelectorPath = ".status.anotherLabelSelector"
 
-func NewNoxuSubresourcesCRD(scope apiextensionsv1beta1.ResourceScope) *apiextensionsv1beta1.CustomResourceDefinition {
-	return &apiextensionsv1beta1.CustomResourceDefinition{
+func NewNoxuSubresourcesCRDs(scope apiextensionsv1beta1.ResourceScope) []*apiextensionsv1beta1.CustomResourceDefinition {
+	return []*apiextensionsv1beta1.CustomResourceDefinition{
+		// CRD that uses top-level subresources
+		{
 			ObjectMeta: metav1.ObjectMeta{Name: "noxus.mygroup.example.com"},
 			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
 				Group:   "mygroup.example.com",
@@ -51,11 +58,19 @@ func NewNoxuSubresourcesCRD(scope apiextensionsv1beta1.ResourceScope) *apiextens
 					ShortNames: []string{"foo", "bar", "abc", "def"},
 					ListKind:   "NoxuItemList",
 				},
-			Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{
-				{Name: "v1beta1", Served: true, Storage: false},
-				{Name: "v1", Served: true, Storage: true},
-			},
 				Scope: scope,
+				Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{
+					{
+						Name:    "v1beta1",
+						Served:  true,
+						Storage: true,
+					},
+					{
+						Name:    "v1",
+						Served:  true,
+						Storage: false,
+					},
+				},
 				Subresources: &apiextensionsv1beta1.CustomResourceSubresources{
 					Status: &apiextensionsv1beta1.CustomResourceSubresourceStatus{},
 					Scale: &apiextensionsv1beta1.CustomResourceSubresourceScale{
@@ -65,13 +80,58 @@ func NewNoxuSubresourcesCRD(scope apiextensionsv1beta1.ResourceScope) *apiextens
 					},
 				},
 			},
+		},
+		// CRD that uses per-version subresources
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "noxus.mygroup.example.com"},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group:   "mygroup.example.com",
+				Version: "v1beta1",
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Plural:     "noxus",
+					Singular:   "nonenglishnoxu",
+					Kind:       "WishIHadChosenNoxu",
+					ShortNames: []string{"foo", "bar", "abc", "def"},
+					ListKind:   "NoxuItemList",
+				},
+				Scope: scope,
+				Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{
+					{
+						Name:    "v1beta1",
+						Served:  true,
+						Storage: true,
+						Subresources: &apiextensionsv1beta1.CustomResourceSubresources{
+							Status: &apiextensionsv1beta1.CustomResourceSubresourceStatus{},
+							Scale: &apiextensionsv1beta1.CustomResourceSubresourceScale{
+								SpecReplicasPath:   ".spec.replicas",
+								StatusReplicasPath: ".status.replicas",
+								LabelSelectorPath:  &labelSelectorPath,
+							},
+						},
+					},
+					{
+						Name:    "v1",
+						Served:  true,
+						Storage: false,
+						Subresources: &apiextensionsv1beta1.CustomResourceSubresources{
+							Status: &apiextensionsv1beta1.CustomResourceSubresourceStatus{},
+							Scale: &apiextensionsv1beta1.CustomResourceSubresourceScale{
+								SpecReplicasPath:   ".spec.replicas",
+								StatusReplicasPath: ".status.replicas",
+								LabelSelectorPath:  &anotherLabelSelectorPath,
+							},
+						},
+					},
+				},
+			},
+		},
 	}
 }
 
-func NewNoxuSubresourceInstance(namespace, name string) *unstructured.Unstructured {
+func NewNoxuSubresourceInstance(namespace, name, version string) *unstructured.Unstructured {
 	return &unstructured.Unstructured{
 		Object: map[string]interface{}{
-			"apiVersion": "mygroup.example.com/v1beta1",
+			"apiVersion": fmt.Sprintf("mygroup.example.com/%s", version),
 			"kind":       "WishIHadChosenNoxu",
 			"metadata": map[string]interface{}{
 				"namespace": namespace,
@@ -89,30 +149,31 @@ func NewNoxuSubresourceInstance(namespace, name string) *unstructured.Unstructur
 }
 
 func TestStatusSubresource(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)()
 	tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t)
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer tearDown()
 
-	noxuDefinition := NewNoxuSubresourcesCRD(apiextensionsv1beta1.NamespaceScoped)
+	noxuDefinitions := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.NamespaceScoped)
+	for _, noxuDefinition := range noxuDefinitions {
 		noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
 		if err != nil {
 			t.Fatal(err)
 		}
 
 		ns := "not-the-default"
-	noxuResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition)
-	_, err = instantiateCustomResource(t, NewNoxuSubresourceInstance(ns, "foo"), noxuResourceClient, noxuDefinition)
+		for _, v := range noxuDefinition.Spec.Versions {
+			noxuResourceClient := newNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name)
+			_, err = instantiateVersionedCustomResource(t, NewNoxuSubresourceInstance(ns, "foo", v.Name), noxuResourceClient, noxuDefinition, v.Name)
 			if err != nil {
 				t.Fatalf("unable to create noxu instance: %v", err)
 			}
-
 			gottenNoxuInstance, err := noxuResourceClient.Get("foo", metav1.GetOptions{})
 			if err != nil {
 				t.Fatal(err)
 			}
-
 			// status should not be set after creation
 			if val, ok := gottenNoxuInstance.Object["status"]; ok {
 				t.Fatalf("status should not be set after creation, got %v", val)
@@ -192,9 +253,16 @@ func TestStatusSubresource(t *testing.T) {
 			if statusNum != int64(20) {
 				t.Fatalf(".status.num: expected: %v, got: %v", int64(20), statusNum)
 			}
+			noxuResourceClient.Delete("foo", &metav1.DeleteOptions{})
+		}
+		if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil {
+			t.Fatal(err)
+		}
+	}
 }
 
 func TestScaleSubresource(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)()
 	groupResource := schema.GroupResource{
 		Group:    "mygroup.example.com",
 		Resource: "noxus",
@@ -215,29 +283,37 @@ func TestScaleSubresource(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	noxuDefinition := NewNoxuSubresourcesCRD(apiextensionsv1beta1.NamespaceScoped)
+	noxuDefinitions := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.NamespaceScoped)
+	for _, noxuDefinition := range noxuDefinitions {
+		for _, v := range noxuDefinition.Spec.Versions {
+			// Start with a new CRD, so that the object doesn't have resourceVersion
+			noxuDefinition := noxuDefinition.DeepCopy()
 
+			subresources, err := getSubresourcesForVersion(noxuDefinition, v.Name)
+			if err != nil {
+				t.Fatal(err)
+			}
 			// set invalid json path for specReplicasPath
-	noxuDefinition.Spec.Subresources.Scale.SpecReplicasPath = "foo,bar"
+			subresources.Scale.SpecReplicasPath = "foo,bar"
 			_, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
 			if err == nil {
 				t.Fatalf("unexpected non-error: specReplicasPath should be a valid json path under .spec")
 			}
 
-	noxuDefinition.Spec.Subresources.Scale.SpecReplicasPath = ".spec.replicas"
+			subresources.Scale.SpecReplicasPath = ".spec.replicas"
 			noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
 			if err != nil {
 				t.Fatal(err)
 			}
 
 			ns := "not-the-default"
-	noxuResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition)
-	_, err = instantiateCustomResource(t, NewNoxuSubresourceInstance(ns, "foo"), noxuResourceClient, noxuDefinition)
+			noxuResourceClient := newNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name)
+			_, err = instantiateVersionedCustomResource(t, NewNoxuSubresourceInstance(ns, "foo", v.Name), noxuResourceClient, noxuDefinition, v.Name)
 			if err != nil {
 				t.Fatalf("unable to create noxu instance: %v", err)
 			}
 
-	scaleClient, err := fixtures.CreateNewScaleClient(noxuDefinition, config)
+			scaleClient, err := fixtures.CreateNewVersionedScaleClient(noxuDefinition, config, v.Name)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -247,7 +323,7 @@ func TestScaleSubresource(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-	err = unstructured.SetNestedField(gottenNoxuInstance.Object, "bar", "status", "labelSelector")
+			err = unstructured.SetNestedField(gottenNoxuInstance.Object, "bar", strings.Split((*subresources.Scale.LabelSelectorPath)[1:], ".")...)
 			if err != nil {
 				t.Fatalf("unexpected error: %v", err)
 			}
@@ -269,7 +345,7 @@ func TestScaleSubresource(t *testing.T) {
 			}
 
 			// check self link
-	expectedSelfLink := "/apis/mygroup.example.com/v1beta1/namespaces/not-the-default/noxus/foo/scale"
+			expectedSelfLink := fmt.Sprintf("/apis/mygroup.example.com/%s/namespaces/not-the-default/noxus/foo/scale", v.Name)
 			if gottenScale.GetSelfLink() != expectedSelfLink {
 				t.Fatalf("Scale.Metadata.SelfLink: expected: %v, got: %v", expectedSelfLink, gottenScale.GetSelfLink())
 			}
@@ -301,9 +377,9 @@ func TestScaleSubresource(t *testing.T) {
 			if specReplicas != 5 {
 				t.Fatalf("replicas: expected: %v, got: %v", 5, specReplicas)
 			}
-	statusLabelSelector, found, err := unstructured.NestedString(updatedNoxuInstance.Object, "status", "labelSelector")
+			statusLabelSelector, found, err := unstructured.NestedString(updatedNoxuInstance.Object, strings.Split((*subresources.Scale.LabelSelectorPath)[1:], ".")...)
 			if !found || err != nil {
-		t.Fatalf("unable to get .status.labelSelector")
+				t.Fatalf("unable to get %s", *subresources.Scale.LabelSelectorPath)
 			}
 			if statusLabelSelector != "bar" {
 				t.Fatalf("scale should not update status: expected %v, got: %v", "bar", statusLabelSelector)
@@ -323,9 +399,16 @@ func TestScaleSubresource(t *testing.T) {
 			if err == nil {
 				t.Fatalf("unexpected non-error: .spec.replicas should be less than 2147483647")
 			}
+			noxuResourceClient.Delete("foo", &metav1.DeleteOptions{})
+			if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
 }
 
 func TestValidationSchemaWithStatus(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)()
 	tearDown, config, _, err := fixtures.StartDefaultServer(t)
 	if err != nil {
 		t.Fatal(err)
@@ -342,7 +425,7 @@ func TestValidationSchemaWithStatus(t *testing.T) {
 	}
 
 	// fields other than properties in root schema are not allowed
-	noxuDefinition := newNoxuValidationCRD(apiextensionsv1beta1.NamespaceScoped)
+	noxuDefinition := newNoxuValidationCRDs(apiextensionsv1beta1.NamespaceScoped)[0]
 	noxuDefinition.Spec.Subresources = &apiextensionsv1beta1.CustomResourceSubresources{
 		Status: &apiextensionsv1beta1.CustomResourceSubresourceStatus{},
 	}
@@ -373,6 +456,7 @@ func TestValidationSchemaWithStatus(t *testing.T) {
 }
 
 func TestValidateOnlyStatus(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)()
 	tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t)
 	if err != nil {
 		t.Fatal(err)
@@ -407,26 +491,39 @@ func TestValidateOnlyStatus(t *testing.T) {
 		},
 	}
 
-	noxuDefinition := NewNoxuSubresourcesCRD(apiextensionsv1beta1.NamespaceScoped)
+	noxuDefinitions := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.NamespaceScoped)
+	for i, noxuDefinition := range noxuDefinitions {
+		if i == 0 {
 			noxuDefinition.Spec.Validation = &apiextensionsv1beta1.CustomResourceValidation{
 				OpenAPIV3Schema: schema,
 			}
+		} else {
+			noxuDefinition.Spec.Versions[0].Schema = &apiextensionsv1beta1.CustomResourceValidation{
+				OpenAPIV3Schema: schema,
+			}
+			schemaWithDescription := schema.DeepCopy()
+			schemaWithDescription.Description = "test"
+			noxuDefinition.Spec.Versions[1].Schema = &apiextensionsv1beta1.CustomResourceValidation{
+				OpenAPIV3Schema: schemaWithDescription,
+			}
+		}
 
 		noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
 		if err != nil {
 			t.Fatal(err)
 		}
 		ns := "not-the-default"
-	noxuResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition)
+		for _, v := range noxuDefinition.Spec.Versions {
+			noxuResourceClient := newNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name)
 
 			// set .spec.num = 10 and .status.num = 10
-	noxuInstance := NewNoxuSubresourceInstance(ns, "foo")
+			noxuInstance := NewNoxuSubresourceInstance(ns, "foo", v.Name)
 			err = unstructured.SetNestedField(noxuInstance.Object, int64(10), "status", "num")
 			if err != nil {
 				t.Fatalf("unexpected error: %v", err)
 			}
 
-	createdNoxuInstance, err := instantiateCustomResource(t, noxuInstance, noxuResourceClient, noxuDefinition)
+			createdNoxuInstance, err := instantiateVersionedCustomResource(t, noxuInstance, noxuResourceClient, noxuDefinition, v.Name)
 			if err != nil {
 				t.Fatalf("unable to create noxu instance: %v", err)
 			}
@@ -457,9 +554,16 @@ func TestValidateOnlyStatus(t *testing.T) {
 			if !strings.Contains(statusError.Error(), "Invalid value") {
 				t.Fatalf("expected 'Invalid value' in error, got: %v", err)
 			}
+			noxuResourceClient.Delete("foo", &metav1.DeleteOptions{})
+		}
+		if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil {
+			t.Fatal(err)
+		}
+	}
 }
 
 func TestSubresourcesDiscovery(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)()
 	tearDown, config, _, err := fixtures.StartDefaultServer(t)
 	if err != nil {
 		t.Fatal(err)
@@ -475,14 +579,16 @@ func TestSubresourcesDiscovery(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	noxuDefinition := NewNoxuSubresourcesCRD(apiextensionsv1beta1.NamespaceScoped)
+	noxuDefinitions := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.NamespaceScoped)
+	for _, noxuDefinition := range noxuDefinitions {
 		noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
 		if err != nil {
 			t.Fatal(err)
 		}
 
+		for _, v := range noxuDefinition.Spec.Versions {
 			group := "mygroup.example.com"
-	version := "v1beta1"
+			version := v.Name
 
 			resources, err := apiExtensionClient.Discovery().ServerResourcesForGroupVersion(group + "/" + version)
 			if err != nil {
@@ -541,24 +647,32 @@ func TestSubresourcesDiscovery(t *testing.T) {
 			if !reflect.DeepEqual([]string(scale.Verbs), expectedVerbs) {
 				t.Fatalf("incorrect scale via discovery: expected: %v, got: %v", expectedVerbs, scale.Verbs)
 			}
+		}
+		if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil {
+			t.Fatal(err)
+		}
+	}
 }
 
 func TestGeneration(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)()
 	tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t)
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer tearDown()
 
-	noxuDefinition := NewNoxuSubresourcesCRD(apiextensionsv1beta1.NamespaceScoped)
+	noxuDefinitions := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.NamespaceScoped)
+	for _, noxuDefinition := range noxuDefinitions {
 		noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
 		if err != nil {
 			t.Fatal(err)
 		}
 
 		ns := "not-the-default"
-	noxuResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition)
-	_, err = instantiateCustomResource(t, NewNoxuSubresourceInstance(ns, "foo"), noxuResourceClient, noxuDefinition)
+		for _, v := range noxuDefinition.Spec.Versions {
+			noxuResourceClient := newNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name)
+			_, err = instantiateVersionedCustomResource(t, NewNoxuSubresourceInstance(ns, "foo", v.Name), noxuResourceClient, noxuDefinition, v.Name)
 			if err != nil {
 				t.Fatalf("unable to create noxu instance: %v", err)
 			}
@@ -606,9 +720,16 @@ func TestGeneration(t *testing.T) {
 			if updatedInstance.GetGeneration() != 2 {
 				t.Fatalf("updating spec should increment .metadata.generation: expected: %v, got: %v", 2, updatedStatusInstance.GetGeneration())
 			}
+			noxuResourceClient.Delete("foo", &metav1.DeleteOptions{})
+		}
+		if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil {
+			t.Fatal(err)
+		}
+	}
 }
 
 func TestSubresourcePatch(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)()
 	groupResource := schema.GroupResource{
 		Group:    "mygroup.example.com",
 		Resource: "noxus",
@@ -629,22 +750,24 @@ func TestSubresourcePatch(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	noxuDefinition := NewNoxuSubresourcesCRD(apiextensionsv1beta1.NamespaceScoped)
+	noxuDefinitions := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.NamespaceScoped)
+	for _, noxuDefinition := range noxuDefinitions {
 		noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
 		if err != nil {
 			t.Fatal(err)
 		}
 
 		ns := "not-the-default"
-	noxuResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition)
+		for _, v := range noxuDefinition.Spec.Versions {
+			noxuResourceClient := newNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name)
 
 			t.Logf("Creating foo")
-	_, err = instantiateCustomResource(t, NewNoxuSubresourceInstance(ns, "foo"), noxuResourceClient, noxuDefinition)
+			_, err = instantiateVersionedCustomResource(t, NewNoxuSubresourceInstance(ns, "foo", v.Name), noxuResourceClient, noxuDefinition, v.Name)
 			if err != nil {
 				t.Fatalf("unable to create noxu instance: %v", err)
 			}
 
-	scaleClient, err := fixtures.CreateNewScaleClient(noxuDefinition, config)
+			scaleClient, err := fixtures.CreateNewVersionedScaleClient(noxuDefinition, config, v.Name)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -694,6 +817,7 @@ func TestSubresourcePatch(t *testing.T) {
 			if err != nil {
 				t.Fatalf("unexpected error: %v", err)
 			}
+
 			// an empty patch is a no-op patch. make sure it does not increment resourceVersion
 			expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 999, "status", "num")
 			expectInt64(t, patchedNoxuInstance.UnstructuredContent(), 10, "spec", "num")
@@ -771,4 +895,10 @@ func TestSubresourcePatch(t *testing.T) {
 			if err == nil {
 				t.Fatalf("unexpected non-error: strategic merge patch is not supported for custom resources")
 			}
+			noxuResourceClient.Delete("foo", &metav1.DeleteOptions{})
+		}
+		if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil {
+			t.Fatal(err)
+		}
+	}
 }

staging/src/k8s.io/apiextensions-apiserver/test/integration/table_test.go

@@ -28,10 +28,14 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/types"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/rest"
 
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features"
 	"k8s.io/apiextensions-apiserver/test/integration/fixtures"
 )
 
@@ -48,6 +52,11 @@ func newTableCRD() *apiextensionsv1beta1.CustomResourceDefinition {
 				ListKind: "TablemList",
 			},
 			Scope: apiextensionsv1beta1.ClusterScoped,
+			Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{
+				{
+					Name:    "v1beta1",
+					Served:  true,
+					Storage: false,
 					AdditionalPrinterColumns: []apiextensionsv1beta1.CustomResourceColumnDefinition{
 						{Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp"},
 						{Name: "Alpha", Type: "string", JSONPath: ".spec.alpha"},
@@ -56,13 +65,28 @@ func newTableCRD() *apiextensionsv1beta1.CustomResourceDefinition {
 						{Name: "Epsilon", Type: "string", Description: "an array of integers as string", JSONPath: ".spec.epsilon"},
 					},
 				},
+				{
+					Name:    "v1",
+					Served:  true,
+					Storage: true,
+					AdditionalPrinterColumns: []apiextensionsv1beta1.CustomResourceColumnDefinition{
+						{Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp"},
+						{Name: "Alpha", Type: "string", JSONPath: ".spec.alpha"},
+						{Name: "Beta", Type: "integer", Description: "the beta field", Format: "int64", Priority: 42, JSONPath: ".spec.beta"},
+						{Name: "Gamma", Type: "integer", Description: "a column with wrongly typed values", JSONPath: ".spec.gamma"},
+						{Name: "Epsilon", Type: "string", Description: "an array of integers as string", JSONPath: ".spec.epsilon"},
+						{Name: "Zeta", Type: "integer", Description: "the zeta field", Format: "int64", Priority: 42, JSONPath: ".spec.zeta"},
+					},
+				},
+			},
+		},
 	}
 }
 
 func newTableInstance(name string) *unstructured.Unstructured {
 	return &unstructured.Unstructured{
 		Object: map[string]interface{}{
-			"apiVersion": "mygroup.example.com/v1beta1",
+			"apiVersion": "mygroup.example.com/v1",
 			"kind":       "Table",
 			"metadata": map[string]interface{}{
 				"name": name,
@@ -73,12 +97,14 @@ func newTableInstance(name string) *unstructured.Unstructured {
 				"gamma":   "bar",
 				"delta":   "hello",
 				"epsilon": []int64{1, 2, 3},
+				"zeta":    5,
 			},
 		},
 	}
 }
 
 func TestTableGet(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)()
 	tearDown, config, _, err := fixtures.StartDefaultServer(t)
 	if err != nil {
 		t.Fatal(err)
@@ -107,14 +133,15 @@ func TestTableGet(t *testing.T) {
 	}
 	t.Logf("table crd created: %#v", crd)
 
-	crClient := newNamespacedCustomResourceClient("", dynamicClient, crd)
+	crClient := newNamespacedCustomResourceVersionedClient("", dynamicClient, crd, "v1")
 	foo, err := crClient.Create(newTableInstance("foo"), metav1.CreateOptions{})
 	if err != nil {
 		t.Fatalf("unable to create noxu instance: %v", err)
 	}
 	t.Logf("foo created: %#v", foo.UnstructuredContent())
 
-	gv := schema.GroupVersion{Group: crd.Spec.Group, Version: crd.Spec.Version}
+	for i, v := range crd.Spec.Versions {
+		gv := schema.GroupVersion{Group: crd.Spec.Group, Version: v.Name}
 		gvk := gv.WithKind(crd.Spec.Names.Kind)
 
 		scheme := runtime.NewScheme()
@@ -149,7 +176,12 @@ func TestTableGet(t *testing.T) {
 		}
 		t.Logf("%v table list: %#v", gvk, tbl)
 
-	if got, expected := len(tbl.ColumnDefinitions), 6; got != expected {
+		columns, err := getColumnsForVersion(crd, v.Name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		expectColumnNum := len(columns) + 1
+		if got, expected := len(tbl.ColumnDefinitions), expectColumnNum; got != expected {
 			t.Errorf("expected %d headers, got %d", expected, got)
 		} else {
 			age := metav1beta1.TableColumnDefinition{Name: "Age", Type: "date", Format: "", Description: "Custom resource definition column (in JSONPath format): .metadata.creationTimestamp", Priority: 0}
@@ -176,10 +208,18 @@ func TestTableGet(t *testing.T) {
 			if got, expected := tbl.ColumnDefinitions[5], epsilon; got != expected {
 				t.Errorf("expected column definition %#v, got %#v", expected, got)
 			}
+
+			// Validate extra column for v1
+			if i == 1 {
+				zeta := metav1beta1.TableColumnDefinition{Name: "Zeta", Type: "integer", Format: "int64", Description: "the zeta field", Priority: 42}
+				if got, expected := tbl.ColumnDefinitions[6], zeta; got != expected {
+					t.Errorf("expected column definition %#v, got %#v", expected, got)
+				}
+			}
 		}
 		if got, expected := len(tbl.Rows), 1; got != expected {
 			t.Errorf("expected %d rows, got %d", expected, got)
-	} else if got, expected := len(tbl.Rows[0].Cells), 6; got != expected {
+		} else if got, expected := len(tbl.Rows[0].Cells), expectColumnNum; got != expected {
 			t.Errorf("expected %d cells, got %d", expected, got)
 		} else {
 			if got, expected := tbl.Rows[0].Cells[0], "foo"; got != expected {
@@ -207,7 +247,105 @@ func TestTableGet(t *testing.T) {
 			if got, expected := tbl.Rows[0].Cells[5], "[1 2 3]"; got != expected {
 				t.Errorf("expected cell[5] to equal %q, got %q", expected, got)
 			}
+			// Validate extra column for v1
+			if i == 1 {
+				if got, expected := tbl.Rows[0].Cells[6], int64(5); got != expected {
+					t.Errorf("expected cell[6] to equal %q, got %q", expected, got)
 				}
+			}
+		}
+	}
+}
+
+// TestColumnsPatch tests the case that a CRD was created with no top-level or
+// per-version columns. One should be able to PATCH the CRD setting per-version columns.
+func TestColumnsPatch(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)()
+	tearDown, config, _, err := fixtures.StartDefaultServer(t)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tearDown()
+
+	apiExtensionClient, err := clientset.NewForConfig(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dynamicClient, err := dynamic.NewForConfig(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// CRD with no top-level and per-version columns should be created successfully
+	crd := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.NamespaceScoped)[0]
+	crd, err = fixtures.CreateNewCustomResourceDefinition(crd, apiExtensionClient, dynamicClient)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// One should be able to patch the CRD to use per-version columns. The top-level columns
+	// should not be defaulted during creation, and apiserver should not return validation
+	// error about top-level and per-version columns being mutual exclusive.
+	patch := []byte(`{"spec":{"versions":[{"name":"v1beta1","served":true,"storage":true,"additionalPrinterColumns":[{"name":"Age","type":"date","JSONPath":".metadata.creationTimestamp"}]},{"name":"v1","served":true,"storage":false,"additionalPrinterColumns":[{"name":"Age2","type":"date","JSONPath":".metadata.creationTimestamp"}]}]}}`)
+
+	_, err = apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(crd.Name, types.MergePatchType, patch)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	crd, err = apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("columns crd patched: %#v", crd)
+}
+
+// TestPatchCleanTopLevelColumns tests the case that a CRD was created with top-level columns.
+// One should be able to PATCH the CRD cleaning the top-level columns and setting per-version
+// columns.
+func TestPatchCleanTopLevelColumns(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)()
+	tearDown, config, _, err := fixtures.StartDefaultServer(t)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tearDown()
+
+	apiExtensionClient, err := clientset.NewForConfig(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dynamicClient, err := dynamic.NewForConfig(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	crd := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.NamespaceScoped)[0]
+	crd.Spec.AdditionalPrinterColumns = []apiextensionsv1beta1.CustomResourceColumnDefinition{
+		{Name: "Age", Type: "date", Description: swaggerMetadataDescriptions["creationTimestamp"], JSONPath: ".metadata.creationTimestamp"},
+	}
+	crd, err = fixtures.CreateNewCustomResourceDefinition(crd, apiExtensionClient, dynamicClient)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("columns crd created: %#v", crd)
+
+	// One should be able to patch the CRD to use per-version columns by cleaning
+	// the top-level columns.
+	patch := []byte(`{"spec":{"additionalPrinterColumns":null,"versions":[{"name":"v1beta1","served":true,"storage":true,"additionalPrinterColumns":[{"name":"Age","type":"date","JSONPath":".metadata.creationTimestamp"}]},{"name":"v1","served":true,"storage":false,"additionalPrinterColumns":[{"name":"Age2","type":"date","JSONPath":".metadata.creationTimestamp"}]}]}}`)
+
+	_, err = apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(crd.Name, types.MergePatchType, patch)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	crd, err = apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("columns crd patched: %#v", crd)
 }
 
 func abs(x float64) float64 {

staging/src/k8s.io/apiextensions-apiserver/test/integration/validation_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package integration
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 	"time"
@@ -25,8 +26,11 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/wait"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
 
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features"
 	"k8s.io/apiextensions-apiserver/test/integration/fixtures"
 )
 
@@ -84,22 +88,8 @@ func TestForProperValidationErrors(t *testing.T) {
 	}
 }
 
-func newNoxuValidationCRD(scope apiextensionsv1beta1.ResourceScope) *apiextensionsv1beta1.CustomResourceDefinition {
-	return &apiextensionsv1beta1.CustomResourceDefinition{
-		ObjectMeta: metav1.ObjectMeta{Name: "noxus.mygroup.example.com"},
-		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
-			Group:   "mygroup.example.com",
-			Version: "v1beta1",
-			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
-				Plural:     "noxus",
-				Singular:   "nonenglishnoxu",
-				Kind:       "WishIHadChosenNoxu",
-				ShortNames: []string{"foo", "bar", "abc", "def"},
-				ListKind:   "NoxuItemList",
-			},
-			Scope: apiextensionsv1beta1.NamespaceScoped,
-			Validation: &apiextensionsv1beta1.CustomResourceValidation{
-				OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
+func newNoxuValidationCRDs(scope apiextensionsv1beta1.ResourceScope) []*apiextensionsv1beta1.CustomResourceDefinition {
+	validationSchema := &apiextensionsv1beta1.JSONSchemaProps{
 		Required: []string{"alpha", "beta"},
 		AdditionalProperties: &apiextensionsv1beta1.JSONSchemaPropsOrBool{
 			Allows: true,
@@ -144,6 +134,70 @@ func newNoxuValidationCRD(scope apiextensionsv1beta1.ResourceScope) *apiextensio
 				},
 			},
 		},
+	}
+	validationSchemaWithDescription := validationSchema.DeepCopy()
+	validationSchemaWithDescription.Description = "test"
+	return []*apiextensionsv1beta1.CustomResourceDefinition{
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "noxus.mygroup.example.com"},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group:   "mygroup.example.com",
+				Version: "v1beta1",
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Plural:     "noxus",
+					Singular:   "nonenglishnoxu",
+					Kind:       "WishIHadChosenNoxu",
+					ShortNames: []string{"foo", "bar", "abc", "def"},
+					ListKind:   "NoxuItemList",
+				},
+				Scope: apiextensionsv1beta1.NamespaceScoped,
+				Validation: &apiextensionsv1beta1.CustomResourceValidation{
+					OpenAPIV3Schema: validationSchema,
+				},
+				Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{
+					{
+						Name:    "v1beta1",
+						Served:  true,
+						Storage: true,
+					},
+					{
+						Name:    "v1",
+						Served:  true,
+						Storage: false,
+					},
+				},
+			},
+		},
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "noxus.mygroup.example.com"},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group:   "mygroup.example.com",
+				Version: "v1beta1",
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Plural:     "noxus",
+					Singular:   "nonenglishnoxu",
+					Kind:       "WishIHadChosenNoxu",
+					ShortNames: []string{"foo", "bar", "abc", "def"},
+					ListKind:   "NoxuItemList",
+				},
+				Scope: apiextensionsv1beta1.NamespaceScoped,
+				Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{
+					{
+						Name:    "v1beta1",
+						Served:  true,
+						Storage: true,
+						Schema: &apiextensionsv1beta1.CustomResourceValidation{
+							OpenAPIV3Schema: validationSchema,
+						},
+					},
+					{
+						Name:    "v1",
+						Served:  true,
+						Storage: false,
+						Schema: &apiextensionsv1beta1.CustomResourceValidation{
+							OpenAPIV3Schema: validationSchemaWithDescription,
+						},
+					},
 				},
 			},
 		},
@@ -168,42 +222,58 @@ func newNoxuValidationInstance(namespace, name string) *unstructured.Unstructure
 }
 
 func TestCustomResourceValidation(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)()
 	tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t)
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer tearDown()
 
-	noxuDefinition := newNoxuValidationCRD(apiextensionsv1beta1.NamespaceScoped)
+	noxuDefinitions := newNoxuValidationCRDs(apiextensionsv1beta1.NamespaceScoped)
+	for _, noxuDefinition := range noxuDefinitions {
 		noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
 		if err != nil {
 			t.Fatal(err)
 		}
 
 		ns := "not-the-default"
-	noxuResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition)
-	_, err = instantiateCustomResource(t, newNoxuValidationInstance(ns, "foo"), noxuResourceClient, noxuDefinition)
+		for _, v := range noxuDefinition.Spec.Versions {
+			noxuResourceClient := newNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name)
+			instanceToCreate := newNoxuValidationInstance(ns, "foo")
+			instanceToCreate.Object["apiVersion"] = fmt.Sprintf("%s/%s", noxuDefinition.Spec.Group, v.Name)
+			_, err = instantiateVersionedCustomResource(t, instanceToCreate, noxuResourceClient, noxuDefinition, v.Name)
 			if err != nil {
 				t.Fatalf("unable to create noxu instance: %v", err)
 			}
+			noxuResourceClient.Delete("foo", &metav1.DeleteOptions{})
+		}
+		if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil {
+			t.Fatal(err)
+		}
+	}
 }
 
 func TestCustomResourceUpdateValidation(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)()
 	tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t)
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer tearDown()
 
-	noxuDefinition := newNoxuValidationCRD(apiextensionsv1beta1.NamespaceScoped)
+	noxuDefinitions := newNoxuValidationCRDs(apiextensionsv1beta1.NamespaceScoped)
+	for _, noxuDefinition := range noxuDefinitions {
 		noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
 		if err != nil {
 			t.Fatal(err)
 		}
 
 		ns := "not-the-default"
-	noxuResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition)
-	_, err = instantiateCustomResource(t, newNoxuValidationInstance(ns, "foo"), noxuResourceClient, noxuDefinition)
+		for _, v := range noxuDefinition.Spec.Versions {
+			noxuResourceClient := newNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name)
+			instanceToCreate := newNoxuValidationInstance(ns, "foo")
+			instanceToCreate.Object["apiVersion"] = fmt.Sprintf("%s/%s", noxuDefinition.Spec.Group, v.Name)
+			_, err = instantiateVersionedCustomResource(t, instanceToCreate, noxuResourceClient, noxuDefinition, v.Name)
 			if err != nil {
 				t.Fatalf("unable to create noxu instance: %v", err)
 			}
@@ -229,23 +299,30 @@ func TestCustomResourceUpdateValidation(t *testing.T) {
 			if err == nil {
 				t.Fatalf("unexpected non-error: alpha and beta should be present while updating %v", gottenNoxuInstance)
 			}
+			noxuResourceClient.Delete("foo", &metav1.DeleteOptions{})
+		}
+		if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil {
+			t.Fatal(err)
+		}
+	}
 }
 
 func TestCustomResourceValidationErrors(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)()
 	tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t)
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer tearDown()
 
-	noxuDefinition := newNoxuValidationCRD(apiextensionsv1beta1.NamespaceScoped)
+	noxuDefinitions := newNoxuValidationCRDs(apiextensionsv1beta1.NamespaceScoped)
+	for _, noxuDefinition := range noxuDefinitions {
 		noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
 		if err != nil {
 			t.Fatal(err)
 		}
 
 		ns := "not-the-default"
-	noxuResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition)
 
 		tests := []struct {
 			name          string
@@ -309,7 +386,11 @@ func TestCustomResourceValidationErrors(t *testing.T) {
 		}
 
 		for _, tc := range tests {
-		_, err := noxuResourceClient.Create(tc.instanceFn(), metav1.CreateOptions{})
+			for _, v := range noxuDefinition.Spec.Versions {
+				noxuResourceClient := newNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name)
+				instanceToCreate := tc.instanceFn()
+				instanceToCreate.Object["apiVersion"] = fmt.Sprintf("%s/%s", noxuDefinition.Spec.Group, v.Name)
+				_, err := noxuResourceClient.Create(instanceToCreate, metav1.CreateOptions{})
 				if err == nil {
 					t.Errorf("%v: expected %v", tc.name, tc.expectedError)
 					continue
@@ -320,36 +401,56 @@ func TestCustomResourceValidationErrors(t *testing.T) {
 					continue
 				}
 			}
+		}
+		if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil {
+			t.Fatal(err)
+		}
+	}
 }
 
 func TestCRValidationOnCRDUpdate(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)()
 	tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t)
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer tearDown()
 
-	noxuDefinition := newNoxuValidationCRD(apiextensionsv1beta1.NamespaceScoped)
+	noxuDefinitions := newNoxuValidationCRDs(apiextensionsv1beta1.NamespaceScoped)
+	for i, noxuDefinition := range noxuDefinitions {
+		for _, v := range noxuDefinition.Spec.Versions {
+			// Re-define the CRD to make sure we start with a clean CRD
+			noxuDefinition := newNoxuValidationCRDs(apiextensionsv1beta1.NamespaceScoped)[i]
+			validationSchema, err := getSchemaForVersion(noxuDefinition, v.Name)
+			if err != nil {
+				t.Fatal(err)
+			}
 
 			// set stricter schema
-	noxuDefinition.Spec.Validation.OpenAPIV3Schema.Required = []string{"alpha", "beta", "epsilon"}
+			validationSchema.OpenAPIV3Schema.Required = []string{"alpha", "beta", "epsilon"}
 
 			noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
 			if err != nil {
 				t.Fatal(err)
 			}
 			ns := "not-the-default"
-	noxuResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition)
+			noxuResourceClient := newNamespacedCustomResourceVersionedClient(ns, dynamicClient, noxuDefinition, v.Name)
+			instanceToCreate := newNoxuValidationInstance(ns, "foo")
+			instanceToCreate.Object["apiVersion"] = fmt.Sprintf("%s/%s", noxuDefinition.Spec.Group, v.Name)
 
 			// CR is rejected
-	_, err = instantiateCustomResource(t, newNoxuValidationInstance(ns, "foo"), noxuResourceClient, noxuDefinition)
+			_, err = instantiateVersionedCustomResource(t, instanceToCreate, noxuResourceClient, noxuDefinition, v.Name)
 			if err == nil {
 				t.Fatalf("unexpected non-error: CR should be rejected")
 			}
 
 			// update the CRD to a less stricter schema
 			_, err = updateCustomResourceDefinitionWithRetry(apiExtensionClient, "noxus.mygroup.example.com", func(crd *apiextensionsv1beta1.CustomResourceDefinition) {
-		crd.Spec.Validation.OpenAPIV3Schema.Required = []string{"alpha", "beta"}
+				validationSchema, err := getSchemaForVersion(crd, v.Name)
+				if err != nil {
+					t.Fatal(err)
+				}
+				validationSchema.OpenAPIV3Schema.Required = []string{"alpha", "beta"}
 			})
 			if err != nil {
 				t.Fatal(err)
@@ -357,9 +458,9 @@ func TestCRValidationOnCRDUpdate(t *testing.T) {
 
 			// CR is now accepted
 			err = wait.Poll(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
-		_, err := noxuResourceClient.Create(newNoxuValidationInstance(ns, "foo"), metav1.CreateOptions{})
-		if statusError, isStatus := err.(*apierrors.StatusError); isStatus {
-			if strings.Contains(statusError.Error(), "is invalid") {
+				_, err := noxuResourceClient.Create(instanceToCreate, metav1.CreateOptions{})
+				if _, isStatus := err.(*apierrors.StatusError); isStatus {
+					if apierrors.IsInvalid(err) {
 						return false, nil
 					}
 				}
@@ -371,36 +472,51 @@ func TestCRValidationOnCRDUpdate(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
+			noxuResourceClient.Delete("foo", &metav1.DeleteOptions{})
+			if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
 }
 
 func TestForbiddenFieldsInSchema(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)()
 	tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t)
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer tearDown()
 
-	noxuDefinition := newNoxuValidationCRD(apiextensionsv1beta1.NamespaceScoped)
-	noxuDefinition.Spec.Validation.OpenAPIV3Schema.AdditionalProperties.Allows = false
+	noxuDefinitions := newNoxuValidationCRDs(apiextensionsv1beta1.NamespaceScoped)
+	for i, noxuDefinition := range noxuDefinitions {
+		for _, v := range noxuDefinition.Spec.Versions {
+			// Re-define the CRD to make sure we start with a clean CRD
+			noxuDefinition := newNoxuValidationCRDs(apiextensionsv1beta1.NamespaceScoped)[i]
+			validationSchema, err := getSchemaForVersion(noxuDefinition, v.Name)
+			if err != nil {
+				t.Fatal(err)
+			}
+			validationSchema.OpenAPIV3Schema.AdditionalProperties.Allows = false
 
 			_, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
 			if err == nil {
 				t.Fatalf("unexpected non-error: additionalProperties cannot be set to false")
 			}
 
-	noxuDefinition.Spec.Validation.OpenAPIV3Schema.Properties["zeta"] = apiextensionsv1beta1.JSONSchemaProps{
+			validationSchema.OpenAPIV3Schema.Properties["zeta"] = apiextensionsv1beta1.JSONSchemaProps{
 				Type:        "array",
 				UniqueItems: true,
 			}
-	noxuDefinition.Spec.Validation.OpenAPIV3Schema.AdditionalProperties.Allows = true
+			validationSchema.OpenAPIV3Schema.AdditionalProperties.Allows = true
 
 			_, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
 			if err == nil {
 				t.Fatalf("unexpected non-error: uniqueItems cannot be set to true")
 			}
 
-	noxuDefinition.Spec.Validation.OpenAPIV3Schema.Ref = strPtr("#/definition/zeta")
-	noxuDefinition.Spec.Validation.OpenAPIV3Schema.Properties["zeta"] = apiextensionsv1beta1.JSONSchemaProps{
+			validationSchema.OpenAPIV3Schema.Ref = strPtr("#/definition/zeta")
+			validationSchema.OpenAPIV3Schema.Properties["zeta"] = apiextensionsv1beta1.JSONSchemaProps{
 				Type:        "array",
 				UniqueItems: false,
 			}
@@ -410,12 +526,17 @@ func TestForbiddenFieldsInSchema(t *testing.T) {
 				t.Fatal("unexpected non-error: $ref cannot be non-empty string")
 			}
 
-	noxuDefinition.Spec.Validation.OpenAPIV3Schema.Ref = nil
+			validationSchema.OpenAPIV3Schema.Ref = nil
 
 			noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
 			if err != nil {
 				t.Fatal(err)
 			}
+			if err := fixtures.DeleteCustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
 }
 
 func float64Ptr(f float64) *float64 {

staging/src/k8s.io/apiextensions-apiserver/test/integration/yaml_test.go

@@ -27,10 +27,13 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/types"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
 	"k8s.io/client-go/dynamic"
 
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features"
 	"k8s.io/apiextensions-apiserver/test/integration/fixtures"
 )
 
@@ -354,6 +357,7 @@ values:
 }
 
 func TestYAMLSubresource(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)()
 	tearDown, config, _, err := fixtures.StartDefaultServer(t)
 	if err != nil {
 		t.Fatal(err)
@@ -369,7 +373,7 @@ func TestYAMLSubresource(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	noxuDefinition := NewNoxuSubresourcesCRD(apiextensionsv1beta1.ClusterScoped)
+	noxuDefinition := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.ClusterScoped)[0]
 	noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
 	if err != nil {
 		t.Fatal(err)