Merge pull request #71302 from liggitt/verify-unit-test-feature-gates

Split mutable and read-only access to feature gates, limit tests to readonly access

cmd/cloud-controller-manager/app/options/options.go

@@ -143,7 +143,7 @@ func (o *CloudControllerManagerOptions) Flags() apiserverflag.NamedFlagSets {
 	fs.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
 	fs.DurationVar(&o.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", o.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.")
 
-	utilfeature.DefaultFeatureGate.AddFlag(fss.FlagSet("generic"))
+	utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"))
 
 	return fss
 }

cmd/kube-controller-manager/app/options/options.go

@@ -252,7 +252,7 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy
 	fs := fss.FlagSet("misc")
 	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
 	fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
-	utilfeature.DefaultFeatureGate.AddFlag(fss.FlagSet("generic"))
+	utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"))
 
 	return fss
 }

cmd/kube-proxy/app/server.go

@@ -208,7 +208,7 @@ func (o *Options) Complete() error {
 		return err
 	}
 
-	if err := utilfeature.DefaultFeatureGate.SetFromMap(o.config.FeatureGates); err != nil {
+	if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(o.config.FeatureGates); err != nil {
 		return err
 	}
 

cmd/kube-scheduler/app/options/options.go

@@ -152,7 +152,7 @@ func (o *Options) Flags() (nfs apiserverflag.NamedFlagSets) {
 	o.Deprecated.AddFlags(nfs.FlagSet("deprecated"), &o.ComponentConfig)
 
 	leaderelectionconfig.BindFlags(&o.ComponentConfig.LeaderElection.LeaderElectionConfiguration, nfs.FlagSet("leader election"))
-	utilfeature.DefaultFeatureGate.AddFlag(nfs.FlagSet("feature gate"))
+	utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate"))
 
 	return nfs
 }

cmd/kubelet/app/server.go

@@ -169,7 +169,7 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API
 			utilflag.PrintFlags(cleanFlagSet)
 
 			// set feature gates from initial flags-based config
-			if err := utilfeature.DefaultFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil {
+			if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil {
 				klog.Fatal(err)
 			}
 
@@ -195,7 +195,7 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API
 					klog.Fatal(err)
 				}
 				// update feature gates based on new config
-				if err := utilfeature.DefaultFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil {
+				if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil {
 					klog.Fatal(err)
 				}
 			}
@@ -226,7 +226,7 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API
 					kubeletConfig = dynamicKubeletConfig
 					// Note: flag precedence was already enforced in the controller, prior to validation,
 					// by our above transform function. Now we simply update feature gates from the new config.
-					if err := utilfeature.DefaultFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil {
+					if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil {
 						klog.Fatal(err)
 					}
 				}
@@ -467,7 +467,7 @@ func makeEventRecorder(kubeDeps *kubelet.Dependencies, nodeName types.NodeName)
 
 func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan struct{}) (err error) {
 	// Set global feature gates based on the value on the initial KubeletServer
-	err = utilfeature.DefaultFeatureGate.SetFromMap(s.KubeletConfiguration.FeatureGates)
+	err = utilfeature.DefaultMutableFeatureGate.SetFromMap(s.KubeletConfiguration.FeatureGates)
 	if err != nil {
 		return err
 	}

hack/make-rules/verify.sh

@@ -48,6 +48,7 @@ QUICK_PATTERNS+=(
   "verify-spelling.sh"
   "verify-staging-client-go.sh"
   "verify-staging-meta-files.sh"
+  "verify-test-featuregates.sh"
   "verify-test-images.sh"
   "verify-test-owners.sh"
 )

hack/verify-test-featuregates.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+cd "${KUBE_ROOT}"
+
+rc=0
+
+# find test files accessing the mutable global feature gate or interface
+direct_sets=$(grep -n --include *_test.go -R 'MutableFeatureGate' . 2>/dev/null) || true
+if [[ -n "${direct_sets}" ]]; then
+  echo "Test files may not access mutable global feature gates directly:" >&2
+  echo "${direct_sets}" >&2
+  echo >&2
+  echo "Use this invocation instead:" >&2
+  echo "  defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.<FeatureName>, <value>)()" >&2
+  echo >&2
+  rc=1
+fi
+
+# find test files calling SetFeatureGateDuringTest and not calling the result
+missing_defers=$(grep -n --include *_test.go -R 'SetFeatureGateDuringTest' . 2>/dev/null | egrep -v "defer .*\\)\\(\\)$") || true
+if [[ -n "${missing_defers}" ]]; then
+  echo "Invalid invocations of utilfeaturetesting.SetFeatureGateDuringTest():" >&2
+  echo "${missing_defers}" >&2
+  echo >&2
+  echo "Always make a deferred call to the returned function to ensure the feature gate is reset:" >&2
+  echo "  defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.<FeatureName>, <value>)()" >&2
+  echo >&2
+  rc=1
+fi
+
+exit $rc

pkg/api/persistentvolumeclaim/BUILD

@@ -32,10 +32,7 @@ filegroup(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["util_test.go"],
-        "main_test.go",
-        "util_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/core:go_default_library",

pkg/api/persistentvolumeclaim/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package persistentvolumeclaim
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/api/pod/BUILD

@@ -33,10 +33,7 @@ filegroup(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["util_test.go"],
-        "main_test.go",
-        "util_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/core:go_default_library",

pkg/api/pod/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package pod
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/api/podsecuritypolicy/BUILD

@@ -32,10 +32,7 @@ filegroup(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["util_test.go"],
-        "main_test.go",
-        "util_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/core:go_default_library",

pkg/api/podsecuritypolicy/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package podsecuritypolicy
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/api/podsecuritypolicy/util_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/apis/policy"
 	"k8s.io/kubernetes/pkg/features"
@@ -34,10 +35,7 @@ func TestDropAlphaProcMountType(t *testing.T) {
 	}
 
 	// Enable alpha feature ProcMountType
-	err1 := utilfeature.DefaultFeatureGate.Set("ProcMountType=true")
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ProcMountType, true)()
-	if err1 != nil {
-		t.Fatalf("Failed to enable feature gate for ProcMountType: %v", err1)
-	}
 
 	// now test dropping the fields - should not be dropped
 	DropDisabledAlphaFields(&psp.Spec)
@@ -51,10 +49,7 @@ func TestDropAlphaProcMountType(t *testing.T) {
 	}
 
 	// Disable alpha feature ProcMountType
-	err := utilfeature.DefaultFeatureGate.Set("ProcMountType=false")
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ProcMountType, false)()
-	if err != nil {
-		t.Fatalf("Failed to disable feature gate for ProcMountType: %v", err)
-	}
 
 	// now test dropping the fields
 	DropDisabledAlphaFields(&psp.Spec)

pkg/apis/batch/validation/BUILD

@@ -27,10 +27,7 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["validation_test.go"],
-        "main_test.go",
-        "validation_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/batch:go_default_library",

pkg/apis/batch/validation/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package validation
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/apis/batch/validation/validation_test.go

@@ -24,6 +24,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
 	"k8s.io/kubernetes/pkg/apis/batch"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/features"
@@ -74,14 +75,6 @@ func featureToggle(feature utilfeature.Feature) []string {
 }
 
 func TestValidateJob(t *testing.T) {
-	ttlEnabled := utilfeature.DefaultFeatureGate.Enabled(features.TTLAfterFinished)
-	defer func() {
-		err := utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=%t", features.TTLAfterFinished, ttlEnabled))
-		if err != nil {
-			t.Fatalf("Failed to set feature gate for %s: %v", features.TTLAfterFinished, err)
-		}
-	}()
-
 	validManualSelector := getValidManualSelector()
 	validPodTemplateSpecForManual := getValidPodTemplateSpecForManual(validManualSelector)
 	validGeneratedSelector := getValidGeneratedSelector()
@@ -231,11 +224,8 @@ func TestValidateJob(t *testing.T) {
 		},
 	}
 
-	for _, setFeature := range featureToggle(features.TTLAfterFinished) {
+	for _, setFeature := range []bool{true, false} {
-		// Set error cases based on if TTLAfterFinished feature is enabled or not
+		defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.TTLAfterFinished, setFeature)()
-		if err := utilfeature.DefaultFeatureGate.Set(setFeature); err != nil {
-			t.Fatalf("Failed to set feature gate for %s: %v", features.TTLAfterFinished, err)
-		}
 		ttlCase := "spec.ttlSecondsAfterFinished:must be greater than or equal to 0"
 		if utilfeature.DefaultFeatureGate.Enabled(features.TTLAfterFinished) {
 			errorCases[ttlCase] = batch.Job{

pkg/apis/core/v1/BUILD

@@ -36,7 +36,6 @@ go_test(
     srcs = [
         "conversion_test.go",
         "defaults_test.go",
-        "main_test.go",
     ],
     embed = [":go_default_library"],
     deps = [

pkg/apis/core/v1/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package v1
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/apis/core/validation/BUILD

@@ -47,7 +47,6 @@ go_test(
     name = "go_default_test",
     srcs = [
         "events_test.go",
-        "main_test.go",
         "validation_test.go",
     ],
     embed = [":go_default_library"],

pkg/apis/core/validation/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package validation
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/apis/core/validation/validation_test.go

@@ -18,7 +18,6 @@ package validation
 
 import (
 	"bytes"
-	"fmt"
 	"math"
 	"reflect"
 	"strings"
@@ -784,11 +783,7 @@ func TestAlphaVolumeSnapshotDataSource(t *testing.T) {
 	}
 
 	// Enable alpha feature VolumeSnapshotDataSource
-	err := utilfeature.DefaultFeatureGate.Set("VolumeSnapshotDataSource=true")
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSnapshotDataSource, true)()
-	if err != nil {
-		t.Errorf("Failed to enable feature gate for VolumeSnapshotDataSource: %v", err)
-		return
-	}
 	for _, tc := range successTestCases {
 		if errs := ValidatePersistentVolumeClaimSpec(&tc, field.NewPath("spec")); len(errs) != 0 {
 			t.Errorf("expected success: %v", errs)
@@ -800,11 +795,7 @@ func TestAlphaVolumeSnapshotDataSource(t *testing.T) {
 		}
 	}
 	// Disable alpha feature VolumeSnapshotDataSource
-	err = utilfeature.DefaultFeatureGate.Set("VolumeSnapshotDataSource=false")
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSnapshotDataSource, false)()
-	if err != nil {
-		t.Errorf("Failed to disable feature gate for VolumeSnapshotDataSource: %v", err)
-		return
-	}
 	for _, tc := range successTestCases {
 		if errs := ValidatePersistentVolumeClaimSpec(&tc, field.NewPath("spec")); len(errs) == 0 {
 			t.Errorf("expected failure: %v", errs)
@@ -4897,8 +4888,7 @@ func TestValidateVolumeMounts(t *testing.T) {
 }
 
 func TestValidateDisabledSubpath(t *testing.T) {
-	utilfeature.DefaultFeatureGate.Set("VolumeSubpath=false")
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSubpath, false)()
-	defer utilfeature.DefaultFeatureGate.Set("VolumeSubpath=true")
 
 	volumes := []core.Volume{
 		{Name: "abc", VolumeSource: core.VolumeSource{PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{ClaimName: "testclaim1"}}},
@@ -5734,16 +5724,7 @@ func TestValidateRestartPolicy(t *testing.T) {
 }
 
 func TestValidateDNSPolicy(t *testing.T) {
-	customDNSEnabled := utilfeature.DefaultFeatureGate.Enabled("CustomPodDNS")
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CustomPodDNS, true)()
-	defer func() {
-		// Restoring the old value.
-		if err := utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("CustomPodDNS=%v", customDNSEnabled)); err != nil {
-			t.Errorf("Failed to restore CustomPodDNS feature gate: %v", err)
-		}
-	}()
-	if err := utilfeature.DefaultFeatureGate.Set("CustomPodDNS=true"); err != nil {
-		t.Errorf("Failed to enable CustomPodDNS feature gate: %v", err)
-	}
 
 	successCases := []core.DNSPolicy{core.DNSClusterFirst, core.DNSDefault, core.DNSPolicy(core.DNSClusterFirst), core.DNSNone}
 	for _, policy := range successCases {
@@ -5761,16 +5742,7 @@ func TestValidateDNSPolicy(t *testing.T) {
 }
 
 func TestValidatePodDNSConfig(t *testing.T) {
-	customDNSEnabled := utilfeature.DefaultFeatureGate.Enabled("CustomPodDNS")
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CustomPodDNS, true)()
-	defer func() {
-		// Restoring the old value.
-		if err := utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("CustomPodDNS=%v", customDNSEnabled)); err != nil {
-			t.Errorf("Failed to restore CustomPodDNS feature gate: %v", err)
-		}
-	}()
-	if err := utilfeature.DefaultFeatureGate.Set("CustomPodDNS=true"); err != nil {
-		t.Errorf("Failed to enable CustomPodDNS feature gate: %v", err)
-	}
 
 	generateTestSearchPathFunc := func(numChars int) string {
 		res := ""
@@ -5932,16 +5904,7 @@ func TestValidatePodDNSConfig(t *testing.T) {
 }
 
 func TestValidatePodReadinessGates(t *testing.T) {
-	podReadinessGatesEnabled := utilfeature.DefaultFeatureGate.Enabled(features.PodReadinessGates)
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodReadinessGates, true)()
-	defer func() {
-		// Restoring the old value.
-		if err := utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=%v", features.PodReadinessGates, podReadinessGatesEnabled)); err != nil {
-			t.Errorf("Failed to restore PodReadinessGates feature gate: %v", err)
-		}
-	}()
-	if err := utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.PodReadinessGates)); err != nil {
-		t.Errorf("Failed to enable PodReadinessGates feature gate: %v", err)
-	}
 
 	successCases := []struct {
 		desc           string
@@ -6420,8 +6383,7 @@ func TestValidatePodSpec(t *testing.T) {
 		}
 	}
 
-	// original value will be restored by previous defer
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodShareProcessNamespace, false)()
-	utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodShareProcessNamespace, false)
 
 	featuregatedCases := map[string]core.PodSpec{
 		"set ShareProcessNamespace": {

pkg/apis/storage/util/BUILD

@@ -36,10 +36,7 @@ filegroup(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["util_test.go"],
-        "main_test.go",
-        "util_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/core:go_default_library",

pkg/apis/storage/util/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package util
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/apis/storage/v1/BUILD

@@ -47,10 +47,7 @@ filegroup(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["defaults_test.go"],
-        "defaults_test.go",
-        "main_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/api/legacyscheme:go_default_library",

pkg/apis/storage/v1/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package v1
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/apis/storage/v1beta1/BUILD

@@ -47,10 +47,7 @@ filegroup(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["defaults_test.go"],
-        "defaults_test.go",
-        "main_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/api/legacyscheme:go_default_library",

pkg/apis/storage/v1beta1/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package v1beta1
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/apis/storage/validation/BUILD

@@ -26,10 +26,7 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["validation_test.go"],
-        "main_test.go",
-        "validation_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/core:go_default_library",

pkg/apis/storage/validation/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package validation
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/controller/cloud/BUILD

@@ -50,7 +50,6 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
-        "main_test.go",
         "node_controller_test.go",
         "pvlcontroller_test.go",
     ],

pkg/controller/cloud/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package cloud
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/controller/daemon/BUILD

@@ -60,7 +60,6 @@ go_test(
     name = "go_default_test",
     srcs = [
         "daemon_controller_test.go",
-        "main_test.go",
         "update_test.go",
     ],
     embed = [":go_default_library"],

pkg/controller/daemon/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package daemon
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/controller/daemon/util/BUILD

@@ -36,10 +36,7 @@ filegroup(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["daemonset_util_test.go"],
-        "daemonset_util_test.go",
-        "main_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/api/testapi:go_default_library",

pkg/controller/daemon/util/daemonset_util_test.go

@@ -25,6 +25,7 @@ import (
 	extensions "k8s.io/api/extensions/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
 	"k8s.io/kubernetes/pkg/api/testapi"
 	"k8s.io/kubernetes/pkg/features"
 	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
@@ -482,17 +483,12 @@ func TestReplaceDaemonSetPodNodeNameNodeAffinity(t *testing.T) {
 
 func forEachFeatureGate(t *testing.T, tf func(t *testing.T), gates ...utilfeature.Feature) {
 	for _, fg := range gates {
-		func() {
+		for _, f := range []bool{true, false} {
-			enabled := utilfeature.DefaultFeatureGate.Enabled(fg)
+			func() {
-			defer func() {
+				defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, fg, f)()
-				utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%v=%t", fg, enabled))
-			}()
-
-			for _, f := range []bool{true, false} {
-				utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%v=%t", fg, f))
 				t.Run(fmt.Sprintf("%v (%t)", fg, f), tf)
-			}
+			}()
-		}()
+		}
 	}
 }
 

pkg/controller/daemon/util/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package util
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/controller/volume/persistentvolume/BUILD

@@ -70,7 +70,6 @@ go_test(
         "delete_test.go",
         "framework_test.go",
         "index_test.go",
-        "main_test.go",
         "provision_test.go",
         "pv_controller_test.go",
         "recycle_test.go",

pkg/controller/volume/persistentvolume/index_test.go

@@ -1161,19 +1161,20 @@ func TestVolumeModeCheck(t *testing.T) {
 	}
 
 	for name, scenario := range scenarios {
-		recover := utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, scenario.enableBlock)
+		t.Run(name, func(t *testing.T) {
-		expectedMismatch, err := checkVolumeModeMismatches(&scenario.pvc.Spec, &scenario.vol.Spec)
+			defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, scenario.enableBlock)()
-		if err != nil {
+			expectedMismatch, err := checkVolumeModeMismatches(&scenario.pvc.Spec, &scenario.vol.Spec)
-			t.Errorf("Unexpected failure for checkVolumeModeMismatches: %v", err)
+			if err != nil {
-		}
+				t.Errorf("Unexpected failure for checkVolumeModeMismatches: %v", err)
-		// expected to match but either got an error or no returned pvmatch
+			}
-		if expectedMismatch && !scenario.isExpectedMismatch {
+			// expected to match but either got an error or no returned pvmatch
-			t.Errorf("Unexpected failure for scenario, expected not to mismatch on modes but did: %s", name)
+			if expectedMismatch && !scenario.isExpectedMismatch {
-		}
+				t.Errorf("Unexpected failure for scenario, expected not to mismatch on modes but did: %s", name)
-		if !expectedMismatch && scenario.isExpectedMismatch {
+			}
-			t.Errorf("Unexpected failure for scenario, did not mismatch on mode when expected to mismatch: %s", name)
+			if !expectedMismatch && scenario.isExpectedMismatch {
-		}
+				t.Errorf("Unexpected failure for scenario, did not mismatch on mode when expected to mismatch: %s", name)
-		recover()
+			}
+		})
 	}
 }
 
@@ -1252,23 +1253,24 @@ func TestFilteringVolumeModes(t *testing.T) {
 	}
 
 	for name, scenario := range scenarios {
-		recover := utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, scenario.enableBlock)
+		t.Run(name, func(t *testing.T) {
-		pvmatch, err := scenario.vol.findBestMatchForClaim(scenario.pvc, false)
+			defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BlockVolume, scenario.enableBlock)()
-		// expected to match but either got an error or no returned pvmatch
+			pvmatch, err := scenario.vol.findBestMatchForClaim(scenario.pvc, false)
-		if pvmatch == nil && scenario.isExpectedMatch {
+			// expected to match but either got an error or no returned pvmatch
-			t.Errorf("Unexpected failure for scenario, no matching volume: %s", name)
+			if pvmatch == nil && scenario.isExpectedMatch {
-		}
+				t.Errorf("Unexpected failure for scenario, no matching volume: %s", name)
-		if err != nil && scenario.isExpectedMatch {
+			}
-			t.Errorf("Unexpected failure for scenario: %s - %+v", name, err)
+			if err != nil && scenario.isExpectedMatch {
-		}
+				t.Errorf("Unexpected failure for scenario: %s - %+v", name, err)
-		// expected to not match but either got an error or a returned pvmatch
+			}
-		if pvmatch != nil && !scenario.isExpectedMatch {
+			// expected to not match but either got an error or a returned pvmatch
-			t.Errorf("Unexpected failure for scenario, expected no matching volume: %s", name)
+			if pvmatch != nil && !scenario.isExpectedMatch {
-		}
+				t.Errorf("Unexpected failure for scenario, expected no matching volume: %s", name)
-		if err != nil && !scenario.isExpectedMatch {
+			}
-			t.Errorf("Unexpected failure for scenario: %s - %+v", name, err)
+			if err != nil && !scenario.isExpectedMatch {
-		}
+				t.Errorf("Unexpected failure for scenario: %s - %+v", name, err)
-		recover()
+			}
+		})
 	}
 }
 

pkg/controller/volume/persistentvolume/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package persistentvolume
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/features/kube_features.go

@@ -396,7 +396,7 @@ const (
 )
 
 func init() {
-	utilfeature.DefaultFeatureGate.Add(defaultKubernetesFeatureGates)
+	utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates)
 }
 
 // defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys.

pkg/kubelet/BUILD

@@ -166,7 +166,6 @@ go_test(
         "kubelet_resources_test.go",
         "kubelet_test.go",
         "kubelet_volumes_test.go",
-        "main_test.go",
         "oom_watcher_test.go",
         "pod_container_deletor_test.go",
         "pod_workers_test.go",

pkg/kubelet/cadvisor/BUILD

@@ -50,19 +50,17 @@ go_test(
     name = "go_default_test",
     srcs = [
         "cadvisor_linux_test.go",
-        "main_test.go",
         "util_test.go",
     ],
     embed = [":go_default_library"],
-    deps = [
+    deps = select({
-        "//pkg/features:go_default_library",
-        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
-        "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
-    ] + select({
         "@io_bazel_rules_go//go/platform:linux": [
             "//pkg/apis/core/v1/helper:go_default_library",
+            "//pkg/features:go_default_library",
             "//pkg/kubelet/types:go_default_library",
             "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
             "//vendor/github.com/google/cadvisor/info/v1:go_default_library",
             "//vendor/github.com/google/cadvisor/metrics:go_default_library",
             "//vendor/github.com/stretchr/testify/assert:go_default_library",

pkg/kubelet/cadvisor/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package cadvisor
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/kubelet/cm/helpers_linux_test.go

@@ -274,8 +274,7 @@ func TestResourceConfigForPodWithCustomCPUCFSQuotaPeriod(t *testing.T) {
 	tunedQuotaPeriod := uint64(5 * time.Millisecond / time.Microsecond)
 	tunedQuota := int64(1 * time.Millisecond / time.Microsecond)
 
-	utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.CPUCFSQuotaPeriod, true)
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.CPUCFSQuotaPeriod, true)()
-	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.CPUCFSQuotaPeriod, false)
 
 	minShares := uint64(MinShares)
 	burstableShares := MilliCPUToShares(100)

pkg/kubelet/eviction/BUILD

@@ -11,7 +11,6 @@ go_test(
     srcs = [
         "eviction_manager_test.go",
         "helpers_test.go",
-        "main_test.go",
         "memory_threshold_notifier_test.go",
         "mock_threshold_notifier_test.go",
     ],

pkg/kubelet/eviction/eviction_manager_test.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/clock"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
 	"k8s.io/client-go/tools/record"
 	kubeapi "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/features"
@@ -181,7 +182,7 @@ type podToMake struct {
 
 // TestMemoryPressure
 func TestMemoryPressure(t *testing.T) {
-	utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{string(features.PodPriority): true})
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodPriority, true)()
 	podMaker := makePodWithMemoryStats
 	summaryStatsMaker := makeMemoryStats
 	podsToMake := []podToMake{
@@ -399,10 +400,9 @@ func parseQuantity(value string) resource.Quantity {
 }
 
 func TestDiskPressureNodeFs(t *testing.T) {
-	utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LocalStorageCapacityIsolation, true)()
-		string(features.LocalStorageCapacityIsolation): true,
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodPriority, true)()
-		string(features.PodPriority):                   true,
+
-	})
 	podMaker := makePodWithDiskStats
 	summaryStatsMaker := makeDiskStats
 	podsToMake := []podToMake{
@@ -600,7 +600,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
 
 // TestMinReclaim verifies that min-reclaim works as desired.
 func TestMinReclaim(t *testing.T) {
-	utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{string(features.PodPriority): true})
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodPriority, true)()
 	podMaker := makePodWithMemoryStats
 	summaryStatsMaker := makeMemoryStats
 	podsToMake := []podToMake{
@@ -739,10 +739,9 @@ func TestMinReclaim(t *testing.T) {
 }
 
 func TestNodeReclaimFuncs(t *testing.T) {
-	utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LocalStorageCapacityIsolation, true)()
-		string(features.PodPriority):                   true,
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodPriority, true)()
-		string(features.LocalStorageCapacityIsolation): true,
+
-	})
 	podMaker := makePodWithDiskStats
 	summaryStatsMaker := makeDiskStats
 	podsToMake := []podToMake{
@@ -918,7 +917,7 @@ func TestNodeReclaimFuncs(t *testing.T) {
 }
 
 func TestInodePressureNodeFsInodes(t *testing.T) {
-	utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{string(features.PodPriority): true})
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodPriority, true)()
 	podMaker := func(name string, priority int32, requests v1.ResourceList, limits v1.ResourceList, rootInodes, logInodes, volumeInodes string) (*v1.Pod, statsapi.PodStats) {
 		pod := newPod(name, priority, []v1.Container{
 			newContainer(name, requests, limits),
@@ -1140,7 +1139,7 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
 
 // TestCriticalPodsAreNotEvicted
 func TestCriticalPodsAreNotEvicted(t *testing.T) {
-	utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{string(features.PodPriority): true})
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodPriority, true)()
 	podMaker := makePodWithMemoryStats
 	summaryStatsMaker := makeMemoryStats
 	podsToMake := []podToMake{
@@ -1210,7 +1209,7 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
 	}
 
 	// Enable critical pod annotation feature gate
-	utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{string(features.ExperimentalCriticalPodAnnotation): true})
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)()
 	// induce soft threshold
 	fakeClock.Step(1 * time.Minute)
 	summaryProvider.result = summaryStatsMaker("1500Mi", podStats)
@@ -1255,7 +1254,7 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
 	}
 
 	// Disable critical pod annotation feature gate
-	utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{string(features.ExperimentalCriticalPodAnnotation): false})
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, false)()
 
 	// induce memory pressure!
 	fakeClock.Step(1 * time.Minute)
@@ -1275,7 +1274,7 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
 
 // TestAllocatableMemoryPressure
 func TestAllocatableMemoryPressure(t *testing.T) {
-	utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{string(features.PodPriority): true})
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodPriority, true)()
 	podMaker := makePodWithMemoryStats
 	summaryStatsMaker := makeMemoryStats
 	podsToMake := []podToMake{

pkg/kubelet/eviction/helpers_test.go

@@ -28,6 +28,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
 	"k8s.io/kubernetes/pkg/features"
 	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
 	evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api"
@@ -423,7 +424,8 @@ func thresholdEqual(a evictionapi.Threshold, b evictionapi.Threshold) bool {
 }
 
 func TestOrderedByExceedsRequestMemory(t *testing.T) {
-	utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.PodPriority))
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodPriority, true)()
+
 	below := newPod("below-requests", -1, []v1.Container{
 		newContainer("below-requests", newResourceList("", "200Mi", ""), newResourceList("", "", "")),
 	}, nil)
@@ -450,8 +452,8 @@ func TestOrderedByExceedsRequestMemory(t *testing.T) {
 }
 
 func TestOrderedByExceedsRequestDisk(t *testing.T) {
-	utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.PodPriority))
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodPriority, true)()
-	utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.LocalStorageCapacityIsolation))
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LocalStorageCapacityIsolation, true)()
 	below := newPod("below-requests", -1, []v1.Container{
 		newContainer("below-requests", v1.ResourceList{v1.ResourceEphemeralStorage: resource.MustParse("200Mi")}, newResourceList("", "", "")),
 	}, nil)
@@ -478,7 +480,7 @@ func TestOrderedByExceedsRequestDisk(t *testing.T) {
 }
 
 func TestOrderedByPriority(t *testing.T) {
-	utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.PodPriority))
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodPriority, true)()
 	low := newPod("low-priority", -134, []v1.Container{
 		newContainer("low-priority", newResourceList("", "", ""), newResourceList("", "", "")),
 	}, nil)
@@ -501,7 +503,7 @@ func TestOrderedByPriority(t *testing.T) {
 }
 
 func TestOrderedByPriorityDisabled(t *testing.T) {
-	utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=false", features.PodPriority))
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodPriority, false)()
 	low := newPod("low-priority", lowPriority, []v1.Container{
 		newContainer("low-priority", newResourceList("", "", ""), newResourceList("", "", "")),
 	}, nil)
@@ -525,7 +527,7 @@ func TestOrderedByPriorityDisabled(t *testing.T) {
 }
 
 func TestOrderedbyDisk(t *testing.T) {
-	utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.LocalStorageCapacityIsolation))
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LocalStorageCapacityIsolation, true)()
 	pod1 := newPod("best-effort-high", defaultPriority, []v1.Container{
 		newContainer("best-effort-high", newResourceList("", "", ""), newResourceList("", "", "")),
 	}, []v1.Volume{
@@ -592,7 +594,7 @@ func TestOrderedbyDisk(t *testing.T) {
 
 // Tests that we correctly ignore disk requests when the local storage feature gate is disabled.
 func TestOrderedbyDiskDisableLocalStorage(t *testing.T) {
-	utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=false", features.LocalStorageCapacityIsolation))
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LocalStorageCapacityIsolation, false)()
 	pod1 := newPod("best-effort-high", defaultPriority, []v1.Container{
 		newContainer("best-effort-high", newResourceList("", "", ""), newResourceList("", "", "")),
 	}, []v1.Volume{
@@ -658,8 +660,8 @@ func TestOrderedbyDiskDisableLocalStorage(t *testing.T) {
 }
 
 func TestOrderedbyInodes(t *testing.T) {
-	utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.PodPriority))
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodPriority, true)()
-	utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.LocalStorageCapacityIsolation))
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LocalStorageCapacityIsolation, true)()
 	low := newPod("low", defaultPriority, []v1.Container{
 		newContainer("low", newResourceList("", "", ""), newResourceList("", "", "")),
 	}, []v1.Volume{
@@ -702,8 +704,8 @@ func TestOrderedbyInodes(t *testing.T) {
 
 // TestOrderedByPriorityDisk ensures we order pods by priority and then greediest resource consumer
 func TestOrderedByPriorityDisk(t *testing.T) {
-	utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.PodPriority))
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodPriority, true)()
-	utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.LocalStorageCapacityIsolation))
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LocalStorageCapacityIsolation, true)()
 	pod1 := newPod("above-requests-low-priority-high-usage", lowPriority, []v1.Container{
 		newContainer("above-requests-low-priority-high-usage", newResourceList("", "", ""), newResourceList("", "", "")),
 	}, []v1.Volume{
@@ -787,7 +789,7 @@ func TestOrderedByPriorityDisk(t *testing.T) {
 
 // TestOrderedByPriorityInodes ensures we order pods by priority and then greediest resource consumer
 func TestOrderedByPriorityInodes(t *testing.T) {
-	utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.PodPriority))
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodPriority, true)()
 	pod1 := newPod("low-priority-high-usage", lowPriority, []v1.Container{
 		newContainer("low-priority-high-usage", newResourceList("", "", ""), newResourceList("", "", "")),
 	}, []v1.Volume{
@@ -880,7 +882,7 @@ func TestOrderedByMemory(t *testing.T) {
 
 // TestOrderedByPriorityMemory ensures we order by priority and then memory consumption relative to request.
 func TestOrderedByPriorityMemory(t *testing.T) {
-	utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.PodPriority))
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodPriority, true)()
 	pod1 := newPod("above-requests-low-priority-high-usage", lowPriority, []v1.Container{
 		newContainer("above-requests-low-priority-high-usage", newResourceList("", "", ""), newResourceList("", "", "")),
 	}, nil)

pkg/kubelet/eviction/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package eviction
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/kubelet/kubelet_test.go

@@ -36,6 +36,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
 	"k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/flowcontrol"
@@ -2254,17 +2255,12 @@ func runVolumeManager(kubelet *Kubelet) chan struct{} {
 
 func forEachFeatureGate(t *testing.T, fs []utilfeature.Feature, tf func(t *testing.T)) {
 	for _, fg := range fs {
-		func() {
+		for _, f := range []bool{true, false} {
-			enabled := utilfeature.DefaultFeatureGate.Enabled(fg)
+			func() {
-			defer func() {
+				defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, fg, f)()
-				utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%v=%t", fg, enabled))
-			}()
-
-			for _, f := range []bool{true, false} {
-				utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%v=%t", fg, f))
 				t.Run(fmt.Sprintf("%v(%t)", fg, f), tf)
-			}
+			}()
-		}()
+		}
 	}
 }
 

pkg/kubelet/kuberuntime/BUILD

@@ -95,7 +95,6 @@ go_test(
         "kuberuntime_sandbox_test.go",
         "labels_test.go",
         "legacy_test.go",
-        "main_test.go",
         "security_context_test.go",
     ],
     embed = [":go_default_library"],

pkg/kubelet/kuberuntime/helpers_linux_test.go

@@ -89,8 +89,7 @@ func TestMilliCPUToQuota(t *testing.T) {
 }
 
 func TestMilliCPUToQuotaWithCustomCPUCFSQuotaPeriod(t *testing.T) {
-	utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CPUCFSQuotaPeriod, true)
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CPUCFSQuotaPeriod, true)()
-	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CPUCFSQuotaPeriod, false)
 
 	for _, testCase := range []struct {
 		msg      string

pkg/kubelet/kuberuntime/helpers_test.go

@@ -351,8 +351,7 @@ func TestNamespacesForPod(t *testing.T) {
 		assert.Equal(t, test.expected, actual)
 	}
 
-	// Test ShareProcessNamespace feature disabled, feature gate restored by previous defer
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodShareProcessNamespace, false)()
-	utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodShareProcessNamespace, false)
 
 	for desc, test := range map[string]struct {
 		input    *v1.Pod

pkg/kubelet/kuberuntime/labels_test.go

@@ -24,6 +24,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
+	"k8s.io/kubernetes/pkg/features"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 )
 
@@ -67,15 +69,15 @@ func TestContainerLabels(t *testing.T) {
 
 	var tests = []struct {
 		description     string
-		featuresCreated string // Features enabled when container is created
+		featuresCreated bool // Features enabled when container is created
-		featuresStatus  string // Features enabled when container status is read
+		featuresStatus  bool // Features enabled when container status is read
 		typeLabel       kubecontainer.ContainerType
 		expected        *labeledContainerInfo
 	}{
 		{
 			"Debug containers disabled",
-			"DebugContainers=False",
+			false,
-			"DebugContainers=False",
+			false,
 			"ignored",
 			&labeledContainerInfo{
 				PodName:       pod.Name,
@@ -87,8 +89,8 @@ func TestContainerLabels(t *testing.T) {
 		},
 		{
 			"Regular containers",
-			"DebugContainers=True",
+			true,
-			"DebugContainers=True",
+			true,
 			kubecontainer.ContainerTypeRegular,
 			&labeledContainerInfo{
 				PodName:       pod.Name,
@@ -100,8 +102,8 @@ func TestContainerLabels(t *testing.T) {
 		},
 		{
 			"Init containers",
-			"DebugContainers=True",
+			true,
-			"DebugContainers=True",
+			true,
 			kubecontainer.ContainerTypeInit,
 			&labeledContainerInfo{
 				PodName:       pod.Name,
@@ -113,8 +115,8 @@ func TestContainerLabels(t *testing.T) {
 		},
 		{
 			"Created without type label",
-			"DebugContainers=False",
+			false,
-			"DebugContainers=True",
+			true,
 			"ignored",
 			&labeledContainerInfo{
 				PodName:       pod.Name,
@@ -126,8 +128,8 @@ func TestContainerLabels(t *testing.T) {
 		},
 		{
 			"Created with type label, subsequently disabled",
-			"DebugContainers=True",
+			true,
-			"DebugContainers=False",
+			false,
 			kubecontainer.ContainerTypeRegular,
 			&labeledContainerInfo{
 				PodName:       pod.Name,
@@ -141,15 +143,16 @@ func TestContainerLabels(t *testing.T) {
 
 	// Test whether we can get right information from label
 	for _, test := range tests {
-		utilfeature.DefaultFeatureGate.Set(test.featuresCreated)
+		func() {
-		labels := newContainerLabels(container, pod, test.typeLabel)
+			defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DebugContainers, test.featuresCreated)()
-		utilfeature.DefaultFeatureGate.Set(test.featuresStatus)
+			labels := newContainerLabels(container, pod, test.typeLabel)
-		containerInfo := getContainerInfoFromLabels(labels)
+			defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DebugContainers, test.featuresStatus)()
-		if !reflect.DeepEqual(containerInfo, test.expected) {
+			containerInfo := getContainerInfoFromLabels(labels)
-			t.Errorf("%v: expected %v, got %v", test.description, test.expected, containerInfo)
+			if !reflect.DeepEqual(containerInfo, test.expected) {
-		}
+				t.Errorf("%v: expected %v, got %v", test.description, test.expected, containerInfo)
+			}
+		}()
 	}
-	utilfeature.DefaultFeatureGate.Set("DebugContainers=False")
 }
 
 func TestContainerAnnotations(t *testing.T) {

pkg/kubelet/kuberuntime/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package kuberuntime
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/kubelet/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package kubelet
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/kubelet/network/dns/BUILD

@@ -21,10 +21,7 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["dns_test.go"],
-        "dns_test.go",
-        "main_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/features:go_default_library",

pkg/kubelet/network/dns/dns_test.go

@@ -29,7 +29,9 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
 	"k8s.io/client-go/tools/record"
+	"k8s.io/kubernetes/pkg/features"
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 
 	"github.com/stretchr/testify/assert"
@@ -266,14 +268,6 @@ func TestMergeDNSOptions(t *testing.T) {
 }
 
 func TestGetPodDNSType(t *testing.T) {
-	customDNSEnabled := utilfeature.DefaultFeatureGate.Enabled("CustomPodDNS")
-	defer func() {
-		// Restoring the old value.
-		if err := utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("CustomPodDNS=%v", customDNSEnabled)); err != nil {
-			t.Errorf("Failed to set CustomPodDNS feature gate: %v", err)
-		}
-	}()
-
 	recorder := record.NewFakeRecorder(20)
 	nodeRef := &v1.ObjectReference{
 		Kind:      "Node",
@@ -361,28 +355,28 @@ func TestGetPodDNSType(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		if err := utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("CustomPodDNS=%v", tc.customPodDNSFeatureGate)); err != nil {
+		t.Run(tc.desc, func(t *testing.T) {
-			t.Errorf("Failed to set CustomPodDNS feature gate: %v", err)
+			defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CustomPodDNS, tc.customPodDNSFeatureGate)()
-		}
 
-		if tc.hasClusterDNS {
+			if tc.hasClusterDNS {
-			configurer.clusterDNS = testClusterDNS
+				configurer.clusterDNS = testClusterDNS
-		} else {
+			} else {
-			configurer.clusterDNS = nil
+				configurer.clusterDNS = nil
-		}
-		pod.Spec.DNSPolicy = tc.dnsPolicy
-		pod.Spec.HostNetwork = tc.hostNetwork
-
-		resType, err := getPodDNSType(pod)
-		if tc.expectedError {
-			if err == nil {
-				t.Errorf("%s: GetPodDNSType(%v) got no error, want error", tc.desc, pod)
 			}
-			continue
+			pod.Spec.DNSPolicy = tc.dnsPolicy
-		}
+			pod.Spec.HostNetwork = tc.hostNetwork
-		if resType != tc.expectedDNSType {
+
-			t.Errorf("%s: GetPodDNSType(%v)=%v, want %v", tc.desc, pod, resType, tc.expectedDNSType)
+			resType, err := getPodDNSType(pod)
-		}
+			if tc.expectedError {
+				if err == nil {
+					t.Errorf("%s: GetPodDNSType(%v) got no error, want error", tc.desc, pod)
+				}
+				return
+			}
+			if resType != tc.expectedDNSType {
+				t.Errorf("%s: GetPodDNSType(%v)=%v, want %v", tc.desc, pod, resType, tc.expectedDNSType)
+			}
+		})
 	}
 }
 
@@ -482,14 +476,6 @@ func TestGetPodDNS(t *testing.T) {
 }
 
 func TestGetPodDNSCustom(t *testing.T) {
-	customDNSEnabled := utilfeature.DefaultFeatureGate.Enabled("CustomPodDNS")
-	defer func() {
-		// Restoring the old value.
-		if err := utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("CustomPodDNS=%v", customDNSEnabled)); err != nil {
-			t.Errorf("Failed to set CustomPodDNS feature gate: %v", err)
-		}
-	}()
-
 	recorder := record.NewFakeRecorder(20)
 	nodeRef := &v1.ObjectReference{
 		Kind:      "Node",
@@ -628,21 +614,21 @@ func TestGetPodDNSCustom(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		if err := utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("CustomPodDNS=%v", tc.customPodDNSFeatureGate)); err != nil {
+		t.Run(tc.desc, func(t *testing.T) {
-			t.Errorf("Failed to set CustomPodDNS feature gate: %v", err)
+			defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CustomPodDNS, tc.customPodDNSFeatureGate)()
-		}
 
-		testPod.Spec.HostNetwork = tc.hostnetwork
+			testPod.Spec.HostNetwork = tc.hostnetwork
-		testPod.Spec.DNSConfig = tc.dnsConfig
+			testPod.Spec.DNSConfig = tc.dnsConfig
-		testPod.Spec.DNSPolicy = tc.dnsPolicy
+			testPod.Spec.DNSPolicy = tc.dnsPolicy
 
-		resDNSConfig, err := configurer.GetPodDNS(testPod)
+			resDNSConfig, err := configurer.GetPodDNS(testPod)
-		if err != nil {
+			if err != nil {
-			t.Errorf("%s: GetPodDNS(%v), unexpected error: %v", tc.desc, testPod, err)
+				t.Errorf("%s: GetPodDNS(%v), unexpected error: %v", tc.desc, testPod, err)
-		}
+			}
-		if !dnsConfigsAreEqual(resDNSConfig, tc.expectedDNSConfig) {
+			if !dnsConfigsAreEqual(resDNSConfig, tc.expectedDNSConfig) {
-			t.Errorf("%s: GetPodDNS(%v)=%v, want %v", tc.desc, testPod, resDNSConfig, tc.expectedDNSConfig)
+				t.Errorf("%s: GetPodDNS(%v)=%v, want %v", tc.desc, testPod, resDNSConfig, tc.expectedDNSConfig)
-		}
+			}
+		})
 	}
 }
 

pkg/kubelet/network/dns/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package dns
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/kubelet/preemption/BUILD

@@ -41,10 +41,7 @@ filegroup(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["preemption_test.go"],
-        "main_test.go",
-        "preemption_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/core:go_default_library",

pkg/kubelet/preemption/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package preemption
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/kubelet/types/BUILD

@@ -32,7 +32,6 @@ go_test(
     name = "go_default_test",
     srcs = [
         "labels_test.go",
-        "main_test.go",
         "pod_status_test.go",
         "pod_update_test.go",
         "types_test.go",

pkg/kubelet/types/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package types
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/kubelet/volumemanager/populator/BUILD

@@ -47,10 +47,7 @@ filegroup(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["desired_state_of_world_populator_test.go"],
-        "desired_state_of_world_populator_test.go",
-        "main_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/features:go_default_library",

pkg/kubelet/volumemanager/populator/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package populator
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/kubelet/volumemanager/reconciler/BUILD

@@ -35,10 +35,7 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["reconciler_test.go"],
-        "main_test.go",
-        "reconciler_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/features:go_default_library",

pkg/kubelet/volumemanager/reconciler/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package reconciler
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/scheduler/BUILD

@@ -40,10 +40,7 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["scheduler_test.go"],
-        "main_test.go",
-        "scheduler_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/api/legacyscheme:go_default_library",

pkg/scheduler/algorithm/predicates/BUILD

@@ -49,7 +49,6 @@ go_test(
     name = "go_default_test",
     srcs = [
         "csi_volume_predicate_test.go",
-        "main_test.go",
         "max_attachable_volume_predicate_test.go",
         "metadata_test.go",
         "predicates_test.go",

pkg/scheduler/algorithm/predicates/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package predicates
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/scheduler/algorithm/priorities/BUILD

@@ -55,7 +55,6 @@ go_test(
         "image_locality_test.go",
         "interpod_affinity_test.go",
         "least_requested_test.go",
-        "main_test.go",
         "metadata_test.go",
         "most_requested_test.go",
         "node_affinity_test.go",

pkg/scheduler/algorithm/priorities/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package priorities
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/scheduler/algorithmprovider/BUILD

@@ -15,10 +15,7 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["plugins_test.go"],
-        "main_test.go",
-        "plugins_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/features:go_default_library",

pkg/scheduler/algorithmprovider/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package algorithmprovider
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/scheduler/internal/cache/BUILD

@@ -26,7 +26,6 @@ go_test(
     name = "go_default_test",
     srcs = [
         "cache_test.go",
-        "main_test.go",
         "node_tree_test.go",
     ],
     embed = [":go_default_library"],

pkg/scheduler/internal/cache/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package cache
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/scheduler/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package scheduler
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/volume/csi/BUILD

@@ -46,7 +46,6 @@ go_test(
         "csi_client_test.go",
         "csi_mounter_test.go",
         "csi_plugin_test.go",
-        "main_test.go",
     ],
     embed = [":go_default_library"],
     deps = [

pkg/volume/csi/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package csi
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

pkg/volume/util/BUILD

@@ -52,7 +52,6 @@ go_test(
         "atomic_writer_test.go",
         "attach_limit_test.go",
         "device_util_linux_test.go",
-        "main_test.go",
         "nested_volumes_test.go",
         "resize_util_test.go",
         "util_test.go",

pkg/volume/util/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package util
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

plugin/pkg/admission/podtolerationrestriction/BUILD

@@ -8,10 +8,7 @@ load(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["admission_test.go"],
-        "admission_test.go",
-        "main_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/core:go_default_library",

plugin/pkg/admission/podtolerationrestriction/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package podtolerationrestriction
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

plugin/pkg/admission/priority/BUILD

@@ -8,10 +8,7 @@ load(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["admission_test.go"],
-        "admission_test.go",
-        "main_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/core:go_default_library",

plugin/pkg/admission/priority/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package priority
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

plugin/pkg/admission/resourcequota/BUILD

@@ -51,10 +51,7 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["admission_test.go"],
-        "admission_test.go",
-        "main_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/core:go_default_library",

plugin/pkg/admission/resourcequota/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package resourcequota
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

plugin/pkg/admission/storage/persistentvolume/label/BUILD

@@ -32,10 +32,7 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["admission_test.go"],
-        "admission_test.go",
-        "main_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/core:go_default_library",

plugin/pkg/admission/storage/persistentvolume/label/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package label
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

plugin/pkg/admission/storage/storageobjectinuseprotection/BUILD

@@ -17,10 +17,7 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["admission_test.go"],
-        "admission_test.go",
-        "main_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/core:go_default_library",

plugin/pkg/admission/storage/storageobjectinuseprotection/main_test.go

@@ -1,29 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package storageobjectinuseprotection
-
-import (
-	"testing"
-
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	_ "k8s.io/kubernetes/pkg/features"
-)
-
-func TestMain(m *testing.M) {
-	utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-}

staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go

@@ -49,7 +49,7 @@ const (
 )
 
 func init() {
-	utilfeature.DefaultFeatureGate.Add(defaultKubernetesFeatureGates)
+	utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates)
 }
 
 // defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys.

staging/src/k8s.io/apiserver/pkg/admission/plugin/initialization/initialization.go

@@ -85,7 +85,7 @@ func (i *initializer) ValidateInitialization() error {
 	}
 
 	if !utilfeature.DefaultFeatureGate.Enabled(features.Initializers) {
-		if err := utilfeature.DefaultFeatureGate.Set(string(features.Initializers) + "=true"); err != nil {
+		if err := utilfeature.DefaultMutableFeatureGate.Set(string(features.Initializers) + "=true"); err != nil {
 			klog.Errorf("error enabling Initializers feature as part of admission plugin setup: %v", err)
 		} else {
 			klog.Infof("enabled Initializers feature as part of admission plugin setup")

staging/src/k8s.io/apiserver/pkg/features/kube_features.go

@@ -91,7 +91,7 @@ const (
 )
 
 func init() {
-	utilfeature.DefaultFeatureGate.Add(defaultKubernetesFeatureGates)
+	utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates)
 }
 
 // defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys.

staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go

@@ -154,5 +154,5 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"handler, which picks a randomized value above this number as the connection timeout, "+
 		"to spread out load.")
 
-	utilfeature.DefaultFeatureGate.AddFlag(fs)
+	utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
 }

staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go

@@ -51,8 +51,15 @@ var (
 		allAlphaGate: setUnsetAlphaGates,
 	}
 
+	// DefaultMutableFeatureGate is a mutable version of DefaultFeatureGate.
+	// Only top-level commands/options setup and the k8s.io/apiserver/pkg/util/feature/testing package should make use of this.
+	// Tests that need to modify feature gates for the duration of their test should use:
+	//   defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.<FeatureName>, <value>)()
+	DefaultMutableFeatureGate MutableFeatureGate = NewFeatureGate()
+
 	// DefaultFeatureGate is a shared global FeatureGate.
-	DefaultFeatureGate FeatureGate = NewFeatureGate()
+	// Top-level commands/options setup that needs to modify this feature gate should use DefaultMutableFeatureGate.
+	DefaultFeatureGate FeatureGate = DefaultMutableFeatureGate
 )
 
 type FeatureSpec struct {
@@ -72,9 +79,23 @@ const (
 	Deprecated = prerelease("DEPRECATED")
 )
 
-// FeatureGate parses and stores flag gates for known features from
+// FeatureGate indicates whether a given feature is enabled or not
-// a string like feature1=true,feature2=false,...
 type FeatureGate interface {
+	// Enabled returns true if the key is enabled.
+	Enabled(key Feature) bool
+	// KnownFeatures returns a slice of strings describing the FeatureGate's known features.
+	KnownFeatures() []string
+	// DeepCopy returns a deep copy of the FeatureGate object, such that gates can be
+	// set on the copy without mutating the original. This is useful for validating
+	// config against potential feature gate changes before committing those changes.
+	DeepCopy() MutableFeatureGate
+}
+
+// MutableFeatureGate parses and stores flag gates for known features from
+// a string like feature1=true,feature2=false,...
+type MutableFeatureGate interface {
+	FeatureGate
+
 	// AddFlag adds a flag for setting global feature gates to the specified FlagSet.
 	AddFlag(fs *pflag.FlagSet)
 	// Set parses and stores flag gates for known features
@@ -82,16 +103,8 @@ type FeatureGate interface {
 	Set(value string) error
 	// SetFromMap stores flag gates for known features from a map[string]bool or returns an error
 	SetFromMap(m map[string]bool) error
-	// Enabled returns true if the key is enabled.
-	Enabled(key Feature) bool
 	// Add adds features to the featureGate.
 	Add(features map[Feature]FeatureSpec) error
-	// KnownFeatures returns a slice of strings describing the FeatureGate's known features.
-	KnownFeatures() []string
-	// DeepCopy returns a deep copy of the FeatureGate object, such that gates can be
-	// set on the copy without mutating the original. This is useful for validating
-	// config against potential feature gate changes before committing those changes.
-	DeepCopy() FeatureGate
 }
 
 // featureGate implements FeatureGate as well as pflag.Value for flag parsing.
@@ -294,7 +307,7 @@ func (f *featureGate) KnownFeatures() []string {
 // DeepCopy returns a deep copy of the FeatureGate object, such that gates can be
 // set on the copy without mutating the original. This is useful for validating
 // config against potential feature gate changes before committing those changes.
-func (f *featureGate) DeepCopy() FeatureGate {
+func (f *featureGate) DeepCopy() MutableFeatureGate {
 	// Copy existing state.
 	known := map[Feature]FeatureSpec{}
 	for k, v := range f.known.Load().(map[Feature]FeatureSpec) {

staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate_test.go

@@ -148,7 +148,7 @@ func TestFeatureGateOverride(t *testing.T) {
 	const testBetaGate Feature = "TestBeta"
 
 	// Don't parse the flag, assert defaults are used.
-	var f FeatureGate = NewFeatureGate()
+	var f *featureGate = NewFeatureGate()
 	f.Add(map[Feature]FeatureSpec{
 		testAlphaGate: {Default: false, PreRelease: Alpha},
 		testBetaGate:  {Default: false, PreRelease: Beta},
@@ -177,7 +177,7 @@ func TestFeatureGateFlagDefaults(t *testing.T) {
 	const testBetaGate Feature = "TestBeta"
 
 	// Don't parse the flag, assert defaults are used.
-	var f FeatureGate = NewFeatureGate()
+	var f *featureGate = NewFeatureGate()
 	f.Add(map[Feature]FeatureSpec{
 		testAlphaGate: {Default: false, PreRelease: Alpha},
 		testBetaGate:  {Default: true, PreRelease: Beta},
@@ -201,7 +201,7 @@ func TestFeatureGateKnownFeatures(t *testing.T) {
 	)
 
 	// Don't parse the flag, assert defaults are used.
-	var f FeatureGate = NewFeatureGate()
+	var f *featureGate = NewFeatureGate()
 	f.Add(map[Feature]FeatureSpec{
 		testAlphaGate:      {Default: false, PreRelease: Alpha},
 		testBetaGate:       {Default: true, PreRelease: Beta},

staging/src/k8s.io/apiserver/pkg/util/feature/testing/feature_gate_testing.go

@@ -18,67 +18,27 @@ package testing
 
 import (
 	"fmt"
-	"os"
-	"strings"
 	"testing"
 
 	"k8s.io/apiserver/pkg/util/feature"
 )
 
-// VerifyFeatureGatesUnchanged ensures the provided gate does not change any values when tests() are completed.
-// Intended to be placed into unit test packages that mess with feature gates.
-//
-// Example use:
-//
-// import (
-//   "testing"
-//
-//   utilfeature "k8s.io/apiserver/pkg/util/feature"
-//   utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-//   _ "k8s.io/kubernetes/pkg/features"
-// )
-//
-// func TestMain(m *testing.M) {
-//   utilfeaturetesting.VerifyFeatureGatesUnchanged(utilfeature.DefaultFeatureGate, m.Run)
-// }
-func VerifyFeatureGatesUnchanged(gate feature.FeatureGate, tests func() int) {
-	originalGates := gate.DeepCopy()
-	originalSet := fmt.Sprint(gate)
-
-	rc := tests()
-
-	finalSet := fmt.Sprint(gate)
-	if finalSet != originalSet {
-		for _, kv := range strings.Split(finalSet, ",") {
-			k := strings.Split(kv, "=")[0]
-			if originalGates.Enabled(feature.Feature(k)) != gate.Enabled(feature.Feature(k)) {
-				fmt.Println(fmt.Sprintf("VerifyFeatureGatesUnchanged: mutated %s feature gate from %v to %v", k, originalGates.Enabled(feature.Feature(k)), gate.Enabled(feature.Feature(k))))
-				rc = 1
-			}
-		}
-	}
-
-	if rc != 0 {
-		os.Exit(rc)
-	}
-}
-
 // SetFeatureGateDuringTest sets the specified gate to the specified value, and returns a function that restores the original value.
 // Failures to set or restore cause the test to fail.
 //
 // Example use:
 //
 // defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.<FeatureName>, true)()
-func SetFeatureGateDuringTest(t *testing.T, gate feature.FeatureGate, feature feature.Feature, value bool) func() {
+func SetFeatureGateDuringTest(t *testing.T, gate feature.FeatureGate, f feature.Feature, value bool) func() {
-	originalValue := gate.Enabled(feature)
+	originalValue := gate.Enabled(f)
 
-	if err := gate.Set(fmt.Sprintf("%s=%v", feature, value)); err != nil {
+	if err := gate.(feature.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, value)); err != nil {
-		t.Errorf("error setting %s=%v: %v", feature, value, err)
+		t.Errorf("error setting %s=%v: %v", f, value, err)
 	}
 
 	return func() {
-		if err := gate.Set(fmt.Sprintf("%s=%v", feature, originalValue)); err != nil {
+		if err := gate.(feature.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, originalValue)); err != nil {
-			t.Errorf("error restoring %s=%v: %v", feature, originalValue, err)
+			t.Errorf("error restoring %s=%v: %v", f, originalValue, err)
 		}
 	}
 }

test/e2e_node/services/kubelet.go

@@ -108,7 +108,7 @@ func (e *E2EServices) startKubelet() (*server, error) {
 	klog.Info("Starting kubelet")
 
 	// set feature gates so we can check which features are enabled and pass the appropriate flags
-	utilfeature.DefaultFeatureGate.SetFromMap(framework.TestContext.FeatureGates)
+	utilfeature.DefaultMutableFeatureGate.SetFromMap(framework.TestContext.FeatureGates)
 
 	// Build kubeconfig
 	kubeconfigPath, err := createKubeconfigCWD()

test/e2e_node/services/services.go

@@ -109,7 +109,7 @@ func (e *E2EServices) Stop() {
 func RunE2EServices(t *testing.T) {
 	// Populate global DefaultFeatureGate with value from TestContext.FeatureGates.
 	// This way, statically-linked components see the same feature gate config as the test context.
-	utilfeature.DefaultFeatureGate.SetFromMap(framework.TestContext.FeatureGates)
+	utilfeature.DefaultMutableFeatureGate.SetFromMap(framework.TestContext.FeatureGates)
 	e := newE2EServices()
 	if err := e.run(t); err != nil {
 		klog.Fatalf("Failed to run e2e services: %v", err)