From 379676c4bef48e5d2add28851302b55b41fcabcf Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Mon, 10 Jun 2024 17:50:22 +0000 Subject: [PATCH] add DefaultComponentGlobalsRegistry flags in ServerRunOptions Signed-off-by: Siyuan Zhang --- cmd/kube-apiserver/app/options/options.go | 6 +- .../app/options/options_test.go | 24 +- cmd/kube-apiserver/app/options/validation.go | 10 + cmd/kube-apiserver/app/server.go | 6 +- cmd/kube-apiserver/app/testing/testserver.go | 6 +- pkg/controlplane/apiserver/config_test.go | 6 +- pkg/controlplane/apiserver/options/options.go | 6 +- .../apiserver/options/options_test.go | 23 +- .../apiserver/options/validation_test.go | 4 +- .../pkg/cmd/server/options/options.go | 6 +- .../pkg/cmd/server/server.go | 9 +- .../pkg/cmd/server/testing/testserver.go | 4 +- .../src/k8s.io/apiserver/pkg/server/config.go | 8 +- .../apiserver/pkg/server/config_test.go | 2 + .../apiserver/pkg/server/genericapiserver.go | 9 +- .../pkg/server/options/api_enablement_test.go | 1 - .../pkg/server/options/server_run_options.go | 50 ++-- .../server/options/server_run_options_test.go | 66 +++--- .../storage/resource_encoding_config.go | 6 +- .../apiserver/pkg/util/version/registry.go | 18 ++ .../apiserver/pkg/util/version/version.go | 9 +- .../pkg/util/version/version_test.go | 19 -- .../featuregate/feature_gate.go | 151 ++++++------ .../featuregate/feature_gate_test.go | 200 ++++++++++++++-- .../featuregate/testing/feature_gate.go | 92 +++++++- .../featuregate/testing/feature_gate_test.go | 220 +++++++++++++++++- .../src/k8s.io/component-base/version/base.go | 7 + .../kube-aggregator/pkg/cmd/server/start.go | 13 +- staging/src/k8s.io/sample-apiserver/go.mod | 2 + staging/src/k8s.io/sample-apiserver/go.sum | 1 + .../sample-apiserver/pkg/cmd/server/start.go | 2 +- test/e2e_node/services/apiserver.go | 6 +- .../integration/etcd/etcd_cross_group_test.go | 8 +- .../etcd/etcd_storage_path_test.go | 7 +- test/integration/etcd/server.go | 6 +- test/integration/framework/test_server.go | 6 +- 36 files changed, 734 insertions(+), 285 deletions(-) diff --git a/cmd/kube-apiserver/app/options/options.go b/cmd/kube-apiserver/app/options/options.go index 53a5b3c06ee..ebed12af1d6 100644 --- a/cmd/kube-apiserver/app/options/options.go +++ b/cmd/kube-apiserver/app/options/options.go @@ -26,8 +26,6 @@ import ( utilnet "k8s.io/apimachinery/pkg/util/net" cliflag "k8s.io/component-base/cli/flag" - utilversion "k8s.io/apiserver/pkg/util/version" - "k8s.io/component-base/featuregate" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/cluster/ports" controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" @@ -66,9 +64,9 @@ type Extra struct { } // NewServerRunOptions creates and returns ServerRunOptions according to the given featureGate and effectiveVersion of the server binary to run. -func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions { +func NewServerRunOptions() *ServerRunOptions { s := ServerRunOptions{ - Options: controlplaneapiserver.NewOptions(featureGate, effectiveVersion), + Options: controlplaneapiserver.NewOptions(), CloudProvider: kubeoptions.NewCloudProviderOptions(), Extra: Extra{ diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index fa53175c9ad..6668fd19054 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -48,17 +48,17 @@ import ( ) func TestAddFlags(t *testing.T) { + componentGlobalsRegistry := utilversion.DefaultComponentGlobalsRegistry + t.Cleanup(func() { + componentGlobalsRegistry.Reset() + }) fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError) - featureGate := featuregate.NewFeatureGate() - componentRegistry := utilversion.NewComponentGlobalsRegistry() - effectiveVersion := utilversion.NewEffectiveVersion("1.32") - utilruntime.Must(componentRegistry.Register("test", effectiveVersion, featureGate)) - s := NewServerRunOptions(featureGate, effectiveVersion) + utilruntime.Must(componentGlobalsRegistry.Register("test", utilversion.NewEffectiveVersion("1.32"), featuregate.NewFeatureGate())) + s := NewServerRunOptions() for _, f := range s.Flags().FlagSets { fs.AddFlagSet(f) } - componentRegistry.AddFlags(fs) args := []string{ "--enable-admission-plugins=AlwaysDeny", @@ -133,7 +133,7 @@ func TestAddFlags(t *testing.T) { "--emulated-version=test=1.31", } fs.Parse(args) - utilruntime.Must(componentRegistry.Set()) + utilruntime.Must(componentGlobalsRegistry.Set()) // This is a snapshot of expected options parsed by args. expected := &ServerRunOptions{ @@ -147,8 +147,8 @@ func TestAddFlags(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024), MaxRequestBodyBytes: int64(3 * 1024 * 1024), - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: componentGlobalsRegistry, + ComponentName: utilversion.DefaultKubeComponent, }, Admission: &kubeoptions.AdmissionOptions{ GenericAdmission: &apiserveroptions.AdmissionOptions{ @@ -350,8 +350,8 @@ func TestAddFlags(t *testing.T) { if !reflect.DeepEqual(expected, s) { t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{}))) } - - if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" { - t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31") + testEffectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor("test") + if testEffectiveVersion.EmulationVersion().String() != "1.31" { + t.Errorf("Got emulation version %s, wanted %s", testEffectiveVersion.EmulationVersion().String(), "1.31") } } diff --git a/cmd/kube-apiserver/app/options/validation.go b/cmd/kube-apiserver/app/options/validation.go index cc64a288635..4c36f2bce17 100644 --- a/cmd/kube-apiserver/app/options/validation.go +++ b/cmd/kube-apiserver/app/options/validation.go @@ -26,6 +26,7 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" netutils "k8s.io/utils/net" + "k8s.io/apimachinery/pkg/util/version" controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" "k8s.io/kubernetes/pkg/controlplane/reconcilers" "k8s.io/kubernetes/pkg/features" @@ -139,5 +140,14 @@ func (s CompletedOptions) Validate() []error { errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount)) } + // TODO: remove in 1.32 + // emulationVersion is introduced in 1.31, so it is only allowed to be equal to the binary version at 1.31. + effectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor(s.GenericServerRunOptions.ComponentName) + binaryVersion := version.MajorMinor(effectiveVersion.BinaryVersion().Major(), effectiveVersion.BinaryVersion().Minor()) + if binaryVersion.EqualTo(version.MajorMinor(1, 31)) && !effectiveVersion.EmulationVersion().EqualTo(binaryVersion) { + errs = append(errs, fmt.Errorf("emulation version needs to be equal to binary version(%s) in compatibility-version alpha, got %s", + binaryVersion.String(), effectiveVersion.EmulationVersion().String())) + } + return errs } diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 9d32d738f84..fad0f1e9579 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -64,9 +64,9 @@ func init() { // NewAPIServerCommand creates a *cobra.Command object with default parameters func NewAPIServerCommand() *cobra.Command { - effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( + _, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( utilversion.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) - s := options.NewServerRunOptions(featureGate, effectiveVersion) + s := options.NewServerRunOptions() cmd := &cobra.Command{ Use: "kube-apiserver", @@ -124,8 +124,6 @@ cluster's shared state through which all other components interact.`, fs := cmd.Flags() namedFlagSets := s.Flags() verflag.AddFlags(namedFlagSets.FlagSet("global")) - utilversion.DefaultComponentGlobalsRegistry.AddFlags(namedFlagSets.FlagSet("global")) - globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags()) options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic")) for _, f := range namedFlagSets.FlagSets { diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go index 0e6411fe869..785724e7448 100644 --- a/cmd/kube-apiserver/app/testing/testserver.go +++ b/cmd/kube-apiserver/app/testing/testserver.go @@ -55,6 +55,7 @@ import ( clientgotransport "k8s.io/client-go/transport" "k8s.io/client-go/util/cert" "k8s.io/client-go/util/keyutil" + featuregatetesting "k8s.io/component-base/featuregate/testing" logsapi "k8s.io/component-base/logs/api/v1" "k8s.io/klog/v2" "k8s.io/kube-aggregator/pkg/apiserver" @@ -187,15 +188,16 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, if instanceOptions.BinaryVersion != "" { effectiveVersion = utilversion.NewEffectiveVersion(instanceOptions.BinaryVersion) } + // need to call SetFeatureGateEmulationVersionDuringTest to reset the feature gate emulation version at the end of the test. + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, featureGate, effectiveVersion.EmulationVersion()) utilversion.DefaultComponentGlobalsRegistry.Reset() utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate)) - s := options.NewServerRunOptions(featureGate, effectiveVersion) + s := options.NewServerRunOptions() for _, f := range s.Flags().FlagSets { fs.AddFlagSet(f) } - utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs) s.SecureServing.Listener, s.SecureServing.BindPort, err = createLocalhostListenerOnFreePort() if err != nil { diff --git a/pkg/controlplane/apiserver/config_test.go b/pkg/controlplane/apiserver/config_test.go index c5ff6a79b83..0e6516e62ca 100644 --- a/pkg/controlplane/apiserver/config_test.go +++ b/pkg/controlplane/apiserver/config_test.go @@ -24,8 +24,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" apiserveroptions "k8s.io/apiserver/pkg/server/options" - utilversion "k8s.io/apiserver/pkg/util/version" - "k8s.io/component-base/featuregate" aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/controlplane/apiserver/options" @@ -34,9 +32,7 @@ import ( ) func TestBuildGenericConfig(t *testing.T) { - featureGate := featuregate.NewFeatureGate() - effectiveVersion := utilversion.DefaultKubeEffectiveVersion() - opts := options.NewOptions(featureGate, effectiveVersion) + opts := options.NewOptions() s := (&apiserveroptions.SecureServingOptions{ BindAddress: netutils.ParseIPSloppy("127.0.0.1"), }).WithLoopback() diff --git a/pkg/controlplane/apiserver/options/options.go b/pkg/controlplane/apiserver/options/options.go index bdaa56ffede..2da89f9555d 100644 --- a/pkg/controlplane/apiserver/options/options.go +++ b/pkg/controlplane/apiserver/options/options.go @@ -28,10 +28,8 @@ import ( peerreconcilers "k8s.io/apiserver/pkg/reconcilers" genericoptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/storage/storagebackend" - utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/util/keyutil" cliflag "k8s.io/component-base/cli/flag" - "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" logsapi "k8s.io/component-base/logs/api/v1" "k8s.io/component-base/metrics" @@ -100,9 +98,9 @@ type CompletedOptions struct { } // NewOptions creates a new ServerRunOptions object with default parameters -func NewOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *Options { +func NewOptions() *Options { s := Options{ - GenericServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion), + GenericServerRunOptions: genericoptions.NewServerRunOptions(), Etcd: genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)), SecureServing: kubeoptions.NewSecureServingOptions(), Audit: genericoptions.NewAuditOptions(), diff --git a/pkg/controlplane/apiserver/options/options_test.go b/pkg/controlplane/apiserver/options/options_test.go index 557983c3679..6e3ea1a52d6 100644 --- a/pkg/controlplane/apiserver/options/options_test.go +++ b/pkg/controlplane/apiserver/options/options_test.go @@ -44,18 +44,18 @@ import ( ) func TestAddFlags(t *testing.T) { + componentGlobalsRegistry := utilversion.DefaultComponentGlobalsRegistry + t.Cleanup(func() { + componentGlobalsRegistry.Reset() + }) fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError) - featureGate := featuregate.NewFeatureGate() - effectiveVersion := utilversion.NewEffectiveVersion("1.32") - componentRegistry := utilversion.NewComponentGlobalsRegistry() - utilruntime.Must(componentRegistry.Register("test", effectiveVersion, featureGate)) - s := NewOptions(featureGate, effectiveVersion) + utilruntime.Must(componentGlobalsRegistry.Register("test", utilversion.NewEffectiveVersion("1.32"), featuregate.NewFeatureGate())) + s := NewOptions() var fss cliflag.NamedFlagSets s.AddFlags(&fss) for _, f := range fss.FlagSets { fs.AddFlagSet(f) } - componentRegistry.AddFlags(fs) args := []string{ "--enable-admission-plugins=AlwaysDeny", @@ -119,7 +119,7 @@ func TestAddFlags(t *testing.T) { "--emulated-version=test=1.31", } fs.Parse(args) - utilruntime.Must(componentRegistry.Set()) + utilruntime.Must(componentGlobalsRegistry.Set()) // This is a snapshot of expected options parsed by args. expected := &Options{ @@ -132,8 +132,8 @@ func TestAddFlags(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024), MaxRequestBodyBytes: int64(3 * 1024 * 1024), - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: componentGlobalsRegistry, + ComponentName: utilversion.DefaultKubeComponent, }, Admission: &kubeoptions.AdmissionOptions{ GenericAdmission: &apiserveroptions.AdmissionOptions{ @@ -305,7 +305,8 @@ func TestAddFlags(t *testing.T) { t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{}))) } - if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" { - t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31") + testEffectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor("test") + if testEffectiveVersion.EmulationVersion().String() != "1.31" { + t.Errorf("Got emulation version %s, wanted %s", testEffectiveVersion.EmulationVersion().String(), "1.31") } } diff --git a/pkg/controlplane/apiserver/options/validation_test.go b/pkg/controlplane/apiserver/options/validation_test.go index 9dddc971a42..518d8a547d2 100644 --- a/pkg/controlplane/apiserver/options/validation_test.go +++ b/pkg/controlplane/apiserver/options/validation_test.go @@ -201,7 +201,7 @@ func TestValidateOptions(t *testing.T) { name: "validate master count equal 0", expectErrors: true, options: &Options{ - GenericServerRunOptions: &genericoptions.ServerRunOptions{FeatureGate: utilfeature.DefaultFeatureGate.DeepCopy(), EffectiveVersion: utilversion.NewEffectiveVersion("1.32")}, + GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry}, Etcd: &genericoptions.EtcdOptions{}, SecureServing: &genericoptions.SecureServingOptionsWithLoopback{}, Audit: &genericoptions.AuditOptions{}, @@ -228,7 +228,7 @@ func TestValidateOptions(t *testing.T) { name: "validate token request enable not attempted", expectErrors: true, options: &Options{ - GenericServerRunOptions: &genericoptions.ServerRunOptions{FeatureGate: utilfeature.DefaultFeatureGate.DeepCopy(), EffectiveVersion: utilversion.NewEffectiveVersion("1.32")}, + GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry}, Etcd: &genericoptions.EtcdOptions{}, SecureServing: &genericoptions.SecureServingOptionsWithLoopback{}, Audit: &genericoptions.AuditOptions{}, diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go index 07fdcb940f5..8ceab525415 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go @@ -39,11 +39,9 @@ import ( flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request" "k8s.io/apiserver/pkg/util/openapi" "k8s.io/apiserver/pkg/util/proxy" - utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/apiserver/pkg/util/webhook" scheme "k8s.io/client-go/kubernetes/scheme" corev1 "k8s.io/client-go/listers/core/v1" - "k8s.io/component-base/featuregate" netutils "k8s.io/utils/net" ) @@ -60,9 +58,9 @@ type CustomResourceDefinitionsServerOptions struct { } // NewCustomResourceDefinitionsServerOptions creates default options of an apiextensions-apiserver. -func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer, featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *CustomResourceDefinitionsServerOptions { +func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer) *CustomResourceDefinitionsServerOptions { o := &CustomResourceDefinitionsServerOptions{ - ServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion), + ServerRunOptions: genericoptions.NewServerRunOptions(), RecommendedOptions: genericoptions.NewRecommendedOptions( defaultEtcdPathPrefix, apiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion), diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go index 6aa64f0f071..39be3d19a0e 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go @@ -24,17 +24,11 @@ import ( "k8s.io/apiextensions-apiserver/pkg/cmd/server/options" genericapiserver "k8s.io/apiserver/pkg/server" - utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" ) func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command { - // effectiveVersion is used to set what apis and feature gates the generic api server is compatible with. - // You can also have the flag setting the effectiveVersion of the apiextensions apiserver, and - // having a mapping from the apiextensions apiserver version to generic apiserver version. - effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( - utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) - o := options.NewCustomResourceDefinitionsServerOptions(out, errOut, featureGate, effectiveVersion) + o := options.NewCustomResourceDefinitionsServerOptions(out, errOut) cmd := &cobra.Command{ Short: "Launch an API extensions API server", @@ -58,7 +52,6 @@ func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command cmd.SetContext(ctx) fs := cmd.Flags() - utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs) o.AddFlags(fs) return cmd } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go index ef5fce2fead..4c4156d14be 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go @@ -127,9 +127,7 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin effectiveVersion := utilversion.DefaultKubeEffectiveVersion() utilversion.DefaultComponentGlobalsRegistry.Reset() utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate)) - s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr, featureGate, effectiveVersion) - - utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs) + s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr) s.AddFlags(fs) s.RecommendedOptions.SecureServing.Listener, s.RecommendedOptions.SecureServing.BindPort, err = createLocalhostListenerOnFreePort() diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index 931226bb678..6f0ca1bcac4 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -832,7 +832,7 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{}, } - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) { + if c.FeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) { manager := c.AggregatedDiscoveryGroupManager if manager == nil { manager = discoveryendpoint.NewResourceManager("apis") @@ -1047,14 +1047,14 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler { handler = genericfilters.WithRetryAfter(handler, c.lifecycleSignals.NotAcceptingNewRequest.Signaled()) } handler = genericfilters.WithHTTPLogging(handler) - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) { + if c.FeatureGate.Enabled(genericfeatures.APIServerTracing) { handler = genericapifilters.WithTracing(handler, c.TracerProvider) } handler = genericapifilters.WithLatencyTrackers(handler) // WithRoutine will execute future handlers in a separate goroutine and serving // handler in current goroutine to minimize the stack memory usage. It must be // after WithPanicRecover() to be protected from panics. - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServingWithRoutine) { + if c.FeatureGate.Enabled(genericfeatures.APIServingWithRoutine) { handler = genericfilters.WithRoutine(handler, c.LongRunningFunc) } handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver) @@ -1098,7 +1098,7 @@ func installAPI(s *GenericAPIServer, c *Config) { routes.Version{Version: c.EffectiveVersion.BinaryVersion().Info()}.Install(s.Handler.GoRestfulContainer) if c.EnableDiscovery { - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) { + if c.FeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) { wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(s.DiscoveryGroupManager, s.AggregatedDiscoveryGroupManager) s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/apis", metav1.APIGroupList{})) } else { diff --git a/staging/src/k8s.io/apiserver/pkg/server/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/config_test.go index 6d25272b0a8..a7f76d0142e 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config_test.go @@ -40,6 +40,7 @@ import ( "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/server/healthz" + utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" @@ -308,6 +309,7 @@ func TestAuthenticationAuditAnnotationsDefaultChain(t *testing.T) { LongRunningFunc: func(_ *http.Request, _ *request.RequestInfo) bool { return false }, lifecycleSignals: newLifecycleSignals(), TracerProvider: tracing.NewNoopTracerProvider(), + FeatureGate: utilfeature.DefaultFeatureGate, } h := DefaultBuildHandlerChain(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go index 7b8e13da256..fed94bf4abd 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go @@ -51,7 +51,6 @@ import ( "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/server/routes" "k8s.io/apiserver/pkg/storageversion" - utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" restclient "k8s.io/client-go/rest" "k8s.io/component-base/featuregate" @@ -780,7 +779,7 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A } resourceInfos = append(resourceInfos, r...) - if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) { + if s.FeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) { // Aggregated discovery only aggregates resources under /apis if apiPrefix == APIGroupPrefix { s.AggregatedDiscoveryGroupManager.AddGroupVersion( @@ -808,8 +807,8 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A s.RegisterDestroyFunc(apiGroupInfo.destroyStorage) - if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionAPI) && - utilfeature.DefaultFeatureGate.Enabled(features.APIServerIdentity) { + if s.FeatureGate.Enabled(features.StorageVersionAPI) && + s.FeatureGate.Enabled(features.APIServerIdentity) { // API installation happens before we start listening on the handlers, // therefore it is safe to register ResourceInfos here. The handler will block // write requests until the storage versions of the targeting resources are updated. @@ -839,7 +838,7 @@ func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo // Install the version handler. // Add a handler at / to enumerate the supported api versions. legacyRootAPIHandler := discovery.NewLegacyRootAPIHandler(s.discoveryAddresses, s.Serializer, apiPrefix) - if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) { + if s.FeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) { wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(legacyRootAPIHandler, s.AggregatedLegacyDiscoveryGroupManager) s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/api", metav1.APIVersions{})) } else { diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement_test.go index 7d60244004d..a14319e5373 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement_test.go @@ -33,7 +33,6 @@ func (f fakeGroupRegistry) IsGroupRegistered(group string) bool { func TestAPIEnablementOptionsValidate(t *testing.T) { testCases := []struct { name string - testOptions *APIEnablementOptions runtimeConfig cliflag.ConfigurationMap expectErr string }{ diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go index 88dc3802f7f..593d591638e 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go @@ -25,9 +25,10 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/server" + utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" - "k8s.io/component-base/featuregate" "github.com/spf13/pflag" ) @@ -91,12 +92,23 @@ type ServerRunOptions struct { // it is not overridden by any other grace period. ShutdownWatchTerminationGracePeriod time.Duration - // FeatureGate are the featuregate to install on the CLI - FeatureGate featuregate.FeatureGate - EffectiveVersion utilversion.EffectiveVersion + // ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored. + ComponentGlobalsRegistry utilversion.ComponentGlobalsRegistry + // ComponentName is name under which the server's global variabled are registered in the ComponentGlobalsRegistry. + ComponentName string } -func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions { +func NewServerRunOptions() *ServerRunOptions { + if utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) == nil { + featureGate := utilfeature.DefaultMutableFeatureGate + effectiveVersion := utilversion.DefaultKubeEffectiveVersion() + utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate)) + } + + return NewServerRunOptionsForComponent(utilversion.DefaultKubeComponent, utilversion.DefaultComponentGlobalsRegistry) +} + +func NewServerRunOptionsForComponent(componentName string, componentGlobalsRegistry utilversion.ComponentGlobalsRegistry) *ServerRunOptions { defaults := server.NewConfig(serializer.CodecFactory{}) return &ServerRunOptions{ MaxRequestsInFlight: defaults.MaxRequestsInFlight, @@ -109,13 +121,16 @@ func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion u JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes, MaxRequestBodyBytes: defaults.MaxRequestBodyBytes, ShutdownSendRetryAfter: false, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentName: componentName, + ComponentGlobalsRegistry: componentGlobalsRegistry, } } // ApplyTo applies the run options to the method receiver and returns self func (s *ServerRunOptions) ApplyTo(c *server.Config) error { + if err := s.ComponentGlobalsRegistry.SetFallback(); err != nil { + return err + } c.CorsAllowedOriginList = s.CorsAllowedOriginList c.HSTSDirectives = s.HSTSDirectives c.ExternalAddress = s.ExternalHost @@ -131,8 +146,8 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error { c.PublicAddress = s.AdvertiseAddress c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter c.ShutdownWatchTerminationGracePeriod = s.ShutdownWatchTerminationGracePeriod - c.EffectiveVersion = s.EffectiveVersion - c.FeatureGate = s.FeatureGate + c.EffectiveVersion = s.ComponentGlobalsRegistry.EffectiveVersionFor(s.ComponentName) + c.FeatureGate = s.ComponentGlobalsRegistry.FeatureGateFor(s.ComponentName) return nil } @@ -205,12 +220,7 @@ func (s *ServerRunOptions) Validate() []error { if err := validateCorsAllowedOriginList(s.CorsAllowedOriginList); err != nil { errors = append(errors, err) } - if s.FeatureGate != nil { - if errs := s.FeatureGate.Validate(); len(errs) != 0 { - errors = append(errors, errs...) - } - } - if errs := s.EffectiveVersion.Validate(); len(errs) != 0 { + if errs := s.ComponentGlobalsRegistry.Validate(); len(errs) != 0 { errors = append(errors, errs...) } return errors @@ -353,15 +363,11 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { fs.DurationVar(&s.ShutdownWatchTerminationGracePeriod, "shutdown-watch-termination-grace-period", s.ShutdownWatchTerminationGracePeriod, ""+ "This option, if set, represents the maximum amount of grace period the apiserver will wait "+ "for active watch request(s) to drain during the graceful server shutdown window.") + + s.ComponentGlobalsRegistry.AddFlags(fs) } // Complete fills missing fields with defaults. func (s *ServerRunOptions) Complete() error { - if s.FeatureGate == nil { - return fmt.Errorf("nil FeatureGate in ServerRunOptions") - } - if s.EffectiveVersion == nil { - return fmt.Errorf("nil EffectiveVersion in ServerRunOptions") - } - return nil + return s.ComponentGlobalsRegistry.SetFallback() } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go index 0c62a4d6893..7bfcf0245f1 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go @@ -23,14 +23,21 @@ import ( "time" utilerrors "k8s.io/apimachinery/pkg/util/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/version" utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" netutils "k8s.io/utils/net" ) func TestServerRunOptionsValidate(t *testing.T) { + testRegistry := utilversion.NewComponentGlobalsRegistry() featureGate := utilfeature.DefaultFeatureGate.DeepCopy() effectiveVersion := utilversion.NewEffectiveVersion("1.30") + effectiveVersion.SetEmulationVersion(version.MajorMinor(1, 32)) + testComponent := "test" + utilruntime.Must(testRegistry.Register(testComponent, effectiveVersion, featureGate)) + testCases := []struct { name string testOptions *ServerRunOptions @@ -47,8 +54,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--max-requests-inflight can not be negative value", }, @@ -63,8 +69,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--max-mutating-requests-inflight can not be negative value", }, @@ -79,8 +84,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--request-timeout can not be negative value", }, @@ -95,8 +99,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: -1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--min-request-timeout can not be negative value", }, @@ -111,8 +114,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: -10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value", }, @@ -127,8 +129,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: -10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "ServerRunOptions.MaxRequestBodyBytes can not be negative value", }, @@ -144,8 +145,7 @@ func TestServerRunOptionsValidate(t *testing.T) { JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, LivezGracePeriod: -time.Second, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--livez-grace-period can not be a negative value", }, @@ -161,8 +161,7 @@ func TestServerRunOptionsValidate(t *testing.T) { JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, ShutdownDelayDuration: -time.Second, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--shutdown-delay-duration can not be negative value", }, @@ -178,11 +177,27 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--strict-transport-security-directives invalid, allowed values: max-age=expireTime, includeSubDomains, preload. see https://tools.ietf.org/html/rfc6797#section-6.1 for more information", }, + { + name: "Test when emulation version is invalid", + testOptions: &ServerRunOptions{ + AdvertiseAddress: netutils.ParseIPSloppy("192.168.10.10"), + CorsAllowedOriginList: []string{"^10.10.10.100$", "^10.10.10.200$"}, + HSTSDirectives: []string{"max-age=31536000", "includeSubDomains", "preload"}, + MaxRequestsInFlight: 400, + MaxMutatingRequestsInFlight: 200, + RequestTimeout: time.Duration(2) * time.Minute, + MinRequestTimeout: 1800, + JSONPatchMaxCopyBytes: 10 * 1024 * 1024, + MaxRequestBodyBytes: 10 * 1024 * 1024, + ComponentName: testComponent, + ComponentGlobalsRegistry: testRegistry, + }, + expectErr: "emulation version 1.32 is not between [1.29, 1.30.0]", + }, { name: "Test when ServerRunOptions is valid", testOptions: &ServerRunOptions{ @@ -195,8 +210,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, }, } @@ -216,8 +230,6 @@ func TestServerRunOptionsValidate(t *testing.T) { } func TestValidateCorsAllowedOriginList(t *testing.T) { - featureGate := utilfeature.DefaultFeatureGate.DeepCopy() - effectiveVersion := utilversion.NewEffectiveVersion("1.30") tests := []struct { regexp [][]string errShouldContain string @@ -265,7 +277,7 @@ func TestValidateCorsAllowedOriginList(t *testing.T) { for _, test := range tests { for _, regexp := range test.regexp { t.Run(fmt.Sprintf("regexp/%s", regexp), func(t *testing.T) { - options := NewServerRunOptions(featureGate, effectiveVersion) + options := NewServerRunOptions() if errs := options.Validate(); len(errs) != 0 { t.Fatalf("wrong test setup: %#v", errs) } @@ -289,8 +301,6 @@ func TestValidateCorsAllowedOriginList(t *testing.T) { } func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) { - featureGate := utilfeature.DefaultFeatureGate.DeepCopy() - effectiveVersion := utilversion.NewEffectiveVersion("1.30") tests := []struct { name string optionsFn func() *ServerRunOptions @@ -299,13 +309,13 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) { { name: "default should be valid", optionsFn: func() *ServerRunOptions { - return NewServerRunOptions(featureGate, effectiveVersion) + return NewServerRunOptions() }, }, { name: "negative not allowed", optionsFn: func() *ServerRunOptions { - o := NewServerRunOptions(featureGate, effectiveVersion) + o := NewServerRunOptions() o.ShutdownWatchTerminationGracePeriod = -time.Second return o }, @@ -332,7 +342,7 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) { } t.Run("default should be zero", func(t *testing.T) { - options := NewServerRunOptions(featureGate, effectiveVersion) + options := NewServerRunOptions() if options.ShutdownWatchTerminationGracePeriod != time.Duration(0) { t.Errorf("expected default of ShutdownWatchTerminationGracePeriod to be zero, but got: %s", options.ShutdownWatchTerminationGracePeriod) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go b/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go index 7339d17df50..d73c8e62c61 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go @@ -181,9 +181,9 @@ func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example // was being served which didn't exist at all in min-compatibility-version. // // In the alpha case - we do not support compatibility versioning of - // alpha types and recommend users do not mix the two. + // alpha types and recommend users do not mix the two. // In the skip-level case - The version of apiserver we are retaining - // compatibility with has no knowledge of the type, - // so storing it in another type is no issue. + // compatibility with has no knowledge of the type, + // so storing it in another type is no issue. return binaryVersionOfResource, nil } diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/registry.go b/staging/src/k8s.io/apiserver/pkg/util/version/registry.go index 170ac28045e..e6655277c0e 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/registry.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/registry.go @@ -101,6 +101,8 @@ type ComponentGlobalsRegistry interface { AddFlags(fs *pflag.FlagSet) // Set sets the flags for all global variables for all components registered. Set() error + // SetFallback calls Set() if it has never been called. + SetFallback() error // Validate calls the Validate() function for all the global variables for all components registered. Validate() []error // Reset removes all stored ComponentGlobals, configurations, and version mappings. @@ -120,6 +122,8 @@ type componentGlobalsRegistry struct { emulationVersionConfig []string // map of component name to the list of feature gates set from the flag. featureGatesConfig map[string][]string + // set stores if the Set() function for the registry is already called. + set bool } func NewComponentGlobalsRegistry() *componentGlobalsRegistry { @@ -136,6 +140,7 @@ func (r *componentGlobalsRegistry) Reset() { r.componentGlobals = make(map[string]*ComponentGlobals) r.emulationVersionConfig = nil r.featureGatesConfig = nil + r.set = false } func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion { @@ -330,9 +335,22 @@ func toVersionMap(versionConfig []string) (map[string]*version.Version, error) { return m, nil } +func (r *componentGlobalsRegistry) SetFallback() error { + r.mutex.Lock() + set := r.set + r.mutex.Unlock() + if set { + return nil + } + klog.Warning("setting componentGlobalsRegistry in SetFallback. We recommend calling componentGlobalsRegistry.Set()" + + " right after parsing flags to avoid using feature gates before their final values are set by the flags.") + return r.Set() +} + func (r *componentGlobalsRegistry) Set() error { r.mutex.Lock() defer r.mutex.Unlock() + r.set = true emulationVersionConfigMap, err := toVersionMap(r.emulationVersionConfig) if err != nil { return err diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/version.go b/staging/src/k8s.io/apiserver/pkg/util/version/version.go index 1596aef389b..a7a5fda87c7 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/version.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/version.go @@ -111,12 +111,6 @@ func (m *effectiveVersion) Validate() []error { // emulationVersion can only be 1.{binaryMinor-1}...1.{binaryMinor}. maxEmuVer := binaryVersion minEmuVer := binaryVersion.SubtractMinor(1) - // TODO: remove in 1.32 - // emulationVersion is introduced in 1.31, so it cannot be lower than that. - // binaryVersion could be lower than 1.31 in tests. So we are only checking 1.31. - if binaryVersion.EqualTo(version.MajorMinor(1, 31)) { - minEmuVer = version.MajorMinor(1, 31) - } if emulationVersion.GreaterThan(maxEmuVer) || emulationVersion.LessThan(minEmuVer) { errs = append(errs, fmt.Errorf("emulation version %s is not between [%s, %s]", emulationVersion.String(), minEmuVer.String(), maxEmuVer.String())) } @@ -157,8 +151,7 @@ func DefaultBuildEffectiveVersion() MutableEffectiveVersion { // DefaultKubeEffectiveVersion returns the MutableEffectiveVersion based on the // latest K8s release. -// Should update for each minor release! func DefaultKubeEffectiveVersion() MutableEffectiveVersion { - binaryVersion := version.MustParse("1.31").WithInfo(baseversion.Get()) + binaryVersion := version.MustParse(baseversion.DefaultKubeBinaryVersion).WithInfo(baseversion.Get()) return newEffectiveVersion(binaryVersion) } diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go b/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go index 784e3e4e186..aff8a8e4a0b 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go @@ -42,25 +42,6 @@ func TestValidate(t *testing.T) { emulationVersion: "v1.31.0", minCompatibilityVersion: "v1.31.0", }, - { - name: "binary version 1.31, emulation version lower than 1.31", - binaryVersion: "v1.31.2", - emulationVersion: "v1.30.0", - minCompatibilityVersion: "v1.30.0", - expectErrors: true, - }, - { - name: "binary version 1.31, emulation version 1.31", - binaryVersion: "v1.31.2", - emulationVersion: "v1.31.0", - minCompatibilityVersion: "v1.30.0", - }, - { - name: "binary version lower than 1.31", - binaryVersion: "v1.30.2", - emulationVersion: "v1.29.0", - minCompatibilityVersion: "v1.29.0", - }, { name: "emulation version two minor lower than binary not ok", binaryVersion: "v1.33.2", diff --git a/staging/src/k8s.io/component-base/featuregate/feature_gate.go b/staging/src/k8s.io/component-base/featuregate/feature_gate.go index 680d92d3d83..e3ca5e91dae 100644 --- a/staging/src/k8s.io/component-base/featuregate/feature_gate.go +++ b/staging/src/k8s.io/component-base/featuregate/feature_gate.go @@ -114,6 +114,9 @@ type FeatureGate interface { // set on the copy without mutating the original. This is useful for validating // config against potential feature gate changes before committing those changes. DeepCopy() MutableVersionedFeatureGate + // CopyKnownFeatures returns a partial copy of the FeatureGate object, with all the known features and overrides. + // This is useful for creating a new instance of feature gate without inheriting all the enabled configurations of the base feature gate. + CopyKnownFeatures() MutableVersionedFeatureGate // Validate checks if the flag gates are valid at the emulated version. Validate() []error } @@ -127,9 +130,6 @@ type MutableFeatureGate interface { AddFlag(fs *pflag.FlagSet) // Close sets closed to true, and prevents subsequent calls to Add Close() - // OpenForModification sets closedForModification to false, and allows subsequent calls to SetEmulationVersion to change enabled features - // before the next Enabled is called. - OpenForModification() // Set parses and stores flag gates for known features // from a string like feature1=true,feature2=false,... Set(value string) error @@ -166,8 +166,6 @@ type MutableVersionedFeatureGate interface { // SetEmulationVersion overrides the emulationVersion of the feature gate. // Otherwise, the emulationVersion will be the same as the binary version. // If set, the feature defaults and availability will be as if the binary is at the emulated version. - // Returns error if the new emulationVersion will change the enablement state of a feature that has already been queried. - // If you have to use featureGate.Enabled before parsing the flags, call featureGate.OpenForModification following featureGate.Enabled. SetEmulationVersion(emulationVersion *version.Version) error // GetAll returns a copy of the map of known feature names to versioned feature specs. GetAllVersioned() map[Feature]VersionedSpecs @@ -185,15 +183,11 @@ type MutableVersionedFeatureGate interface { // overriding its default to true for a limited number of components without simultaneously // changing its default for all consuming components. OverrideDefaultAtVersion(name Feature, override bool, ver *version.Version) error -} - -// MutableVersionedFeatureGateForTests is a feature gate interface that should only be used in tests. -type MutableVersionedFeatureGateForTests interface { - MutableVersionedFeatureGate - // Reset sets the enabled and enabledRaw to the input map. - Reset(m map[string]bool) - // EnabledRawMap returns the raw enable map from the feature gate. - EnabledRawMap() map[string]bool + // ExplicitlySet returns true if the feature value is explicitly set instead of + // being derived from the default values or special features. + ExplicitlySet(name Feature) bool + // ResetFeatureValueToDefault resets the value of the feature back to the default value. + ResetFeatureValueToDefault(name Feature) error } // featureGate implements FeatureGate as well as pflag.Value for flag parsing. @@ -214,12 +208,8 @@ type featureGate struct { enabledRaw atomic.Value // closed is set to true when AddFlag is called, and prevents subsequent calls to Add closed bool - // closedForModification is set to true when Enabled is called, and prevents subsequent calls to SetEmulationVersion to change the enabled features. - // TODO: after all feature gates have migrated to versioned feature gates, - // closedForModification should also prevents subsequent calls to Set and SetFromMap to change the enabled features - closedForModification atomic.Bool // queriedFeatures stores all the features that have been queried through the Enabled interface. - // It is reset when closedForModification is reset. + // It is reset when SetEmulationVersion is called. queriedFeatures atomic.Value emulationVersion atomic.Pointer[version.Version] } @@ -282,7 +272,7 @@ func NewVersionedFeatureGate(emulationVersion *version.Version) *featureGate { // NewFeatureGate creates a feature gate with the current binary version. func NewFeatureGate() *featureGate { - binaryVersison := version.MustParse(baseversion.Get().String()) + binaryVersison := version.MustParse(baseversion.DefaultKubeBinaryVersion) return NewVersionedFeatureGate(binaryVersison) } @@ -311,6 +301,8 @@ func (f *featureGate) Set(value string) error { // Validate checks if the flag gates are valid at the emulated version. func (f *featureGate) Validate() []error { + f.lock.Lock() + defer f.lock.Unlock() m, ok := f.enabledRaw.Load().(map[string]bool) if !ok { return []error{fmt.Errorf("cannot cast enabledRaw to map[string]bool")} @@ -497,13 +489,17 @@ func (f *featureGate) OverrideDefaultAtVersion(name Feature, override bool, ver // GetAll returns a copy of the map of known feature names to feature specs for the current emulationVersion. func (f *featureGate) GetAll() map[Feature]FeatureSpec { retval := map[Feature]FeatureSpec{} - for k, v := range f.GetAllVersioned() { - spec := f.featureSpecAtEmulationVersion(v) + f.lock.Lock() + versionedSpecs := f.GetAllVersioned() + emuVer := f.EmulationVersion() + f.lock.Unlock() + for k, v := range versionedSpecs { + spec := featureSpecAtEmulationVersion(v, emuVer) if spec.PreRelease == PreAlpha { // The feature is not available at the emulation version. continue } - retval[k] = *f.featureSpecAtEmulationVersion(v) + retval[k] = *spec } return retval } @@ -518,6 +514,9 @@ func (f *featureGate) GetAllVersioned() map[Feature]VersionedSpecs { } func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) error { + if emulationVersion.EqualTo(f.EmulationVersion()) { + return nil + } f.lock.Lock() defer f.lock.Unlock() klog.V(1).Infof("set feature gate emulationVersion to %s", emulationVersion.String()) @@ -531,22 +530,21 @@ func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) err enabled := map[Feature]bool{} errs := f.unsafeSetFromMap(enabled, enabledRaw, emulationVersion) - if f.closedForModification.Load() { - queriedFeatures := f.queriedFeatures.Load().(map[Feature]struct{}) - known := f.known.Load().(map[Feature]VersionedSpecs) - for feature := range queriedFeatures { - newVal := featureEnabled(feature, enabled, known, emulationVersion) - oldVal := f.Enabled(feature) - // it is ok to modify emulation version if it does not result in feature enablemennt change for features that have already been queried. - if newVal != oldVal { - errs = append(errs, fmt.Errorf("SetEmulationVersion will change already queried feature:%s from %v to %v\ncall featureGate.OpenForModification() first to override", feature, oldVal, newVal)) - } + queriedFeatures := f.queriedFeatures.Load().(map[Feature]struct{}) + known := f.known.Load().(map[Feature]VersionedSpecs) + for feature := range queriedFeatures { + newVal := featureEnabled(feature, enabled, known, emulationVersion) + oldVal := featureEnabled(feature, f.enabled.Load().(map[Feature]bool), known, f.EmulationVersion()) + if newVal != oldVal { + klog.Warningf("SetEmulationVersion will change already queried feature:%s from %v to %v", feature, oldVal, newVal) } } + if len(errs) == 0 { // Persist changes f.enabled.Store(enabled) f.emulationVersion.Store(emulationVersion) + f.queriedFeatures.Store(map[Feature]struct{}{}) } return utilerrors.NewAggregate(errs) } @@ -555,9 +553,9 @@ func (f *featureGate) EmulationVersion() *version.Version { return f.emulationVersion.Load() } -// FeatureSpec returns the FeatureSpec at the EmulationVersion if the key exists, an error otherwise. +// featureSpec returns the featureSpec at the EmulationVersion if the key exists, an error otherwise. // This is useful to keep multiple implementations of a feature based on the PreRelease or Version info. -func (f *featureGate) FeatureSpec(key Feature) (FeatureSpec, error) { +func (f *featureGate) featureSpec(key Feature) (FeatureSpec, error) { if v, ok := f.known.Load().(map[Feature]VersionedSpecs)[key]; ok { featureSpec := f.featureSpecAtEmulationVersion(v) return *featureSpec, nil @@ -565,14 +563,16 @@ func (f *featureGate) FeatureSpec(key Feature) (FeatureSpec, error) { return FeatureSpec{}, fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName) } -func (f *featureGate) recordQueried(key Feature) { +func (f *featureGate) unsafeRecordQueried(key Feature) { queriedFeatures := map[Feature]struct{}{} for k := range f.queriedFeatures.Load().(map[Feature]struct{}) { queriedFeatures[k] = struct{}{} } + if _, ok := queriedFeatures[key]; ok { + return + } queriedFeatures[key] = struct{}{} f.queriedFeatures.Store(queriedFeatures) - f.closedForModification.Store(true) } func featureEnabled(key Feature, enabled map[Feature]bool, known map[Feature]VersionedSpecs, emulationVersion *version.Version) bool { @@ -589,8 +589,9 @@ func featureEnabled(key Feature, enabled map[Feature]bool, known map[Feature]Ver // Enabled returns true if the key is enabled. If the key is not known, this call will panic. func (f *featureGate) Enabled(key Feature) bool { + // TODO: ideally we should lock the feature gate in this call to be safe, need to evaluate how much performance impact locking would have. v := featureEnabled(key, f.enabled.Load().(map[Feature]bool), f.known.Load().(map[Feature]VersionedSpecs), f.EmulationVersion()) - f.recordQueried(key) + f.unsafeRecordQueried(key) return v } @@ -620,18 +621,6 @@ func (f *featureGate) Close() { f.lock.Unlock() } -// OpenForModification sets closedForModification to false, and allows subsequent calls to SetEmulationVersion to change enabled features -// before the next Enabled is called. -func (f *featureGate) OpenForModification() { - queriedFeatures := []Feature{} - for feature := range f.queriedFeatures.Load().(map[Feature]struct{}) { - queriedFeatures = append(queriedFeatures, feature) - } - klog.Warningf("open feature gate for modification after querying features: %v.", queriedFeatures) - f.closedForModification.Store(false) - f.queriedFeatures.Store(map[Feature]struct{}{}) -} - // AddFlag adds a flag for setting global feature gates to the specified FlagSet. func (f *featureGate) AddFlag(fs *pflag.FlagSet) { // TODO(mtaufen): Shouldn't we just close it on the first Set/SetFromMap instead? @@ -671,10 +660,21 @@ func (f *featureGate) KnownFeatures() []string { return known } +// CopyKnownFeatures returns a partial copy of the FeatureGate object, with all the known features and overrides. +// This is useful for creating a new instance of feature gate without inheriting all the enabled configurations of the base feature gate. +func (f *featureGate) CopyKnownFeatures() MutableVersionedFeatureGate { + fg := NewVersionedFeatureGate(f.EmulationVersion()) + known := f.GetAllVersioned() + fg.known.Store(known) + return fg +} + // DeepCopy returns a deep copy of the FeatureGate object, such that gates can be // set on the copy without mutating the original. This is useful for validating // config against potential feature gate changes before committing those changes. func (f *featureGate) DeepCopy() MutableVersionedFeatureGate { + f.lock.Lock() + defer f.lock.Unlock() // Copy existing state. known := map[Feature]VersionedSpecs{} for k, v := range f.known.Load().(map[Feature]VersionedSpecs) { @@ -691,7 +691,7 @@ func (f *featureGate) DeepCopy() MutableVersionedFeatureGate { // Construct a new featureGate around the copied state. // Note that specialFeatures is treated as immutable by convention, - // and we maintain the value of f.closed across the copy, but resets closedForModification. + // and we maintain the value of f.closed across the copy. fg := &featureGate{ special: specialFeatures, closed: f.closed, @@ -704,21 +704,40 @@ func (f *featureGate) DeepCopy() MutableVersionedFeatureGate { return fg } -// Reset sets the enabled and enabledRaw to the input map. -func (f *featureGate) Reset(m map[string]bool) { - enabled := map[Feature]bool{} - enabledRaw := map[string]bool{} - queriedFeatures := map[Feature]struct{}{} - f.enabled.Store(enabled) - f.enabledRaw.Store(enabledRaw) - _ = f.SetFromMap(m) - f.closedForModification.Store(false) - f.queriedFeatures.Store(queriedFeatures) - f.lock.Lock() - defer f.lock.Unlock() - f.closed = false +// ExplicitlySet returns true if the feature value is explicitly set instead of +// being derived from the default values or special features. +func (f *featureGate) ExplicitlySet(name Feature) bool { + enabledRaw := f.enabledRaw.Load().(map[string]bool) + _, ok := enabledRaw[string(name)] + return ok } -func (f *featureGate) EnabledRawMap() map[string]bool { - return f.enabledRaw.Load().(map[string]bool) +// ResetFeatureValueToDefault resets the value of the feature back to the default value. +func (f *featureGate) ResetFeatureValueToDefault(name Feature) error { + f.lock.Lock() + defer f.lock.Unlock() + enabled := map[Feature]bool{} + for k, v := range f.enabled.Load().(map[Feature]bool) { + enabled[k] = v + } + enabledRaw := map[string]bool{} + for k, v := range f.enabledRaw.Load().(map[string]bool) { + enabledRaw[k] = v + } + _, inEnabled := enabled[name] + if inEnabled { + delete(enabled, name) + } + _, inEnabledRaw := enabledRaw[string(name)] + if inEnabledRaw { + delete(enabledRaw, string(name)) + } + // some features could be in enabled map but not enabledRaw map, + // for example some Alpha feature when AllAlpha is set. + if inEnabledRaw && !inEnabled { + return fmt.Errorf("feature:%s was explicitly set, but not in enabled map", name) + } + f.enabled.Store(enabled) + f.enabledRaw.Store(enabledRaw) + return nil } diff --git a/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go b/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go index 85f99d25787..cd6eeb83ecb 100644 --- a/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go +++ b/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go @@ -596,6 +596,19 @@ func TestFeatureGateOverrideDefault(t *testing.T) { } }) + t.Run("overrides are preserved across CopyKnownFeatures", func(t *testing.T) { + f := NewFeatureGate() + require.NoError(t, f.Add(map[Feature]FeatureSpec{"TestFeature": {Default: false}})) + require.NoError(t, f.OverrideDefault("TestFeature", true)) + fcopy := f.CopyKnownFeatures() + if !f.Enabled("TestFeature") { + t.Error("TestFeature should be enabled by override") + } + if !fcopy.Enabled("TestFeature") { + t.Error("default override was not preserved by CopyKnownFeatures") + } + }) + t.Run("reflected in known features", func(t *testing.T) { f := NewFeatureGate() if err := f.Add(map[Feature]FeatureSpec{"TestFeature": { @@ -1119,22 +1132,22 @@ func TestVersionedFeatureGateFlagDefaults(t *testing.T) { if f.Enabled(testAlphaGate) != false { t.Errorf("Expected false") } - if fs, _ := f.FeatureSpec(testAlphaGate); fs.PreRelease != PreAlpha || fs.Version.String() != "0.0" { + if fs, _ := f.featureSpec(testAlphaGate); fs.PreRelease != PreAlpha || fs.Version.String() != "0.0" { t.Errorf("Expected (PreAlpha, 0.0)") } if f.Enabled(testBetaGate) != false { t.Errorf("Expected false") } - if fs, _ := f.FeatureSpec(testBetaGate); fs.PreRelease != Beta || fs.Version.String() != "1.28" { + if fs, _ := f.featureSpec(testBetaGate); fs.PreRelease != Beta || fs.Version.String() != "1.28" { t.Errorf("Expected (Beta, 1.28)") } if f.Enabled(testGAGate) != true { t.Errorf("Expected true") } - if fs, _ := f.FeatureSpec(testGAGate); fs.PreRelease != Beta || fs.Version.String() != "1.27" { + if fs, _ := f.featureSpec(testGAGate); fs.PreRelease != Beta || fs.Version.String() != "1.27" { t.Errorf("Expected (Beta, 1.27)") } - if _, err := f.FeatureSpec("NonExist"); err == nil { + if _, err := f.featureSpec("NonExist"); err == nil { t.Errorf("Expected Error") } allFeatures := f.GetAll() @@ -1304,7 +1317,6 @@ func TestVersionedFeatureGateOverrideDefault(t *testing.T) { if !f.Enabled("TestFeature2") { t.Error("expected TestFeature2 to have effective default of true") } - f.OpenForModification() require.NoError(t, f.SetEmulationVersion(version.MustParse("1.29"))) if !f.Enabled("TestFeature1") { t.Error("expected TestFeature1 to have effective default of true") @@ -1312,7 +1324,6 @@ func TestVersionedFeatureGateOverrideDefault(t *testing.T) { if f.Enabled("TestFeature2") { t.Error("expected TestFeature2 to have effective default of false") } - f.OpenForModification() require.NoError(t, f.SetEmulationVersion(version.MustParse("1.26"))) if f.Enabled("TestFeature1") { t.Error("expected TestFeature1 to have effective default of false") @@ -1520,32 +1531,173 @@ func TestFeatureSpecAtEmulationVersion(t *testing.T) { } } -func TestOpenForModification(t *testing.T) { - const testBetaGate Feature = "testBetaGate" - f := NewVersionedFeatureGate(version.MustParse("1.29")) +func TestCopyKnownFeatures(t *testing.T) { + f := NewFeatureGate() + require.NoError(t, f.Add(map[Feature]FeatureSpec{"FeatureA": {Default: false}, "FeatureB": {Default: false}})) + require.NoError(t, f.Set("FeatureA=true")) + require.NoError(t, f.OverrideDefault("FeatureB", true)) + fcopy := f.CopyKnownFeatures() + require.NoError(t, f.Add(map[Feature]FeatureSpec{"FeatureC": {Default: false}})) + assert.True(t, f.Enabled("FeatureA")) + assert.True(t, f.Enabled("FeatureB")) + assert.False(t, f.Enabled("FeatureC")) + + assert.False(t, fcopy.Enabled("FeatureA")) + assert.True(t, fcopy.Enabled("FeatureB")) + + require.NoError(t, fcopy.Set("FeatureB=false")) + assert.True(t, f.Enabled("FeatureB")) + assert.False(t, fcopy.Enabled("FeatureB")) + if err := fcopy.Set("FeatureC=true"); err == nil { + t.Error("expected FeatureC not registered in the copied feature gate") + } +} + +func TestExplicitlySet(t *testing.T) { + // gates for testing + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" + + tests := []struct { + arg string + expectedFeatureValue map[Feature]bool + expectedExplicitlySet map[Feature]bool + }{ + { + arg: "", + expectedFeatureValue: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + }, + expectedExplicitlySet: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + }, + }, + { + arg: "AllAlpha=true,TestBeta=false", + expectedFeatureValue: map[Feature]bool{ + allAlphaGate: true, + allBetaGate: false, + testAlphaGate: true, + testBetaGate: false, + }, + expectedExplicitlySet: map[Feature]bool{ + allAlphaGate: true, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: true, + }, + }, + { + arg: "AllAlpha=true,AllBeta=false", + expectedFeatureValue: map[Feature]bool{ + allAlphaGate: true, + allBetaGate: false, + testAlphaGate: true, + testBetaGate: false, + }, + expectedExplicitlySet: map[Feature]bool{ + allAlphaGate: true, + allBetaGate: true, + testAlphaGate: false, + testBetaGate: false, + }, + }, + } + for i, test := range tests { + t.Run(test.arg, func(t *testing.T) { + fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError) + f := NewVersionedFeatureGate(version.MustParse("1.29")) + err := f.AddVersioned(map[Feature]VersionedSpecs{ + testAlphaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + }, + testBetaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha}, + }, + }) + require.NoError(t, err) + f.AddFlag(fs) + + var errs []error + err = fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)}) + if err != nil { + errs = append(errs, err) + } + err = utilerrors.NewAggregate(errs) + require.NoError(t, err) + for k, v := range test.expectedFeatureValue { + if actual := f.Enabled(k); actual != v { + t.Errorf("%d: expected %s=%v, Got %v", i, k, v, actual) + } + } + for k, v := range test.expectedExplicitlySet { + if actual := f.ExplicitlySet(k); actual != v { + t.Errorf("%d: expected ExplicitlySet(%s)=%v, Got %v", i, k, v, actual) + } + } + }) + } +} + +func TestResetFeatureValueToDefault(t *testing.T) { + // gates for testing + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" + + f := NewVersionedFeatureGate(version.MustParse("1.29")) err := f.AddVersioned(map[Feature]VersionedSpecs{ + testAlphaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + }, testBetaGate: { {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta}, - {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, - {Version: version.MustParse("1.26"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha}, }, }) require.NoError(t, err) - if f.Enabled(testBetaGate) != true { - t.Errorf("Expected true") + fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError) + assert.False(t, f.Enabled("AllAlpha")) + assert.False(t, f.Enabled("AllBeta")) + assert.False(t, f.Enabled("TestAlpha")) + assert.True(t, f.Enabled("TestBeta")) + + f.AddFlag(fs) + var errs []error + err = fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, "AllAlpha=true,TestBeta=false")}) + if err != nil { + errs = append(errs, err) } - err = f.SetEmulationVersion(version.MustParse("1.28")) - if err == nil { - t.Fatalf("Expected error when SetEmulationVersion after querying features") - } - if f.Enabled(testBetaGate) != true { - t.Errorf("Expected true") - } - f.OpenForModification() + err = utilerrors.NewAggregate(errs) + require.NoError(t, err) + assert.True(t, f.Enabled("AllAlpha")) + assert.False(t, f.Enabled("AllBeta")) + assert.True(t, f.Enabled("TestAlpha")) + assert.False(t, f.Enabled("TestBeta")) + + require.NoError(t, f.ResetFeatureValueToDefault("AllAlpha")) + assert.False(t, f.Enabled("AllAlpha")) + assert.False(t, f.Enabled("AllBeta")) + assert.True(t, f.Enabled("TestAlpha")) + assert.False(t, f.Enabled("TestBeta")) + + require.NoError(t, f.ResetFeatureValueToDefault("TestBeta")) + assert.False(t, f.Enabled("AllAlpha")) + assert.False(t, f.Enabled("AllBeta")) + assert.True(t, f.Enabled("TestAlpha")) + assert.True(t, f.Enabled("TestBeta")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) - if f.Enabled(testBetaGate) != false { - t.Errorf("Expected false at 1.28") - } + assert.False(t, f.Enabled("AllAlpha")) + assert.False(t, f.Enabled("AllBeta")) + assert.False(t, f.Enabled("TestAlpha")) + assert.False(t, f.Enabled("TestBeta")) } diff --git a/staging/src/k8s.io/component-base/featuregate/testing/feature_gate.go b/staging/src/k8s.io/component-base/featuregate/testing/feature_gate.go index 47c0f927738..eb28217aff5 100644 --- a/staging/src/k8s.io/component-base/featuregate/testing/feature_gate.go +++ b/staging/src/k8s.io/component-base/featuregate/testing/feature_gate.go @@ -20,14 +20,16 @@ import ( "fmt" "strings" "sync" - "testing" + "k8s.io/apimachinery/pkg/util/version" "k8s.io/component-base/featuregate" ) var ( - overrideLock sync.Mutex - featureFlagOverride map[featuregate.Feature]string + overrideLock sync.Mutex + featureFlagOverride map[featuregate.Feature]string + emulationVersionOverride string + emulationVersionOverrideValue *version.Version ) func init() { @@ -42,10 +44,12 @@ func init() { // Example use: // // featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features., true) -func SetFeatureGateDuringTest(tb testing.TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) { +func SetFeatureGateDuringTest(tb TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) { tb.Helper() detectParallelOverrideCleanup := detectParallelOverride(tb, f) - originalEnabled := gate.(featuregate.MutableVersionedFeatureGateForTests).EnabledRawMap() + originalValue := gate.Enabled(f) + originalEmuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion() + originalExplicitlySet := gate.(featuregate.MutableVersionedFeatureGate).ExplicitlySet(f) // Specially handle AllAlpha and AllBeta if f == "AllAlpha" || f == "AllBeta" { @@ -67,11 +71,46 @@ func SetFeatureGateDuringTest(tb testing.TB, gate featuregate.FeatureGate, f fea tb.Cleanup(func() { tb.Helper() detectParallelOverrideCleanup() - gate.(featuregate.MutableVersionedFeatureGateForTests).Reset(originalEnabled) + emuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion() + if !emuVer.EqualTo(originalEmuVer) { + tb.Fatalf("change of feature gate emulation version from %s to %s in the chain of SetFeatureGateDuringTest is not allowed\nuse SetFeatureGateEmulationVersionDuringTest to change emulation version in tests", + originalEmuVer.String(), emuVer.String()) + } + if originalExplicitlySet { + if err := gate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, originalValue)); err != nil { + tb.Errorf("error restoring %s=%v: %v", f, originalValue, err) + } + } else { + if err := gate.(featuregate.MutableVersionedFeatureGate).ResetFeatureValueToDefault(f); err != nil { + tb.Errorf("error restoring %s=%v: %v", f, originalValue, err) + } + } }) } -func detectParallelOverride(tb testing.TB, f featuregate.Feature) func() { +// SetFeatureGateEmulationVersionDuringTest sets the specified gate to the specified emulation version for duration of the test. +// Fails when it detects second call to set a different emulation version or is unable to set or restore emulation version. +// WARNING: Can leak set variable when called in test calling t.Parallel(), however second attempt to set a different emulation version will cause fatal. +// Example use: + +// featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) +func SetFeatureGateEmulationVersionDuringTest(tb TB, gate featuregate.FeatureGate, ver *version.Version) { + tb.Helper() + detectParallelOverrideCleanup := detectParallelOverrideEmulationVersion(tb, ver) + originalEmuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion() + if err := gate.(featuregate.MutableVersionedFeatureGate).SetEmulationVersion(ver); err != nil { + tb.Fatalf("failed to set emulation version to %s during test", ver.String()) + } + tb.Cleanup(func() { + tb.Helper() + detectParallelOverrideCleanup() + if err := gate.(featuregate.MutableVersionedFeatureGate).SetEmulationVersion(originalEmuVer); err != nil { + tb.Fatalf("failed to restore emulation version to %s during test", originalEmuVer.String()) + } + }) +} + +func detectParallelOverride(tb TB, f featuregate.Feature) func() { tb.Helper() overrideLock.Lock() defer overrideLock.Unlock() @@ -92,7 +131,44 @@ func detectParallelOverride(tb testing.TB, f featuregate.Feature) func() { } } -func sameTestOrSubtest(tb testing.TB, testName string) bool { +func detectParallelOverrideEmulationVersion(tb TB, ver *version.Version) func() { + tb.Helper() + overrideLock.Lock() + defer overrideLock.Unlock() + beforeOverrideTestName := emulationVersionOverride + beforeOverrideValue := emulationVersionOverrideValue + if ver.EqualTo(beforeOverrideValue) { + return func() {} + } + if beforeOverrideTestName != "" && !sameTestOrSubtest(tb, beforeOverrideTestName) { + tb.Fatalf("Detected parallel setting of a feature gate emulation version by both %q and %q", beforeOverrideTestName, tb.Name()) + } + emulationVersionOverride = tb.Name() + emulationVersionOverrideValue = ver + + return func() { + tb.Helper() + overrideLock.Lock() + defer overrideLock.Unlock() + if afterOverrideTestName := emulationVersionOverride; afterOverrideTestName != tb.Name() { + tb.Fatalf("Detected parallel setting of a feature gate emulation version between both %q and %q", afterOverrideTestName, tb.Name()) + } + emulationVersionOverride = beforeOverrideTestName + emulationVersionOverrideValue = beforeOverrideValue + } +} + +func sameTestOrSubtest(tb TB, testName string) bool { // Assumes that "/" is not used in test names. return tb.Name() == testName || strings.HasPrefix(tb.Name(), testName+"/") } + +type TB interface { + Cleanup(func()) + Error(args ...any) + Errorf(format string, args ...any) + Fatal(args ...any) + Fatalf(format string, args ...any) + Helper() + Name() string +} diff --git a/staging/src/k8s.io/component-base/featuregate/testing/feature_gate_test.go b/staging/src/k8s.io/component-base/featuregate/testing/feature_gate_test.go index cec7309eb2e..189efcb11f0 100644 --- a/staging/src/k8s.io/component-base/featuregate/testing/feature_gate_test.go +++ b/staging/src/k8s.io/component-base/featuregate/testing/feature_gate_test.go @@ -21,6 +21,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/util/version" "k8s.io/component-base/featuregate" ) @@ -71,6 +72,7 @@ func TestSpecialGates(t *gotest.T) { expect(t, gate, before) t.Cleanup(func() { expect(t, gate, before) + cleanup() }) SetFeatureGateDuringTest(t, gate, "AllAlpha", true) @@ -156,10 +158,107 @@ func TestSetFeatureGateInTest(t *gotest.T) { assert.True(t, gate.Enabled("feature")) } -func TestDetectLeakToMainTest(t *gotest.T) { - t.Cleanup(func() { - featureFlagOverride = map[featuregate.Feature]string{} +func TestSpecialGatesVersioned(t *gotest.T) { + originalEmulationVersion := version.MustParse("1.31") + gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion) + + err := gate.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ + "alpha_default_on": { + {Version: version.MustParse("1.27"), Default: true, PreRelease: featuregate.Alpha}, + }, + "alpha_default_off": { + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, + }, + "beta_default_on": { + {Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta}, + }, + "beta_default_on_set_off": { + {Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, + }, + "beta_default_off": { + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Beta}, + }, + "beta_default_off_set_on": { + {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, + }, }) + require.NoError(t, err) + + require.NoError(t, gate.Set("beta_default_on_set_off=false")) + require.NoError(t, gate.Set("beta_default_off_set_on=true")) + + before := map[featuregate.Feature]bool{ + "AllAlpha": false, + "AllBeta": false, + + "alpha_default_on": true, + "alpha_default_off": false, + + "beta_default_on": true, + "beta_default_on_set_off": false, + "beta_default_off": false, + "beta_default_off_set_on": true, + } + expect(t, gate, before) + t.Cleanup(func() { + expect(t, gate, before) + cleanup() + }) + + t.Run("OverwriteInSubtest", func(t *gotest.T) { + SetFeatureGateDuringTest(t, gate, "AllAlpha", true) + expect(t, gate, map[featuregate.Feature]bool{ + "AllAlpha": true, + "AllBeta": false, + + "alpha_default_on": true, + "alpha_default_off": true, + + "beta_default_on": true, + "beta_default_on_set_off": false, + "beta_default_off": false, + "beta_default_off_set_on": true, + }) + + SetFeatureGateDuringTest(t, gate, "AllBeta", true) + expect(t, gate, map[featuregate.Feature]bool{ + "AllAlpha": true, + "AllBeta": true, + + "alpha_default_on": true, + "alpha_default_off": true, + + "beta_default_on": true, + "beta_default_on_set_off": true, + "beta_default_off": true, + "beta_default_off_set_on": true, + }) + }) + expect(t, gate, before) + + t.Run("OverwriteInSubtestAtDifferentVersion", func(t *gotest.T) { + SetFeatureGateEmulationVersionDuringTest(t, gate, version.MustParse("1.28")) + SetFeatureGateDuringTest(t, gate, "AllAlpha", true) + expect(t, gate, map[featuregate.Feature]bool{ + "AllAlpha": true, + "AllBeta": false, + + "alpha_default_on": true, + "alpha_default_off": true, + + "beta_default_on": false, + "beta_default_on_set_off": true, + "beta_default_off": false, + "beta_default_off_set_on": true, + }) + }) + +} + +func TestDetectLeakToMainTest(t *gotest.T) { + t.Cleanup(cleanup) gate := featuregate.NewFeatureGate() err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{ "feature": {PreRelease: featuregate.Alpha, Default: false}, @@ -183,9 +282,7 @@ func TestDetectLeakToMainTest(t *gotest.T) { } func TestDetectLeakToOtherSubtest(t *gotest.T) { - t.Cleanup(func() { - featureFlagOverride = map[featuregate.Feature]string{} - }) + t.Cleanup(cleanup) gate := featuregate.NewFeatureGate() err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{ "feature": {PreRelease: featuregate.Alpha, Default: false}, @@ -211,9 +308,7 @@ func TestDetectLeakToOtherSubtest(t *gotest.T) { } func TestCannotDetectLeakFromSubtest(t *gotest.T) { - t.Cleanup(func() { - featureFlagOverride = map[featuregate.Feature]string{} - }) + t.Cleanup(cleanup) gate := featuregate.NewFeatureGate() err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{ "feature": {PreRelease: featuregate.Alpha, Default: false}, @@ -230,6 +325,107 @@ func TestCannotDetectLeakFromSubtest(t *gotest.T) { assert.True(t, gate.Enabled("feature")) } +func TestCannotDetectLeakFromTwoSubtestsWithDifferentFeatures(t *gotest.T) { + t.Cleanup(cleanup) + gate := featuregate.NewFeatureGate() + err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{ + "feature1": {PreRelease: featuregate.Alpha, Default: false}, + "feature2": {PreRelease: featuregate.Alpha, Default: false}, + }) + require.NoError(t, err) + + assert.False(t, gate.Enabled("feature1")) + assert.False(t, gate.Enabled("feature2")) + subtestName := "Subtest" + // Subtest setting feature gate and calling parallel will leak it out + t.Run(subtestName, func(t *gotest.T) { + SetFeatureGateDuringTest(t, gate, "feature1", true) + t.Parallel() + assert.True(t, gate.Enabled("feature1")) + assert.False(t, gate.Enabled("feature2")) + }) + // Leaked true + assert.True(t, gate.Enabled("feature1")) + assert.False(t, gate.Enabled("feature2")) + // Add suffix to name to prevent tests with the same prefix. + t.Run(subtestName+"Suffix", func(t *gotest.T) { + // Leaked true + assert.True(t, gate.Enabled("feature1")) + assert.False(t, gate.Enabled("feature2")) + SetFeatureGateDuringTest(t, gate, "feature2", true) + assert.True(t, gate.Enabled("feature1")) + assert.True(t, gate.Enabled("feature2")) + }) +} + +func TestDetectEmulationVersionLeakToMainTest(t *gotest.T) { + t.Cleanup(cleanup) + originalEmulationVersion := version.MustParse("1.31") + newEmulationVersion := version.MustParse("1.30") + gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion) + assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion)) + + // Subtest setting feature gate and calling parallel will leak it out + t.Run("LeakingSubtest", func(t *gotest.T) { + fakeT := &ignoreFatalT{T: t} + SetFeatureGateEmulationVersionDuringTest(fakeT, gate, newEmulationVersion) + // Calling t.Parallel in subtest will resume the main test body + t.Parallel() + // Leaked from main test + assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion)) + }) + // Leaked from subtest + assert.True(t, gate.EmulationVersion().EqualTo(newEmulationVersion)) + fakeT := &ignoreFatalT{T: t} + SetFeatureGateEmulationVersionDuringTest(fakeT, gate, originalEmulationVersion) + assert.True(t, fakeT.fatalRecorded) +} + +func TestNoLeakFromSameEmulationVersionToMainTest(t *gotest.T) { + t.Cleanup(cleanup) + originalEmulationVersion := version.MustParse("1.31") + newEmulationVersion := version.MustParse("1.31") + gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion) + assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion)) + + // Subtest setting feature gate and calling parallel will leak it out + t.Run("LeakingSubtest", func(t *gotest.T) { + SetFeatureGateEmulationVersionDuringTest(t, gate, newEmulationVersion) + // Calling t.Parallel in subtest will resume the main test body + t.Parallel() + // Leaked from main test + assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion)) + }) + // Leaked from subtest + assert.True(t, gate.EmulationVersion().EqualTo(newEmulationVersion)) + SetFeatureGateEmulationVersionDuringTest(t, gate, originalEmulationVersion) +} + +func TestDetectEmulationVersionLeakToOtherSubtest(t *gotest.T) { + t.Cleanup(cleanup) + originalEmulationVersion := version.MustParse("1.31") + newEmulationVersion := version.MustParse("1.30") + gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion) + assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion)) + + subtestName := "Subtest" + // Subtest setting feature gate and calling parallel will leak it out + t.Run(subtestName, func(t *gotest.T) { + fakeT := &ignoreFatalT{T: t} + SetFeatureGateEmulationVersionDuringTest(fakeT, gate, newEmulationVersion) + t.Parallel() + }) + // Add suffix to name to prevent tests with the same prefix. + t.Run(subtestName+"Suffix", func(t *gotest.T) { + // Leaked newEmulationVersion + assert.True(t, gate.EmulationVersion().EqualTo(newEmulationVersion)) + + fakeT := &ignoreFatalT{T: t} + SetFeatureGateEmulationVersionDuringTest(fakeT, gate, originalEmulationVersion) + assert.True(t, fakeT.fatalRecorded) + }) +} + type ignoreFatalT struct { *gotest.T fatalRecorded bool @@ -248,3 +444,9 @@ func (f *ignoreFatalT) Fatalf(format string, args ...any) { f.fatalRecorded = true f.T.Logf("[IGNORED] "+format, args...) } + +func cleanup() { + featureFlagOverride = map[featuregate.Feature]string{} + emulationVersionOverride = "" + emulationVersionOverrideValue = nil +} diff --git a/staging/src/k8s.io/component-base/version/base.go b/staging/src/k8s.io/component-base/version/base.go index b753b7d191f..60154678249 100644 --- a/staging/src/k8s.io/component-base/version/base.go +++ b/staging/src/k8s.io/component-base/version/base.go @@ -61,3 +61,10 @@ var ( buildDate = "1970-01-01T00:00:00Z" // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ') ) + +const ( + // DefaultKubeBinaryVersion is the hard coded k8 binary version based on the latest K8s release. + // It is supposed to be consistent with gitMajor and gitMinor, except for local tests, where gitMajor and gitMinor are "". + // Should update for each minor release! + DefaultKubeBinaryVersion = "1.31" +) diff --git a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go index 6e3c02bfef8..d081702d246 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go @@ -31,7 +31,6 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/filters" genericoptions "k8s.io/apiserver/pkg/server/options" - utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" "k8s.io/kube-aggregator/pkg/apiserver" @@ -81,10 +80,7 @@ func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions) } cmd.SetContext(ctx) - fs := cmd.Flags() - utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs) - - o.AddFlags(fs) + o.AddFlags(cmd.Flags()) return cmd } @@ -99,13 +95,8 @@ func (o *AggregatorOptions) AddFlags(fs *pflag.FlagSet) { // NewDefaultOptions builds a "normal" set of options. You wouldn't normally expose this, but hyperkube isn't cobra compatible func NewDefaultOptions(out, err io.Writer) *AggregatorOptions { - // effectiveVersion is used to set what apis and feature gates the generic api server is compatible with. - // You can also have the flag setting the effectiveVersion of the aggregator apiserver, and - // having a mapping from the aggregator apiserver version to generic apiserver version. - effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( - utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) o := &AggregatorOptions{ - ServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion), + ServerRunOptions: genericoptions.NewServerRunOptions(), RecommendedOptions: genericoptions.NewRecommendedOptions( defaultEtcdPathPrefix, aggregatorscheme.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion), diff --git a/staging/src/k8s.io/sample-apiserver/go.mod b/staging/src/k8s.io/sample-apiserver/go.mod index 32f73176861..8f2dee1a0a2 100644 --- a/staging/src/k8s.io/sample-apiserver/go.mod +++ b/staging/src/k8s.io/sample-apiserver/go.mod @@ -7,6 +7,7 @@ go 1.22.0 require ( github.com/google/gofuzz v1.2.0 github.com/spf13/cobra v1.8.1 + github.com/stretchr/testify v1.9.0 k8s.io/apimachinery v0.0.0 k8s.io/apiserver v0.0.0 k8s.io/client-go v0.0.0 @@ -55,6 +56,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.19.0 // indirect github.com/prometheus/client_model v0.6.0 // indirect github.com/prometheus/common v0.48.0 // indirect diff --git a/staging/src/k8s.io/sample-apiserver/go.sum b/staging/src/k8s.io/sample-apiserver/go.sum index dc64c5fca30..bfb8e18b984 100644 --- a/staging/src/k8s.io/sample-apiserver/go.sum +++ b/staging/src/k8s.io/sample-apiserver/go.sum @@ -170,6 +170,7 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go index 6152d0b84ca..5e30e2bb319 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go @@ -113,7 +113,7 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti o.RecommendedOptions.AddFlags(flags) wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2") - wardleFeatureGate := featuregate.NewVersionedFeatureGate(version.MustParse("1.2")) + wardleFeatureGate := utilfeature.DefaultFeatureGate.CopyKnownFeatures() utilruntime.Must(wardleFeatureGate.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ "BanFlunder": { {Version: version.MustParse("1.2"), Default: true, PreRelease: featuregate.GA}, diff --git a/test/e2e_node/services/apiserver.go b/test/e2e_node/services/apiserver.go index c092d9d69b6..4488715b3bd 100644 --- a/test/e2e_node/services/apiserver.go +++ b/test/e2e_node/services/apiserver.go @@ -29,8 +29,6 @@ import ( netutils "k8s.io/utils/net" utilerrors "k8s.io/apimachinery/pkg/util/errors" - utilfeature "k8s.io/apiserver/pkg/util/feature" - utilversion "k8s.io/apiserver/pkg/util/version" apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/test/e2e/framework" @@ -65,9 +63,7 @@ func NewAPIServer(storageConfig storagebackend.Config) *APIServer { func (a *APIServer) Start(ctx context.Context) error { const tokenFilePath = "known_tokens.csv" - featureGate := utilfeature.DefaultFeatureGate - effectiveVersion := utilversion.DefaultKubeEffectiveVersion() - o := options.NewServerRunOptions(featureGate, effectiveVersion) + o := options.NewServerRunOptions() o.Etcd.StorageConfig = a.storageConfig _, ipnet, err := netutils.ParseCIDRSloppy(clusterIPRange) if err != nil { diff --git a/test/integration/etcd/etcd_cross_group_test.go b/test/integration/etcd/etcd_cross_group_test.go index c2592fc0a49..efc4fcc4e6c 100644 --- a/test/integration/etcd/etcd_cross_group_test.go +++ b/test/integration/etcd/etcd_cross_group_test.go @@ -27,9 +27,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/storage" + utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/dynamic" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" @@ -37,9 +39,13 @@ import ( // TestCrossGroupStorage tests to make sure that all objects stored in an expected location in etcd can be converted/read. func TestCrossGroupStorage(t *testing.T) { + testRegistry := utilversion.NewComponentGlobalsRegistry() + utilruntime.Must(testRegistry.Register("test", utilversion.NewEffectiveVersion("0.0"), utilfeature.DefaultFeatureGate.DeepCopy())) + apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { // force enable all resources so we can check storage. - opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("0.0") + opts.GenericServerRunOptions.ComponentName = "test" + opts.GenericServerRunOptions.ComponentGlobalsRegistry = testRegistry }) defer apiServer.Cleanup() diff --git a/test/integration/etcd/etcd_storage_path_test.go b/test/integration/etcd/etcd_storage_path_test.go index eda94aee93f..43cbedbbfdc 100644 --- a/test/integration/etcd/etcd_storage_path_test.go +++ b/test/integration/etcd/etcd_storage_path_test.go @@ -34,6 +34,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" @@ -75,8 +76,12 @@ var allowMissingTestdataFixtures = map[schema.GroupVersionKind]bool{ func TestEtcdStoragePath(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true) featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true) + testRegistry := utilversion.NewComponentGlobalsRegistry() + utilruntime.Must(testRegistry.Register("test", utilversion.NewEffectiveVersion("0.0"), feature.DefaultFeatureGate.DeepCopy())) + apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { - opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("0.0") + opts.GenericServerRunOptions.ComponentName = "test" + opts.GenericServerRunOptions.ComponentGlobalsRegistry = testRegistry }) defer apiServer.Cleanup() defer dumpEtcdKVOnFailure(t, apiServer.KV) diff --git a/test/integration/etcd/server.go b/test/integration/etcd/server.go index fb57211ea43..ed69e3d532f 100644 --- a/test/integration/etcd/server.go +++ b/test/integration/etcd/server.go @@ -40,8 +40,6 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" genericapiserveroptions "k8s.io/apiserver/pkg/server/options" - utilfeature "k8s.io/apiserver/pkg/util/feature" - utilversion "k8s.io/apiserver/pkg/util/version" cacheddiscovery "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" @@ -93,9 +91,7 @@ func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRu t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err) } - featureGate := utilfeature.DefaultFeatureGate - effectiveVersion := utilversion.DefaultBuildEffectiveVersion() - opts := options.NewServerRunOptions(featureGate, effectiveVersion) + opts := options.NewServerRunOptions() opts.Options.SecureServing.Listener = listener opts.Options.SecureServing.ServerCert.CertDirectory = certDir opts.Options.ServiceAccountSigningKeyFile = saSigningKeyFile.Name() diff --git a/test/integration/framework/test_server.go b/test/integration/framework/test_server.go index 5d514b360d4..bb43850466d 100644 --- a/test/integration/framework/test_server.go +++ b/test/integration/framework/test_server.go @@ -35,8 +35,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" genericapiserver "k8s.io/apiserver/pkg/server" genericapiserveroptions "k8s.io/apiserver/pkg/server/options" - utilfeature "k8s.io/apiserver/pkg/util/feature" - utilversion "k8s.io/apiserver/pkg/util/version" client "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/util/cert" @@ -137,9 +135,7 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) ( t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err) } - featureGate := utilfeature.DefaultFeatureGate - effectiveVersion := utilversion.DefaultKubeEffectiveVersion() - opts := options.NewServerRunOptions(featureGate, effectiveVersion) + opts := options.NewServerRunOptions() opts.SecureServing.Listener = listener opts.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1") opts.SecureServing.ServerCert.CertDirectory = certDir