From 403301bfdf2c7312591077827abd2e72f445a53a Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Fri, 19 Jan 2024 16:07:00 -0800 Subject: [PATCH 1/4] apiserver: Add API emulation versioning. Co-authored-by: Siyuan Zhang Co-authored-by: Joe Betz Co-authored-by: Alex Zielenski Signed-off-by: Siyuan Zhang --- cmd/kube-apiserver/app/options/options.go | 8 +- .../app/options/options_test.go | 16 +- cmd/kube-apiserver/app/server.go | 17 +- cmd/kube-apiserver/app/testing/testserver.go | 21 +- .../app/options/options.go | 2 +- cmd/kube-scheduler/app/options/options.go | 2 +- pkg/controlplane/apiserver/apiextensions.go | 2 +- pkg/controlplane/apiserver/apis.go | 2 +- pkg/controlplane/apiserver/config.go | 2 + pkg/controlplane/apiserver/config_test.go | 8 +- pkg/controlplane/apiserver/options/options.go | 10 +- .../apiserver/options/options_test.go | 15 +- .../apiserver/options/validation.go | 1 + .../apiserver/options/validation_test.go | 5 +- pkg/controlplane/apiserver/peer.go | 2 +- pkg/controlplane/instance.go | 2 +- pkg/controlplane/instance_test.go | 3 + pkg/features/client_adapter_test.go | 5 +- pkg/features/kube_features.go | 1 + pkg/features/kube_features_test.go | 3 + pkg/features/versioned_kube_features.go | 34 + .../default_storage_factory_builder.go | 2 + pkg/registry/core/rest/storage_core.go | 2 +- pkg/registry/core/rest/storage_core_test.go | 3 +- pkg/registry/registrytest/etcd.go | 2 +- .../pkg/apiserver/customresource_handler.go | 5 +- .../pkg/cmd/server/options/options.go | 8 +- .../pkg/cmd/server/server.go | 15 +- .../pkg/cmd/server/testing/testserver.go | 14 +- .../integration/conversion/conversion_test.go | 2 +- .../test/integration/defaulting_test.go | 2 +- .../test/integration/fixtures/server.go | 2 +- .../test/integration/objectmeta_test.go | 2 +- .../test/integration/pruning_test.go | 2 +- .../apimachinery/pkg/util/version/version.go | 71 ++ .../pkg/util/version/version_test.go | 91 ++ .../apiserver/pkg/cel/environment/base.go | 21 +- .../apiserver/pkg/features/kube_features.go | 11 + .../apiserver/pkg/registry/generic/options.go | 8 +- .../pkg/registry/generic/registry/store.go | 2 +- .../src/k8s.io/apiserver/pkg/server/config.go | 34 +- .../apiserver/pkg/server/deleted_kinds.go | 65 +- .../pkg/server/deleted_kinds_test.go | 173 +-- .../apiserver/pkg/server/genericapiserver.go | 7 + .../pkg/server/genericapiserver_test.go | 4 +- .../pkg/server/options/api_enablement_test.go | 42 +- .../apiserver/pkg/server/options/etcd.go | 10 +- .../apiserver/pkg/server/options/etcd_test.go | 2 +- .../pkg/server/options/server_run_options.go | 34 +- .../server/options/server_run_options_test.go | 36 +- .../pkg/server/options/serving_test.go | 11 +- .../storage/resource_encoding_config.go | 110 +- .../pkg/server/storage/storage_factory.go | 19 +- .../server/storage/storage_factory_test.go | 246 +++- .../pkg/util/feature/feature_gate.go | 2 +- .../apiserver/pkg/util/version/registry.go | 142 +++ .../pkg/util/version/registry_test.go | 48 + .../apiserver/pkg/util/version/version.go | 195 +++ .../pkg/util/version/version_test.go | 180 +++ .../k8s.io/cloud-provider/options/options.go | 2 +- .../featuregate/feature_gate.go | 398 +++++-- .../featuregate/feature_gate_test.go | 1055 +++++++++++++++-- .../featuregate/testing/feature_gate.go | 6 +- .../component-base/logs/example/cmd/logger.go | 2 +- .../logs/example/k8s2slog/k8s2slog.go | 2 +- .../pkg/apiserver/apiserver.go | 2 +- .../kube-aggregator/pkg/cmd/server/start.go | 23 +- .../pkg/apis/wardle/v1alpha1/doc.go | 1 + .../pkg/apis/wardle/v1alpha1/types.go | 8 + .../zz_generated.prerelease-lifecycle.go | 94 ++ .../pkg/apiserver/apiserver.go | 12 +- .../sample-apiserver/pkg/cmd/server/start.go | 46 +- .../pkg/cmd/server/start_test.go | 66 ++ test/e2e/dra/test-driver/app/server.go | 2 +- test/e2e_node/services/apiserver.go | 6 +- test/integration/apiserver/apiserver_test.go | 296 +++++ test/integration/auth/rbac_test.go | 5 +- .../kmsv2_transformation_test.go | 2 +- .../transformation/transformation_test.go | 4 +- .../etcd/crd_overlap_storage_test.go | 10 +- .../integration/etcd/etcd_cross_group_test.go | 2 + .../etcd/etcd_storage_path_test.go | 2 + test/integration/etcd/server.go | 6 +- test/integration/examples/apiserver_test.go | 1 + test/integration/framework/test_server.go | 6 +- .../integration/servicecidr/allocator_test.go | 2 + 86 files changed, 3420 insertions(+), 427 deletions(-) create mode 100644 pkg/features/versioned_kube_features.go create mode 100644 staging/src/k8s.io/apiserver/pkg/util/version/registry.go create mode 100644 staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go create mode 100644 staging/src/k8s.io/apiserver/pkg/util/version/version.go create mode 100644 staging/src/k8s.io/apiserver/pkg/util/version/version_test.go create mode 100644 staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/zz_generated.prerelease-lifecycle.go create mode 100644 staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go diff --git a/cmd/kube-apiserver/app/options/options.go b/cmd/kube-apiserver/app/options/options.go index afdb9e73ff3..53a5b3c06ee 100644 --- a/cmd/kube-apiserver/app/options/options.go +++ b/cmd/kube-apiserver/app/options/options.go @@ -26,6 +26,8 @@ import ( utilnet "k8s.io/apimachinery/pkg/util/net" cliflag "k8s.io/component-base/cli/flag" + utilversion "k8s.io/apiserver/pkg/util/version" + "k8s.io/component-base/featuregate" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/cluster/ports" controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" @@ -63,10 +65,10 @@ type Extra struct { MasterCount int } -// NewServerRunOptions creates a new ServerRunOptions object with default parameters -func NewServerRunOptions() *ServerRunOptions { +// NewServerRunOptions creates and returns ServerRunOptions according to the given featureGate and effectiveVersion of the server binary to run. +func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions { s := ServerRunOptions{ - Options: controlplaneapiserver.NewOptions(), + Options: controlplaneapiserver.NewOptions(featureGate, effectiveVersion), CloudProvider: kubeoptions.NewCloudProviderOptions(), Extra: Extra{ diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index 2fa5da605a6..56cbcbebd03 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -31,9 +31,11 @@ import ( apiserveroptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/storage/etcd3" "k8s.io/apiserver/pkg/storage/storagebackend" + utilversion "k8s.io/apiserver/pkg/util/version" auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered" audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate" cliflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" "k8s.io/component-base/metrics" kapi "k8s.io/kubernetes/pkg/apis/core" @@ -46,10 +48,15 @@ import ( func TestAddFlags(t *testing.T) { fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError) - s := NewServerRunOptions() + + featureGate := featuregate.NewFeatureGate() + effectiveVersion := utilversion.NewEffectiveVersion("1.32") + s := NewServerRunOptions(featureGate, effectiveVersion) for _, f := range s.Flags().FlagSets { fs.AddFlagSet(f) } + featureGate.AddFlag(fs, "") + effectiveVersion.AddFlags(fs, "") args := []string{ "--enable-admission-plugins=AlwaysDeny", @@ -121,6 +128,7 @@ func TestAddFlags(t *testing.T) { "--storage-backend=etcd3", "--service-cluster-ip-range=192.168.128.0/17", "--lease-reuse-duration-seconds=100", + "--emulated-version=1.31", } fs.Parse(args) @@ -136,6 +144,8 @@ func TestAddFlags(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024), MaxRequestBodyBytes: int64(3 * 1024 * 1024), + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, Admission: &kubeoptions.AdmissionOptions{ GenericAdmission: &apiserveroptions.AdmissionOptions{ @@ -337,4 +347,8 @@ func TestAddFlags(t *testing.T) { if !reflect.DeepEqual(expected, s) { t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{}))) } + + if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" { + t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31") + } } diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 81f826ac876..3489f2130ae 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -36,6 +36,7 @@ import ( serverstorage "k8s.io/apiserver/pkg/server/storage" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/notfoundhandler" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/apiserver/pkg/util/webhook" clientgoinformers "k8s.io/client-go/informers" "k8s.io/client-go/rest" @@ -63,7 +64,10 @@ func init() { // NewAPIServerCommand creates a *cobra.Command object with default parameters func NewAPIServerCommand() *cobra.Command { - s := options.NewServerRunOptions() + effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( + utilversion.ComponentGenericAPIServer, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) + s := options.NewServerRunOptions(featureGate, effectiveVersion) + cmd := &cobra.Command{ Use: "kube-apiserver", Long: `The Kubernetes API server validates and configures data @@ -83,9 +87,13 @@ cluster's shared state through which all other components interact.`, verflag.PrintAndExitIfRequested() fs := cmd.Flags() + if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { + return err + } + // Activate logging as soon as possible, after that // show flags with the final logging configuration. - if err := logsapi.ValidateAndApply(s.Logs, utilfeature.DefaultFeatureGate); err != nil { + if err := logsapi.ValidateAndApply(s.Logs, featureGate); err != nil { return err } cliflag.PrintFlags(fs) @@ -101,7 +109,7 @@ cluster's shared state through which all other components interact.`, return utilerrors.NewAggregate(errs) } // add feature enablement metrics - utilfeature.DefaultMutableFeatureGate.AddMetrics() + featureGate.AddMetrics() return Run(cmd.Context(), completedOptions) }, Args: func(cmd *cobra.Command, args []string) error { @@ -118,6 +126,9 @@ cluster's shared state through which all other components interact.`, fs := cmd.Flags() namedFlagSets := s.Flags() verflag.AddFlags(namedFlagSets.FlagSet("global")) + featureGate.AddFlag(namedFlagSets.FlagSet("global"), "") + effectiveVersion.AddFlags(namedFlagSets.FlagSet("global"), "") + globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags()) options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic")) for _, f := range namedFlagSets.FlagSets { diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go index 3d00006e38a..7eb7dce0496 100644 --- a/cmd/kube-apiserver/app/testing/testserver.go +++ b/cmd/kube-apiserver/app/testing/testserver.go @@ -48,6 +48,7 @@ import ( "k8s.io/apiserver/pkg/storage/storagebackend" "k8s.io/apiserver/pkg/storageversion" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" clientgotransport "k8s.io/client-go/transport" @@ -98,6 +99,9 @@ type TestServerInstanceOptions struct { // We specify this as on option to pass a common proxyCA to multiple apiservers to simulate // an apiserver version skew scenario where all apiservers use the same proxyCA to verify client connections. ProxyCA *ProxyCA + // Set the BinaryVersion of server effective version. + // Default to 1.31 + BinaryVersion string } // TestServer return values supplied by kube-test-ApiServer @@ -177,10 +181,21 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, fs := pflag.NewFlagSet("test", pflag.PanicOnError) - s := options.NewServerRunOptions() + featureGate := utilfeature.DefaultMutableFeatureGate + binaryVersion := utilversion.DefaultKubeEffectiveVersion().BinaryVersion().String() + if instanceOptions.BinaryVersion != "" { + binaryVersion = instanceOptions.BinaryVersion + } + effectiveVersion := utilversion.NewEffectiveVersion(binaryVersion) + _ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.ComponentGenericAPIServer, effectiveVersion, featureGate, true) + + s := options.NewServerRunOptions(featureGate, effectiveVersion) + for _, f := range s.Flags().FlagSets { fs.AddFlagSet(f) } + featureGate.AddFlag(fs, "") + effectiveVersion.AddFlags(fs, "") s.SecureServing.Listener, s.SecureServing.BindPort, err = createLocalhostListenerOnFreePort() if err != nil { @@ -321,6 +336,10 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, return result, err } + if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { + return result, err + } + saSigningKeyFile, err := os.CreateTemp("/tmp", "insecure_test_key") if err != nil { t.Fatalf("create temp file failed: %v", err) diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index 1c3e4edb178..d72920d0cd1 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -273,7 +273,7 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy fs := fss.FlagSet("misc") fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") fs.StringVar(&s.Generic.ClientConnection.Kubeconfig, "kubeconfig", s.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).") - utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic")) + utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"), "") return fss } diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go index e039b0d05c9..2b1233c435b 100644 --- a/cmd/kube-scheduler/app/options/options.go +++ b/cmd/kube-scheduler/app/options/options.go @@ -189,7 +189,7 @@ func (o *Options) initFlags() { o.Authorization.AddFlags(nfs.FlagSet("authorization")) o.Deprecated.AddFlags(nfs.FlagSet("deprecated")) options.BindLeaderElectionFlags(o.LeaderElection, nfs.FlagSet("leader election")) - utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate")) + utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate"), "") o.Metrics.AddFlags(nfs.FlagSet("metrics")) logsapi.AddFlags(o.Logs, nfs.FlagSet("logs")) diff --git a/pkg/controlplane/apiserver/apiextensions.go b/pkg/controlplane/apiserver/apiextensions.go index d203d2757e3..1a0b8910c29 100644 --- a/pkg/controlplane/apiserver/apiextensions.go +++ b/pkg/controlplane/apiserver/apiextensions.go @@ -17,7 +17,6 @@ limitations under the License. package apiserver import ( - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" apiextensionsoptions "k8s.io/apiextensions-apiserver/pkg/cmd/server/options" @@ -27,6 +26,7 @@ import ( "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/util/webhook" "k8s.io/client-go/informers" + v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" "k8s.io/kubernetes/pkg/controlplane/apiserver/options" ) diff --git a/pkg/controlplane/apiserver/apis.go b/pkg/controlplane/apiserver/apis.go index 1ccfe787466..9ebb7e8f4fe 100644 --- a/pkg/controlplane/apiserver/apis.go +++ b/pkg/controlplane/apiserver/apis.go @@ -88,7 +88,7 @@ func (s *Server) InstallAPIs(restStorageProviders ...RESTStorageProvider) error nonLegacy := []*genericapiserver.APIGroupInfo{} // used later in the loop to filter the served resource by those that have expired. - resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(*s.GenericAPIServer.Version) + resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(s.GenericAPIServer.EffectiveVersion.EmulationVersion()) if err != nil { return err } diff --git a/pkg/controlplane/apiserver/config.go b/pkg/controlplane/apiserver/config.go index b413fac34c6..7933f076ec6 100644 --- a/pkg/controlplane/apiserver/config.go +++ b/pkg/controlplane/apiserver/config.go @@ -185,7 +185,9 @@ func BuildGenericConfig( } storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig() + storageFactoryConfig.CurrentVersion = genericConfig.EffectiveVersion storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig + storageFactoryConfig.DefaultResourceEncoding.SetEffectiveVersion(genericConfig.EffectiveVersion) storageFactory, lastErr = storageFactoryConfig.Complete(s.Etcd).New() if lastErr != nil { return diff --git a/pkg/controlplane/apiserver/config_test.go b/pkg/controlplane/apiserver/config_test.go index 80bff6f50a8..c5ff6a79b83 100644 --- a/pkg/controlplane/apiserver/config_test.go +++ b/pkg/controlplane/apiserver/config_test.go @@ -24,6 +24,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" apiserveroptions "k8s.io/apiserver/pkg/server/options" + utilversion "k8s.io/apiserver/pkg/util/version" + "k8s.io/component-base/featuregate" aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/controlplane/apiserver/options" @@ -32,7 +34,9 @@ import ( ) func TestBuildGenericConfig(t *testing.T) { - opts := options.NewOptions() + featureGate := featuregate.NewFeatureGate() + effectiveVersion := utilversion.DefaultKubeEffectiveVersion() + opts := options.NewOptions(featureGate, effectiveVersion) s := (&apiserveroptions.SecureServingOptions{ BindAddress: netutils.ParseIPSloppy("127.0.0.1"), }).WithLoopback() @@ -66,7 +70,7 @@ func TestBuildGenericConfig(t *testing.T) { t.Errorf("There are different StorageObjectCountTracker in genericConfig and storageFactory") } - restOptions, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}) + restOptions, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}, nil) if err != nil { t.Fatal(err) } diff --git a/pkg/controlplane/apiserver/options/options.go b/pkg/controlplane/apiserver/options/options.go index 98396cd8450..bdaa56ffede 100644 --- a/pkg/controlplane/apiserver/options/options.go +++ b/pkg/controlplane/apiserver/options/options.go @@ -28,8 +28,10 @@ import ( peerreconcilers "k8s.io/apiserver/pkg/reconcilers" genericoptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/storage/storagebackend" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/util/keyutil" cliflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" logsapi "k8s.io/component-base/logs/api/v1" "k8s.io/component-base/metrics" @@ -98,9 +100,9 @@ type CompletedOptions struct { } // NewOptions creates a new ServerRunOptions object with default parameters -func NewOptions() *Options { +func NewOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *Options { s := Options{ - GenericServerRunOptions: genericoptions.NewServerRunOptions(), + GenericServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion), Etcd: genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)), SecureServing: kubeoptions.NewSecureServingOptions(), Audit: genericoptions.NewAuditOptions(), @@ -202,6 +204,10 @@ func (o *Options) Complete(alternateDNS []string, alternateIPs []net.IP) (Comple Options: *o, } + if err := completed.GenericServerRunOptions.Complete(); err != nil { + return CompletedOptions{}, err + } + // set defaults if err := completed.GenericServerRunOptions.DefaultAdvertiseAddress(completed.SecureServing.SecureServingOptions); err != nil { return CompletedOptions{}, err diff --git a/pkg/controlplane/apiserver/options/options_test.go b/pkg/controlplane/apiserver/options/options_test.go index 88fe5b961dd..8660231d2d0 100644 --- a/pkg/controlplane/apiserver/options/options_test.go +++ b/pkg/controlplane/apiserver/options/options_test.go @@ -30,9 +30,11 @@ import ( apiserveroptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/storage/etcd3" "k8s.io/apiserver/pkg/storage/storagebackend" + utilversion "k8s.io/apiserver/pkg/util/version" auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered" audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate" cliflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" "k8s.io/component-base/metrics" netutils "k8s.io/utils/net" @@ -42,12 +44,16 @@ import ( func TestAddFlags(t *testing.T) { fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError) - s := NewOptions() + featureGate := featuregate.NewFeatureGate() + effectiveVersion := utilversion.NewEffectiveVersion("1.32") + s := NewOptions(featureGate, effectiveVersion) var fss cliflag.NamedFlagSets s.AddFlags(&fss) for _, f := range fss.FlagSets { fs.AddFlagSet(f) } + featureGate.AddFlag(fs, "") + effectiveVersion.AddFlags(fs, "") args := []string{ "--enable-admission-plugins=AlwaysDeny", @@ -108,6 +114,7 @@ func TestAddFlags(t *testing.T) { "--request-timeout=2m", "--storage-backend=etcd3", "--lease-reuse-duration-seconds=100", + "--emulated-version=1.31", } fs.Parse(args) @@ -122,6 +129,8 @@ func TestAddFlags(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024), MaxRequestBodyBytes: int64(3 * 1024 * 1024), + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, Admission: &kubeoptions.AdmissionOptions{ GenericAdmission: &apiserveroptions.AdmissionOptions{ @@ -292,4 +301,8 @@ func TestAddFlags(t *testing.T) { if !reflect.DeepEqual(expected, s) { t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{}))) } + + if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" { + t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31") + } } diff --git a/pkg/controlplane/apiserver/options/validation.go b/pkg/controlplane/apiserver/options/validation.go index 1cae7323ab1..cca855be43f 100644 --- a/pkg/controlplane/apiserver/options/validation.go +++ b/pkg/controlplane/apiserver/options/validation.go @@ -100,6 +100,7 @@ func validateUnknownVersionInteroperabilityProxyFlags(options *Options) []error func (s *Options) Validate() []error { var errs []error + errs = append(errs, s.GenericServerRunOptions.Validate()...) errs = append(errs, s.Etcd.Validate()...) errs = append(errs, validateAPIPriorityAndFairness(s)...) errs = append(errs, s.SecureServing.Validate()...) diff --git a/pkg/controlplane/apiserver/options/validation_test.go b/pkg/controlplane/apiserver/options/validation_test.go index de7456c43a7..9dddc971a42 100644 --- a/pkg/controlplane/apiserver/options/validation_test.go +++ b/pkg/controlplane/apiserver/options/validation_test.go @@ -23,6 +23,7 @@ import ( kubeapiserveradmission "k8s.io/apiserver/pkg/admission" genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/component-base/featuregate" basemetrics "k8s.io/component-base/metrics" "k8s.io/kubernetes/pkg/features" @@ -200,7 +201,7 @@ func TestValidateOptions(t *testing.T) { name: "validate master count equal 0", expectErrors: true, options: &Options{ - GenericServerRunOptions: &genericoptions.ServerRunOptions{}, + GenericServerRunOptions: &genericoptions.ServerRunOptions{FeatureGate: utilfeature.DefaultFeatureGate.DeepCopy(), EffectiveVersion: utilversion.NewEffectiveVersion("1.32")}, Etcd: &genericoptions.EtcdOptions{}, SecureServing: &genericoptions.SecureServingOptionsWithLoopback{}, Audit: &genericoptions.AuditOptions{}, @@ -227,7 +228,7 @@ func TestValidateOptions(t *testing.T) { name: "validate token request enable not attempted", expectErrors: true, options: &Options{ - GenericServerRunOptions: &genericoptions.ServerRunOptions{}, + GenericServerRunOptions: &genericoptions.ServerRunOptions{FeatureGate: utilfeature.DefaultFeatureGate.DeepCopy(), EffectiveVersion: utilversion.NewEffectiveVersion("1.32")}, Etcd: &genericoptions.EtcdOptions{}, SecureServing: &genericoptions.SecureServingOptionsWithLoopback{}, Audit: &genericoptions.AuditOptions{}, diff --git a/pkg/controlplane/apiserver/peer.go b/pkg/controlplane/apiserver/peer.go index 8d04f520c66..e15b8980c8d 100644 --- a/pkg/controlplane/apiserver/peer.go +++ b/pkg/controlplane/apiserver/peer.go @@ -82,7 +82,7 @@ func BuildPeerProxy(versionedInformer clientgoinformers.SharedInformerFactory, s // The peer endpoint leases are used to find network locations of apiservers for peer proxy func CreatePeerEndpointLeaseReconciler(c genericapiserver.Config, storageFactory serverstorage.StorageFactory) (reconcilers.PeerEndpointLeaseReconciler, error) { ttl := DefaultPeerEndpointReconcilerTTL - config, err := storageFactory.NewConfig(api.Resource("apiServerPeerIPInfo")) + config, err := storageFactory.NewConfig(api.Resource("apiServerPeerIPInfo"), &api.Endpoints{}) if err != nil { return nil, fmt.Errorf("error creating storage factory config: %w", err) } diff --git a/pkg/controlplane/instance.go b/pkg/controlplane/instance.go index d51f15ec982..5f1b732de5b 100644 --- a/pkg/controlplane/instance.go +++ b/pkg/controlplane/instance.go @@ -214,7 +214,7 @@ func (c *Config) createLeaseReconciler() reconcilers.EndpointReconciler { endpointsAdapter := reconcilers.NewEndpointsAdapter(endpointClient, endpointSliceClient) ttl := c.Extra.MasterEndpointReconcileTTL - config, err := c.ControlPlane.StorageFactory.NewConfig(api.Resource("apiServerIPInfo")) + config, err := c.ControlPlane.StorageFactory.NewConfig(api.Resource("apiServerIPInfo"), &api.Endpoints{}) if err != nil { klog.Fatalf("Error creating storage factory config: %v", err) } diff --git a/pkg/controlplane/instance_test.go b/pkg/controlplane/instance_test.go index d4c415101bc..32d1811d82c 100644 --- a/pkg/controlplane/instance_test.go +++ b/pkg/controlplane/instance_test.go @@ -55,6 +55,7 @@ import ( serverstorage "k8s.io/apiserver/pkg/server/storage" etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing" "k8s.io/apiserver/pkg/util/openapi" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/discovery" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" @@ -103,7 +104,9 @@ func setUp(t *testing.T) (*etcd3testing.EtcdTestServer, Config, *assert.Assertio }, } + config.ControlPlane.Generic.EffectiveVersion = utilversion.DefaultKubeEffectiveVersion() storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig() + storageFactoryConfig.DefaultResourceEncoding.SetEffectiveVersion(config.ControlPlane.Generic.EffectiveVersion) storageConfig.StorageObjectCountTracker = config.ControlPlane.Generic.StorageObjectCountTracker resourceEncoding := resourceconfig.MergeResourceEncodingConfigs(storageFactoryConfig.DefaultResourceEncoding, storageFactoryConfig.ResourceEncodingOverrides) storageFactory := serverstorage.NewDefaultStorageFactory(*storageConfig, "application/vnd.kubernetes.protobuf", storageFactoryConfig.Serializer, resourceEncoding, DefaultAPIResourceConfigSource(), nil) diff --git a/pkg/features/client_adapter_test.go b/pkg/features/client_adapter_test.go index 13ec29094ec..cae067ca99c 100644 --- a/pkg/features/client_adapter_test.go +++ b/pkg/features/client_adapter_test.go @@ -19,6 +19,8 @@ package features import ( "testing" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" clientfeatures "k8s.io/client-go/features" "k8s.io/component-base/featuregate" ) @@ -78,8 +80,7 @@ func TestClientAdapterAdd(t *testing.T) { t.Errorf("expected feature %q not found", name) continue } - - if actual != expected { + if diff := cmp.Diff(actual, expected, cmpopts.IgnoreFields(featuregate.FeatureSpec{}, "Version")); diff != "" { t.Errorf("expected feature %q spec %#v, got spec %#v", name, expected, actual) } } diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index a83aa2a27f3..f84825e8e9f 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -946,6 +946,7 @@ const ( func init() { runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates)) + runtime.Must(utilfeature.DefaultMutableFeatureGate.AddVersioned(defaultVersionedKubernetesFeatureGates)) // Register all client-go features with kube's feature gate instance and make all client-go // feature checks use kube's instance. The effect is that for kube binaries, client-go diff --git a/pkg/features/kube_features_test.go b/pkg/features/kube_features_test.go index 81bfbc421d2..be847a1f040 100644 --- a/pkg/features/kube_features_test.go +++ b/pkg/features/kube_features_test.go @@ -62,6 +62,9 @@ func TestAllRegisteredFeaturesExpected(t *testing.T) { if err := knownFeatureGates.Add(defaultKubernetesFeatureGates); err != nil { t.Fatal(err) } + if err := knownFeatureGates.AddVersioned(defaultVersionedKubernetesFeatureGates); err != nil { + t.Fatal(err) + } knownFeatures := knownFeatureGates.GetAll() for registeredFeature := range registeredFeatures { diff --git a/pkg/features/versioned_kube_features.go b/pkg/features/versioned_kube_features.go new file mode 100644 index 00000000000..7ef53861822 --- /dev/null +++ b/pkg/features/versioned_kube_features.go @@ -0,0 +1,34 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + "k8s.io/component-base/featuregate" +) + +// defaultVersionedKubernetesFeatureGates consists of all known Kubernetes-specific feature keys with VersionedSpecs. +// To add a new feature, define a key for it and add it here. The features will be +// available throughout Kubernetes binaries. +// +// Entries are separated from each other with blank lines to avoid sweeping gofmt changes +// when adding or removing one entry. +var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{ + // Example: + // genericfeatures.EmulationVersion: { + // {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha}, + // }, +} diff --git a/pkg/kubeapiserver/default_storage_factory_builder.go b/pkg/kubeapiserver/default_storage_factory_builder.go index f73289ba761..0622862d596 100644 --- a/pkg/kubeapiserver/default_storage_factory_builder.go +++ b/pkg/kubeapiserver/default_storage_factory_builder.go @@ -25,6 +25,7 @@ import ( "k8s.io/apiserver/pkg/server/resourceconfig" serverstorage "k8s.io/apiserver/pkg/server/storage" "k8s.io/apiserver/pkg/storage/storagebackend" + "k8s.io/apiserver/pkg/util/version" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/apps" "k8s.io/kubernetes/pkg/apis/certificates" @@ -93,6 +94,7 @@ type StorageFactoryConfig struct { Serializer runtime.StorageSerializer ResourceEncodingOverrides []schema.GroupVersionResource EtcdServersOverrides []string + CurrentVersion version.EffectiveVersion } // Complete completes the StorageFactoryConfig with provided etcdOptions returning completedStorageFactoryConfig. diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go index f808615ecd1..4e57732e8f3 100644 --- a/pkg/registry/core/rest/storage_core.go +++ b/pkg/registry/core/rest/storage_core.go @@ -321,7 +321,7 @@ func (p *legacyProvider) NewRESTStorage(apiResourceConfigSource serverstorage.AP func (c *Config) newServiceIPAllocators() (registries rangeRegistries, primaryClusterIPAllocator ipallocator.Interface, clusterIPAllocators map[api.IPFamily]ipallocator.Interface, nodePortAllocator *portallocator.PortAllocator, err error) { clusterIPAllocators = map[api.IPFamily]ipallocator.Interface{} - serviceStorageConfig, err := c.StorageFactory.NewConfig(api.Resource("services")) + serviceStorageConfig, err := c.StorageFactory.NewConfig(api.Resource("services"), &api.Service{}) if err != nil { return rangeRegistries{}, nil, nil, nil, err } diff --git a/pkg/registry/core/rest/storage_core_test.go b/pkg/registry/core/rest/storage_core_test.go index 411efa211dc..4c74d034162 100644 --- a/pkg/registry/core/rest/storage_core_test.go +++ b/pkg/registry/core/rest/storage_core_test.go @@ -19,6 +19,7 @@ package rest import ( "testing" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/server/storage" "k8s.io/apiserver/pkg/storage/storagebackend" @@ -40,7 +41,7 @@ func TestGetServersToValidate(t *testing.T) { type fakeStorageFactory struct{} -func (f fakeStorageFactory) NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error) { +func (f fakeStorageFactory) NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) { return nil, nil } diff --git a/pkg/registry/registrytest/etcd.go b/pkg/registry/registrytest/etcd.go index 990d6b3d277..226c35c1e7f 100644 --- a/pkg/registry/registrytest/etcd.go +++ b/pkg/registry/registrytest/etcd.go @@ -44,7 +44,7 @@ func NewEtcdStorageForResource(t *testing.T, resource schema.GroupResource) (*st if err != nil { t.Fatalf("Error while making storage factory: %v", err) } - resourceConfig, err := factory.NewConfig(resource) + resourceConfig, err := factory.NewConfig(resource, nil) if err != nil { t.Fatalf("Error while finding storage destination: %v", err) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go index 9c0ee416f52..e0440f7a185 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go @@ -1178,8 +1178,9 @@ type crdConversionRESTOptionsGetter struct { preserveUnknownFields bool } -func (t crdConversionRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) { - ret, err := t.RESTOptionsGetter.GetRESTOptions(resource) +func (t crdConversionRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) { + // Explicitly ignore example, we override storageconfig below + ret, err := t.RESTOptionsGetter.GetRESTOptions(resource, nil) if err == nil { d := schemaCoercingDecoder{delegate: ret.StorageConfig.Codec, validator: unstructuredSchemaCoercer{ // drop invalid fields while decoding old CRs (before we haven't had any ObjectMeta validation) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go index 3d42acf8b3d..07fdcb940f5 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go @@ -39,9 +39,11 @@ import ( flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request" "k8s.io/apiserver/pkg/util/openapi" "k8s.io/apiserver/pkg/util/proxy" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/apiserver/pkg/util/webhook" scheme "k8s.io/client-go/kubernetes/scheme" corev1 "k8s.io/client-go/listers/core/v1" + "k8s.io/component-base/featuregate" netutils "k8s.io/utils/net" ) @@ -58,9 +60,9 @@ type CustomResourceDefinitionsServerOptions struct { } // NewCustomResourceDefinitionsServerOptions creates default options of an apiextensions-apiserver. -func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer) *CustomResourceDefinitionsServerOptions { +func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer, featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *CustomResourceDefinitionsServerOptions { o := &CustomResourceDefinitionsServerOptions{ - ServerRunOptions: genericoptions.NewServerRunOptions(), + ServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion), RecommendedOptions: genericoptions.NewRecommendedOptions( defaultEtcdPathPrefix, apiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion), @@ -92,7 +94,7 @@ func (o CustomResourceDefinitionsServerOptions) Validate() error { // Complete fills in missing options. func (o *CustomResourceDefinitionsServerOptions) Complete() error { - return nil + return o.ServerRunOptions.Complete() } // Config returns an apiextensions-apiserver configuration. diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go index 0509a6aa6df..b59b12f6da9 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go @@ -24,15 +24,26 @@ import ( "k8s.io/apiextensions-apiserver/pkg/cmd/server/options" genericapiserver "k8s.io/apiserver/pkg/server" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" ) func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command { - o := options.NewCustomResourceDefinitionsServerOptions(out, errOut) + // effectiveVersion is used to set what apis and feature gates the generic api server is compatible with. + // You can also have the flag setting the effectiveVersion of the apiextensions apiserver, and + // having a mapping from the apiextensions apiserver version to generic apiserver version. + effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( + utilversion.ComponentGenericAPIServer, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) + o := options.NewCustomResourceDefinitionsServerOptions(out, errOut, featureGate, effectiveVersion) cmd := &cobra.Command{ Short: "Launch an API extensions API server", Long: "Launch an API extensions API server", RunE: func(c *cobra.Command, args []string) error { + if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { + return err + } + if err := o.Complete(); err != nil { return err } @@ -48,6 +59,8 @@ func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command cmd.SetContext(ctx) fs := cmd.Flags() + featureGate.AddFlag(fs, "") + effectiveVersion.AddFlags(fs, "") o.AddFlags(fs) return cmd } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go index 3f2e7812bf4..b1e5b9dd8d6 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go @@ -35,7 +35,9 @@ import ( openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/storage/storagebackend" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/openapi" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" logsapi "k8s.io/component-base/logs/api/v1" @@ -120,7 +122,13 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin fs := pflag.NewFlagSet("test", pflag.PanicOnError) - s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr) + featureGate := utilfeature.DefaultMutableFeatureGate + effectiveVersion := utilversion.DefaultKubeEffectiveVersion() + _ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.ComponentGenericAPIServer, effectiveVersion, featureGate, true) + s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr, featureGate, effectiveVersion) + + featureGate.AddFlag(fs, "") + effectiveVersion.AddFlags(fs, "") s.AddFlags(fs) s.RecommendedOptions.SecureServing.Listener, s.RecommendedOptions.SecureServing.BindPort, err = createLocalhostListenerOnFreePort() @@ -143,6 +151,10 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin fs.Parse(customFlags) + if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { + return result, err + } + if err := s.Complete(); err != nil { return result, fmt.Errorf("failed to set default options: %v", err) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go index 01a15e02c48..71e14f2cbd5 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go @@ -183,7 +183,7 @@ func testWebhookConverter(t *testing.T, watchCache bool) { crd := multiVersionFixture.DeepCopy() RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd, nil, nil) - restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}) + restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}, nil) if err != nil { t.Fatal(err) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/defaulting_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/defaulting_test.go index dcfdf2f7525..eff48b4f847 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/defaulting_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/defaulting_test.go @@ -659,7 +659,7 @@ func TestCustomResourceDefaultingOfMetaFields(t *testing.T) { // get persisted object RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd, nil, nil) - restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}) + restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}, nil) if err != nil { t.Fatal(err) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/server.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/server.go index 8be6a88e54c..60a98f335a9 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/server.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/server.go @@ -156,7 +156,7 @@ func StartDefaultServerWithClientsAndEtcd(t servertesting.Logger, extraFlags ... } RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd, resourceTransformers, nil) - restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "hopefully-ignored-group", Resource: "hopefully-ignored-resources"}) + restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "hopefully-ignored-group", Resource: "hopefully-ignored-resources"}, nil) if err != nil { return nil, nil, nil, nil, "", err } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/objectmeta_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/objectmeta_test.go index 9ac7387d7ab..67e85bcc125 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/objectmeta_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/objectmeta_test.go @@ -133,7 +133,7 @@ func TestInvalidObjectMetaInStorage(t *testing.T) { } RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd, nil, nil) - restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: noxuDefinition.Spec.Group, Resource: noxuDefinition.Spec.Names.Plural}) + restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: noxuDefinition.Spec.Group, Resource: noxuDefinition.Spec.Names.Plural}, nil) if err != nil { t.Fatal(err) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/pruning_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/pruning_test.go index 003a9b55ac5..18042c3a973 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/pruning_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/pruning_test.go @@ -326,7 +326,7 @@ func TestPruningFromStorage(t *testing.T) { t.Fatal(err) } - restOptions, err := completedConfig.GenericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}) + restOptions, err := completedConfig.GenericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}, nil) if err != nil { t.Fatal(err) } diff --git a/staging/src/k8s.io/apimachinery/pkg/util/version/version.go b/staging/src/k8s.io/apimachinery/pkg/util/version/version.go index 2292ba13765..d327f00b3be 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/version/version.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/version/version.go @@ -145,6 +145,43 @@ func MustParseGeneric(str string) *Version { return v } +// Parse tries to do ParseSemantic first to keep more information. +// If ParseSemantic fails, it would just do ParseGeneric. +func Parse(str string) (*Version, error) { + v, err := parse(str, true) + if err != nil { + return parse(str, false) + } + return v, err +} + +// MustParse is like Parse except that it panics on error +func MustParse(str string) *Version { + v, err := Parse(str) + if err != nil { + panic(err) + } + return v +} + +// ParseMajorMinor parses a "generic" version string and returns a version with the major and minor version. +func ParseMajorMinor(str string) (*Version, error) { + v, err := ParseGeneric(str) + if err != nil { + return nil, err + } + return MajorMinor(v.Major(), v.Minor()), nil +} + +// MustParseMajorMinor is like ParseMajorMinor except that it panics on error +func MustParseMajorMinor(str string) *Version { + v, err := ParseMajorMinor(str) + if err != nil { + panic(err) + } + return v +} + // ParseSemantic parses a version string that exactly obeys the syntax and semantics of // the "Semantic Versioning" specification (http://semver.org/) (although it ignores // leading and trailing whitespace, and allows the version to be preceded by "v"). For @@ -215,6 +252,21 @@ func (v *Version) WithMinor(minor uint) *Version { return &result } +// SubtractMinor returns the version diff minor versions back, with the same major and no patch. +// If diff >= current minor, the minor would be 0. +func (v *Version) SubtractMinor(diff uint) *Version { + var minor uint + if diff < v.Minor() { + minor = v.Minor() - diff + } + return MajorMinor(v.Major(), minor) +} + +// AddMinor returns the version diff minor versions forward, with the same major and no patch. +func (v *Version) AddMinor(diff uint) *Version { + return MajorMinor(v.Major(), v.Minor()+diff) +} + // WithPatch returns copy of the version object with requested patch number func (v *Version) WithPatch(patch uint) *Version { result := *v @@ -224,6 +276,9 @@ func (v *Version) WithPatch(patch uint) *Version { // WithPreRelease returns copy of the version object with requested prerelease func (v *Version) WithPreRelease(preRelease string) *Version { + if len(preRelease) == 0 { + return v + } result := *v result.components = []uint{v.Major(), v.Minor(), v.Patch()} result.preRelease = preRelease @@ -345,6 +400,17 @@ func onlyZeros(array []uint) bool { return true } +// EqualTo tests if a version is equal to a given version. +func (v *Version) EqualTo(other *Version) bool { + if v == nil { + return other == nil + } + if other == nil { + return false + } + return v.compareInternal(other) == 0 +} + // AtLeast tests if a version is at least equal to a given minimum version. If both // Versions are Semantic Versions, this will use the Semantic Version comparison // algorithm. Otherwise, it will compare only the numeric components, with non-present @@ -360,6 +426,11 @@ func (v *Version) LessThan(other *Version) bool { return v.compareInternal(other) == -1 } +// GreaterThan tests if a version is greater than a given version. +func (v *Version) GreaterThan(other *Version) bool { + return v.compareInternal(other) == 1 +} + // Compare compares v against a version string (which will be parsed as either Semantic // or non-Semantic depending on v). On success it returns -1 if v is less than other, 1 if // it is greater than other, or 0 if they are equal. diff --git a/staging/src/k8s.io/apimachinery/pkg/util/version/version_test.go b/staging/src/k8s.io/apimachinery/pkg/util/version/version_test.go index 4d36bf3c121..c8625fb8091 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/version/version_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/version/version_test.go @@ -452,3 +452,94 @@ func TestHighestSupportedVersion(t *testing.T) { } } } + +func TestSubtractMinor(t *testing.T) { + var tests = []struct { + version string + diff uint + expectedComponents []uint + }{ + { + version: "1.0.2", + diff: 3, + expectedComponents: []uint{1, 0}, + }, + { + version: "1.3.2-alpha+001", + diff: 2, + expectedComponents: []uint{1, 1}, + }, + { + version: "1.3.2-alpha+001", + diff: 3, + expectedComponents: []uint{1, 0}, + }, + { + version: "1.20", + diff: 5, + expectedComponents: []uint{1, 15}, + }, + } + + for _, test := range tests { + version, _ := ParseGeneric(test.version) + if !reflect.DeepEqual(test.expectedComponents, version.SubtractMinor(test.diff).Components()) { + t.Error("parse returned un'expected components") + } + } +} + +func TestParse(t *testing.T) { + + var tests = []struct { + version string + expectErr bool + expectedComponents []uint + expectedPreRelease string + expectedBuildMetadata string + }{ + { + version: "1.0.2", + expectedComponents: []uint{1, 0, 2}, + }, + { + version: "1.0.2-alpha+001", + expectedComponents: []uint{1, 0, 2}, + expectedPreRelease: "alpha", + expectedBuildMetadata: "001", + }, + { + version: "1.2", + expectedComponents: []uint{1, 2}, + }, + { + version: "1.0.2-beta+exp.sha.5114f85", + expectedComponents: []uint{1, 0, 2}, + expectedPreRelease: "beta", + expectedBuildMetadata: "exp.sha.5114f85", + }, + { + version: "a.b.c", + expectErr: true, + }, + } + + for _, test := range tests { + version, err := Parse(test.version) + if test.expectErr { + if err == nil { + t.Fatalf("got no err, expected err") + } + continue + } + if !reflect.DeepEqual(test.expectedComponents, version.Components()) { + t.Error("parse returned un'expected components") + } + if test.expectedPreRelease != version.PreRelease() { + t.Errorf("parse returned version.PreRelease %s, expected %s", test.expectedPreRelease, version.PreRelease()) + } + if test.expectedBuildMetadata != version.BuildMetadata() { + t.Errorf("parse returned version.BuildMetadata %s, expected %s", test.expectedBuildMetadata, version.BuildMetadata()) + } + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go b/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go index 9153869995b..837cbacddb0 100644 --- a/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go +++ b/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go @@ -30,20 +30,27 @@ import ( "k8s.io/apimachinery/pkg/util/version" celconfig "k8s.io/apiserver/pkg/apis/cel" "k8s.io/apiserver/pkg/cel/library" + utilversion "k8s.io/apiserver/pkg/util/version" ) // DefaultCompatibilityVersion returns a default compatibility version for use with EnvSet // that guarantees compatibility with CEL features/libraries/parameters understood by -// an n-1 version +// the api server min compatibility version // -// This default will be set to no more than n-1 the current Kubernetes major.minor version. +// This default will be set to no more than the current Kubernetes major.minor version. // -// Note that a default version number less than n-1 indicates a wider range of version -// compatibility than strictly required for rollback. A wide range of compatibility is -// desirable because it means that CEL expressions are portable across a wider range -// of Kubernetes versions. +// Note that a default version number less than n-1 the current Kubernetes major.minor version +// indicates a wider range of version compatibility than strictly required for rollback. +// A wide range of compatibility is desirable because it means that CEL expressions are portable +// across a wider range of Kubernetes versions. +// A default version number equal to the current Kubernetes major.minor version +// indicates fast forward CEL features that can be used when rollback is no longer needed. func DefaultCompatibilityVersion() *version.Version { - return version.MajorMinor(1, 30) + effectiveVer := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer) + if effectiveVer == nil { + effectiveVer = utilversion.DefaultKubeEffectiveVersion() + } + return effectiveVer.MinCompatibilityVersion() } var baseOpts = append(baseOptsWithoutStrictCost, StrictCostOpt) diff --git a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go index 80dc25cc6b1..8448e36ec6a 100644 --- a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go +++ b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go @@ -322,6 +322,17 @@ const ( func init() { runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates)) + runtime.Must(utilfeature.DefaultMutableFeatureGate.AddVersioned(defaultVersionedKubernetesFeatureGates)) +} + +// defaultVersionedKubernetesFeatureGates consists of all known Kubernetes-specific feature keys with VersionedSpecs. +// To add a new feature, define a key for it above and add it here. The features will be +// available throughout Kubernetes binaries. +var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{ + // Example: + // EmulationVersion: { + // {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha}, + // }, } // defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys. diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/options.go b/staging/src/k8s.io/apiserver/pkg/registry/generic/options.go index d675a258f5e..44d07c0e241 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/options.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/options.go @@ -19,6 +19,7 @@ package generic import ( "time" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/storagebackend" @@ -39,12 +40,15 @@ type RESTOptions struct { } // Implement RESTOptionsGetter so that RESTOptions can directly be used when available (i.e. tests) -func (opts RESTOptions) GetRESTOptions(schema.GroupResource) (RESTOptions, error) { +func (opts RESTOptions) GetRESTOptions(schema.GroupResource, runtime.Object) (RESTOptions, error) { return opts, nil } type RESTOptionsGetter interface { - GetRESTOptions(resource schema.GroupResource) (RESTOptions, error) + // GetRESTOptions returns the RESTOptions for the given resource and example object. + // The example object is used to determine the storage version for the resource. + // If the example object is nil, the storage version will be determined by the resource's default storage version. + GetRESTOptions(resource schema.GroupResource, example runtime.Object) (RESTOptions, error) } // StoreOptions is set of configuration options used to complete generic registries. diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go index a8e01708a30..98efac93339 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go @@ -1518,7 +1518,7 @@ func (e *Store) CompleteWithOptions(options *generic.StoreOptions) error { return err } - opts, err := options.RESTOptions.GetRESTOptions(e.DefaultQualifiedResource) + opts, err := options.RESTOptions.GetRESTOptions(e.DefaultQualifiedResource, e.NewFunc()) if err != nil { return err } diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index 0502448d6eb..b266cb2e494 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -42,8 +42,9 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/version" utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup" - "k8s.io/apimachinery/pkg/version" + apimachineryversion "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/audit" "k8s.io/apiserver/pkg/authentication/authenticator" @@ -70,8 +71,10 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol" flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/informers" restclient "k8s.io/client-go/rest" + "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" "k8s.io/component-base/metrics/features" "k8s.io/component-base/metrics/prometheus/slis" @@ -148,7 +151,12 @@ type Config struct { PostStartHooks map[string]PostStartHookConfigEntry // Version will enable the /version endpoint if non-nil - Version *version.Info + Version *apimachineryversion.Info + // EffectiveVersion determines which apis and features are available + // based on when the api/feature lifecyle. + EffectiveVersion utilversion.EffectiveVersion + // FeatureGate is a way to plumb feature gate through if you have them. + FeatureGate featuregate.FeatureGate // AuditBackend is where audit events are sent to. AuditBackend audit.Backend // AuditPolicyRuleEvaluator makes the decision of whether and how to audit log a request. @@ -585,7 +593,7 @@ func (c *Config) AddPostStartHookOrDie(name string, hook PostStartHookFunc) { } } -func completeOpenAPI(config *openapicommon.Config, version *version.Info) { +func completeOpenAPI(config *openapicommon.Config, version *version.Version) { if config == nil { return } @@ -624,7 +632,7 @@ func completeOpenAPI(config *openapicommon.Config, version *version.Info) { } } -func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Info) { +func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Version) { if config == nil { return } @@ -676,6 +684,9 @@ func (c *Config) ShutdownInitiatedNotify() <-chan struct{} { // Complete fills in any fields not set that are required to have valid data and can be derived // from other fields. If you're going to `ApplyOptions`, do that first. It's mutating the receiver. func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedConfig { + if c.FeatureGate == nil { + c.FeatureGate = utilfeature.DefaultFeatureGate + } if len(c.ExternalAddress) == 0 && c.PublicAddress != nil { c.ExternalAddress = c.PublicAddress.String() } @@ -691,9 +702,12 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo } c.ExternalAddress = net.JoinHostPort(c.ExternalAddress, strconv.Itoa(port)) } - - completeOpenAPI(c.OpenAPIConfig, c.Version) - completeOpenAPIV3(c.OpenAPIV3Config, c.Version) + var ver *version.Version + if c.EffectiveVersion != nil { + ver = c.EffectiveVersion.EmulationVersion() + } + completeOpenAPI(c.OpenAPIConfig, ver) + completeOpenAPIV3(c.OpenAPIV3Config, ver) if c.DiscoveryAddresses == nil { c.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: c.ExternalAddress} @@ -711,7 +725,7 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo } else { c.EquivalentResourceRegistry = runtime.NewEquivalentResourceRegistryWithIdentity(func(groupResource schema.GroupResource) string { // use the storage prefix as the key if possible - if opts, err := c.RESTOptionsGetter.GetRESTOptions(groupResource); err == nil { + if opts, err := c.RESTOptionsGetter.GetRESTOptions(groupResource, nil); err == nil { return opts.ResourcePrefix } // otherwise return "" to use the default key (parent GV name) @@ -819,7 +833,9 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G APIServerID: c.APIServerID, StorageVersionManager: c.StorageVersionManager, - Version: c.Version, + EffectiveVersion: c.EffectiveVersion, + Version: c.Version, + FeatureGate: c.FeatureGate, muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{}, } diff --git a/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go index f14ed9213d1..841c1f5729c 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go +++ b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go @@ -17,6 +17,7 @@ limitations under the License. package server import ( + "fmt" "os" "strconv" "strings" @@ -25,16 +26,15 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" - apimachineryversion "k8s.io/apimachinery/pkg/version" + apimachineryversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apiserver/pkg/registry/rest" "k8s.io/klog/v2" ) // resourceExpirationEvaluator holds info for deciding if a particular rest.Storage needs to excluded from the API type resourceExpirationEvaluator struct { - currentMajor int - currentMinor int - isAlpha bool + currentVersion *apimachineryversion.Version + isAlpha bool // This is usually set for testing for which tests need to be removed. This prevent insta-failing CI. // Set KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA to see what will be removed when we tag beta strictRemovedHandlingInAlpha bool @@ -53,30 +53,17 @@ type ResourceExpirationEvaluator interface { ShouldServeForVersion(majorRemoved, minorRemoved int) bool } -func NewResourceExpirationEvaluator(currentVersion apimachineryversion.Info) (ResourceExpirationEvaluator, error) { +func NewResourceExpirationEvaluator(currentVersion *apimachineryversion.Version) (ResourceExpirationEvaluator, error) { + if currentVersion == nil { + return nil, fmt.Errorf("empty NewResourceExpirationEvaluator currentVersion") + } + klog.V(1).Infof("NewResourceExpirationEvaluator with currentVersion: %s.", currentVersion) ret := &resourceExpirationEvaluator{ strictRemovedHandlingInAlpha: false, } - if len(currentVersion.Major) > 0 { - currentMajor64, err := strconv.ParseInt(currentVersion.Major, 10, 32) - if err != nil { - return nil, err - } - ret.currentMajor = int(currentMajor64) - } - if len(currentVersion.Minor) > 0 { - // split the "normal" + and - for semver stuff - minorString := strings.Split(currentVersion.Minor, "+")[0] - minorString = strings.Split(minorString, "-")[0] - minorString = strings.Split(minorString, ".")[0] - currentMinor64, err := strconv.ParseInt(minorString, 10, 32) - if err != nil { - return nil, err - } - ret.currentMinor = int(currentMinor64) - } - - ret.isAlpha = strings.Contains(currentVersion.GitVersion, "alpha") + // Only keeps the major and minor versions from input version. + ret.currentVersion = apimachineryversion.MajorMinor(currentVersion.Major(), currentVersion.Minor()) + ret.isAlpha = strings.Contains(currentVersion.PreRelease(), "alpha") if envString, ok := os.LookupEnv("KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA"); !ok { // do nothing @@ -112,6 +99,16 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio return false } + introduced, ok := versionedPtr.(introducedInterface) + // skip the introduced check for test where currentVersion is 0.0 + if ok && (e.currentVersion.Major() > 0 || e.currentVersion.Minor() > 0) { + majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced() + verIntroduced := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced)) + if e.currentVersion.LessThan(verIntroduced) { + return false + } + } + removed, ok := versionedPtr.(removedInterface) if !ok { return true @@ -121,16 +118,11 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio } func (e *resourceExpirationEvaluator) ShouldServeForVersion(majorRemoved, minorRemoved int) bool { - if e.currentMajor < majorRemoved { + removedVer := apimachineryversion.MajorMinor(uint(majorRemoved), uint(minorRemoved)) + if removedVer.GreaterThan(e.currentVersion) { return true } - if e.currentMajor > majorRemoved { - return false - } - if e.currentMinor < minorRemoved { - return true - } - if e.currentMinor > minorRemoved { + if removedVer.LessThan(e.currentVersion) { return false } // at this point major and minor are equal, so this API should be removed when the current release GAs. @@ -152,6 +144,11 @@ type removedInterface interface { APILifecycleRemoved() (major, minor int) } +// Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go. +type introducedInterface interface { + APILifecycleIntroduced() (major, minor int) +} + // removeDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted. // versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage. func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage) { @@ -171,6 +168,8 @@ func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versi } klog.V(1).Infof("Removing resource %v.%v.%v because it is time to stop serving it per APILifecycle.", resourceName, apiVersion, groupName) + storage := versionToResource[resourceName] + storage.Destroy() delete(versionToResource, resourceName) } versionedResourcesStorageMap[apiVersion] = versionToResource diff --git a/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds_test.go b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds_test.go index c033990b405..84403fd1587 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds_test.go @@ -25,57 +25,41 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/dump" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/version" + apimachineryversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apiserver/pkg/registry/rest" ) func Test_newResourceExpirationEvaluator(t *testing.T) { tests := []struct { name string - currentVersion version.Info + currentVersion string expected resourceExpirationEvaluator expectedErr string }{ { - name: "beta", - currentVersion: version.Info{ - Major: "1", - Minor: "20+", - GitVersion: "v1.20.0-beta.0.62+a5d22854a2ac21", - }, - expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20}, + name: "beta", + currentVersion: "v1.20.0-beta.0.62+a5d22854a2ac21", + expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)}, }, { - name: "alpha", - currentVersion: version.Info{ - Major: "1", - Minor: "20+", - GitVersion: "v1.20.0-alpha.0.62+a5d22854a2ac21", - }, - expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20, isAlpha: true}, + name: "alpha", + currentVersion: "v1.20.0-alpha.0.62+a5d22854a2ac21", + expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20), isAlpha: true}, }, { - name: "maintenance", - currentVersion: version.Info{ - Major: "1", - Minor: "20+", - GitVersion: "v1.20.1", - }, - expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20}, + name: "maintenance", + currentVersion: "v1.20.1", + expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)}, }, { - name: "bad", - currentVersion: version.Info{ - Major: "1", - Minor: "20something+", - GitVersion: "v1.20.1", - }, - expectedErr: `strconv.ParseInt: parsing "20something": invalid syntax`, + name: "no v prefix", + currentVersion: "1.20.1", + expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - actual, actualErr := NewResourceExpirationEvaluator(tt.currentVersion) + actual, actualErr := NewResourceExpirationEvaluator(apimachineryversion.MustParse(tt.currentVersion)) checkErr(t, actualErr, tt.expectedErr) if actualErr != nil { @@ -90,12 +74,12 @@ func Test_newResourceExpirationEvaluator(t *testing.T) { } } -func storageRemovedIn(major, minor int) removedInStorage { - return removedInStorage{major: major, minor: minor} +func storageRemovedIn(major, minor int) *removedInStorage { + return &removedInStorage{major: major, minor: minor} } -func storageNeverRemoved() removedInStorage { - return removedInStorage{neverRemoved: true} +func storageNeverRemoved() *removedInStorage { + return &removedInStorage{neverRemoved: true} } type removedInStorage struct { @@ -103,23 +87,23 @@ type removedInStorage struct { neverRemoved bool } -func (r removedInStorage) New() runtime.Object { +func (r *removedInStorage) New() runtime.Object { if r.neverRemoved { - return neverRemovedObj{} + return &defaultObj{} } - return removedInObj{major: r.major, minor: r.minor} + return &removedInObj{major: r.major, minor: r.minor} } -func (r removedInStorage) Destroy() { +func (r *removedInStorage) Destroy() { } -type neverRemovedObj struct { +type defaultObj struct { } -func (r neverRemovedObj) GetObjectKind() schema.ObjectKind { +func (r *defaultObj) GetObjectKind() schema.ObjectKind { panic("don't do this") } -func (r neverRemovedObj) DeepCopyObject() runtime.Object { +func (r *defaultObj) DeepCopyObject() runtime.Object { panic("don't do this either") } @@ -127,13 +111,45 @@ type removedInObj struct { major, minor int } -func (r removedInObj) GetObjectKind() schema.ObjectKind { +func (r *removedInObj) GetObjectKind() schema.ObjectKind { panic("don't do this") } -func (r removedInObj) DeepCopyObject() runtime.Object { +func (r *removedInObj) DeepCopyObject() runtime.Object { panic("don't do this either") } -func (r removedInObj) APILifecycleRemoved() (major, minor int) { +func (r *removedInObj) APILifecycleRemoved() (major, minor int) { + return r.major, r.minor +} + +func storageIntroducedIn(major, minor int) *introducedInStorage { + return &introducedInStorage{major: major, minor: minor} +} + +type introducedInStorage struct { + major, minor int +} + +func (r *introducedInStorage) New() runtime.Object { + if r.major == 0 && r.minor == 0 { + return &defaultObj{} + } + return &IntroducedInObj{major: r.major, minor: r.minor} +} + +func (r *introducedInStorage) Destroy() { +} + +type IntroducedInObj struct { + major, minor int +} + +func (r *IntroducedInObj) GetObjectKind() schema.ObjectKind { + panic("don't do this") +} +func (r *IntroducedInObj) DeepCopyObject() runtime.Object { + panic("don't do this either") +} +func (r *IntroducedInObj) APILifecycleIntroduced() (major, minor int) { return r.major, r.minor } @@ -147,8 +163,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-curr", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), }, restStorage: storageRemovedIn(1, 20), expected: false, @@ -156,8 +171,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-curr-but-deferred", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), serveRemovedAPIsOneMoreRelease: true, }, restStorage: storageRemovedIn(1, 20), @@ -166,9 +180,8 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-curr-but-alpha", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, - isAlpha: true, + currentVersion: apimachineryversion.MajorMinor(1, 20), + isAlpha: true, }, restStorage: storageRemovedIn(1, 20), expected: true, @@ -176,8 +189,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-curr-but-alpha-but-strict", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), isAlpha: true, strictRemovedHandlingInAlpha: true, }, @@ -187,8 +199,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-prev-deferral-does-not-help", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 21, + currentVersion: apimachineryversion.MajorMinor(1, 21), serveRemovedAPIsOneMoreRelease: true, }, restStorage: storageRemovedIn(1, 20), @@ -197,8 +208,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-prev-major", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 2, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(2, 20), serveRemovedAPIsOneMoreRelease: true, }, restStorage: storageRemovedIn(1, 20), @@ -207,8 +217,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "removed-in-future", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), }, restStorage: storageRemovedIn(1, 21), expected: true, @@ -216,12 +225,43 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) { { name: "never-removed", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), }, restStorage: storageNeverRemoved(), expected: true, }, + { + name: "introduced-in-curr", + resourceExpirationEvaluator: resourceExpirationEvaluator{ + currentVersion: apimachineryversion.MajorMinor(1, 20), + }, + restStorage: storageIntroducedIn(1, 20), + expected: true, + }, + { + name: "introduced-in-prev-major", + resourceExpirationEvaluator: resourceExpirationEvaluator{ + currentVersion: apimachineryversion.MajorMinor(1, 20), + }, + restStorage: storageIntroducedIn(1, 19), + expected: true, + }, + { + name: "introduced-in-future", + resourceExpirationEvaluator: resourceExpirationEvaluator{ + currentVersion: apimachineryversion.MajorMinor(1, 20), + }, + restStorage: storageIntroducedIn(1, 21), + expected: false, + }, + { + name: "missing-introduced", + resourceExpirationEvaluator: resourceExpirationEvaluator{ + currentVersion: apimachineryversion.MajorMinor(1, 20), + }, + restStorage: storageIntroducedIn(0, 0), + expected: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -269,8 +309,7 @@ func Test_removeDeletedKinds(t *testing.T) { { name: "remove-one-of-two", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), }, versionedResourcesStorageMap: map[string]map[string]rest.Storage{ "v1": { @@ -287,8 +326,7 @@ func Test_removeDeletedKinds(t *testing.T) { { name: "remove-nested-not-expired", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), }, versionedResourcesStorageMap: map[string]map[string]rest.Storage{ "v1": { @@ -306,8 +344,7 @@ func Test_removeDeletedKinds(t *testing.T) { { name: "remove-all-of-version", resourceExpirationEvaluator: resourceExpirationEvaluator{ - currentMajor: 1, - currentMinor: 20, + currentVersion: apimachineryversion.MajorMinor(1, 20), }, versionedResourcesStorageMap: map[string]map[string]rest.Storage{ "v1": { diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go index e0dcbf75849..c47fc5b23f0 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go @@ -53,7 +53,9 @@ import ( "k8s.io/apiserver/pkg/server/routes" "k8s.io/apiserver/pkg/storageversion" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" restclient "k8s.io/client-go/rest" + "k8s.io/component-base/featuregate" "k8s.io/klog/v2" openapibuilder3 "k8s.io/kube-openapi/pkg/builder3" openapicommon "k8s.io/kube-openapi/pkg/common" @@ -238,6 +240,11 @@ type GenericAPIServer struct { // Version will enable the /version endpoint if non-nil Version *version.Info + // EffectiveVersion determines which apis and features are available + // based on when the api/feature lifecyle. + EffectiveVersion utilversion.EffectiveVersion + // FeatureGate is a way to plumb feature gate through if you have them. + FeatureGate featuregate.FeatureGate // lifecycleSignals provides access to the various signals that happen during the life cycle of the apiserver. lifecycleSignals lifecycleSignals diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go index 3549b238c9f..c2bfef4dda4 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go @@ -48,6 +48,7 @@ import ( openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" "k8s.io/apiserver/pkg/registry/rest" genericfilters "k8s.io/apiserver/pkg/server/filters" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/apiserver/pkg/warning" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" @@ -460,6 +461,7 @@ func TestNotRestRoutesHaveAuth(t *testing.T) { kubeVersion := fakeVersion() config.Version = &kubeVersion + config.EffectiveVersion = utilversion.NewEffectiveVersion(kubeVersion.String()) s, err := config.Complete(nil).New("test", NewEmptyDelegate()) if err != nil { @@ -586,7 +588,7 @@ func fakeVersion() version.Info { return version.Info{ Major: "42", Minor: "42", - GitVersion: "42", + GitVersion: "42.42", GitCommit: "34973274ccef6ab4dfaaf86599792fa9c3fe4689", GitTreeState: "Dirty", BuildDate: time.Now().String(), diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement_test.go index b37931eb28c..7d60244004d 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement_test.go @@ -32,46 +32,42 @@ func (f fakeGroupRegistry) IsGroupRegistered(group string) bool { func TestAPIEnablementOptionsValidate(t *testing.T) { testCases := []struct { - name string - testOptions *APIEnablementOptions - expectErr string + name string + testOptions *APIEnablementOptions + runtimeConfig cliflag.ConfigurationMap + expectErr string }{ { name: "test when options is nil", }, { - name: "test when invalid key with only api/all=false", - testOptions: &APIEnablementOptions{ - RuntimeConfig: cliflag.ConfigurationMap{"api/all": "false"}, - }, - expectErr: "invalid key with only api/all=false", + name: "test when invalid key with only api/all=false", + runtimeConfig: cliflag.ConfigurationMap{"api/all": "false"}, + expectErr: "invalid key with only api/all=false", }, { - name: "test when ConfigurationMap key is invalid", - testOptions: &APIEnablementOptions{ - RuntimeConfig: cliflag.ConfigurationMap{"apiall": "false"}, - }, - expectErr: "runtime-config invalid key", + name: "test when ConfigurationMap key is invalid", + runtimeConfig: cliflag.ConfigurationMap{"apiall": "false"}, + expectErr: "runtime-config invalid key", }, { - name: "test when unknown api groups", - testOptions: &APIEnablementOptions{ - RuntimeConfig: cliflag.ConfigurationMap{"api/v1": "true"}, - }, - expectErr: "unknown api groups", + name: "test when unknown api groups", + runtimeConfig: cliflag.ConfigurationMap{"api/v1": "true"}, + expectErr: "unknown api groups", }, { - name: "test when valid api groups", - testOptions: &APIEnablementOptions{ - RuntimeConfig: cliflag.ConfigurationMap{"apiregistration.k8s.io/v1beta1": "true"}, - }, + name: "test when valid api groups", + runtimeConfig: cliflag.ConfigurationMap{"apiregistration.k8s.io/v1beta1": "true"}, }, } testGroupRegistry := fakeGroupRegistry{} for _, testcase := range testCases { t.Run(testcase.name, func(t *testing.T) { - errs := testcase.testOptions.Validate(testGroupRegistry) + testOptions := &APIEnablementOptions{ + RuntimeConfig: testcase.runtimeConfig, + } + errs := testOptions.Validate(testGroupRegistry) if len(testcase.expectErr) != 0 && !strings.Contains(utilerrors.NewAggregate(errs).Error(), testcase.expectErr) { t.Errorf("got err: %v, expected err: %s", errs, testcase.expectErr) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go b/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go index 10f9775efcc..af7696c40ae 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go @@ -383,8 +383,8 @@ type StorageFactoryRestOptionsFactory struct { StorageFactory serverstorage.StorageFactory } -func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) { - storageConfig, err := f.StorageFactory.NewConfig(resource) +func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) { + storageConfig, err := f.StorageFactory.NewConfig(resource, example) if err != nil { return generic.RESTOptions{}, fmt.Errorf("unable to find storage destination for %v, due to %v", resource, err.Error()) } @@ -469,7 +469,7 @@ type SimpleStorageFactory struct { StorageConfig storagebackend.Config } -func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) { +func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) { return s.StorageConfig.ForResource(resource), nil } @@ -493,8 +493,8 @@ type transformerStorageFactory struct { resourceTransformers storagevalue.ResourceTransformers } -func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) { - config, err := t.delegate.NewConfig(resource) +func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) { + config, err := t.delegate.NewConfig(resource, example) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/etcd_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/etcd_test.go index 06d0cae0a6c..ea4464b24f8 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/etcd_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/etcd_test.go @@ -437,7 +437,7 @@ func TestRestOptionsStorageObjectCountTracker(t *testing.T) { if err := etcdOptions.ApplyTo(serverConfig); err != nil { t.Fatalf("Failed to apply etcd options error: %v", err) } - restOptions, err := serverConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}) + restOptions, err := serverConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}, nil) if err != nil { t.Fatal(err) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go index 1373d8a4d73..88dc3802f7f 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go @@ -26,7 +26,8 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apiserver/pkg/server" - utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" + "k8s.io/component-base/featuregate" "github.com/spf13/pflag" ) @@ -89,9 +90,13 @@ type ServerRunOptions struct { // This grace period is orthogonal to other grace periods, and // it is not overridden by any other grace period. ShutdownWatchTerminationGracePeriod time.Duration + + // FeatureGate are the featuregate to install on the CLI + FeatureGate featuregate.FeatureGate + EffectiveVersion utilversion.EffectiveVersion } -func NewServerRunOptions() *ServerRunOptions { +func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions { defaults := server.NewConfig(serializer.CodecFactory{}) return &ServerRunOptions{ MaxRequestsInFlight: defaults.MaxRequestsInFlight, @@ -104,6 +109,8 @@ func NewServerRunOptions() *ServerRunOptions { JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes, MaxRequestBodyBytes: defaults.MaxRequestBodyBytes, ShutdownSendRetryAfter: false, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, } } @@ -124,6 +131,8 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error { c.PublicAddress = s.AdvertiseAddress c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter c.ShutdownWatchTerminationGracePeriod = s.ShutdownWatchTerminationGracePeriod + c.EffectiveVersion = s.EffectiveVersion + c.FeatureGate = s.FeatureGate return nil } @@ -196,6 +205,14 @@ func (s *ServerRunOptions) Validate() []error { if err := validateCorsAllowedOriginList(s.CorsAllowedOriginList); err != nil { errors = append(errors, err) } + if s.FeatureGate != nil { + if errs := s.FeatureGate.Validate(); len(errs) != 0 { + errors = append(errors, errs...) + } + } + if errs := s.EffectiveVersion.Validate(); len(errs) != 0 { + errors = append(errors, errs...) + } return errors } @@ -336,6 +353,15 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { fs.DurationVar(&s.ShutdownWatchTerminationGracePeriod, "shutdown-watch-termination-grace-period", s.ShutdownWatchTerminationGracePeriod, ""+ "This option, if set, represents the maximum amount of grace period the apiserver will wait "+ "for active watch request(s) to drain during the graceful server shutdown window.") - - utilfeature.DefaultMutableFeatureGate.AddFlag(fs) +} + +// Complete fills missing fields with defaults. +func (s *ServerRunOptions) Complete() error { + if s.FeatureGate == nil { + return fmt.Errorf("nil FeatureGate in ServerRunOptions") + } + if s.EffectiveVersion == nil { + return fmt.Errorf("nil EffectiveVersion in ServerRunOptions") + } + return nil } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go index d1f67fe0719..0c62a4d6893 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go @@ -23,10 +23,14 @@ import ( "time" utilerrors "k8s.io/apimachinery/pkg/util/errors" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" netutils "k8s.io/utils/net" ) func TestServerRunOptionsValidate(t *testing.T) { + featureGate := utilfeature.DefaultFeatureGate.DeepCopy() + effectiveVersion := utilversion.NewEffectiveVersion("1.30") testCases := []struct { name string testOptions *ServerRunOptions @@ -43,6 +47,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--max-requests-inflight can not be negative value", }, @@ -57,6 +63,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--max-mutating-requests-inflight can not be negative value", }, @@ -71,6 +79,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--request-timeout can not be negative value", }, @@ -85,6 +95,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: -1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--min-request-timeout can not be negative value", }, @@ -99,6 +111,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: -10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value", }, @@ -113,6 +127,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: -10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "ServerRunOptions.MaxRequestBodyBytes can not be negative value", }, @@ -128,6 +144,8 @@ func TestServerRunOptionsValidate(t *testing.T) { JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, LivezGracePeriod: -time.Second, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--livez-grace-period can not be a negative value", }, @@ -143,6 +161,8 @@ func TestServerRunOptionsValidate(t *testing.T) { JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, ShutdownDelayDuration: -time.Second, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--shutdown-delay-duration can not be negative value", }, @@ -158,6 +178,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, expectErr: "--strict-transport-security-directives invalid, allowed values: max-age=expireTime, includeSubDomains, preload. see https://tools.ietf.org/html/rfc6797#section-6.1 for more information", }, @@ -173,6 +195,8 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, + FeatureGate: featureGate, + EffectiveVersion: effectiveVersion, }, }, } @@ -192,6 +216,8 @@ func TestServerRunOptionsValidate(t *testing.T) { } func TestValidateCorsAllowedOriginList(t *testing.T) { + featureGate := utilfeature.DefaultFeatureGate.DeepCopy() + effectiveVersion := utilversion.NewEffectiveVersion("1.30") tests := []struct { regexp [][]string errShouldContain string @@ -239,7 +265,7 @@ func TestValidateCorsAllowedOriginList(t *testing.T) { for _, test := range tests { for _, regexp := range test.regexp { t.Run(fmt.Sprintf("regexp/%s", regexp), func(t *testing.T) { - options := NewServerRunOptions() + options := NewServerRunOptions(featureGate, effectiveVersion) if errs := options.Validate(); len(errs) != 0 { t.Fatalf("wrong test setup: %#v", errs) } @@ -263,6 +289,8 @@ func TestValidateCorsAllowedOriginList(t *testing.T) { } func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) { + featureGate := utilfeature.DefaultFeatureGate.DeepCopy() + effectiveVersion := utilversion.NewEffectiveVersion("1.30") tests := []struct { name string optionsFn func() *ServerRunOptions @@ -271,13 +299,13 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) { { name: "default should be valid", optionsFn: func() *ServerRunOptions { - return NewServerRunOptions() + return NewServerRunOptions(featureGate, effectiveVersion) }, }, { name: "negative not allowed", optionsFn: func() *ServerRunOptions { - o := NewServerRunOptions() + o := NewServerRunOptions(featureGate, effectiveVersion) o.ShutdownWatchTerminationGracePeriod = -time.Second return o }, @@ -304,7 +332,7 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) { } t.Run("default should be zero", func(t *testing.T) { - options := NewServerRunOptions() + options := NewServerRunOptions(featureGate, effectiveVersion) if options.ShutdownWatchTerminationGracePeriod != time.Duration(0) { t.Errorf("expected default of ShutdownWatchTerminationGracePeriod to be zero, but got: %s", options.ShutdownWatchTerminationGracePeriod) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go index a08ce2b3bd3..40feeaf62c1 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go @@ -43,6 +43,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/server" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/discovery" restclient "k8s.io/client-go/rest" cliflag "k8s.io/component-base/cli/flag" @@ -276,9 +277,9 @@ func TestServerRunWithSNI(t *testing.T) { // launch server config := setUp(t) - v := fakeVersion() config.Version = &v + config.EffectiveVersion = utilversion.NewEffectiveVersion(v.String()) config.EnableIndex = true secureOptions := (&SecureServingOptions{ @@ -463,11 +464,9 @@ func certSignature(cert tls.Certificate) (string, error) { func fakeVersion() version.Info { return version.Info{ - Major: "42", - Minor: "42", - GitVersion: "42", - GitCommit: "34973274ccef6ab4dfaaf86599792fa9c3fe4689", - GitTreeState: "Dirty", + Major: "42", + Minor: "42", + GitVersion: "42.42", } } diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go b/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go index efb22fbc8d6..5d5cfccc7fd 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go @@ -21,6 +21,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + apimachineryversion "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apiserver/pkg/util/version" ) type ResourceEncodingConfig interface { @@ -33,10 +35,15 @@ type ResourceEncodingConfig interface { InMemoryEncodingFor(schema.GroupResource) (schema.GroupVersion, error) } +type CompatibilityResourceEncodingConfig interface { + BackwardCompatibileStorageEncodingFor(schema.GroupResource, runtime.Object) (schema.GroupVersion, error) +} + type DefaultResourceEncodingConfig struct { // resources records the overriding encoding configs for individual resources. - resources map[schema.GroupResource]*OverridingResourceEncoding - scheme *runtime.Scheme + resources map[schema.GroupResource]*OverridingResourceEncoding + scheme *runtime.Scheme + effectiveVersion version.EffectiveVersion } type OverridingResourceEncoding struct { @@ -47,7 +54,7 @@ type OverridingResourceEncoding struct { var _ ResourceEncodingConfig = &DefaultResourceEncodingConfig{} func NewDefaultResourceEncodingConfig(scheme *runtime.Scheme) *DefaultResourceEncodingConfig { - return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme} + return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme, effectiveVersion: version.DefaultKubeEffectiveVersion()} } func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored schema.GroupResource, externalEncodingVersion, internalVersion schema.GroupVersion) { @@ -57,6 +64,10 @@ func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored } } +func (o *DefaultResourceEncodingConfig) SetEffectiveVersion(effectiveVersion version.EffectiveVersion) { + o.effectiveVersion = effectiveVersion +} + func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) { if !o.scheme.IsGroupRegistered(resource.Group) { return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group) @@ -71,6 +82,24 @@ func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.Group return o.scheme.PrioritizedVersionsForGroup(resource.Group)[0], nil } +func (o *DefaultResourceEncodingConfig) BackwardCompatibileStorageEncodingFor(resource schema.GroupResource, example runtime.Object) (schema.GroupVersion, error) { + if !o.scheme.IsGroupRegistered(resource.Group) { + return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group) + } + + // Always respect overrides + resourceOverride, resourceExists := o.resources[resource] + if resourceExists { + return resourceOverride.ExternalResourceEncoding, nil + } + + return emulatedStorageVersion( + o.scheme.PrioritizedVersionsForGroup(resource.Group)[0], + example, + o.effectiveVersion, + o.scheme) +} + func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) { if !o.scheme.IsGroupRegistered(resource.Group) { return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group) @@ -82,3 +111,78 @@ func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.Grou } return schema.GroupVersion{Group: resource.Group, Version: runtime.APIVersionInternal}, nil } + +// Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go. +type introducedInterface interface { + APILifecycleIntroduced() (major, minor int) +} + +func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example runtime.Object, effectiveVersion version.EffectiveVersion, scheme *runtime.Scheme) (schema.GroupVersion, error) { + if example == nil || effectiveVersion == nil { + return binaryVersionOfResource, nil + } + + // Look up example in scheme to find all objects of the same Group-Kind + // Use the highest priority version for that group-kind whose lifecycle window + // includes the current emulation version. + // If no version is found, use the binary version + // (in this case the API should be disabled anyway) + gvks, _, err := scheme.ObjectKinds(example) + if err != nil { + return schema.GroupVersion{}, err + } else if len(gvks) == 0 { + // Probably shouldn't happen if err is non-nil + return schema.GroupVersion{}, fmt.Errorf("object %T has no GVKs registered in scheme", example) + } + + // VersionsForGroupKind returns versions in priority order + versions := scheme.VersionsForGroupKind(schema.GroupKind{Group: gvks[0].Group, Kind: gvks[0].Kind}) + + compatibilityVersion := effectiveVersion.MinCompatibilityVersion() + + for _, gv := range versions { + if gv.Version == runtime.APIVersionInternal { + continue + } + + gvk := schema.GroupVersionKind{ + Group: gv.Group, + Version: gv.Version, + Kind: gvks[0].Kind, + } + + exampleOfGVK, err := scheme.New(gvk) + if err != nil { + return schema.GroupVersion{}, err + } + + // If it was introduced after current compatibility version, don't use it + if introduced, hasIntroduced := exampleOfGVK.(introducedInterface); hasIntroduced && (compatibilityVersion.Major() > 0 || compatibilityVersion.Minor() > 0) { + // API resource lifecycles should be relative to k8s api version + majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced() + introducedVer := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced)) + if introducedVer.GreaterThan(compatibilityVersion) { + continue + } + } + + // versions is returned in priority order, so just use first result + return gvk.GroupVersion(), nil + } + + // Getting here means we're serving a version that is unknown to the + // min-compatibility-version server. + // + // This is only expected to happen when serving an alpha API type due + // to missing pre-release lifecycle information + // (which doesn't happen by default), or when emulation-version and + // min-compatibility-version are several versions apart so a beta or GA API + // was being served which didn't exist at all in min-compatibility-version. + // + // In the alpha case - we do not support compatibility versioning of + // alpha types and recommend users do not mix the two. + // In the skip-level case - The version of apiserver we are retaining + // compatibility with has no knowledge of the type, + // so storing it in another type is no issue. + return binaryVersionOfResource, nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go index 0dc50cea61d..ad01c5a5d89 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go @@ -42,7 +42,7 @@ type Backend struct { type StorageFactory interface { // New finds the storage destination for the given group and resource. It will // return an error if the group has no storage destination configured. - NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error) + NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) // ResourcePrefix returns the overridden resource prefix for the GroupResource // This allows for cohabitation of resources with different native types and provides @@ -226,7 +226,7 @@ func (s *DefaultStorageFactory) getStorageGroupResource(groupResource schema.Gro // New finds the storage destination for the given group and resource. It will // return an error if the group has no storage destination configured. -func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error) { +func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) { chosenStorageResource := s.getStorageGroupResource(groupResource) // operate on copy @@ -244,14 +244,23 @@ func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (* } var err error - codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource) - if err != nil { - return nil, err + if backwardCompatibleInterface, ok := s.ResourceEncodingConfig.(CompatibilityResourceEncodingConfig); ok { + codecConfig.StorageVersion, err = backwardCompatibleInterface.BackwardCompatibileStorageEncodingFor(groupResource, example) + if err != nil { + return nil, err + } + } else { + codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource) + if err != nil { + return nil, err + } } + codecConfig.MemoryVersion, err = s.ResourceEncodingConfig.InMemoryEncodingFor(groupResource) if err != nil { return nil, err } + codecConfig.Config = storageConfig storageConfig.Codec, storageConfig.EncodeVersioner, err = s.newStorageCodecFn(codecConfig) diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory_test.go b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory_test.go index c52049ca0ec..a12e865f059 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory_test.go @@ -26,10 +26,12 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" + apimachineryversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apiserver/pkg/apis/example" exampleinstall "k8s.io/apiserver/pkg/apis/example/install" examplev1 "k8s.io/apiserver/pkg/apis/example/v1" "k8s.io/apiserver/pkg/storage/storagebackend" + "k8s.io/apiserver/pkg/util/version" ) var ( @@ -118,7 +120,7 @@ func TestConfigurableStorageFactory(t *testing.T) { f.SetEtcdLocation(example.Resource("*"), []string{"/server2"}) f.SetEtcdPrefix(example.Resource("test"), "/prefix_for_test") - config, err := f.NewConfig(example.Resource("test")) + config, err := f.NewConfig(example.Resource("test"), nil) if err != nil { t.Fatal(err) } @@ -163,7 +165,7 @@ func TestUpdateEtcdOverrides(t *testing.T) { storageFactory.SetEtcdLocation(test.resource, test.servers) var err error - config, err := storageFactory.NewConfig(test.resource) + config, err := storageFactory.NewConfig(test.resource, nil) if err != nil { t.Errorf("%d: unexpected error %v", i, err) continue @@ -173,7 +175,7 @@ func TestUpdateEtcdOverrides(t *testing.T) { continue } - config, err = storageFactory.NewConfig(schema.GroupResource{Group: examplev1.GroupName, Resource: "unlikely"}) + config, err = storageFactory.NewConfig(schema.GroupResource{Group: examplev1.GroupName, Resource: "unlikely"}, nil) if err != nil { t.Errorf("%d: unexpected error %v", i, err) continue @@ -244,3 +246,241 @@ func TestConfigs(t *testing.T) { } } } + +var introducedLifecycles = map[reflect.Type]*apimachineryversion.Version{} +var removedLifecycles = map[reflect.Type]*apimachineryversion.Version{} + +type fakeLifecycler[T, V any] struct { + metav1.TypeMeta + metav1.ObjectMeta +} + +type removedLifecycler[T, V any] struct { + fakeLifecycler[T, V] +} + +func (f *fakeLifecycler[T, V]) GetObjectKind() schema.ObjectKind { return f } +func (f *fakeLifecycler[T, V]) DeepCopyObject() runtime.Object { return f } +func (f *fakeLifecycler[T, V]) APILifecycleIntroduced() (major, minor int) { + if introduced, ok := introducedLifecycles[reflect.TypeOf(f)]; ok { + return int(introduced.Major()), int(introduced.Minor()) + } + panic("no lifecycle version set") +} +func (f *removedLifecycler[T, V]) APILifecycleRemoved() (major, minor int) { + if removed, ok := removedLifecycles[reflect.TypeOf(f)]; ok { + return int(removed.Major()), int(removed.Minor()) + } + panic("no lifecycle version set") +} + +func registerFakeLifecycle[T, V any](sch *runtime.Scheme, group, introduced, removed string) { + f := fakeLifecycler[T, V]{} + + introducedLifecycles[reflect.TypeOf(&f)] = apimachineryversion.MustParseSemantic(introduced) + + var res runtime.Object + if removed != "" { + removedLifecycles[reflect.TypeOf(&f)] = apimachineryversion.MustParseSemantic(removed) + res = &removedLifecycler[T, V]{fakeLifecycler: f} + } else { + res = &f + } + + var v V + var t T + sch.AddKnownTypeWithName( + schema.GroupVersionKind{ + Group: group, + Version: strings.ToLower(reflect.TypeOf(v).Name()), + Kind: reflect.TypeOf(t).Name(), + }, + res, + ) + + // Also ensure internal version is registered + // If it is registertd multiple times, it will ignore subsequent registrations + internalInstance := &fakeLifecycler[T, struct{}]{} + sch.AddKnownTypeWithName( + schema.GroupVersionKind{ + Group: group, + Version: runtime.APIVersionInternal, + Kind: reflect.TypeOf(t).Name(), + }, + internalInstance, + ) +} + +func TestStorageFactoryCompatibilityVersion(t *testing.T) { + // Creates a scheme with stub types for unit test + sch := runtime.NewScheme() + codecs := serializer.NewCodecFactory(sch) + + type Internal = struct{} + type V1beta1 struct{} + type V1beta2 struct{} + type V1beta3 struct{} + type V1 struct{} + + type Pod struct{} + type FlowSchema struct{} + type ValidatingAdmisisonPolicy struct{} + type CronJob struct{} + + // Order dictates priority order + registerFakeLifecycle[FlowSchema, V1](sch, "flowcontrol.apiserver.k8s.io", "1.29.0", "") + registerFakeLifecycle[FlowSchema, V1beta3](sch, "flowcontrol.apiserver.k8s.io", "1.26.0", "1.32.0") + registerFakeLifecycle[FlowSchema, V1beta2](sch, "flowcontrol.apiserver.k8s.io", "1.23.0", "1.29.0") + registerFakeLifecycle[FlowSchema, V1beta1](sch, "flowcontrol.apiserver.k8s.io", "1.20.0", "1.26.0") + registerFakeLifecycle[CronJob, V1](sch, "batch", "1.21.0", "") + registerFakeLifecycle[CronJob, V1beta1](sch, "batch", "1.8.0", "1.21.0") + registerFakeLifecycle[ValidatingAdmisisonPolicy, V1](sch, "admissionregistration.k8s.io", "1.30.0", "") + registerFakeLifecycle[ValidatingAdmisisonPolicy, V1beta1](sch, "admissionregistration.k8s.io", "1.28.0", "1.34.0") + registerFakeLifecycle[Pod, V1](sch, "", "1.31.0", "") + + // FlowSchema + // - v1beta1: 1.20.0 - 1.23.0 + // - v1beta2: 1.23.0 - 1.26.0 + // - v1beta3: 1.26.0 - 1.30.0 + // - v1: 1.29.0+ + // CronJob + // - v1beta1: 1.8.0 - 1.21.0 + // - v1: 1.21.0+ + // ValidatingAdmissionPolicy + // - v1beta1: 1.28.0 - 1.31.0 + // - v1: 1.30.0+ + + testcases := []struct { + effectiveVersion string + example runtime.Object + expectedVersion schema.GroupVersion + }{ + { + // Basic case. Beta version for long time + effectiveVersion: "1.14.0", + example: &fakeLifecycler[CronJob, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1beta1"}, + }, + { + // Basic case. Beta version for long time + effectiveVersion: "1.20.0", + example: &fakeLifecycler[CronJob, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1beta1"}, + }, + { + // Basic case. GA version for long time + effectiveVersion: "1.28.0", + example: &fakeLifecycler[CronJob, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1"}, + }, + { + // Basic core/v1 + effectiveVersion: "1.31.0", + example: &fakeLifecycler[Pod, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "", Version: "v1"}, + }, + { + // Corner case: 1.1.0 has no flowcontrol. Options are to error + // out or to use the latest version. This test assumes the latter. + effectiveVersion: "1.1.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1"}, + }, + { + effectiveVersion: "1.21.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1"}, + }, + { + // v2Beta1 introduced this version, but minCompatibility should + // force v1beta1 + effectiveVersion: "1.23.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1"}, + }, + { + effectiveVersion: "1.24.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2"}, + }, + { + effectiveVersion: "1.26.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2"}, + }, + { + effectiveVersion: "1.27.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta3"}, + }, + { + // GA API introduced 1.29 but must keep storing in v1beta3 for downgrades + effectiveVersion: "1.29.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta3"}, + }, + { + // Version after GA api is introduced + effectiveVersion: "1.30.0", + example: &fakeLifecycler[FlowSchema, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1"}, + }, + { + effectiveVersion: "1.30.0", + example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1beta1"}, + }, + { + effectiveVersion: "1.31.0", + example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1"}, + }, + { + effectiveVersion: "1.29.0", + example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{}, + expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1beta1"}, + }, + } + + for _, tc := range testcases { + gvks, _, err := sch.ObjectKinds(tc.example) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + gvk := gvks[0] + t.Run(gvk.GroupKind().String()+"@"+tc.effectiveVersion, func(t *testing.T) { + config := NewDefaultResourceEncodingConfig(sch) + config.SetEffectiveVersion(version.NewEffectiveVersion(tc.effectiveVersion)) + f := NewDefaultStorageFactory( + storagebackend.Config{}, + "", + codecs, + config, + NewResourceConfig(), + nil) + + cfg, err := f.NewConfig(schema.GroupResource{ + Group: gvk.Group, + Resource: gvk.Kind, // doesnt really matter here + }, tc.example) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + gvks, _, err := sch.ObjectKinds(tc.example) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + expectEncodeVersioner := runtime.NewMultiGroupVersioner(tc.expectedVersion, + schema.GroupKind{ + Group: gvks[0].Group, + }, schema.GroupKind{ + Group: gvks[0].Group, + }) + if cfg.EncodeVersioner.Identifier() != expectEncodeVersioner.Identifier() { + t.Errorf("expected %v, got %v", expectEncodeVersioner, cfg.EncodeVersioner) + } + }) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go b/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go index 7dd3df589e3..00a9e099ba7 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go +++ b/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go @@ -25,7 +25,7 @@ var ( // Only top-level commands/options setup and the k8s.io/component-base/featuregate/testing package should make use of this. // Tests that need to modify feature gates for the duration of their test should use: // featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features., ) - DefaultMutableFeatureGate featuregate.MutableFeatureGate = featuregate.NewFeatureGate() + DefaultMutableFeatureGate featuregate.MutableVersionedFeatureGate = featuregate.NewFeatureGate() // DefaultFeatureGate is a shared global FeatureGate. // Top-level commands/options setup that needs to modify this feature gate should use DefaultMutableFeatureGate. diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/registry.go b/staging/src/k8s.io/apiserver/pkg/util/version/registry.go new file mode 100644 index 00000000000..f8b4ef5cd41 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/version/registry.go @@ -0,0 +1,142 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +import ( + "fmt" + "sync" + + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/component-base/featuregate" +) + +var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGlobalsRegistry() + +const ( + ComponentGenericAPIServer = "k8s.io/apiserver" +) + +// ComponentGlobals stores the global variables for a component for easy access. +type ComponentGlobals struct { + effectiveVersion MutableEffectiveVersion + featureGate featuregate.MutableVersionedFeatureGate +} + +type ComponentGlobalsRegistry interface { + // EffectiveVersionFor returns the EffectiveVersion registered under the component. + // Returns nil if the component is not registered. + EffectiveVersionFor(component string) EffectiveVersion + // FeatureGateFor returns the FeatureGate registered under the component. + // Returns nil if the component is not registered. + FeatureGateFor(component string) featuregate.FeatureGate + // Register registers the EffectiveVersion and FeatureGate for a component. + // Overrides existing ComponentGlobals if it is already in the registry if override is true, + // otherwise returns error if the component is already registered. + Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error + // ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry. + // Otherwise, the provided variables would be registered under the component, and the same variables would be returned. + ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) + // SetAllComponents sets the emulation version for other global variables for all components registered. + SetAllComponents() error + // SetAllComponents calls the Validate() function for all the global variables for all components registered. + ValidateAllComponents() []error +} + +type componentGlobalsRegistry struct { + componentGlobals map[string]ComponentGlobals + mutex sync.RWMutex +} + +func NewComponentGlobalsRegistry() ComponentGlobalsRegistry { + return &componentGlobalsRegistry{componentGlobals: map[string]ComponentGlobals{}} +} + +func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion { + r.mutex.RLock() + defer r.mutex.RUnlock() + globals, ok := r.componentGlobals[component] + if !ok { + return nil + } + return globals.effectiveVersion +} + +func (r *componentGlobalsRegistry) FeatureGateFor(component string) featuregate.FeatureGate { + r.mutex.RLock() + defer r.mutex.RUnlock() + globals, ok := r.componentGlobals[component] + if !ok { + return nil + } + return globals.featureGate +} + +func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error { + if _, ok := r.componentGlobals[component]; ok && !override { + return fmt.Errorf("component globals of %s already registered", component) + } + if featureGate != nil { + featureGate.DeferErrorsToValidation(true) + } + c := ComponentGlobals{effectiveVersion: effectiveVersion, featureGate: featureGate} + r.componentGlobals[component] = c + return nil +} + +func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error { + r.mutex.Lock() + defer r.mutex.Unlock() + return r.unsafeRegister(component, effectiveVersion, featureGate, override) +} + +func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) { + r.mutex.Lock() + defer r.mutex.Unlock() + globals, ok := r.componentGlobals[component] + if ok { + return globals.effectiveVersion, globals.featureGate + } + utilruntime.Must(r.unsafeRegister(component, effectiveVersion, featureGate, false)) + return effectiveVersion, featureGate +} + +func (r *componentGlobalsRegistry) SetAllComponents() error { + r.mutex.Lock() + defer r.mutex.Unlock() + for _, globals := range r.componentGlobals { + if globals.featureGate == nil { + continue + } + if err := globals.featureGate.SetEmulationVersion(globals.effectiveVersion.EmulationVersion()); err != nil { + return err + } + } + return nil +} + +func (r *componentGlobalsRegistry) ValidateAllComponents() []error { + var errs []error + r.mutex.Lock() + defer r.mutex.Unlock() + for _, globals := range r.componentGlobals { + errs = append(errs, globals.effectiveVersion.Validate()...) + if globals.featureGate != nil { + errs = append(errs, globals.featureGate.Validate()...) + } + } + return errs +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go b/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go new file mode 100644 index 00000000000..2e19552703f --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go @@ -0,0 +1,48 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +import ( + "testing" +) + +func TestEffectiveVersionRegistry(t *testing.T) { + r := NewComponentGlobalsRegistry() + testComponent := "test" + ver1 := NewEffectiveVersion("1.31") + ver2 := NewEffectiveVersion("1.28") + + if r.EffectiveVersionFor(testComponent) != nil { + t.Fatalf("expected nil EffectiveVersion initially") + } + if err := r.Register(testComponent, ver1, nil, false); err != nil { + t.Fatalf("expected no error to register new component, but got err: %v", err) + } + if !r.EffectiveVersionFor(testComponent).EqualTo(ver1) { + t.Fatalf("expected EffectiveVersionFor to return the version registered") + } + // overwrite + if err := r.Register(testComponent, ver2, nil, false); err == nil { + t.Fatalf("expected error to register existing component when override is false") + } + if err := r.Register(testComponent, ver2, nil, true); err != nil { + t.Fatalf("expected no error to overriding existing component, but got err: %v", err) + } + if !r.EffectiveVersionFor(testComponent).EqualTo(ver2) { + t.Fatalf("expected EffectiveVersionFor to return the version overridden") + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/version.go b/staging/src/k8s.io/apiserver/pkg/util/version/version.go new file mode 100644 index 00000000000..fce927c6c61 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/version/version.go @@ -0,0 +1,195 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +import ( + "fmt" + "strings" + "sync/atomic" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/version" + baseversion "k8s.io/component-base/version" +) + +type EffectiveVersion interface { + BinaryVersion() *version.Version + EmulationVersion() *version.Version + MinCompatibilityVersion() *version.Version + EqualTo(other EffectiveVersion) bool + String() string + Validate() []error +} + +type MutableEffectiveVersion interface { + EffectiveVersion + Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) + SetEmulationVersion(emulationVersion *version.Version) + SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) + // AddFlags adds the "{prefix}-emulated-version" to the flagset. + AddFlags(fs *pflag.FlagSet, prefix string) +} + +type VersionVar struct { + val atomic.Pointer[version.Version] +} + +// Set sets the flag value +func (v *VersionVar) Set(s string) error { + components := strings.Split(s, ".") + if len(components) != 2 { + return fmt.Errorf("version %s is not in the format of major.minor", s) + } + ver, err := version.ParseGeneric(s) + if err != nil { + return err + } + v.val.Store(ver) + return nil +} + +// String returns the flag value +func (v *VersionVar) String() string { + ver := v.val.Load() + return ver.String() +} + +// Type gets the flag type +func (v *VersionVar) Type() string { + return "version" +} + +type effectiveVersion struct { + binaryVersion atomic.Pointer[version.Version] + // If the emulationVersion is set by the users, it could only contain major and minor versions. + // In tests, emulationVersion could be the same as the binary version, or set directly, + // which can have "alpha" as pre-release to continue serving expired apis while we clean up the test. + emulationVersion VersionVar + // minCompatibilityVersion could only contain major and minor versions. + minCompatibilityVersion VersionVar +} + +func (m *effectiveVersion) BinaryVersion() *version.Version { + return m.binaryVersion.Load() +} + +func (m *effectiveVersion) EmulationVersion() *version.Version { + // Emulation version can have "alpha" as pre-release to continue serving expired apis while we clean up the test. + // The pre-release should not be accessible to the users. + return m.emulationVersion.val.Load().WithPreRelease(m.BinaryVersion().PreRelease()) +} + +func (m *effectiveVersion) MinCompatibilityVersion() *version.Version { + return m.minCompatibilityVersion.val.Load() +} + +func (m *effectiveVersion) EqualTo(other EffectiveVersion) bool { + return m.BinaryVersion().EqualTo(other.BinaryVersion()) && m.EmulationVersion().EqualTo(other.EmulationVersion()) && m.MinCompatibilityVersion().EqualTo(other.MinCompatibilityVersion()) +} + +func (m *effectiveVersion) String() string { + if m == nil { + return "" + } + return fmt.Sprintf("{BinaryVersion: %s, EmulationVersion: %s, MinCompatibilityVersion: %s}", + m.BinaryVersion().String(), m.EmulationVersion().String(), m.MinCompatibilityVersion().String()) +} + +func (m *effectiveVersion) Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) { + m.binaryVersion.Store(binaryVersion) + m.emulationVersion.val.Store(version.MajorMinor(emulationVersion.Major(), emulationVersion.Minor())) + m.minCompatibilityVersion.val.Store(version.MajorMinor(minCompatibilityVersion.Major(), minCompatibilityVersion.Minor())) +} + +func (m *effectiveVersion) SetEmulationVersion(emulationVersion *version.Version) { + m.emulationVersion.val.Store(version.MajorMinor(emulationVersion.Major(), emulationVersion.Minor())) +} + +func (m *effectiveVersion) SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) { + m.minCompatibilityVersion.val.Store(version.MajorMinor(minCompatibilityVersion.Major(), minCompatibilityVersion.Minor())) +} + +func (m *effectiveVersion) Validate() []error { + var errs []error + // Validate only checks the major and minor versions. + binaryVersion := m.binaryVersion.Load().WithPatch(0) + emulationVersion := m.emulationVersion.val.Load() + minCompatibilityVersion := m.minCompatibilityVersion.val.Load() + + // emulationVersion can only be 1.{binaryMinor-1}...1.{binaryMinor}. + maxEmuVer := binaryVersion + minEmuVer := binaryVersion.SubtractMinor(1) + // TODO: remove in 1.32 + // emulationVersion is introduced in 1.31, so it cannot be lower than that. + // binaryVersion could be lower than 1.31 in tests. So we are only checking 1.31. + if binaryVersion.EqualTo(version.MajorMinor(1, 31)) { + minEmuVer = version.MajorMinor(1, 31) + } + if emulationVersion.GreaterThan(maxEmuVer) || emulationVersion.LessThan(minEmuVer) { + errs = append(errs, fmt.Errorf("emulation version %s is not between [%s, %s]", emulationVersion.String(), minEmuVer.String(), maxEmuVer.String())) + } + // minCompatibilityVersion can only be 1.{binaryMinor-1} for alpha. + maxCompVer := binaryVersion.SubtractMinor(1) + minCompVer := binaryVersion.SubtractMinor(1) + if minCompatibilityVersion.GreaterThan(maxCompVer) || minCompatibilityVersion.LessThan(minCompVer) { + errs = append(errs, fmt.Errorf("minCompatibilityVersion version %s is not between [%s, %s]", minCompatibilityVersion.String(), minCompVer.String(), maxCompVer.String())) + } + return errs +} + +// AddFlags adds the "{prefix}-emulated-version" to the flagset. +func (m *effectiveVersion) AddFlags(fs *pflag.FlagSet, prefix string) { + if m == nil { + return + } + if len(prefix) > 0 && !strings.HasSuffix(prefix, "-") { + prefix += "-" + } + fs.Var(&m.emulationVersion, prefix+"emulated-version", ""+ + "The version the K8s component emulates its capabilities (APIs, features, ...) of.\n"+ + "If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+ + "Any capabilities present in the binary version that were introduced after the emulated version will be unavailable and any capabilities removed after the emulated version will be available.\n"+ + "This flag applies only to component capabilities, and does not disable bug fixes and performance improvements present in the binary version.\n"+ + "Defaults to the binary version. The value should be between 1.{binaryMinorVersion-1} and 1.{binaryMinorVersion}.\n"+ + "Format could only be major.minor") +} + +func NewEffectiveVersion(binaryVer string) MutableEffectiveVersion { + effective := &effectiveVersion{} + binaryVersion := version.MustParse(binaryVer) + compatVersion := binaryVersion.SubtractMinor(1) + effective.Set(binaryVersion, binaryVersion, compatVersion) + return effective +} + +// DefaultBuildEffectiveVersion returns the MutableEffectiveVersion based on the +// current build information. +func DefaultBuildEffectiveVersion() MutableEffectiveVersion { + verInfo := baseversion.Get() + ver := NewEffectiveVersion(verInfo.String()) + if ver.BinaryVersion().Major() == 0 && ver.BinaryVersion().Minor() == 0 { + ver = DefaultKubeEffectiveVersion() + } + return ver +} + +// DefaultKubeEffectiveVersion returns the MutableEffectiveVersion based on the +// latest K8s release. +// Should update for each minor release! +func DefaultKubeEffectiveVersion() MutableEffectiveVersion { + return NewEffectiveVersion("1.31") +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go b/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go new file mode 100644 index 00000000000..8c61e92c1de --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go @@ -0,0 +1,180 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +import ( + "fmt" + "strings" + "testing" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/version" +) + +func TestValidate(t *testing.T) { + tests := []struct { + name string + binaryVersion string + emulationVersion string + minCompatibilityVersion string + expectErrors bool + }{ + { + name: "patch version diff ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.32.1", + minCompatibilityVersion: "v1.31.5", + }, + { + name: "emulation version one minor lower than binary ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.31.0", + minCompatibilityVersion: "v1.31.0", + }, + { + name: "binary version 1.31, emulation version lower than 1.31", + binaryVersion: "v1.31.2", + emulationVersion: "v1.30.0", + minCompatibilityVersion: "v1.30.0", + expectErrors: true, + }, + { + name: "binary version 1.31, emulation version 1.31", + binaryVersion: "v1.31.2", + emulationVersion: "v1.31.0", + minCompatibilityVersion: "v1.30.0", + }, + { + name: "binary version lower than 1.31", + binaryVersion: "v1.30.2", + emulationVersion: "v1.29.0", + minCompatibilityVersion: "v1.29.0", + }, + { + name: "emulation version two minor lower than binary not ok", + binaryVersion: "v1.33.2", + emulationVersion: "v1.31.0", + minCompatibilityVersion: "v1.32.0", + expectErrors: true, + }, + { + name: "emulation version one minor higher than binary not ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.33.0", + minCompatibilityVersion: "v1.31.0", + expectErrors: true, + }, + { + name: "emulation version two minor higher than binary not ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.34.0", + minCompatibilityVersion: "v1.31.0", + expectErrors: true, + }, + { + name: "compatibility version same as binary not ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.32.0", + minCompatibilityVersion: "v1.32.0", + expectErrors: true, + }, + { + name: "compatibility version two minor lower than binary not ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.32.0", + minCompatibilityVersion: "v1.30.0", + expectErrors: true, + }, + { + name: "compatibility version one minor higher than binary not ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.32.0", + minCompatibilityVersion: "v1.33.0", + expectErrors: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + binaryVersion := version.MustParseGeneric(test.binaryVersion) + effective := &effectiveVersion{} + emulationVersion := version.MustParseGeneric(test.emulationVersion) + minCompatibilityVersion := version.MustParseGeneric(test.minCompatibilityVersion) + effective.Set(binaryVersion, emulationVersion, minCompatibilityVersion) + + errs := effective.Validate() + if len(errs) > 0 && !test.expectErrors { + t.Errorf("expected no errors, errors found %+v", errs) + } + + if len(errs) == 0 && test.expectErrors { + t.Errorf("expected errors, no errors found") + } + }) + } +} + +func TestEffectiveVersionsFlag(t *testing.T) { + tests := []struct { + name string + emulationVerson string + expectedEmulationVersion *version.Version + parseError string + }{ + { + name: "major.minor ok", + emulationVerson: "1.30", + expectedEmulationVersion: version.MajorMinor(1, 30), + }, + { + name: "v prefix ok", + emulationVerson: "v1.30", + expectedEmulationVersion: version.MajorMinor(1, 30), + }, + { + name: "semantic version not ok", + emulationVerson: "1.30.1", + parseError: "version 1.30.1 is not in the format of major.minor", + }, + { + name: "invalid version", + emulationVerson: "1.foo", + parseError: "illegal version string", + }, + } + for i, test := range tests { + t.Run(test.name, func(t *testing.T) { + fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError) + effective := NewEffectiveVersion("1.30") + effective.AddFlags(fs, "test") + + err := fs.Parse([]string{fmt.Sprintf("--test-emulated-version=%s", test.emulationVerson)}) + if test.parseError != "" { + if !strings.Contains(err.Error(), test.parseError) { + t.Fatalf("%d: Parse() Expected %v, Got %v", i, test.parseError, err) + } + return + } + if err != nil { + t.Fatalf("%d: Parse() Expected nil, Got %v", i, err) + } + if !effective.EmulationVersion().EqualTo(test.expectedEmulationVersion) { + t.Errorf("%d: EmulationVersion Expected %s, Got %s", i, test.expectedEmulationVersion.String(), effective.EmulationVersion().String()) + } + }) + } +} diff --git a/staging/src/k8s.io/cloud-provider/options/options.go b/staging/src/k8s.io/cloud-provider/options/options.go index b96948106ca..9d55650aa19 100644 --- a/staging/src/k8s.io/cloud-provider/options/options.go +++ b/staging/src/k8s.io/cloud-provider/options/options.go @@ -164,7 +164,7 @@ func (o *CloudControllerManagerOptions) Flags(allControllers []string, disabledB fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") fs.StringVar(&o.Generic.ClientConnection.Kubeconfig, "kubeconfig", o.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).") fs.DurationVar(&o.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", o.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.") - utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic")) + utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"), "") return fss } diff --git a/staging/src/k8s.io/component-base/featuregate/feature_gate.go b/staging/src/k8s.io/component-base/featuregate/feature_gate.go index 1e441289ea5..c6b4d7627f5 100644 --- a/staging/src/k8s.io/component-base/featuregate/feature_gate.go +++ b/staging/src/k8s.io/component-base/featuregate/feature_gate.go @@ -19,6 +19,7 @@ package featuregate import ( "context" "fmt" + "reflect" "sort" "strconv" "strings" @@ -27,8 +28,11 @@ import ( "github.com/spf13/pflag" + utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/naming" + "k8s.io/apimachinery/pkg/util/version" featuremetrics "k8s.io/component-base/metrics/prometheus/feature" + baseversion "k8s.io/component-base/version" "k8s.io/klog/v2" ) @@ -52,13 +56,13 @@ const ( var ( // The generic features. - defaultFeatures = map[Feature]FeatureSpec{ - allAlphaGate: {Default: false, PreRelease: Alpha}, - allBetaGate: {Default: false, PreRelease: Beta}, + defaultFeatures = map[Feature]VersionedSpecs{ + allAlphaGate: {{Default: false, PreRelease: Alpha, Version: version.MajorMinor(0, 0)}}, + allBetaGate: {{Default: false, PreRelease: Beta, Version: version.MajorMinor(0, 0)}}, } // Special handling for a few gates. - specialFeatures = map[Feature]func(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool){ + specialFeatures = map[Feature]func(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, cVer *version.Version){ allAlphaGate: setUnsetAlphaGates, allBetaGate: setUnsetBetaGates, } @@ -69,13 +73,28 @@ type FeatureSpec struct { Default bool // LockToDefault indicates that the feature is locked to its default and cannot be changed LockToDefault bool - // PreRelease indicates the maturity level of the feature + // PreRelease indicates the current maturity level of the feature PreRelease prerelease + // Version indicates the earliest version from which this FeatureSpec is valid. + // If multiple FeatureSpecs exist for a Feature, the one with the highest version that is less + // than or equal to the effective version of the component is used. + Version *version.Version } +type VersionedSpecs []FeatureSpec + +func (g VersionedSpecs) Len() int { return len(g) } +func (g VersionedSpecs) Less(i, j int) bool { + return g[i].Version.LessThan(g[j].Version) +} +func (g VersionedSpecs) Swap(i, j int) { g[i], g[j] = g[j], g[i] } + +type PromotionVersionMapping map[prerelease]string + type prerelease string const ( + PreAlpha = prerelease("PRE-ALPHA") // Values for PreRelease. Alpha = prerelease("ALPHA") Beta = prerelease("BETA") @@ -94,7 +113,10 @@ type FeatureGate interface { // DeepCopy returns a deep copy of the FeatureGate object, such that gates can be // set on the copy without mutating the original. This is useful for validating // config against potential feature gate changes before committing those changes. - DeepCopy() MutableFeatureGate + DeepCopy() MutableVersionedFeatureGate + // Validate checks if the flag gates are valid at the emulated version. + // Should always be called after Set when DeferErrorsToValidation is set to true. + Validate() []error } // MutableFeatureGate parses and stores flag gates for known features from @@ -103,7 +125,7 @@ type MutableFeatureGate interface { FeatureGate // AddFlag adds a flag for setting global feature gates to the specified FlagSet. - AddFlag(fs *pflag.FlagSet) + AddFlag(fs *pflag.FlagSet, prefix string) // Set parses and stores flag gates for known features // from a string like feature1=true,feature2=false,... Set(value string) error @@ -128,25 +150,70 @@ type MutableFeatureGate interface { OverrideDefault(name Feature, override bool) error } +// MutableVersionedFeatureGate parses and stores flag gates for known features from +// a string like feature1=true,feature2=false,... +// MutableVersionedFeatureGate sets options based on the emulated version of the featured gate. +type MutableVersionedFeatureGate interface { + MutableFeatureGate + // EmulationVersion returns the version the feature gate is set to emulate. + // If set, the feature gate would enable/disable features based on + // feature availability and pre-release at the emulated version instead of the binary version. + EmulationVersion() *version.Version + // SetEmulationVersion overrides the emulationVersion of the feature gate. + // Otherwise, the emulationVersion will be the same as the binary version. + // If set, the feature defaults and availability will be as if the binary is at the emulated version. + SetEmulationVersion(emulationVersion *version.Version) error + // DeferErrorsToValidation defers the errors of Set function to the Validate() function if true. + // This is used when the user wants to set the feature gate flag before the emulationVersion is finalized. + // Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true. + DeferErrorsToValidation(val bool) + // GetAll returns a copy of the map of known feature names to versioned feature specs. + GetAllVersioned() map[Feature]VersionedSpecs + // AddVersioned adds versioned feature specs to the featureGate. + AddVersioned(features map[Feature]VersionedSpecs) error +} + +// MutableVersionedFeatureGateForTests is a feature gate interface that should only be used in tests. +type MutableVersionedFeatureGateForTests interface { + MutableVersionedFeatureGate + // Reset sets the enabled and enabledRaw to the input map. + Reset(m map[string]bool) + // EnabledRawMap returns the raw enable map from the feature gate. + EnabledRawMap() map[string]bool +} + // featureGate implements FeatureGate as well as pflag.Value for flag parsing. type featureGate struct { featureGateName string - special map[Feature]func(map[Feature]FeatureSpec, map[Feature]bool, bool) + special map[Feature]func(map[Feature]VersionedSpecs, map[Feature]bool, bool, *version.Version) - // lock guards writes to known, enabled, and reads/writes of closed + // lock guards writes to all below fields. lock sync.Mutex // known holds a map[Feature]FeatureSpec known atomic.Value // enabled holds a map[Feature]bool enabled atomic.Value + // enabledRaw holds a raw map[string]bool of the parsed flag. + // It keeps the original values of "special" features like "all alpha gates", + // while enabled keeps the values of all resolved features. + enabledRaw atomic.Value // closed is set to true when AddFlag is called, and prevents subsequent calls to Add closed bool + // deferErrorsToValidation could be set to true to defer checking flag setting error, + // because the emulationVersion may not be the final emulationVersion when the flag is set. + // Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true. + deferErrorsToValidation bool + emulationVersion atomic.Pointer[version.Version] } -func setUnsetAlphaGates(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool) { +func setUnsetAlphaGates(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, cVer *version.Version) { for k, v := range known { - if v.PreRelease == Alpha { + if k == "AllAlpha" || k == "AllBeta" { + continue + } + currentVersion := getCurrentVersion(v, cVer) + if currentVersion.PreRelease == Alpha { if _, found := enabled[k]; !found { enabled[k] = val } @@ -154,9 +221,13 @@ func setUnsetAlphaGates(known map[Feature]FeatureSpec, enabled map[Feature]bool, } } -func setUnsetBetaGates(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool) { +func setUnsetBetaGates(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, cVer *version.Version) { for k, v := range known { - if v.PreRelease == Beta { + if k == "AllAlpha" || k == "AllBeta" { + continue + } + currentVersion := getCurrentVersion(v, cVer) + if currentVersion.PreRelease == Beta { if _, found := enabled[k]; !found { enabled[k] = val } @@ -171,8 +242,10 @@ var _ pflag.Value = &featureGate{} // call chains, so they'd be unhelpful as names. var internalPackages = []string{"k8s.io/component-base/featuregate/feature_gate.go"} -func NewFeatureGate() *featureGate { - known := map[Feature]FeatureSpec{} +// NewVersionedFeatureGate creates a feature gate with the emulation version set to the provided version. +// SetEmulationVersion can be called after to change emulation version to a desired value. +func NewVersionedFeatureGate(emulationVersion *version.Version) *featureGate { + known := map[Feature]VersionedSpecs{} for k, v := range defaultFeatures { known[k] = v } @@ -183,10 +256,18 @@ func NewFeatureGate() *featureGate { } f.known.Store(known) f.enabled.Store(map[Feature]bool{}) - + f.enabledRaw.Store(map[string]bool{}) + f.emulationVersion.Store(emulationVersion) + klog.V(1).Infof("new feature gate with emulationVersion=%s", f.emulationVersion.Load().String()) return f } +// NewFeatureGate creates a feature gate with the current binary version. +func NewFeatureGate() *featureGate { + binaryVersison := version.MustParse(baseversion.Get().String()) + return NewVersionedFeatureGate(binaryVersison) +} + // Set parses a string of the form "key1=value1,key2=value2,..." into a // map[string]bool of known keys or returns an error. func (f *featureGate) Set(value string) error { @@ -207,7 +288,68 @@ func (f *featureGate) Set(value string) error { } m[k] = boolValue } - return f.SetFromMap(m) + err := f.SetFromMap(m) + // ignores SetFromMap error, because the emulationVersion may not be the final emulationVersion when the flag is set. + // Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true. + if f.deferErrorsToValidation { + return nil + } + return err +} + +// Validate checks if the flag gates are valid at the emulated version. +// Should always be called after Set when DeferErrorsToValidation is set to true. +func (f *featureGate) Validate() []error { + m, ok := f.enabledRaw.Load().(map[string]bool) + if !ok { + return []error{fmt.Errorf("cannot cast enabledRaw to map[string]bool")} + } + enabled := map[Feature]bool{} + return f.unsafeSetFromMap(enabled, m) +} + +// unsafeSetFromMap stores flag gates for known features from a map[string]bool into an enabled map. +func (f *featureGate) unsafeSetFromMap(enabled map[Feature]bool, m map[string]bool) []error { + var errs []error + // Copy existing state + known := map[Feature]VersionedSpecs{} + for k, v := range f.known.Load().(map[Feature]VersionedSpecs) { + sort.Sort(v) + known[k] = v + } + + for k, v := range m { + key := Feature(k) + versionedSpecs, ok := known[key] + if !ok { + // early return if encounters an unknown feature. + errs = append(errs, fmt.Errorf("unrecognized feature gate: %s", k)) + return errs + } + currentVersion := f.getCurrentVersion(versionedSpecs) + if currentVersion.LockToDefault && currentVersion.Default != v { + errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is locked to %v", k, v, currentVersion.Default)) + continue + } + // Handle "special" features like "all alpha gates" + if fn, found := f.special[key]; found { + fn(known, enabled, v, f.emulationVersion.Load()) + enabled[key] = v + continue + } + if currentVersion.PreRelease == PreAlpha { + errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is PreAlpha at emulated version %s", k, v, f.EmulationVersion().String())) + continue + } + enabled[key] = v + + if currentVersion.PreRelease == Deprecated { + klog.Warningf("Setting deprecated feature gate %s=%t. It will be removed in a future release.", k, v) + } else if currentVersion.PreRelease == GA { + klog.Warningf("Setting GA feature gate %s=%t. It will be removed in a future release.", k, v) + } + } + return errs } // SetFromMap stores flag gates for known features from a map[string]bool or returns an error @@ -216,43 +358,30 @@ func (f *featureGate) SetFromMap(m map[string]bool) error { defer f.lock.Unlock() // Copy existing state - known := map[Feature]FeatureSpec{} - for k, v := range f.known.Load().(map[Feature]FeatureSpec) { - known[k] = v - } enabled := map[Feature]bool{} for k, v := range f.enabled.Load().(map[Feature]bool) { enabled[k] = v } - - for k, v := range m { - k := Feature(k) - featureSpec, ok := known[k] - if !ok { - return fmt.Errorf("unrecognized feature gate: %s", k) - } - if featureSpec.LockToDefault && featureSpec.Default != v { - return fmt.Errorf("cannot set feature gate %v to %v, feature is locked to %v", k, v, featureSpec.Default) - } - enabled[k] = v - // Handle "special" features like "all alpha gates" - if fn, found := f.special[k]; found { - fn(known, enabled, v) - } - - if featureSpec.PreRelease == Deprecated { - klog.Warningf("Setting deprecated feature gate %s=%t. It will be removed in a future release.", k, v) - } else if featureSpec.PreRelease == GA { - klog.Warningf("Setting GA feature gate %s=%t. It will be removed in a future release.", k, v) - } + enabledRaw := map[string]bool{} + for k, v := range f.enabledRaw.Load().(map[string]bool) { + enabledRaw[k] = v } - // Persist changes - f.known.Store(known) - f.enabled.Store(enabled) + // Update enabledRaw first. + // SetFromMap might be called when emulationVersion is not finalized yet, and we do not know the final state of enabled. + // But the flags still need to be saved. + for k, v := range m { + enabledRaw[k] = v + } + f.enabledRaw.Store(enabledRaw) - klog.V(1).Infof("feature gates: %v", f.enabled) - return nil + errs := f.unsafeSetFromMap(enabled, enabledRaw) + if len(errs) == 0 { + // Persist changes + f.enabled.Store(enabled) + klog.V(1).Infof("feature gates: %v", f.enabled) + } + return utilerrors.NewAggregate(errs) } // String returns a string containing all enabled feature gates, formatted as "key1=value1,key2=value2,...". @@ -271,6 +400,17 @@ func (f *featureGate) Type() string { // Add adds features to the featureGate. func (f *featureGate) Add(features map[Feature]FeatureSpec) error { + vs := map[Feature]VersionedSpecs{} + for name, spec := range features { + // if no version is provided for the FeatureSpec, it is defaulted to version 0.0 so that it can be enabled/disabled regardless of emulation version. + spec.Version = version.MajorMinor(0, 0) + vs[name] = VersionedSpecs{spec} + } + return f.AddVersioned(vs) +} + +// AddVersioned adds versioned feature specs to the featureGate. +func (f *featureGate) AddVersioned(features map[Feature]VersionedSpecs) error { f.lock.Lock() defer f.lock.Unlock() @@ -279,20 +419,21 @@ func (f *featureGate) Add(features map[Feature]FeatureSpec) error { } // Copy existing state - known := map[Feature]FeatureSpec{} - for k, v := range f.known.Load().(map[Feature]FeatureSpec) { + known := map[Feature]VersionedSpecs{} + for k, v := range f.known.Load().(map[Feature]VersionedSpecs) { known[k] = v } - for name, spec := range features { + for name, specs := range features { + sort.Sort(specs) if existingSpec, found := known[name]; found { - if existingSpec == spec { + sort.Sort(existingSpec) + if reflect.DeepEqual(existingSpec, specs) { continue } return fmt.Errorf("feature gate %q with different spec already exists: %v", name, existingSpec) } - - known[name] = spec + known[name] = specs } // Persist updated state @@ -309,17 +450,22 @@ func (f *featureGate) OverrideDefault(name Feature, override bool) error { return fmt.Errorf("cannot override default for feature %q: gates already added to a flag set", name) } - known := map[Feature]FeatureSpec{} - for name, spec := range f.known.Load().(map[Feature]FeatureSpec) { - known[name] = spec + known := map[Feature]VersionedSpecs{} + for k, v := range f.known.Load().(map[Feature]VersionedSpecs) { + sort.Sort(v) + known[k] = v } - spec, ok := known[name] - switch { - case !ok: + specs, ok := known[name] + if !ok { return fmt.Errorf("cannot override default: feature %q is not registered", name) + } + spec := f.getCurrentVersion(specs) + switch { case spec.LockToDefault: return fmt.Errorf("cannot override default: feature %q default is locked to %t", name, spec.Default) + case spec.PreRelease == PreAlpha: + return fmt.Errorf("cannot override default: feature %q is not available before emulation version %s", name, f.EmulationVersion().String()) case spec.PreRelease == Deprecated: klog.Warningf("Overriding default of deprecated feature gate %s=%t. It will be removed in a future release.", name, override) case spec.PreRelease == GA: @@ -327,35 +473,112 @@ func (f *featureGate) OverrideDefault(name Feature, override bool) error { } spec.Default = override - known[name] = spec + known[name] = specs f.known.Store(known) return nil } -// GetAll returns a copy of the map of known feature names to feature specs. +// GetAll returns a copy of the map of known feature names to feature specs for the current emulationVersion. func (f *featureGate) GetAll() map[Feature]FeatureSpec { retval := map[Feature]FeatureSpec{} - for k, v := range f.known.Load().(map[Feature]FeatureSpec) { + for k, v := range f.GetAllVersioned() { + spec := f.getCurrentVersion(v) + if spec.PreRelease == PreAlpha { + // The feature is not available at the emulation version. + continue + } + retval[k] = *f.getCurrentVersion(v) + } + return retval +} + +// GetAllVersioned returns a copy of the map of known feature names to versioned feature specs. +func (f *featureGate) GetAllVersioned() map[Feature]VersionedSpecs { + retval := map[Feature]VersionedSpecs{} + for k, v := range f.known.Load().(map[Feature]VersionedSpecs) { retval[k] = v } return retval } +// DeferErrorsToValidation could be used to defer checking flag setting error, +// because the emulationVersion may not be the final emulationVersion when the flag is set. +// Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true. +func (f *featureGate) DeferErrorsToValidation(val bool) { + f.lock.Lock() + defer f.lock.Unlock() + f.deferErrorsToValidation = val +} + +func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) error { + f.lock.Lock() + defer f.lock.Unlock() + klog.V(1).Infof("set feature gate emulationVersion to %s", emulationVersion.String()) + f.emulationVersion.Store(emulationVersion) + + // Copy existing state + enabledRaw := map[string]bool{} + for k, v := range f.enabledRaw.Load().(map[string]bool) { + enabledRaw[k] = v + } + // enabled map should be reset whenever emulationVersion is changed. + enabled := map[Feature]bool{} + errs := f.unsafeSetFromMap(enabled, enabledRaw) + if len(errs) == 0 { + // Persist changes + f.enabled.Store(enabled) + } + return utilerrors.NewAggregate(errs) +} + +func (f *featureGate) EmulationVersion() *version.Version { + return f.emulationVersion.Load() +} + +// FeatureSpec returns the FeatureSpec at the EmulationVersion if the key exists, an error otherwise. +func (f *featureGate) FeatureSpec(key Feature) (FeatureSpec, error) { + if v, ok := f.known.Load().(map[Feature]VersionedSpecs)[key]; ok { + currentVersion := f.getCurrentVersion(v) + return *currentVersion, nil + } + return FeatureSpec{}, fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName) +} + // Enabled returns true if the key is enabled. If the key is not known, this call will panic. func (f *featureGate) Enabled(key Feature) bool { + // fallback to default behavior, since we don't have emulation version set if v, ok := f.enabled.Load().(map[Feature]bool)[key]; ok { return v } - if v, ok := f.known.Load().(map[Feature]FeatureSpec)[key]; ok { - return v.Default + if v, ok := f.known.Load().(map[Feature]VersionedSpecs)[key]; ok { + return f.getCurrentVersion(v).Default } panic(fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName)) } +func (f *featureGate) getCurrentVersion(v VersionedSpecs) *FeatureSpec { + return getCurrentVersion(v, f.EmulationVersion()) +} + +func getCurrentVersion(v VersionedSpecs, emulationVersion *version.Version) *FeatureSpec { + i := len(v) - 1 + for ; i >= 0; i-- { + if v[i].Version.GreaterThan(emulationVersion) { + continue + } + return &v[i] + } + return &FeatureSpec{ + Default: false, + PreRelease: PreAlpha, + Version: version.MajorMinor(0, 0), + } +} + // AddFlag adds a flag for setting global feature gates to the specified FlagSet. -func (f *featureGate) AddFlag(fs *pflag.FlagSet) { +func (f *featureGate) AddFlag(fs *pflag.FlagSet, prefix string) { f.lock.Lock() // TODO(mtaufen): Shouldn't we just close it on the first Set/SetFromMap instead? // Not all components expose a feature gates flag using this AddFlag method, and @@ -365,7 +588,10 @@ func (f *featureGate) AddFlag(fs *pflag.FlagSet) { f.lock.Unlock() known := f.KnownFeatures() - fs.Var(f, flagName, ""+ + if len(prefix) > 0 && !strings.HasSuffix(prefix, "-") { + prefix += "-" + } + fs.Var(f, prefix+flagName, ""+ "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ "Options are:\n"+strings.Join(known, "\n")) } @@ -377,14 +603,19 @@ func (f *featureGate) AddMetrics() { } // KnownFeatures returns a slice of strings describing the FeatureGate's known features. -// Deprecated and GA features are hidden from the list. +// preAlpha, Deprecated and GA features are hidden from the list. func (f *featureGate) KnownFeatures() []string { var known []string - for k, v := range f.known.Load().(map[Feature]FeatureSpec) { - if v.PreRelease == GA || v.PreRelease == Deprecated { + for k, v := range f.known.Load().(map[Feature]VersionedSpecs) { + if k == "AllAlpha" || k == "AllBeta" { + known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, v[0].PreRelease, v[0].Default)) continue } - known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, v.PreRelease, v.Default)) + currentV := f.getCurrentVersion(v) + if currentV.PreRelease == GA || currentV.PreRelease == Deprecated || currentV.PreRelease == PreAlpha { + continue + } + known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, currentV.PreRelease, currentV.Default)) } sort.Strings(known) return known @@ -393,16 +624,20 @@ func (f *featureGate) KnownFeatures() []string { // DeepCopy returns a deep copy of the FeatureGate object, such that gates can be // set on the copy without mutating the original. This is useful for validating // config against potential feature gate changes before committing those changes. -func (f *featureGate) DeepCopy() MutableFeatureGate { +func (f *featureGate) DeepCopy() MutableVersionedFeatureGate { // Copy existing state. - known := map[Feature]FeatureSpec{} - for k, v := range f.known.Load().(map[Feature]FeatureSpec) { + known := map[Feature]VersionedSpecs{} + for k, v := range f.known.Load().(map[Feature]VersionedSpecs) { known[k] = v } enabled := map[Feature]bool{} for k, v := range f.enabled.Load().(map[Feature]bool) { enabled[k] = v } + enabledRaw := map[string]bool{} + for k, v := range f.enabledRaw.Load().(map[string]bool) { + enabledRaw[k] = v + } // Construct a new featureGate around the copied state. // Note that specialFeatures is treated as immutable by convention, @@ -411,9 +646,22 @@ func (f *featureGate) DeepCopy() MutableFeatureGate { special: specialFeatures, closed: f.closed, } - + fg.emulationVersion.Store(f.EmulationVersion()) fg.known.Store(known) fg.enabled.Store(enabled) - + fg.enabledRaw.Store(enabledRaw) return fg } + +// Reset sets the enabled and enabledRaw to the input map. +func (f *featureGate) Reset(m map[string]bool) { + enabled := map[Feature]bool{} + enabledRaw := map[string]bool{} + f.enabled.Store(enabled) + f.enabledRaw.Store(enabledRaw) + _ = f.SetFromMap(m) +} + +func (f *featureGate) EnabledRawMap() map[string]bool { + return f.enabledRaw.Load().(map[string]bool) +} diff --git a/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go b/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go index 063a5a36368..b46d5483fb5 100644 --- a/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go +++ b/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go @@ -18,12 +18,17 @@ package featuregate import ( "fmt" + "reflect" + "sort" "strings" "testing" "github.com/spf13/pflag" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/version" "k8s.io/component-base/metrics/legacyregistry" featuremetrics "k8s.io/component-base/metrics/prometheus/feature" "k8s.io/component-base/metrics/testutil" @@ -33,6 +38,8 @@ func TestFeatureGateFlag(t *testing.T) { // gates for testing const testAlphaGate Feature = "TestAlpha" const testBetaGate Feature = "TestBeta" + const testDeprecatedGate Feature = "TestDeprecated" + const testLockedFalseGate Feature = "TestLockedFalse" tests := []struct { arg string @@ -42,93 +49,126 @@ func TestFeatureGateFlag(t *testing.T) { { arg: "", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testDeprecatedGate: false, + testLockedFalseGate: false, }, }, + { + arg: "TestDeprecated=true", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testDeprecatedGate: true, + testLockedFalseGate: false, + }, + }, + { + arg: "TestLockedFalse=true", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + }, + parseError: "cannot set feature gate TestLockedFalse to true, feature is locked to false", + }, { arg: "fooBarBaz=true", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, }, parseError: "unrecognized feature gate: fooBarBaz", }, { arg: "AllAlpha=false", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, }, }, { arg: "AllAlpha=true", expect: map[Feature]bool{ - allAlphaGate: true, - allBetaGate: false, - testAlphaGate: true, - testBetaGate: false, + allAlphaGate: true, + allBetaGate: false, + testAlphaGate: true, + testBetaGate: false, + testLockedFalseGate: false, }, }, { arg: "AllAlpha=banana", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, }, parseError: "invalid value of AllAlpha", }, { arg: "AllAlpha=false,TestAlpha=true", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: true, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: true, + testBetaGate: false, + testLockedFalseGate: false, }, }, { arg: "TestAlpha=true,AllAlpha=false", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: true, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: true, + testBetaGate: false, + testLockedFalseGate: false, }, }, { arg: "AllAlpha=true,TestAlpha=false", expect: map[Feature]bool{ - allAlphaGate: true, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: false, + allAlphaGate: true, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, }, }, { arg: "TestAlpha=false,AllAlpha=true", expect: map[Feature]bool{ - allAlphaGate: true, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: false, + allAlphaGate: true, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, }, }, { arg: "TestBeta=true,AllAlpha=false", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: true, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, }, }, @@ -144,65 +184,72 @@ func TestFeatureGateFlag(t *testing.T) { { arg: "AllBeta=true", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: true, - testAlphaGate: false, - testBetaGate: true, + allAlphaGate: false, + allBetaGate: true, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, }, }, { arg: "AllBeta=banana", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, }, parseError: "invalid value of AllBeta", }, { arg: "AllBeta=false,TestBeta=true", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: true, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, }, }, { arg: "TestBeta=true,AllBeta=false", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: false, - testBetaGate: true, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, }, }, { arg: "AllBeta=true,TestBeta=false", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: true, - testAlphaGate: false, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: true, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, }, }, { arg: "TestBeta=false,AllBeta=true", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: true, - testAlphaGate: false, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: true, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, }, }, { arg: "TestAlpha=true,AllBeta=false", expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testAlphaGate: true, - testBetaGate: false, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: true, + testBetaGate: false, + testLockedFalseGate: false, }, }, } @@ -210,13 +257,15 @@ func TestFeatureGateFlag(t *testing.T) { t.Run(test.arg, func(t *testing.T) { fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError) f := NewFeatureGate() - f.Add(map[Feature]FeatureSpec{ - testAlphaGate: {Default: false, PreRelease: Alpha}, - testBetaGate: {Default: false, PreRelease: Beta}, + err := f.Add(map[Feature]FeatureSpec{ + testAlphaGate: {Default: false, PreRelease: Alpha}, + testBetaGate: {Default: false, PreRelease: Beta}, + testDeprecatedGate: {Default: false, PreRelease: Deprecated}, + testLockedFalseGate: {Default: false, PreRelease: GA, LockToDefault: true}, }) - f.AddFlag(fs) - - err := fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)}) + require.NoError(t, err) + f.AddFlag(fs, "") + err = fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)}) if test.parseError != "" { if !strings.Contains(err.Error(), test.parseError) { t.Errorf("%d: Parse() Expected %v, Got %v", i, test.parseError, err) @@ -239,12 +288,16 @@ func TestFeatureGateOverride(t *testing.T) { // Don't parse the flag, assert defaults are used. var f *featureGate = NewFeatureGate() - f.Add(map[Feature]FeatureSpec{ + err := f.Add(map[Feature]FeatureSpec{ testAlphaGate: {Default: false, PreRelease: Alpha}, testBetaGate: {Default: false, PreRelease: Beta}, }) + require.NoError(t, err) f.Set("TestAlpha=true,TestBeta=true") + if errs := f.Validate(); len(errs) > 0 { + t.Fatalf("Validate() Expected no error, Got %v", errs) + } if f.Enabled(testAlphaGate) != true { t.Errorf("Expected true") } @@ -253,6 +306,9 @@ func TestFeatureGateOverride(t *testing.T) { } f.Set("TestAlpha=false") + if errs := f.Validate(); len(errs) > 0 { + t.Fatalf("Validate() Expected no error, Got %v", errs) + } if f.Enabled(testAlphaGate) != false { t.Errorf("Expected false") } @@ -268,10 +324,11 @@ func TestFeatureGateFlagDefaults(t *testing.T) { // Don't parse the flag, assert defaults are used. var f *featureGate = NewFeatureGate() - f.Add(map[Feature]FeatureSpec{ + err := f.Add(map[Feature]FeatureSpec{ testAlphaGate: {Default: false, PreRelease: Alpha}, testBetaGate: {Default: true, PreRelease: Beta}, }) + require.NoError(t, err) if f.Enabled(testAlphaGate) != false { t.Errorf("Expected false") @@ -292,13 +349,13 @@ func TestFeatureGateKnownFeatures(t *testing.T) { // Don't parse the flag, assert defaults are used. var f *featureGate = NewFeatureGate() - f.Add(map[Feature]FeatureSpec{ + err := f.Add(map[Feature]FeatureSpec{ testAlphaGate: {Default: false, PreRelease: Alpha}, testBetaGate: {Default: true, PreRelease: Beta}, testGAGate: {Default: true, PreRelease: GA}, testDeprecatedGate: {Default: false, PreRelease: Deprecated}, }) - + require.NoError(t, err) known := strings.Join(f.KnownFeatures(), " ") assert.Contains(t, known, testAlphaGate) @@ -399,13 +456,14 @@ func TestFeatureGateSetFromMap(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("SetFromMap %s", test.name), func(t *testing.T) { f := NewFeatureGate() - f.Add(map[Feature]FeatureSpec{ + err := f.Add(map[Feature]FeatureSpec{ testAlphaGate: {Default: false, PreRelease: Alpha}, testBetaGate: {Default: false, PreRelease: Beta}, testLockedTrueGate: {Default: true, PreRelease: GA, LockToDefault: true}, testLockedFalseGate: {Default: false, PreRelease: GA, LockToDefault: true}, }) - err := f.SetFromMap(test.setmap) + require.NoError(t, err) + err = f.SetFromMap(test.setmap) if test.setmapError != "" { if err == nil { t.Errorf("expected error, got none") @@ -450,8 +508,8 @@ func TestFeatureGateMetrics(t *testing.T) { testBetaGate: {Default: true, PreRelease: Beta}, testBetaDisabled: {Default: true, PreRelease: Alpha}, } - f.Add(fMap) - f.SetFromMap(map[string]bool{"TestAlphaEnabled": true, "TestBetaDisabled": false}) + require.NoError(t, f.Add(fMap)) + require.NoError(t, f.SetFromMap(map[string]bool{"TestAlphaEnabled": true, "TestBetaDisabled": false})) f.AddMetrics() if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedOutput), testedMetrics...); err != nil { t.Fatal(err) @@ -499,8 +557,8 @@ func TestFeatureGateString(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("SetFromMap %s", test.expect), func(t *testing.T) { f := NewFeatureGate() - f.Add(featuremap) - f.SetFromMap(test.setmap) + require.NoError(t, f.Add(featuremap)) + require.NoError(t, f.SetFromMap(test.setmap)) result := f.String() if result != test.expect { t.Errorf("%d: SetFromMap(%#v) Expected %s, Got %s", i, test.setmap, test.expect, result) @@ -518,12 +576,8 @@ func TestFeatureGateOverrideDefault(t *testing.T) { }); err != nil { t.Fatal(err) } - if err := f.OverrideDefault("TestFeature1", false); err != nil { - t.Fatal(err) - } - if err := f.OverrideDefault("TestFeature2", true); err != nil { - t.Fatal(err) - } + require.NoError(t, f.OverrideDefault("TestFeature1", false)) + require.NoError(t, f.OverrideDefault("TestFeature2", true)) if f.Enabled("TestFeature1") { t.Error("expected TestFeature1 to have effective default of false") } @@ -534,12 +588,8 @@ func TestFeatureGateOverrideDefault(t *testing.T) { t.Run("overrides are preserved across deep copies", func(t *testing.T) { f := NewFeatureGate() - if err := f.Add(map[Feature]FeatureSpec{"TestFeature": {Default: false}}); err != nil { - t.Fatal(err) - } - if err := f.OverrideDefault("TestFeature", true); err != nil { - t.Fatal(err) - } + require.NoError(t, f.Add(map[Feature]FeatureSpec{"TestFeature": {Default: false}})) + require.NoError(t, f.OverrideDefault("TestFeature", true)) fcopy := f.DeepCopy() if !fcopy.Enabled("TestFeature") { t.Error("default override was not preserved by deep copy") @@ -554,9 +604,7 @@ func TestFeatureGateOverrideDefault(t *testing.T) { }}); err != nil { t.Fatal(err) } - if err := f.OverrideDefault("TestFeature", true); err != nil { - t.Fatal(err) - } + require.NoError(t, f.OverrideDefault("TestFeature", true)) var found bool for _, s := range f.KnownFeatures() { if !strings.Contains(s, "TestFeature") { @@ -592,15 +640,9 @@ func TestFeatureGateOverrideDefault(t *testing.T) { t.Run("does not supersede explicitly-set value", func(t *testing.T) { f := NewFeatureGate() - if err := f.Add(map[Feature]FeatureSpec{"TestFeature": {Default: true}}); err != nil { - t.Fatal(err) - } - if err := f.OverrideDefault("TestFeature", false); err != nil { - t.Fatal(err) - } - if err := f.SetFromMap(map[string]bool{"TestFeature": true}); err != nil { - t.Fatal(err) - } + require.NoError(t, f.Add(map[Feature]FeatureSpec{"TestFeature": {Default: true}})) + require.NoError(t, f.OverrideDefault("TestFeature", false)) + require.NoError(t, f.SetFromMap(map[string]bool{"TestFeature": true})) if !f.Enabled("TestFeature") { t.Error("expected feature to be effectively enabled despite default override") } @@ -616,9 +658,7 @@ func TestFeatureGateOverrideDefault(t *testing.T) { }); err != nil { t.Fatal(err) } - if err := f.OverrideDefault("TestFeature", false); err != nil { - t.Fatal(err) - } + require.NoError(t, f.OverrideDefault("TestFeature", false)) if err := f.Add(map[Feature]FeatureSpec{ "TestFeature": { Default: true, @@ -639,10 +679,801 @@ func TestFeatureGateOverrideDefault(t *testing.T) { t.Run("returns error if already added to flag set", func(t *testing.T) { f := NewFeatureGate() fs := pflag.NewFlagSet("test", pflag.ContinueOnError) - f.AddFlag(fs) + f.AddFlag(fs, "") if err := f.OverrideDefault("TestFeature", true); err == nil { t.Error("expected a non-nil error to be returned") } }) } + +func TestVersionedFeatureGateFlag(t *testing.T) { + // gates for testing + const testGAGate Feature = "TestGA" + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" + const testLockedFalseGate Feature = "TestLockedFalse" + const testAlphaGateNoVersion Feature = "TestAlphaNoVersion" + const testBetaGateNoVersion Feature = "TestBetaNoVersion" + + tests := []struct { + arg string + expect map[Feature]bool + parseError string + }{ + { + arg: "", + expect: map[Feature]bool{ + testGAGate: false, + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + }, + { + arg: "TestLockedFalse=true", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: true, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + }, + { + arg: "fooBarBaz=true", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + parseError: "unrecognized feature gate: fooBarBaz", + }, + { + arg: "AllAlpha=false", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + }, + { + arg: "AllAlpha=true", + expect: map[Feature]bool{ + allAlphaGate: true, + allBetaGate: false, + testAlphaGate: false, + testGAGate: false, + testBetaGate: true, + testLockedFalseGate: false, + testAlphaGateNoVersion: true, + testBetaGateNoVersion: false, + }, + }, + { + arg: "AllAlpha=banana", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + parseError: "invalid value of AllAlpha", + }, + { + arg: "AllAlpha=false,TestAlpha=true,TestAlphaNoVersion=true", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: true, + testBetaGateNoVersion: false, + }, + parseError: "cannot set feature gate TestAlpha to true, feature is PreAlpha at emulated version 1.28", + }, + { + arg: "AllAlpha=false,TestAlphaNoVersion=true", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: true, + testBetaGateNoVersion: false, + }, + }, + { + arg: "TestAlpha=true,TestAlphaNoVersion=true,AllAlpha=false", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: true, + testBetaGateNoVersion: false, + }, + parseError: "cannot set feature gate TestAlpha to true, feature is PreAlpha at emulated version 1.28", + }, + { + arg: "AllAlpha=true,TestAlpha=false,TestAlphaNoVersion=false", + expect: map[Feature]bool{ + allAlphaGate: true, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + parseError: "cannot set feature gate TestAlpha to false, feature is PreAlpha at emulated version 1.28", + }, + { + arg: "AllAlpha=true,TestAlphaNoVersion=false", + expect: map[Feature]bool{ + allAlphaGate: true, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + }, + { + arg: "TestAlpha=false,TestAlphaNoVersion=false,AllAlpha=true", + expect: map[Feature]bool{ + allAlphaGate: true, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + parseError: "cannot set feature gate TestAlpha to false, feature is PreAlpha at emulated version 1.28", + }, + { + arg: "TestBeta=true,TestBetaNoVersion=true,TestGA=true,AllAlpha=false", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: true, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: true, + }, + }, + + { + arg: "AllBeta=false", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + }, + { + arg: "AllBeta=true", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: true, + testGAGate: true, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: true, + }, + }, + { + arg: "AllBeta=banana", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + parseError: "invalid value of AllBeta", + }, + { + arg: "AllBeta=false,TestBeta=true,TestBetaNoVersion=true,TestGA=true", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: true, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: true, + }, + }, + { + arg: "TestBeta=true,TestBetaNoVersion=true,AllBeta=false", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: false, + testBetaGate: true, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: true, + }, + }, + { + arg: "AllBeta=true,TestBetaNoVersion=false,TestBeta=false,TestGA=false", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: true, + testGAGate: false, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + }, + { + arg: "TestBeta=false,TestBetaNoVersion=false,AllBeta=true", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: true, + testGAGate: true, + testAlphaGate: false, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + }, + { + arg: "TestAlpha=true,AllBeta=false", + expect: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testGAGate: false, + testAlphaGate: true, + testBetaGate: false, + testLockedFalseGate: false, + testAlphaGateNoVersion: false, + testBetaGateNoVersion: false, + }, + parseError: "cannot set feature gate TestAlpha to true, feature is PreAlpha at emulated version 1.28", + }, + } + for i, test := range tests { + t.Run(test.arg, func(t *testing.T) { + fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError) + f := NewVersionedFeatureGate(version.MustParse("1.29")) + f.DeferErrorsToValidation(true) + err := f.AddVersioned(map[Feature]VersionedSpecs{ + testGAGate: { + {Version: version.MustParse("1.29"), Default: true, PreRelease: GA}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.27"), Default: false, PreRelease: Alpha}, + }, + testAlphaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + }, + testBetaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha}, + }, + testLockedFalseGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: GA, LockToDefault: true}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: GA}, + }, + }) + require.NoError(t, err) + err = f.Add(map[Feature]FeatureSpec{ + testAlphaGateNoVersion: {Default: false, PreRelease: Alpha}, + testBetaGateNoVersion: {Default: false, PreRelease: Beta}, + }) + require.NoError(t, err) + f.AddFlag(fs, "") + + var errs []error + err = fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)}) + if err != nil { + errs = append(errs, err) + } else { + errs = append(errs, f.SetEmulationVersion(version.MustParse("1.28"))) + } + err = utilerrors.NewAggregate(errs) + if test.parseError != "" { + if !strings.Contains(err.Error(), test.parseError) { + t.Errorf("%d: Parse() Expected %v, Got %v", i, test.parseError, err) + } + return + } else if err != nil { + t.Errorf("%d: Parse() Expected nil, Got %v", i, err) + } + for k, v := range test.expect { + if actual := f.enabled.Load().(map[Feature]bool)[k]; actual != v { + t.Errorf("%d: expected %s=%v, Got %v", i, k, v, actual) + } + } + }) + } +} + +func TestVersionedFeatureGateOverride(t *testing.T) { + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" + + // Don't parse the flag, assert defaults are used. + f := NewVersionedFeatureGate(version.MustParse("1.29")) + + err := f.AddVersioned(map[Feature]VersionedSpecs{ + testAlphaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + }, + testBetaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha}, + }, + }) + require.NoError(t, err) + if f.Enabled(testAlphaGate) != false { + t.Errorf("Expected false") + } + if f.Enabled(testBetaGate) != false { + t.Errorf("Expected false") + } + if errs := f.Validate(); len(errs) > 0 { + t.Errorf("Expected no errors when emulation version is equal to binary version.") + } + + require.NoError(t, f.Set("TestAlpha=true,TestBeta=true")) + if f.Enabled(testAlphaGate) != true { + t.Errorf("Expected false") + } + if f.Enabled(testBetaGate) != true { + t.Errorf("Expected true") + } + + require.NoError(t, f.Set("TestAlpha=false")) + if f.Enabled(testAlphaGate) != false { + t.Errorf("Expected false") + } + if f.Enabled(testBetaGate) != true { + t.Errorf("Expected true") + } + if errs := f.Validate(); len(errs) > 0 { + t.Errorf("Expected no errors when emulation version is equal to binary version.") + } + + if err := f.SetEmulationVersion(version.MustParse("1.28")); err == nil { + t.Errorf("Expected errors when emulation version is 1.28.") + } +} + +func TestVersionedFeatureGateFlagDefaults(t *testing.T) { + // gates for testing + const testGAGate Feature = "TestGA" + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" + + // Don't parse the flag, assert defaults are used. + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + + err := f.AddVersioned(map[Feature]VersionedSpecs{ + testGAGate: { + {Version: version.MustParse("1.29"), Default: true, PreRelease: GA}, + {Version: version.MustParse("1.27"), Default: true, PreRelease: Beta}, + {Version: version.MustParse("1.25"), Default: true, PreRelease: Alpha}, + }, + testAlphaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + }, + testBetaGate: { + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.26"), Default: false, PreRelease: Alpha}, + }, + }) + require.NoError(t, err) + + if f.Enabled(testAlphaGate) != false { + t.Errorf("Expected false") + } + if fs, _ := f.FeatureSpec(testAlphaGate); fs.PreRelease != PreAlpha || fs.Version.String() != "0.0" { + t.Errorf("Expected (PreAlpha, 0.0)") + } + if f.Enabled(testBetaGate) != false { + t.Errorf("Expected false") + } + if fs, _ := f.FeatureSpec(testBetaGate); fs.PreRelease != Beta || fs.Version.String() != "1.28" { + t.Errorf("Expected (Beta, 1.28)") + } + if f.Enabled(testGAGate) != true { + t.Errorf("Expected true") + } + if fs, _ := f.FeatureSpec(testGAGate); fs.PreRelease != Beta || fs.Version.String() != "1.27" { + t.Errorf("Expected (Beta, 1.27)") + } + if _, err := f.FeatureSpec("NonExist"); err == nil { + t.Errorf("Expected Error") + } + allFeatures := f.GetAll() + expectedAllFeatures := []Feature{testGAGate, testBetaGate, allAlphaGate, allBetaGate} + if len(allFeatures) != 4 { + t.Errorf("Expected 4 features from GetAll(), got %d", len(allFeatures)) + } + for _, feature := range expectedAllFeatures { + if _, ok := allFeatures[feature]; !ok { + t.Errorf("Expected feature %s to be in GetAll()", feature) + } + } +} + +func TestVersionedFeatureGateKnownFeatures(t *testing.T) { + // gates for testing + const ( + testPreAlphaGate Feature = "TestPreAlpha" + testAlphaGate Feature = "TestAlpha" + testBetaGate Feature = "TestBeta" + testGAGate Feature = "TestGA" + testDeprecatedGate Feature = "TestDeprecated" + testGAGateNoVersion Feature = "TestGANoVersion" + testAlphaGateNoVersion Feature = "TestAlphaNoVersion" + testBetaGateNoVersion Feature = "TestBetaNoVersion" + testDeprecatedGateNoVersion Feature = "TestDeprecatedNoVersion" + ) + + // Don't parse the flag, assert defaults are used. + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + err := f.AddVersioned(map[Feature]VersionedSpecs{ + testGAGate: { + {Version: version.MustParse("1.27"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.28"), Default: true, PreRelease: GA}, + }, + testPreAlphaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + }, + testAlphaGate: { + {Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha}, + }, + testBetaGate: { + {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + }, + testDeprecatedGate: { + {Version: version.MustParse("1.28"), Default: true, PreRelease: Deprecated}, + {Version: version.MustParse("1.26"), Default: false, PreRelease: Alpha}, + }, + }) + require.NoError(t, err) + err = f.Add(map[Feature]FeatureSpec{ + testAlphaGateNoVersion: {Default: false, PreRelease: Alpha}, + testBetaGateNoVersion: {Default: false, PreRelease: Beta}, + testGAGateNoVersion: {Default: false, PreRelease: GA}, + testDeprecatedGateNoVersion: {Default: false, PreRelease: Deprecated}, + }) + require.NoError(t, err) + + known := strings.Join(f.KnownFeatures(), " ") + + assert.NotContains(t, known, testPreAlphaGate) + assert.Contains(t, known, testAlphaGate) + assert.Contains(t, known, testBetaGate) + assert.NotContains(t, known, testGAGate) + assert.NotContains(t, known, testDeprecatedGate) + assert.Contains(t, known, testAlphaGateNoVersion) + assert.Contains(t, known, testBetaGateNoVersion) + assert.NotContains(t, known, testGAGateNoVersion) + assert.NotContains(t, known, testDeprecatedGateNoVersion) +} + +func TestVersionedFeatureGateMetrics(t *testing.T) { + // gates for testing + featuremetrics.ResetFeatureInfoMetric() + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" + const testAlphaEnabled Feature = "TestAlphaEnabled" + const testBetaDisabled Feature = "TestBetaDisabled" + testedMetrics := []string{"kubernetes_feature_enabled"} + expectedOutput := ` + # HELP kubernetes_feature_enabled [BETA] This metric records the data about the stage and enablement of a k8s feature. + # TYPE kubernetes_feature_enabled gauge + kubernetes_feature_enabled{name="TestAlpha",stage="ALPHA"} 0 + kubernetes_feature_enabled{name="TestBeta",stage="BETA"} 1 + kubernetes_feature_enabled{name="TestAlphaEnabled",stage="ALPHA"} 1 + kubernetes_feature_enabled{name="AllAlpha",stage="ALPHA"} 0 + kubernetes_feature_enabled{name="AllBeta",stage="BETA"} 0 + kubernetes_feature_enabled{name="TestBetaDisabled",stage="BETA"} 0 +` + + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + err := f.AddVersioned(map[Feature]VersionedSpecs{ + testAlphaGate: { + {Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta}, + }, + testAlphaEnabled: { + {Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta}, + }, + testBetaGate: { + {Version: version.MustParse("1.28"), Default: true, PreRelease: Beta}, + {Version: version.MustParse("1.27"), Default: false, PreRelease: Alpha}, + }, + testBetaDisabled: { + {Version: version.MustParse("1.28"), Default: true, PreRelease: Beta}, + {Version: version.MustParse("1.27"), Default: false, PreRelease: Alpha}, + }, + }) + require.NoError(t, err) + + require.NoError(t, f.SetFromMap(map[string]bool{"TestAlphaEnabled": true, "TestBetaDisabled": false})) + f.AddMetrics() + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedOutput), testedMetrics...); err != nil { + t.Fatal(err) + } +} + +func TestVersionedFeatureGateOverrideDefault(t *testing.T) { + t.Run("overrides take effect", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.AddVersioned(map[Feature]VersionedSpecs{ + "TestFeature1": { + {Version: version.MustParse("1.28"), Default: true}, + }, + "TestFeature2": { + {Version: version.MustParse("1.26"), Default: false}, + {Version: version.MustParse("1.29"), Default: true}, + }, + }); err != nil { + t.Fatal(err) + } + require.NoError(t, f.OverrideDefault("TestFeature1", false)) + require.NoError(t, f.OverrideDefault("TestFeature2", true)) + if f.Enabled("TestFeature1") { + t.Error("expected TestFeature1 to have effective default of false") + } + if !f.Enabled("TestFeature2") { + t.Error("expected TestFeature2 to have effective default of true") + } + }) + + t.Run("overrides are preserved across deep copies", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.AddVersioned(map[Feature]VersionedSpecs{ + "TestFeature": { + {Version: version.MustParse("1.28"), Default: false}, + {Version: version.MustParse("1.29"), Default: true}, + }, + }); err != nil { + t.Fatal(err) + } + require.NoError(t, f.OverrideDefault("TestFeature", true)) + fcopy := f.DeepCopy() + if !fcopy.Enabled("TestFeature") { + t.Error("default override was not preserved by deep copy") + } + }) + + t.Run("reflected in known features", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.AddVersioned(map[Feature]VersionedSpecs{ + "TestFeature": { + {Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta}, + }, + }); err != nil { + t.Fatal(err) + } + require.NoError(t, f.OverrideDefault("TestFeature", true)) + var found bool + for _, s := range f.KnownFeatures() { + if !strings.Contains(s, "TestFeature") { + continue + } + found = true + if !strings.Contains(s, "default=true") { + t.Errorf("expected override of default to be reflected in known feature description %q", s) + } + } + if !found { + t.Error("found no entry for TestFeature in known features") + } + }) + + t.Run("may not change default for specs with locked defaults", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.AddVersioned(map[Feature]VersionedSpecs{ + "LockedFeature": { + {Version: version.MustParse("1.28"), Default: true, LockToDefault: true}, + }, + }); err != nil { + t.Fatal(err) + } + if f.OverrideDefault("LockedFeature", false) == nil { + t.Error("expected error when attempting to override the default for a feature with a locked default") + } + if f.OverrideDefault("LockedFeature", true) == nil { + t.Error("expected error when attempting to override the default for a feature with a locked default") + } + }) + + t.Run("can change default for specs without locked defaults for emulation version", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.AddVersioned(map[Feature]VersionedSpecs{ + "LockedFeature": { + {Version: version.MustParse("1.28"), Default: true}, + {Version: version.MustParse("1.29"), Default: true, LockToDefault: true}, + }, + }); err != nil { + t.Fatal(err) + } + require.NoError(t, f.OverrideDefault("LockedFeature", false)) + if f.Enabled("LockedFeature") { + t.Error("expected LockedFeature to have effective default of false") + } + }) + + t.Run("does not supersede explicitly-set value", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.AddVersioned(map[Feature]VersionedSpecs{ + "TestFeature": { + {Version: version.MustParse("1.28"), Default: true}, + }, + }); err != nil { + t.Fatal(err) + } + require.NoError(t, f.OverrideDefault("TestFeature", false)) + require.NoError(t, f.SetFromMap(map[string]bool{"TestFeature": true})) + if !f.Enabled("TestFeature") { + t.Error("expected feature to be effectively enabled despite default override") + } + }) + + t.Run("prevents re-registration of feature spec after overriding default", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.AddVersioned(map[Feature]VersionedSpecs{ + "TestFeature": { + {Version: version.MustParse("1.28"), Default: true, PreRelease: Alpha}, + }, + }); err != nil { + t.Fatal(err) + } + require.NoError(t, f.OverrideDefault("TestFeature", false)) + if err := f.Add(map[Feature]FeatureSpec{ + "TestFeature": { + Default: true, + PreRelease: Alpha, + }, + }); err == nil { + t.Error("expected re-registration to return a non-nil error after overriding its default") + } + }) + + t.Run("does not allow override for a feature added after emulation version", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.AddVersioned(map[Feature]VersionedSpecs{ + "TestFeature": { + {Version: version.MustParse("1.29"), Default: false}, + }, + }); err != nil { + t.Fatal(err) + } + if err := f.OverrideDefault("TestFeature", true); err == nil { + t.Error("expected an error to be returned in attempt to override default for a feature added after emulation version") + } + }) + + t.Run("does not allow override for an unknown feature", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.OverrideDefault("TestFeature", true); err == nil { + t.Error("expected an error to be returned in attempt to override default for unregistered feature") + } + }) + + t.Run("returns error if already added to flag set", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + fs := pflag.NewFlagSet("test", pflag.ContinueOnError) + f.AddFlag(fs, "") + + if err := f.OverrideDefault("TestFeature", true); err == nil { + t.Error("expected a non-nil error to be returned") + } + }) +} + +func TestGetCurrentVersion(t *testing.T) { + specs := VersionedSpecs{{Version: version.MustParse("1.29"), Default: true, PreRelease: GA}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.25"), Default: false, PreRelease: Alpha}, + } + sort.Sort(specs) + tests := []struct { + cVersion string + expect FeatureSpec + }{ + { + cVersion: "1.30", + expect: FeatureSpec{Version: version.MustParse("1.29"), Default: true, PreRelease: GA}, + }, + { + cVersion: "1.29", + expect: FeatureSpec{Version: version.MustParse("1.29"), Default: true, PreRelease: GA}, + }, + { + cVersion: "1.28", + expect: FeatureSpec{Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + }, + { + cVersion: "1.27", + expect: FeatureSpec{Version: version.MustParse("1.25"), Default: false, PreRelease: Alpha}, + }, + { + cVersion: "1.25", + expect: FeatureSpec{Version: version.MustParse("1.25"), Default: false, PreRelease: Alpha}, + }, + { + cVersion: "1.24", + expect: FeatureSpec{Version: version.MajorMinor(0, 0), Default: false, PreRelease: PreAlpha}, + }, + } + for i, test := range tests { + t.Run(fmt.Sprintf("getCurrentVersion for emulationVersion %s", test.cVersion), func(t *testing.T) { + result := getCurrentVersion(specs, version.MustParse(test.cVersion)) + if !reflect.DeepEqual(*result, test.expect) { + t.Errorf("%d: getCurrentVersion(, %s) Expected %v, Got %v", i, test.cVersion, test.expect, result) + } + }) + } +} diff --git a/staging/src/k8s.io/component-base/featuregate/testing/feature_gate.go b/staging/src/k8s.io/component-base/featuregate/testing/feature_gate.go index 72e62c248c1..47c0f927738 100644 --- a/staging/src/k8s.io/component-base/featuregate/testing/feature_gate.go +++ b/staging/src/k8s.io/component-base/featuregate/testing/feature_gate.go @@ -45,7 +45,7 @@ func init() { func SetFeatureGateDuringTest(tb testing.TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) { tb.Helper() detectParallelOverrideCleanup := detectParallelOverride(tb, f) - originalValue := gate.Enabled(f) + originalEnabled := gate.(featuregate.MutableVersionedFeatureGateForTests).EnabledRawMap() // Specially handle AllAlpha and AllBeta if f == "AllAlpha" || f == "AllBeta" { @@ -67,9 +67,7 @@ func SetFeatureGateDuringTest(tb testing.TB, gate featuregate.FeatureGate, f fea tb.Cleanup(func() { tb.Helper() detectParallelOverrideCleanup() - if err := gate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, originalValue)); err != nil { - tb.Errorf("error restoring %s=%v: %v", f, originalValue, err) - } + gate.(featuregate.MutableVersionedFeatureGateForTests).Reset(originalEnabled) }) } diff --git a/staging/src/k8s.io/component-base/logs/example/cmd/logger.go b/staging/src/k8s.io/component-base/logs/example/cmd/logger.go index b0d4f9baaca..af51c793e30 100644 --- a/staging/src/k8s.io/component-base/logs/example/cmd/logger.go +++ b/staging/src/k8s.io/component-base/logs/example/cmd/logger.go @@ -67,7 +67,7 @@ func NewLoggerCommand() *cobra.Command { }, } logsapi.AddFeatureGates(featureGate) - featureGate.AddFlag(cmd.Flags()) + featureGate.AddFlag(cmd.Flags(), "") logsapi.AddFlags(c, cmd.Flags()) return cmd } diff --git a/staging/src/k8s.io/component-base/logs/example/k8s2slog/k8s2slog.go b/staging/src/k8s.io/component-base/logs/example/k8s2slog/k8s2slog.go index c39ef9fe7a6..6c29c206a3b 100644 --- a/staging/src/k8s.io/component-base/logs/example/k8s2slog/k8s2slog.go +++ b/staging/src/k8s.io/component-base/logs/example/k8s2slog/k8s2slog.go @@ -83,7 +83,7 @@ func NewLoggerCommand() *cobra.Command { // Shouldn't happen. panic(err) } - featureGate.AddFlag(cmd.Flags()) + featureGate.AddFlag(cmd.Flags(), "") logsapi.AddFlags(c, cmd.Flags()) return cmd } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go index bfd1135e787..93ac2b14b31 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go @@ -247,7 +247,7 @@ func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.Deleg } // used later to filter the served resource by those that have expired. - resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(*c.GenericConfig.Version) + resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(s.GenericAPIServer.EffectiveVersion.EmulationVersion()) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go index d4ddf9490ae..a70c4126cf2 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go @@ -31,6 +31,9 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/filters" genericoptions "k8s.io/apiserver/pkg/server/options" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" + "k8s.io/component-base/featuregate" "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" "k8s.io/kube-aggregator/pkg/apiserver" aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" @@ -58,10 +61,15 @@ type AggregatorOptions struct { // with a default AggregatorOptions. func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions) *cobra.Command { o := *defaults + featureGate := o.ServerRunOptions.FeatureGate.(featuregate.MutableVersionedFeatureGate) + effectiveVersion := o.ServerRunOptions.EffectiveVersion.(utilversion.MutableEffectiveVersion) cmd := &cobra.Command{ Short: "Launch a API aggregator and proxy server", Long: "Launch a API aggregator and proxy server", RunE: func(c *cobra.Command, args []string) error { + if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { + return err + } if err := o.Complete(); err != nil { return err } @@ -76,7 +84,11 @@ func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions) } cmd.SetContext(ctx) - o.AddFlags(cmd.Flags()) + fs := cmd.Flags() + featureGate.AddFlag(fs, "") + effectiveVersion.AddFlags(fs, "") + + o.AddFlags(fs) return cmd } @@ -91,8 +103,13 @@ func (o *AggregatorOptions) AddFlags(fs *pflag.FlagSet) { // NewDefaultOptions builds a "normal" set of options. You wouldn't normally expose this, but hyperkube isn't cobra compatible func NewDefaultOptions(out, err io.Writer) *AggregatorOptions { + // effectiveVersion is used to set what apis and feature gates the generic api server is compatible with. + // You can also have the flag setting the effectiveVersion of the aggregator apiserver, and + // having a mapping from the aggregator apiserver version to generic apiserver version. + effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( + utilversion.ComponentGenericAPIServer, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) o := &AggregatorOptions{ - ServerRunOptions: genericoptions.NewServerRunOptions(), + ServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion), RecommendedOptions: genericoptions.NewRecommendedOptions( defaultEtcdPathPrefix, aggregatorscheme.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion), @@ -117,7 +134,7 @@ func (o AggregatorOptions) Validate(args []string) error { // Complete fills in missing Options. func (o *AggregatorOptions) Complete() error { - return nil + return o.ServerRunOptions.Complete() } // RunAggregator runs the API Aggregator. diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/doc.go b/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/doc.go index a8df931cd4f..fc5bdc44fef 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/doc.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/doc.go @@ -18,6 +18,7 @@ limitations under the License. // +k8s:deepcopy-gen=package // +k8s:conversion-gen=k8s.io/sample-apiserver/pkg/apis/wardle // +k8s:defaulter-gen=TypeMeta +// +k8s:prerelease-lifecycle-gen=true // +groupName=wardle.example.com // Package v1alpha1 is the v1alpha1 version of the API. diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/types.go b/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/types.go index 39079e4a8d9..a0b3eed416e 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/types.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/types.go @@ -19,6 +19,8 @@ package v1alpha1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +k8s:prerelease-lifecycle-gen:introduced=1.0 +// +k8s:prerelease-lifecycle-gen:removed=1.10 // FlunderList is a list of Flunder objects. type FlunderList struct { @@ -47,6 +49,8 @@ type FlunderStatus struct { // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +k8s:prerelease-lifecycle-gen:introduced=1.0 +// +k8s:prerelease-lifecycle-gen:removed=1.10 type Flunder struct { metav1.TypeMeta `json:",inline"` @@ -59,6 +63,8 @@ type Flunder struct { // +genclient // +genclient:nonNamespaced // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +k8s:prerelease-lifecycle-gen:introduced=1.0 +// +k8s:prerelease-lifecycle-gen:removed=1.10 type Fischer struct { metav1.TypeMeta `json:",inline"` @@ -71,6 +77,8 @@ type Fischer struct { // +genclient:nonNamespaced // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +k8s:prerelease-lifecycle-gen:introduced=1.0 +// +k8s:prerelease-lifecycle-gen:removed=1.10 // FischerList is a list of Fischer objects. type FischerList struct { diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/zz_generated.prerelease-lifecycle.go b/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/zz_generated.prerelease-lifecycle.go new file mode 100644 index 00000000000..04a9f6b8c16 --- /dev/null +++ b/staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1/zz_generated.prerelease-lifecycle.go @@ -0,0 +1,94 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by prerelease-lifecycle-gen. DO NOT EDIT. + +package v1alpha1 + +// APILifecycleIntroduced is an autogenerated function, returning the release in which the API struct was introduced as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:introduced" tags in types.go. +func (in *Fischer) APILifecycleIntroduced() (major, minor int) { + return 1, 0 +} + +// APILifecycleDeprecated is an autogenerated function, returning the release in which the API struct was or will be deprecated as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:deprecated" tags in types.go or "k8s:prerelease-lifecycle-gen:introduced" plus three minor. +func (in *Fischer) APILifecycleDeprecated() (major, minor int) { + return 1, 3 +} + +// APILifecycleRemoved is an autogenerated function, returning the release in which the API is no longer served as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:removed" tags in types.go or "k8s:prerelease-lifecycle-gen:deprecated" plus three minor. +func (in *Fischer) APILifecycleRemoved() (major, minor int) { + return 1, 10 +} + +// APILifecycleIntroduced is an autogenerated function, returning the release in which the API struct was introduced as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:introduced" tags in types.go. +func (in *FischerList) APILifecycleIntroduced() (major, minor int) { + return 1, 0 +} + +// APILifecycleDeprecated is an autogenerated function, returning the release in which the API struct was or will be deprecated as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:deprecated" tags in types.go or "k8s:prerelease-lifecycle-gen:introduced" plus three minor. +func (in *FischerList) APILifecycleDeprecated() (major, minor int) { + return 1, 3 +} + +// APILifecycleRemoved is an autogenerated function, returning the release in which the API is no longer served as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:removed" tags in types.go or "k8s:prerelease-lifecycle-gen:deprecated" plus three minor. +func (in *FischerList) APILifecycleRemoved() (major, minor int) { + return 1, 10 +} + +// APILifecycleIntroduced is an autogenerated function, returning the release in which the API struct was introduced as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:introduced" tags in types.go. +func (in *Flunder) APILifecycleIntroduced() (major, minor int) { + return 1, 0 +} + +// APILifecycleDeprecated is an autogenerated function, returning the release in which the API struct was or will be deprecated as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:deprecated" tags in types.go or "k8s:prerelease-lifecycle-gen:introduced" plus three minor. +func (in *Flunder) APILifecycleDeprecated() (major, minor int) { + return 1, 3 +} + +// APILifecycleRemoved is an autogenerated function, returning the release in which the API is no longer served as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:removed" tags in types.go or "k8s:prerelease-lifecycle-gen:deprecated" plus three minor. +func (in *Flunder) APILifecycleRemoved() (major, minor int) { + return 1, 10 +} + +// APILifecycleIntroduced is an autogenerated function, returning the release in which the API struct was introduced as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:introduced" tags in types.go. +func (in *FlunderList) APILifecycleIntroduced() (major, minor int) { + return 1, 0 +} + +// APILifecycleDeprecated is an autogenerated function, returning the release in which the API struct was or will be deprecated as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:deprecated" tags in types.go or "k8s:prerelease-lifecycle-gen:introduced" plus three minor. +func (in *FlunderList) APILifecycleDeprecated() (major, minor int) { + return 1, 3 +} + +// APILifecycleRemoved is an autogenerated function, returning the release in which the API is no longer served as int versions of major and minor for comparison. +// It is controlled by "k8s:prerelease-lifecycle-gen:removed" tags in types.go or "k8s:prerelease-lifecycle-gen:deprecated" plus three minor. +func (in *FlunderList) APILifecycleRemoved() (major, minor int) { + return 1, 10 +} diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go index 384c46d4ae3..a968d36177b 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go @@ -17,6 +17,8 @@ limitations under the License. package apiserver import ( + "strconv" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -25,6 +27,7 @@ import ( "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/sample-apiserver/pkg/apis/wardle" "k8s.io/sample-apiserver/pkg/apis/wardle/install" wardleregistry "k8s.io/sample-apiserver/pkg/registry" @@ -37,7 +40,8 @@ var ( Scheme = runtime.NewScheme() // Codecs provides methods for retrieving codecs and serializers for specific // versions and content types. - Codecs = serializer.NewCodecFactory(Scheme) + Codecs = serializer.NewCodecFactory(Scheme) + WardleComponentName = "wardle-server" ) func init() { @@ -90,10 +94,10 @@ func (cfg *Config) Complete() CompletedConfig { cfg.GenericConfig.Complete(), &cfg.ExtraConfig, } - + wardleEffectiveVersion := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(WardleComponentName) c.GenericConfig.Version = &version.Info{ - Major: "1", - Minor: "0", + Major: strconv.Itoa(int(wardleEffectiveVersion.BinaryVersion().Major())), + Minor: strconv.Itoa(int(wardleEffectiveVersion.BinaryVersion().Minor())), } return CompletedConfig{&c} diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go index d6310425a50..275c1a8718f 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go @@ -18,6 +18,7 @@ package server import ( "context" + "errors" "fmt" "io" "net" @@ -27,11 +28,14 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" utilerrors "k8s.io/apimachinery/pkg/util/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/version" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/endpoints/openapi" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/sample-apiserver/pkg/admission/plugin/banflunder" "k8s.io/sample-apiserver/pkg/admission/wardleinitializer" "k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1" @@ -55,6 +59,24 @@ type WardleServerOptions struct { AlternateDNS []string } +func mapWardleEffectiveVersionToKubeEffectiveVersion(registry utilversion.ComponentGlobalsRegistry) error { + wardleVer := registry.EffectiveVersionFor(apiserver.WardleComponentName) + kubeVer := registry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer).(utilversion.MutableEffectiveVersion) + // map from wardle emulation version to kube emulation version. + emulationVersionMap := map[string]string{ + "1.2": kubeVer.BinaryVersion().AddMinor(1).String(), + "1.1": kubeVer.BinaryVersion().String(), + "1.0": kubeVer.BinaryVersion().SubtractMinor(1).String(), + } + wardleEmulationVer := wardleVer.EmulationVersion() + if kubeEmulationVer, ok := emulationVersionMap[wardleEmulationVer.String()]; ok { + kubeVer.SetEmulationVersion(version.MustParse(kubeEmulationVer)) + } else { + return fmt.Errorf("cannot find mapping from wardle emulation version: %s to kube version", wardleVer.EmulationVersion().String()) + } + return nil +} + // NewWardleServerOptions returns a new WardleServerOptions func NewWardleServerOptions(out, errOut io.Writer) *WardleServerOptions { o := &WardleServerOptions{ @@ -94,7 +116,14 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti flags := cmd.Flags() o.RecommendedOptions.AddFlags(flags) - utilfeature.DefaultMutableFeatureGate.AddFlag(flags) + + wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2") + utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, nil, false)) + _, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( + utilversion.ComponentGenericAPIServer, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) + + wardleEffectiveVersion.AddFlags(flags, "wardle-") + featureGate.AddFlag(flags, "") return cmd } @@ -103,6 +132,7 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti func (o WardleServerOptions) Validate(args []string) error { errors := []error{} errors = append(errors, o.RecommendedOptions.Validate()...) + errors = append(errors, utilversion.DefaultComponentGlobalsRegistry.ValidateAllComponents()...) return utilerrors.NewAggregate(errors) } @@ -114,6 +144,17 @@ func (o *WardleServerOptions) Complete() error { // add admission plugins to the RecommendedPluginOrder o.RecommendedOptions.Admission.RecommendedPluginOrder = append(o.RecommendedOptions.Admission.RecommendedPluginOrder, "BanFlunder") + // convert wardle effective version to kube effective version to be used in generic api server, and set the generic api server feature gate. + if err := mapWardleEffectiveVersionToKubeEffectiveVersion(utilversion.DefaultComponentGlobalsRegistry); err != nil { + return err + } + if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { + return err + } + if errs := utilversion.DefaultComponentGlobalsRegistry.ValidateAllComponents(); len(errs) > 0 { + return errors.Join(errs...) + } + return nil } @@ -144,6 +185,9 @@ func (o *WardleServerOptions) Config() (*apiserver.Config, error) { serverConfig.OpenAPIV3Config.Info.Title = "Wardle" serverConfig.OpenAPIV3Config.Info.Version = "0.1" + serverConfig.FeatureGate = utilversion.DefaultComponentGlobalsRegistry.FeatureGateFor(utilversion.ComponentGenericAPIServer) + serverConfig.EffectiveVersion = utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer) + if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil { return nil, err } diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go new file mode 100644 index 00000000000..6ee274c65b6 --- /dev/null +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "testing" + + "k8s.io/apimachinery/pkg/util/version" + utilversion "k8s.io/apiserver/pkg/util/version" + "k8s.io/sample-apiserver/pkg/apiserver" + + "github.com/stretchr/testify/assert" +) + +func TestMapBinaryEffectiveVersionToKubeEffectiveVersion(t *testing.T) { + wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2") + defaultKubeEffectiveVersion := utilversion.DefaultKubeEffectiveVersion() + + testCases := []struct { + desc string + wardleEmulationVer *version.Version + expectedKubeEmulationVer *version.Version + }{ + { + desc: "1 version higher than kube binary", + wardleEmulationVer: version.MajorMinor(1, 2), + expectedKubeEmulationVer: defaultKubeEffectiveVersion.BinaryVersion().AddMinor(1), + }, + { + desc: "no mapping", + wardleEmulationVer: version.MajorMinor(1, 10), + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + registry := utilversion.NewComponentGlobalsRegistry() + _ = registry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, nil, true) + _ = registry.Register(utilversion.ComponentGenericAPIServer, defaultKubeEffectiveVersion, nil, true) + + wardleEffectiveVersion.SetEmulationVersion(tc.wardleEmulationVer) + err := mapWardleEffectiveVersionToKubeEffectiveVersion(registry) + if tc.expectedKubeEmulationVer == nil { + if err == nil { + t.Fatal("expected error, no error found") + } + } else { + assert.True(t, registry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer).EmulationVersion().EqualTo(tc.expectedKubeEmulationVer)) + } + }) + } +} diff --git a/test/e2e/dra/test-driver/app/server.go b/test/e2e/dra/test-driver/app/server.go index 3f55c541463..59e4ac42a99 100644 --- a/test/e2e/dra/test-driver/app/server.go +++ b/test/e2e/dra/test-driver/app/server.go @@ -88,7 +88,7 @@ func NewCommand() *cobra.Command { fs = sharedFlagSets.FlagSet("other") featureGate := featuregate.NewFeatureGate() utilruntime.Must(logsapi.AddFeatureGates(featureGate)) - featureGate.AddFlag(fs) + featureGate.AddFlag(fs, "") fs = cmd.PersistentFlags() for _, f := range sharedFlagSets.FlagSets { diff --git a/test/e2e_node/services/apiserver.go b/test/e2e_node/services/apiserver.go index 4488715b3bd..c092d9d69b6 100644 --- a/test/e2e_node/services/apiserver.go +++ b/test/e2e_node/services/apiserver.go @@ -29,6 +29,8 @@ import ( netutils "k8s.io/utils/net" utilerrors "k8s.io/apimachinery/pkg/util/errors" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/test/e2e/framework" @@ -63,7 +65,9 @@ func NewAPIServer(storageConfig storagebackend.Config) *APIServer { func (a *APIServer) Start(ctx context.Context) error { const tokenFilePath = "known_tokens.csv" - o := options.NewServerRunOptions() + featureGate := utilfeature.DefaultFeatureGate + effectiveVersion := utilversion.DefaultKubeEffectiveVersion() + o := options.NewServerRunOptions(featureGate, effectiveVersion) o.Etcd.StorageConfig = a.storageConfig _, ipnet, err := netutils.ParseCIDRSloppy(clusterIPRange) if err != nil { diff --git a/test/integration/apiserver/apiserver_test.go b/test/integration/apiserver/apiserver_test.go index 63bb662561a..81edfbbc0aa 100644 --- a/test/integration/apiserver/apiserver_test.go +++ b/test/integration/apiserver/apiserver_test.go @@ -32,6 +32,7 @@ import ( "testing" "time" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -55,16 +56,19 @@ import ( "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/endpoints/handlers" "k8s.io/apiserver/pkg/storage/storagebackend" + "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" "k8s.io/client-go/metadata" restclient "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" "k8s.io/client-go/tools/pager" "k8s.io/klog/v2" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/pkg/controlplane" + "k8s.io/kubernetes/pkg/kubeapiserver" "k8s.io/kubernetes/test/integration" "k8s.io/kubernetes/test/integration/etcd" "k8s.io/kubernetes/test/integration/framework" @@ -2929,6 +2933,298 @@ func TestDedupOwnerReferences(t *testing.T) { } } +func TestEmulatedStorageVersion(t *testing.T) { + validVap := &admissionregistrationv1.ValidatingAdmissionPolicy{ + Spec: admissionregistrationv1.ValidatingAdmissionPolicySpec{ + MatchConstraints: &admissionregistrationv1.MatchResources{ + ResourceRules: []admissionregistrationv1.NamedRuleWithOperations{ + { + ResourceNames: []string{"foo"}, + RuleWithOperations: admissionregistrationv1.RuleWithOperations{ + Operations: []admissionregistrationv1.OperationType{"CREATE"}, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{"*"}, + APIVersions: []string{"*"}, + Resources: []string{"*"}, + }, + }, + }, + }, + }, + Validations: []admissionregistrationv1.Validation{ + { + Expression: "true", + Message: "always valid", + }, + }, + }, + } + + type testCase struct { + name string + emulatedVersion string + gvr schema.GroupVersionResource + object runtime.Object + expectedStorageVersion schema.GroupVersion + } + cases := []testCase{ + { + name: "vap first ga release", + emulatedVersion: "1.30", + gvr: schema.GroupVersionResource{ + Group: "admissionregistration.k8s.io", + Version: "v1", + Resource: "validatingadmissionpolicies", + }, + object: validVap, + expectedStorageVersion: schema.GroupVersion{ + Group: "admissionregistration.k8s.io", + Version: "v1beta1", + }, + }, + { + name: "vap after ga release", + emulatedVersion: "1.31", + gvr: schema.GroupVersionResource{ + Group: "admissionregistration.k8s.io", + Version: "v1beta1", + Resource: "validatingadmissionpolicies", + }, + object: validVap, + expectedStorageVersion: schema.GroupVersion{ + Group: "admissionregistration.k8s.io", + Version: "v1", + }, + }, + } + + // Group cases by their emulated version + groupedCases := map[string][]testCase{} + for _, c := range cases { + groupedCases[c.emulatedVersion] = append(groupedCases[c.emulatedVersion], c) + } + + for emulatedVersion, cases := range groupedCases { + t.Run(emulatedVersion, func(t *testing.T) { + server := kubeapiservertesting.StartTestServerOrDie( + t, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: emulatedVersion}, + []string{"--emulated-version=" + emulatedVersion, `--storage-media-type=application/json`}, framework.SharedEtcd()) + defer server.TearDownFn() + + client := clientset.NewForConfigOrDie(server.ClientConfig) + dynamicClient := dynamic.NewForConfigOrDie(server.ClientConfig) + + // create test namespace + testNamespace := "test-emulated-storage-version" + _, err := client.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNamespace, + }, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create test ns: %v", err) + } + + restMapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(client.Discovery())) + + for i, c := range cases { + t.Run(c.name, func(t *testing.T) { + gvk, err := restMapper.KindFor(c.gvr) + if err != nil { + t.Fatalf("failed to get GVK: %v", err) + } + c.object.GetObjectKind().SetGroupVersionKind(gvk) + + mapping, err := restMapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + t.Fatalf("failed to get RESTMapping: %v", err) + } else if mapping.Resource != c.gvr { + t.Fatalf("expected resource %v, got %v", c.gvr, mapping.Resource) + } + + asUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(c.object) + if err != nil { + t.Fatalf("failed to convert object to unstructured: %v", err) + } + + uns := &unstructured.Unstructured{ + Object: asUnstructured, + } + uns.SetName(fmt.Sprintf("test-object%d", i)) + + ns := testNamespace + if mapping.Scope.Name() == meta.RESTScopeNameRoot { + ns = "" + } + + // create object + created, err := dynamicClient.Resource(c.gvr).Namespace(ns).Create(context.TODO(), uns, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create object: %v", err) + } + + // Fetch object from ETCD + // Use this poor man's way to get the etcd path. This wont + // work for all resources, but should work for most we want + // to test against + resourcePrefix := mapping.Resource.Resource + if special, ok := kubeapiserver.SpecialDefaultResourcePrefixes[c.gvr.GroupResource()]; ok { + resourcePrefix = special + } + etcdPathComponents := []string{ + "/", + server.EtcdStoragePrefix, + resourcePrefix, + } + + if len(ns) > 0 { + etcdPathComponents = append(etcdPathComponents, ns) + } + etcdPathComponents = append(etcdPathComponents, created.GetName()) + etcdPath := path.Join(etcdPathComponents...) + fetched, err := server.EtcdClient.Get(context.TODO(), etcdPath) + if err != nil { + t.Fatalf("failed to fetch object from etcd: %v", err) + } else if fetched.More || fetched.Count != 1 || len(fetched.Kvs) != 1 { + t.Fatalf("unexpected fetched response: %v", fetched) + } + + storedObject := &metav1.PartialObjectMetadata{} + err = json.Unmarshal(fetched.Kvs[0].Value, storedObject) + + if err != nil { + t.Fatalf("failed to decode object: %v", err) + } else if storedObject.GroupVersionKind().GroupVersion() != c.expectedStorageVersion { + t.Fatalf("expected storage version %s, got %s", c.expectedStorageVersion.String(), storedObject.GroupVersionKind().GroupVersion().String()) + } + }) + } + }) + } +} + +func TestEnableEmulationVersion(t *testing.T) { + server := kubeapiservertesting.StartTestServerOrDie(t, + &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"}, + []string{"--emulated-version=1.31"}, framework.SharedEtcd()) + defer server.TearDownFn() + + rt, err := restclient.TransportFor(server.ClientConfig) + if err != nil { + t.Fatal(err) + } + + tcs := []struct { + path string + expectedStatusCode int + }{ + { + path: "/", + expectedStatusCode: 200, + }, + { + path: "/apis/apps/v1/deployments", + expectedStatusCode: 200, + }, + { + path: "/apis/flowcontrol.apiserver.k8s.io/v1/flowschemas", + expectedStatusCode: 200, + }, + { + path: "/apis/flowcontrol.apiserver.k8s.io/v1beta1/flowschemas", // introduced at 1.20, removed at 1.26 + expectedStatusCode: 404, + }, + { + path: "/apis/flowcontrol.apiserver.k8s.io/v1beta2/flowschemas", // introduced at 1.23, removed at 1.29 + expectedStatusCode: 404, + }, + { + path: "/apis/flowcontrol.apiserver.k8s.io/v1beta3/flowschemas", // introduced at 1.26, removed at 1.32 + expectedStatusCode: 200, + }, + } + + for _, tc := range tcs { + t.Run(tc.path, func(t *testing.T) { + req, err := http.NewRequest("GET", server.ClientConfig.Host+tc.path, nil) + if err != nil { + t.Fatal(err) + } + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != tc.expectedStatusCode { + t.Errorf("expect status code: %d, got : %d\n", tc.expectedStatusCode, resp.StatusCode) + } + defer func() { + _ = resp.Body.Close() + }() + }) + } +} + +func TestDisableEmulationVersion(t *testing.T) { + server := kubeapiservertesting.StartTestServerOrDie(t, + &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"}, + []string{}, framework.SharedEtcd()) + defer server.TearDownFn() + + rt, err := restclient.TransportFor(server.ClientConfig) + if err != nil { + t.Fatal(err) + } + + tcs := []struct { + path string + expectedStatusCode int + }{ + { + path: "/", + expectedStatusCode: 200, + }, + { + path: "/apis/apps/v1/deployments", + expectedStatusCode: 200, + }, + { + path: "/apis/flowcontrol.apiserver.k8s.io/v1/flowschemas", + expectedStatusCode: 200, + }, + { + path: "/apis/flowcontrol.apiserver.k8s.io/v1beta1/flowschemas", // introduced at 1.20, removed at 1.26 + expectedStatusCode: 404, + }, + { + path: "/apis/flowcontrol.apiserver.k8s.io/v1beta2/flowschemas", // introduced at 1.23, removed at 1.29 + expectedStatusCode: 404, + }, + { + path: "/apis/flowcontrol.apiserver.k8s.io/v1beta3/flowschemas", // introduced at 1.26, removed at 1.32 + expectedStatusCode: 404, + }, + } + + for _, tc := range tcs { + t.Run(tc.path, func(t *testing.T) { + req, err := http.NewRequest("GET", server.ClientConfig.Host+tc.path, nil) + if err != nil { + t.Fatal(err) + } + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != tc.expectedStatusCode { + t.Errorf("expect status code: %d, got : %d\n", tc.expectedStatusCode, resp.StatusCode) + } + defer func() { + _ = resp.Body.Close() + }() + }) + } +} + type dependentClient struct { t *testing.T client dynamic.ResourceInterface diff --git a/test/integration/auth/rbac_test.go b/test/integration/auth/rbac_test.go index f162eb6df2d..f24067e5fc9 100644 --- a/test/integration/auth/rbac_test.go +++ b/test/integration/auth/rbac_test.go @@ -29,6 +29,7 @@ import ( rbacapi "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/watch" @@ -80,8 +81,8 @@ type testRESTOptionsGetter struct { config *controlplane.Config } -func (getter *testRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) { - storageConfig, err := getter.config.ControlPlane.Extra.StorageFactory.NewConfig(resource) +func (getter *testRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) { + storageConfig, err := getter.config.ControlPlane.Extra.StorageFactory.NewConfig(resource, example) if err != nil { return generic.RESTOptions{}, fmt.Errorf("failed to get storage: %v", err) } diff --git a/test/integration/controlplane/transformation/kmsv2_transformation_test.go b/test/integration/controlplane/transformation/kmsv2_transformation_test.go index 1b001681b7b..3d717bf25bc 100644 --- a/test/integration/controlplane/transformation/kmsv2_transformation_test.go +++ b/test/integration/controlplane/transformation/kmsv2_transformation_test.go @@ -1219,7 +1219,7 @@ func getRESTOptionsGetterForSecrets(t testing.TB, test *transformTest) generic.R t.Fatal("not REST options found") } - opts, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: "secrets"}) + opts, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: "secrets"}, nil) if err != nil { t.Fatal(err) } diff --git a/test/integration/controlplane/transformation/transformation_test.go b/test/integration/controlplane/transformation/transformation_test.go index fa9b74157ef..f1c16a9b610 100644 --- a/test/integration/controlplane/transformation/transformation_test.go +++ b/test/integration/controlplane/transformation/transformation_test.go @@ -115,7 +115,9 @@ func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool, return nil, fmt.Errorf("failed to read config file: %w", err) } - if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(tb, nil, e.getEncryptionOptions(reload), e.storageConfig); err != nil { + if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer( + tb, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "0.0"}, + e.getEncryptionOptions(reload), e.storageConfig); err != nil { e.cleanUp() return nil, fmt.Errorf("failed to start KubeAPI server: %w", err) } diff --git a/test/integration/etcd/crd_overlap_storage_test.go b/test/integration/etcd/crd_overlap_storage_test.go index 7aa898a0bf2..ebe1e042dc3 100644 --- a/test/integration/etcd/crd_overlap_storage_test.go +++ b/test/integration/etcd/crd_overlap_storage_test.go @@ -32,9 +32,11 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/dynamic" apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1" + "k8s.io/kubernetes/cmd/kube-apiserver/app/options" ) // TestOverlappingBuiltInResources ensures the list of group-resources the custom resource finalizer should skip is up to date @@ -69,7 +71,9 @@ func TestOverlappingBuiltInResources(t *testing.T) { // TestOverlappingCustomResourceAPIService ensures creating and deleting a custom resource overlapping with APIServices does not destroy APIService data func TestOverlappingCustomResourceAPIService(t *testing.T) { - apiServer := StartRealAPIServerOrDie(t) + apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { + opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.30") + }) defer apiServer.Cleanup() apiServiceClient, err := apiregistrationclient.NewForConfig(apiServer.Config) @@ -231,7 +235,9 @@ func TestOverlappingCustomResourceAPIService(t *testing.T) { // TestOverlappingCustomResourceCustomResourceDefinition ensures creating and deleting a custom resource overlapping with CustomResourceDefinition does not destroy CustomResourceDefinition data func TestOverlappingCustomResourceCustomResourceDefinition(t *testing.T) { - apiServer := StartRealAPIServerOrDie(t) + apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { + opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.30") + }) defer apiServer.Cleanup() crdClient, err := crdclient.NewForConfig(apiServer.Config) diff --git a/test/integration/etcd/etcd_cross_group_test.go b/test/integration/etcd/etcd_cross_group_test.go index d247edeaa5e..f24f0ee39b4 100644 --- a/test/integration/etcd/etcd_cross_group_test.go +++ b/test/integration/etcd/etcd_cross_group_test.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/storage" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/dynamic" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" ) @@ -38,6 +39,7 @@ import ( func TestCrossGroupStorage(t *testing.T) { apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { // force enable all resources so we can check storage. + opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.30") }) defer apiServer.Cleanup() diff --git a/test/integration/etcd/etcd_storage_path_test.go b/test/integration/etcd/etcd_storage_path_test.go index 224be051e8b..eda94aee93f 100644 --- a/test/integration/etcd/etcd_storage_path_test.go +++ b/test/integration/etcd/etcd_storage_path_test.go @@ -36,6 +36,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/dynamic" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" @@ -75,6 +76,7 @@ func TestEtcdStoragePath(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true) featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true) apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { + opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("0.0") }) defer apiServer.Cleanup() defer dumpEtcdKVOnFailure(t, apiServer.KV) diff --git a/test/integration/etcd/server.go b/test/integration/etcd/server.go index ed69e3d532f..fb57211ea43 100644 --- a/test/integration/etcd/server.go +++ b/test/integration/etcd/server.go @@ -40,6 +40,8 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" genericapiserveroptions "k8s.io/apiserver/pkg/server/options" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" cacheddiscovery "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" @@ -91,7 +93,9 @@ func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRu t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err) } - opts := options.NewServerRunOptions() + featureGate := utilfeature.DefaultFeatureGate + effectiveVersion := utilversion.DefaultBuildEffectiveVersion() + opts := options.NewServerRunOptions(featureGate, effectiveVersion) opts.Options.SecureServing.Listener = listener opts.Options.SecureServing.ServerCert.CertDirectory = certDir opts.Options.ServiceAccountSigningKeyFile = saSigningKeyFile.Name() diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index a0c3852618b..2ed1f1c001c 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -292,6 +292,7 @@ func TestAggregatedAPIServer(t *testing.T) { "--etcd-servers", framework.GetEtcdURL(), "--cert-dir", wardleCertDir, "--kubeconfig", wardleToKASKubeConfigFile, + "--wardle-emulated-version", "1.1", }) if err := wardleCmd.Execute(); err != nil { t.Error(err) diff --git a/test/integration/framework/test_server.go b/test/integration/framework/test_server.go index bb43850466d..5d514b360d4 100644 --- a/test/integration/framework/test_server.go +++ b/test/integration/framework/test_server.go @@ -35,6 +35,8 @@ import ( "k8s.io/apimachinery/pkg/util/wait" genericapiserver "k8s.io/apiserver/pkg/server" genericapiserveroptions "k8s.io/apiserver/pkg/server/options" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" client "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/util/cert" @@ -135,7 +137,9 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) ( t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err) } - opts := options.NewServerRunOptions() + featureGate := utilfeature.DefaultFeatureGate + effectiveVersion := utilversion.DefaultKubeEffectiveVersion() + opts := options.NewServerRunOptions(featureGate, effectiveVersion) opts.SecureServing.Listener = listener opts.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1") opts.SecureServing.ServerCert.CertDirectory = certDir diff --git a/test/integration/servicecidr/allocator_test.go b/test/integration/servicecidr/allocator_test.go index dd4b45cc521..ad6d6280dc2 100644 --- a/test/integration/servicecidr/allocator_test.go +++ b/test/integration/servicecidr/allocator_test.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/kubernetes" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" @@ -122,6 +123,7 @@ func TestServiceAllocIPAddress(t *testing.T) { ModifyServerRunOptions: func(opts *options.ServerRunOptions) { opts.ServiceClusterIPRanges = serviceCIDR opts.GenericServerRunOptions.AdvertiseAddress = netutils.ParseIPSloppy("2001:db8::10") + opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.31") opts.APIEnablement.RuntimeConfig.Set("networking.k8s.io/v1alpha1=true") }, }) From 701e5fc3744c2b21ba5b1ca3399b71c9becf06d0 Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Thu, 30 May 2024 12:08:52 -0700 Subject: [PATCH 2/4] Add composition flags for emulation version and feature gate. Signed-off-by: Siyuan Zhang --- .../app/options/options_test.go | 9 +- cmd/kube-apiserver/app/server.go | 13 +- cmd/kube-apiserver/app/testing/testserver.go | 7 +- .../app/options/options.go | 2 +- cmd/kube-scheduler/app/options/options.go | 2 +- .../apiserver/options/options_test.go | 9 +- .../pkg/cmd/server/server.go | 12 +- .../pkg/cmd/server/testing/testserver.go | 7 +- .../apiserver/pkg/cel/environment/base.go | 2 +- .../apiserver/pkg/util/version/registry.go | 153 +++++++++++- .../pkg/util/version/registry_test.go | 225 +++++++++++++++++- .../pkg/util/version/version_test.go | 20 +- .../k8s.io/cloud-provider/options/options.go | 2 +- .../colon_separated_multimap_string_string.go | 24 +- ...n_separated_multimap_string_string_test.go | 15 ++ .../featuregate/feature_gate.go | 53 ++--- .../featuregate/feature_gate_test.go | 14 +- .../component-base/logs/example/cmd/logger.go | 2 +- .../logs/example/k8s2slog/k8s2slog.go | 2 +- .../kube-aggregator/pkg/cmd/server/start.go | 14 +- .../pkg/apiserver/apiserver.go | 2 +- .../sample-apiserver/pkg/cmd/server/start.go | 54 +++-- .../pkg/cmd/server/start_test.go | 4 +- test/e2e/dra/test-driver/app/server.go | 2 +- test/integration/apiserver/apiserver_test.go | 4 +- test/integration/examples/apiserver_test.go | 15 +- 26 files changed, 522 insertions(+), 146 deletions(-) diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index 56cbcbebd03..1e4d5a679f8 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -27,6 +27,7 @@ import ( "github.com/spf13/pflag" noopoteltrace "go.opentelemetry.io/otel/trace/noop" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/admission" apiserveroptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/storage/etcd3" @@ -50,13 +51,14 @@ func TestAddFlags(t *testing.T) { fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError) featureGate := featuregate.NewFeatureGate() + componentRegistry := utilversion.NewComponentGlobalsRegistry() effectiveVersion := utilversion.NewEffectiveVersion("1.32") + _ = componentRegistry.Register("test", effectiveVersion, featureGate, true) s := NewServerRunOptions(featureGate, effectiveVersion) for _, f := range s.Flags().FlagSets { fs.AddFlagSet(f) } - featureGate.AddFlag(fs, "") - effectiveVersion.AddFlags(fs, "") + componentRegistry.AddFlags(fs) args := []string{ "--enable-admission-plugins=AlwaysDeny", @@ -128,9 +130,10 @@ func TestAddFlags(t *testing.T) { "--storage-backend=etcd3", "--service-cluster-ip-range=192.168.128.0/17", "--lease-reuse-duration-seconds=100", - "--emulated-version=1.31", + "--emulated-version=test=1.31", } fs.Parse(args) + utilruntime.Must(componentRegistry.Set()) // This is a snapshot of expected options parsed by args. expected := &ServerRunOptions{ diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 3489f2130ae..9d32d738f84 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -65,7 +65,7 @@ func init() { // NewAPIServerCommand creates a *cobra.Command object with default parameters func NewAPIServerCommand() *cobra.Command { effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( - utilversion.ComponentGenericAPIServer, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) + utilversion.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) s := options.NewServerRunOptions(featureGate, effectiveVersion) cmd := &cobra.Command{ @@ -78,6 +78,9 @@ cluster's shared state through which all other components interact.`, // stop printing usage when the command errors SilenceUsage: true, PersistentPreRunE: func(*cobra.Command, []string) error { + if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil { + return err + } // silence client-go warnings. // kube-apiserver loopback clients should not log self-issued warnings. rest.SetDefaultWarningHandler(rest.NoWarnings{}) @@ -86,11 +89,6 @@ cluster's shared state through which all other components interact.`, RunE: func(cmd *cobra.Command, args []string) error { verflag.PrintAndExitIfRequested() fs := cmd.Flags() - - if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { - return err - } - // Activate logging as soon as possible, after that // show flags with the final logging configuration. if err := logsapi.ValidateAndApply(s.Logs, featureGate); err != nil { @@ -126,8 +124,7 @@ cluster's shared state through which all other components interact.`, fs := cmd.Flags() namedFlagSets := s.Flags() verflag.AddFlags(namedFlagSets.FlagSet("global")) - featureGate.AddFlag(namedFlagSets.FlagSet("global"), "") - effectiveVersion.AddFlags(namedFlagSets.FlagSet("global"), "") + utilversion.DefaultComponentGlobalsRegistry.AddFlags(namedFlagSets.FlagSet("global")) globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags()) options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic")) diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go index 7eb7dce0496..f4268b23a6d 100644 --- a/cmd/kube-apiserver/app/testing/testserver.go +++ b/cmd/kube-apiserver/app/testing/testserver.go @@ -187,15 +187,14 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, binaryVersion = instanceOptions.BinaryVersion } effectiveVersion := utilversion.NewEffectiveVersion(binaryVersion) - _ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.ComponentGenericAPIServer, effectiveVersion, featureGate, true) + _ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate, true) s := options.NewServerRunOptions(featureGate, effectiveVersion) for _, f := range s.Flags().FlagSets { fs.AddFlagSet(f) } - featureGate.AddFlag(fs, "") - effectiveVersion.AddFlags(fs, "") + utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs) s.SecureServing.Listener, s.SecureServing.BindPort, err = createLocalhostListenerOnFreePort() if err != nil { @@ -336,7 +335,7 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, return result, err } - if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { + if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil { return result, err } diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index d72920d0cd1..1c3e4edb178 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -273,7 +273,7 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy fs := fss.FlagSet("misc") fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") fs.StringVar(&s.Generic.ClientConnection.Kubeconfig, "kubeconfig", s.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).") - utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"), "") + utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic")) return fss } diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go index 2b1233c435b..e039b0d05c9 100644 --- a/cmd/kube-scheduler/app/options/options.go +++ b/cmd/kube-scheduler/app/options/options.go @@ -189,7 +189,7 @@ func (o *Options) initFlags() { o.Authorization.AddFlags(nfs.FlagSet("authorization")) o.Deprecated.AddFlags(nfs.FlagSet("deprecated")) options.BindLeaderElectionFlags(o.LeaderElection, nfs.FlagSet("leader election")) - utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate"), "") + utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate")) o.Metrics.AddFlags(nfs.FlagSet("metrics")) logsapi.AddFlags(o.Logs, nfs.FlagSet("logs")) diff --git a/pkg/controlplane/apiserver/options/options_test.go b/pkg/controlplane/apiserver/options/options_test.go index 8660231d2d0..a3f65ebde7f 100644 --- a/pkg/controlplane/apiserver/options/options_test.go +++ b/pkg/controlplane/apiserver/options/options_test.go @@ -26,6 +26,7 @@ import ( "github.com/spf13/pflag" noopoteltrace "go.opentelemetry.io/otel/trace/noop" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/admission" apiserveroptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/storage/etcd3" @@ -46,14 +47,15 @@ func TestAddFlags(t *testing.T) { fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError) featureGate := featuregate.NewFeatureGate() effectiveVersion := utilversion.NewEffectiveVersion("1.32") + componentRegistry := utilversion.NewComponentGlobalsRegistry() + _ = componentRegistry.Register("test", effectiveVersion, featureGate, true) s := NewOptions(featureGate, effectiveVersion) var fss cliflag.NamedFlagSets s.AddFlags(&fss) for _, f := range fss.FlagSets { fs.AddFlagSet(f) } - featureGate.AddFlag(fs, "") - effectiveVersion.AddFlags(fs, "") + componentRegistry.AddFlags(fs) args := []string{ "--enable-admission-plugins=AlwaysDeny", @@ -114,9 +116,10 @@ func TestAddFlags(t *testing.T) { "--request-timeout=2m", "--storage-backend=etcd3", "--lease-reuse-duration-seconds=100", - "--emulated-version=1.31", + "--emulated-version=test=1.31", } fs.Parse(args) + utilruntime.Must(componentRegistry.Set()) // This is a snapshot of expected options parsed by args. expected := &Options{ diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go index b59b12f6da9..6aa64f0f071 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go @@ -33,17 +33,16 @@ func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command // You can also have the flag setting the effectiveVersion of the apiextensions apiserver, and // having a mapping from the apiextensions apiserver version to generic apiserver version. effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( - utilversion.ComponentGenericAPIServer, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) + utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) o := options.NewCustomResourceDefinitionsServerOptions(out, errOut, featureGate, effectiveVersion) cmd := &cobra.Command{ Short: "Launch an API extensions API server", Long: "Launch an API extensions API server", + PersistentPreRunE: func(*cobra.Command, []string) error { + return utilversion.DefaultComponentGlobalsRegistry.Set() + }, RunE: func(c *cobra.Command, args []string) error { - if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { - return err - } - if err := o.Complete(); err != nil { return err } @@ -59,8 +58,7 @@ func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command cmd.SetContext(ctx) fs := cmd.Flags() - featureGate.AddFlag(fs, "") - effectiveVersion.AddFlags(fs, "") + utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs) o.AddFlags(fs) return cmd } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go index b1e5b9dd8d6..547a0ba7712 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go @@ -124,11 +124,10 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin featureGate := utilfeature.DefaultMutableFeatureGate effectiveVersion := utilversion.DefaultKubeEffectiveVersion() - _ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.ComponentGenericAPIServer, effectiveVersion, featureGate, true) + _ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate, true) s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr, featureGate, effectiveVersion) - featureGate.AddFlag(fs, "") - effectiveVersion.AddFlags(fs, "") + utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs) s.AddFlags(fs) s.RecommendedOptions.SecureServing.Listener, s.RecommendedOptions.SecureServing.BindPort, err = createLocalhostListenerOnFreePort() @@ -151,7 +150,7 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin fs.Parse(customFlags) - if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { + if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil { return result, err } diff --git a/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go b/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go index 837cbacddb0..5e2e995c1d1 100644 --- a/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go +++ b/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go @@ -46,7 +46,7 @@ import ( // A default version number equal to the current Kubernetes major.minor version // indicates fast forward CEL features that can be used when rollback is no longer needed. func DefaultCompatibilityVersion() *version.Version { - effectiveVer := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer) + effectiveVer := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) if effectiveVer == nil { effectiveVer = utilversion.DefaultKubeEffectiveVersion() } diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/registry.go b/staging/src/k8s.io/apiserver/pkg/util/version/registry.go index f8b4ef5cd41..589d42a1ade 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/registry.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/registry.go @@ -18,16 +18,47 @@ package version import ( "fmt" + "sort" + "strings" "sync" + "github.com/spf13/pflag" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/version" + cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/featuregate" + "k8s.io/klog/v2" ) +// DefaultComponentGlobalsRegistry is the global var to store the effective versions and feature gates for all components for easy access. +// Example usage: +// // register the component effective version and feature gate first +// _, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) +// wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2") +// wardleFeatureGate := featuregate.NewFeatureGate() +// utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate, false)) +// +// cmd := &cobra.Command{ +// ... +// // call DefaultComponentGlobalsRegistry.Set() in PersistentPreRunE +// PersistentPreRunE: func(*cobra.Command, []string) error { +// if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil { +// return err +// } +// ... +// }, +// RunE: func(c *cobra.Command, args []string) error { +// // call utilversion.DefaultComponentGlobalsRegistry.Validate() somewhere +// }, +// } +// +// flags := cmd.Flags() +// // add flags +// utilversion.DefaultComponentGlobalsRegistry.AddFlags(flags) var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGlobalsRegistry() const ( - ComponentGenericAPIServer = "k8s.io/apiserver" + DefaultKubeComponent = "kube" ) // ComponentGlobals stores the global variables for a component for easy access. @@ -50,19 +81,27 @@ type ComponentGlobalsRegistry interface { // ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry. // Otherwise, the provided variables would be registered under the component, and the same variables would be returned. ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) - // SetAllComponents sets the emulation version for other global variables for all components registered. - SetAllComponents() error + // AddFlags adds flags of "--emulated-version" and "--feature-gates" + AddFlags(fs *pflag.FlagSet) + // Set sets the flags for all global variables for all components registered. + Set() error // SetAllComponents calls the Validate() function for all the global variables for all components registered. - ValidateAllComponents() []error + Validate() []error } type componentGlobalsRegistry struct { componentGlobals map[string]ComponentGlobals mutex sync.RWMutex + // map of component name to emulation version set from the flag. + emulationVersionConfig cliflag.ConfigurationMap + // map of component name to the list of feature gates set from the flag. + featureGatesConfig map[string][]string } func NewComponentGlobalsRegistry() ComponentGlobalsRegistry { - return &componentGlobalsRegistry{componentGlobals: map[string]ComponentGlobals{}} + return &componentGlobalsRegistry{ + componentGlobals: make(map[string]ComponentGlobals), + } } func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion { @@ -90,7 +129,9 @@ func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVer return fmt.Errorf("component globals of %s already registered", component) } if featureGate != nil { - featureGate.DeferErrorsToValidation(true) + if err := featureGate.SetEmulationVersion(effectiveVersion.EmulationVersion()); err != nil { + return err + } } c := ComponentGlobals{effectiveVersion: effectiveVersion, featureGate: featureGate} r.componentGlobals[component] = c @@ -98,6 +139,9 @@ func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVer } func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error { + if effectiveVersion == nil { + return fmt.Errorf("cannot register nil effectiveVersion") + } r.mutex.Lock() defer r.mutex.Unlock() return r.unsafeRegister(component, effectiveVersion, featureGate, override) @@ -114,21 +158,112 @@ func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, return effectiveVersion, featureGate } -func (r *componentGlobalsRegistry) SetAllComponents() error { +func (r *componentGlobalsRegistry) knownFeatures() []string { r.mutex.Lock() defer r.mutex.Unlock() - for _, globals := range r.componentGlobals { + var known []string + for component, globals := range r.componentGlobals { if globals.featureGate == nil { continue } + for _, f := range globals.featureGate.KnownFeatures() { + known = append(known, component+":"+f) + } + } + sort.Strings(known) + return known +} + +func (r *componentGlobalsRegistry) versionFlagOptions(isEmulation bool) []string { + r.mutex.Lock() + defer r.mutex.Unlock() + var vs []string + for component, globals := range r.componentGlobals { + binaryVer := globals.effectiveVersion.BinaryVersion() + if isEmulation { + // emulated version could be between binaryMajor.{binaryMinor} and binaryMajor.{binaryMinor} + // TODO: change to binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} in 1.32 + vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component, + binaryVer.SubtractMinor(0).String(), binaryVer.String(), globals.effectiveVersion.EmulationVersion().String())) + } else { + // min compatibility version could be between binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} + vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component, + binaryVer.SubtractMinor(1).String(), binaryVer.String(), globals.effectiveVersion.MinCompatibilityVersion().String())) + } + } + sort.Strings(vs) + return vs +} + +func (r *componentGlobalsRegistry) AddFlags(fs *pflag.FlagSet) { + if r == nil { + return + } + r.mutex.Lock() + for _, globals := range r.componentGlobals { + if globals.featureGate != nil { + globals.featureGate.Close() + } + } + r.emulationVersionConfig = make(cliflag.ConfigurationMap) + r.featureGatesConfig = make(map[string][]string) + r.mutex.Unlock() + + fs.Var(&r.emulationVersionConfig, "emulated-version", ""+ + "The versions different components emulate their capabilities (APIs, features, ...) of.\n"+ + "If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+ + "Version format could only be major.minor, for example: '--emulated-version=wardle=1.2,kube=1.31'. Options are:\n"+strings.Join(r.versionFlagOptions(true), "\n")) + + fs.Var(cliflag.NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(&r.featureGatesConfig), "feature-gates", "Comma-separated list of component:key=value pairs that describe feature gates for alpha/experimental features of different components.\n"+ + "If the component is not specified, defaults to \"kube\". This flag can be repeatedly invoked. For example: --feature-gates 'wardle:featureA=true,wardle:featureB=false' --feature-gates 'kube:featureC=true'"+ + "Options are:\n"+strings.Join(r.knownFeatures(), "\n")) +} + +func (r *componentGlobalsRegistry) Set() error { + r.mutex.Lock() + defer r.mutex.Unlock() + for comp, emuVer := range r.emulationVersionConfig { + if _, ok := r.componentGlobals[comp]; !ok { + return fmt.Errorf("component not registered: %s", comp) + } + klog.V(2).Infof("setting %s:emulation version to %s\n", comp, emuVer) + v, err := version.Parse(emuVer) + if err != nil { + return err + } + r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(v) + } + // Set feature gate emulation version before setting feature gate flag values. + for comp, globals := range r.componentGlobals { + if globals.featureGate == nil { + continue + } + klog.V(2).Infof("setting %s:feature gate emulation version to %s\n", comp, globals.effectiveVersion.EmulationVersion().String()) if err := globals.featureGate.SetEmulationVersion(globals.effectiveVersion.EmulationVersion()); err != nil { return err } } + for comp, fg := range r.featureGatesConfig { + if comp == "" { + comp = DefaultKubeComponent + } + if _, ok := r.componentGlobals[comp]; !ok { + return fmt.Errorf("component not registered: %s", comp) + } + featureGate := r.componentGlobals[comp].featureGate + if featureGate == nil { + return fmt.Errorf("component featureGate not registered: %s", comp) + } + flagVal := strings.Join(fg, ",") + klog.V(2).Infof("setting %s:feature-gates=%s\n", comp, flagVal) + if err := featureGate.Set(flagVal); err != nil { + return err + } + } return nil } -func (r *componentGlobalsRegistry) ValidateAllComponents() []error { +func (r *componentGlobalsRegistry) Validate() []error { var errs []error r.mutex.Lock() defer r.mutex.Unlock() diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go b/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go index 2e19552703f..68625e526b0 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go @@ -17,12 +17,22 @@ limitations under the License. package version import ( + "fmt" + "strings" "testing" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/version" + cliflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/featuregate" +) + +const ( + testComponent = "test" ) func TestEffectiveVersionRegistry(t *testing.T) { r := NewComponentGlobalsRegistry() - testComponent := "test" ver1 := NewEffectiveVersion("1.31") ver2 := NewEffectiveVersion("1.28") @@ -46,3 +56,216 @@ func TestEffectiveVersionRegistry(t *testing.T) { t.Fatalf("expected EffectiveVersionFor to return the version overridden") } } + +func testRegistry(t *testing.T) *componentGlobalsRegistry { + r := componentGlobalsRegistry{ + componentGlobals: map[string]ComponentGlobals{}, + emulationVersionConfig: make(cliflag.ConfigurationMap), + featureGatesConfig: make(map[string][]string), + } + verKube := NewEffectiveVersion("1.31") + fgKube := featuregate.NewVersionedFeatureGate(version.MustParse("0.0")) + err := fgKube.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ + "kubeA": { + {Version: version.MustParse("1.31"), Default: true, LockToDefault: true, PreRelease: featuregate.GA}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, + }, + "kubeB": { + {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha}, + }, + "commonC": { + {Version: version.MustParse("1.29"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, + }, + }) + if err != nil { + t.Fatal(err) + } + + verTest := NewEffectiveVersion("2.8") + fgTest := featuregate.NewVersionedFeatureGate(version.MustParse("0.0")) + err = fgTest.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ + "testA": { + {Version: version.MustParse("2.10"), Default: true, PreRelease: featuregate.GA}, + {Version: version.MustParse("2.8"), Default: false, PreRelease: featuregate.Beta}, + {Version: version.MustParse("2.7"), Default: false, PreRelease: featuregate.Alpha}, + }, + "testB": { + {Version: version.MustParse("2.9"), Default: false, PreRelease: featuregate.Alpha}, + }, + "commonC": { + {Version: version.MustParse("2.9"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("2.7"), Default: false, PreRelease: featuregate.Alpha}, + }, + }) + if err != nil { + t.Fatal(err) + } + _ = r.Register(DefaultKubeComponent, verKube, fgKube, true) + _ = r.Register(testComponent, verTest, fgTest, true) + return &r +} + +func TestVersionFlagOptions(t *testing.T) { + r := testRegistry(t) + emuVers := strings.Join(r.versionFlagOptions(true), "\n") + expectedEmuVers := "kube=1.31..1.31 (default=1.31)\ntest=2.8..2.8 (default=2.8)" + if emuVers != expectedEmuVers { + t.Errorf("wanted emulation version flag options to be: %s, got %s", expectedEmuVers, emuVers) + } + minCompVers := strings.Join(r.versionFlagOptions(false), "\n") + expectedMinCompVers := "kube=1.30..1.31 (default=1.30)\ntest=2.7..2.8 (default=2.7)" + if minCompVers != expectedMinCompVers { + t.Errorf("wanted min compatibility version flag options to be: %s, got %s", expectedMinCompVers, minCompVers) + } +} + +func TestVersionedFeatureGateFlag(t *testing.T) { + r := testRegistry(t) + known := strings.Join(r.knownFeatures(), "\n") + expectedKnown := "kube:AllAlpha=true|false (ALPHA - default=false)\n" + + "kube:AllBeta=true|false (BETA - default=false)\n" + + "kube:commonC=true|false (BETA - default=true)\n" + + "kube:kubeB=true|false (ALPHA - default=false)\n" + + "test:AllAlpha=true|false (ALPHA - default=false)\n" + + "test:AllBeta=true|false (BETA - default=false)\n" + + "test:commonC=true|false (ALPHA - default=false)\n" + + "test:testA=true|false (BETA - default=false)" + if known != expectedKnown { + t.Errorf("wanted min compatibility version flag options to be:\n%s, got:\n%s", expectedKnown, known) + } +} + +func TestFlags(t *testing.T) { + tests := []struct { + name string + emulationVersionFlag string + featureGatesFlag string + parseError string + expectedKubeEmulationVersion *version.Version + expectedTestEmulationVersion *version.Version + expectedKubeFeatureValues map[featuregate.Feature]bool + expectedTestFeatureValues map[featuregate.Feature]bool + }{ + { + name: "setting kube emulation version", + emulationVersionFlag: "kube=1.30", + expectedKubeEmulationVersion: version.MajorMinor(1, 30), + }, + { + name: "setting kube emulation version, prefix v ok", + emulationVersionFlag: "kube=v1.30", + expectedKubeEmulationVersion: version.MajorMinor(1, 30), + }, + { + name: "setting test emulation version", + emulationVersionFlag: "test=2.7", + expectedKubeEmulationVersion: version.MajorMinor(1, 31), + expectedTestEmulationVersion: version.MajorMinor(2, 7), + }, + { + name: "version missing component", + emulationVersionFlag: "1.31", + parseError: "component not registered: 1.31", + }, + { + name: "version unregistered component", + emulationVersionFlag: "test3=1.31", + parseError: "component not registered: test3", + }, + { + name: "invalid version", + emulationVersionFlag: "test=1.foo", + parseError: "illegal version string \"1.foo\"", + }, + { + name: "setting test feature flag", + emulationVersionFlag: "test=2.7", + featureGatesFlag: "test:testA=true", + expectedKubeEmulationVersion: version.MajorMinor(1, 31), + expectedTestEmulationVersion: version.MajorMinor(2, 7), + expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": true, "kubeB": false, "commonC": true}, + expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": true, "testB": false, "commonC": false}, + }, + { + name: "setting future test feature flag", + emulationVersionFlag: "test=2.7", + featureGatesFlag: "test:testA=true,test:testB=true", + parseError: "cannot set feature gate testB to true, feature is PreAlpha at emulated version 2.7", + }, + { + name: "setting kube feature flag", + emulationVersionFlag: "test=2.7,kube=1.30", + featureGatesFlag: "test:commonC=true,commonC=false,kube:kubeB=true", + expectedKubeEmulationVersion: version.MajorMinor(1, 30), + expectedTestEmulationVersion: version.MajorMinor(2, 7), + expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": false, "kubeB": true, "commonC": false}, + expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": false, "testB": false, "commonC": true}, + }, + { + name: "setting locked kube feature flag", + emulationVersionFlag: "test=2.7", + featureGatesFlag: "kubeA=false", + parseError: "cannot set feature gate kubeA to false, feature is locked to true", + }, + { + name: "setting unknown test feature flag", + emulationVersionFlag: "test=2.7", + featureGatesFlag: "test:testD=true", + parseError: "unrecognized feature gate: testD", + }, + { + name: "setting unknown component feature flag", + emulationVersionFlag: "test=2.7", + featureGatesFlag: "test3:commonC=true", + parseError: "component not registered: test3", + }, + } + for i, test := range tests { + t.Run(test.name, func(t *testing.T) { + fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError) + r := testRegistry(t) + r.AddFlags(fs) + + err := fs.Parse([]string{fmt.Sprintf("--emulated-version=%s", test.emulationVersionFlag), + fmt.Sprintf("--feature-gates=%s", test.featureGatesFlag)}) + if err == nil { + err = r.Set() + } + if test.parseError != "" { + if err == nil || !strings.Contains(err.Error(), test.parseError) { + t.Fatalf("%d: Parse() expected: %v, got: %v", i, test.parseError, err) + } + return + } + if err != nil { + t.Fatalf("%d: Parse() expected: nil, got: %v", i, err) + } + if test.expectedKubeEmulationVersion != nil { + v := r.EffectiveVersionFor("kube").EmulationVersion() + if !v.EqualTo(test.expectedKubeEmulationVersion) { + t.Fatalf("%d: EmulationVersion expected: %s, got: %s", i, test.expectedKubeEmulationVersion.String(), v.String()) + return + } + } + if test.expectedTestEmulationVersion != nil { + v := r.EffectiveVersionFor("test").EmulationVersion() + if !v.EqualTo(test.expectedTestEmulationVersion) { + t.Fatalf("%d: EmulationVersion expected: %s, got: %s", i, test.expectedTestEmulationVersion.String(), v.String()) + return + } + } + for f, v := range test.expectedKubeFeatureValues { + if r.FeatureGateFor(DefaultKubeComponent).Enabled(f) != v { + t.Errorf("%d: expected kube feature Enabled(%s)=%v", i, f, v) + } + } + for f, v := range test.expectedTestFeatureValues { + if r.FeatureGateFor(testComponent).Enabled(f) != v { + t.Errorf("%d: expected test feature Enabled(%s)=%v", i, f, v) + } + } + }) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go b/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go index 8c61e92c1de..24db0318f25 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go @@ -131,29 +131,29 @@ func TestValidate(t *testing.T) { func TestEffectiveVersionsFlag(t *testing.T) { tests := []struct { name string - emulationVerson string + emulationVersion string expectedEmulationVersion *version.Version parseError string }{ { name: "major.minor ok", - emulationVerson: "1.30", + emulationVersion: "1.30", expectedEmulationVersion: version.MajorMinor(1, 30), }, { name: "v prefix ok", - emulationVerson: "v1.30", + emulationVersion: "v1.30", expectedEmulationVersion: version.MajorMinor(1, 30), }, { - name: "semantic version not ok", - emulationVerson: "1.30.1", - parseError: "version 1.30.1 is not in the format of major.minor", + name: "semantic version not ok", + emulationVersion: "1.30.1", + parseError: "version 1.30.1 is not in the format of major.minor", }, { - name: "invalid version", - emulationVerson: "1.foo", - parseError: "illegal version string", + name: "invalid version", + emulationVersion: "1.foo", + parseError: "illegal version string", }, } for i, test := range tests { @@ -162,7 +162,7 @@ func TestEffectiveVersionsFlag(t *testing.T) { effective := NewEffectiveVersion("1.30") effective.AddFlags(fs, "test") - err := fs.Parse([]string{fmt.Sprintf("--test-emulated-version=%s", test.emulationVerson)}) + err := fs.Parse([]string{fmt.Sprintf("--test-emulated-version=%s", test.emulationVersion)}) if test.parseError != "" { if !strings.Contains(err.Error(), test.parseError) { t.Fatalf("%d: Parse() Expected %v, Got %v", i, test.parseError, err) diff --git a/staging/src/k8s.io/cloud-provider/options/options.go b/staging/src/k8s.io/cloud-provider/options/options.go index 9d55650aa19..b96948106ca 100644 --- a/staging/src/k8s.io/cloud-provider/options/options.go +++ b/staging/src/k8s.io/cloud-provider/options/options.go @@ -164,7 +164,7 @@ func (o *CloudControllerManagerOptions) Flags(allControllers []string, disabledB fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") fs.StringVar(&o.Generic.ClientConnection.Kubeconfig, "kubeconfig", o.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).") fs.DurationVar(&o.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", o.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.") - utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"), "") + utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic")) return fss } diff --git a/staging/src/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string.go b/staging/src/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string.go index bd2cf5f8752..728fa520b62 100644 --- a/staging/src/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string.go +++ b/staging/src/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string.go @@ -33,8 +33,9 @@ import ( // while still allowing the distribution of key-value pairs across multiple flag invocations. // For example: `--flag "a:hello" --flag "b:again" --flag "b:beautiful" --flag "c:world"` results in `{"a": ["hello"], "b": ["again", "beautiful"], "c": ["world"]}` type ColonSeparatedMultimapStringString struct { - Multimap *map[string][]string - initialized bool // set to true after the first Set call + Multimap *map[string][]string + initialized bool // set to true after the first Set call + allowDefaultEmptyKey bool } // NewColonSeparatedMultimapStringString takes a pointer to a map[string][]string and returns the @@ -43,6 +44,12 @@ func NewColonSeparatedMultimapStringString(m *map[string][]string) *ColonSeparat return &ColonSeparatedMultimapStringString{Multimap: m} } +// NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey takes a pointer to a map[string][]string and returns the +// ColonSeparatedMultimapStringString flag parsing shim for that map. It allows default empty key with no colon in the flag. +func NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(m *map[string][]string) *ColonSeparatedMultimapStringString { + return &ColonSeparatedMultimapStringString{Multimap: m, allowDefaultEmptyKey: true} +} + // Set implements github.com/spf13/pflag.Value func (m *ColonSeparatedMultimapStringString) Set(value string) error { if m.Multimap == nil { @@ -58,11 +65,16 @@ func (m *ColonSeparatedMultimapStringString) Set(value string) error { continue } kv := strings.SplitN(pair, ":", 2) - if len(kv) != 2 { - return fmt.Errorf("malformed pair, expect string:string") + var k, v string + if m.allowDefaultEmptyKey && len(kv) == 1 { + v = strings.TrimSpace(kv[0]) + } else { + if len(kv) != 2 { + return fmt.Errorf("malformed pair, expect string:string") + } + k = strings.TrimSpace(kv[0]) + v = strings.TrimSpace(kv[1]) } - k := strings.TrimSpace(kv[0]) - v := strings.TrimSpace(kv[1]) (*m.Multimap)[k] = append((*m.Multimap)[k], v) } return nil diff --git a/staging/src/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string_test.go b/staging/src/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string_test.go index 9e77035c3d4..5e5c158ee2d 100644 --- a/staging/src/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string_test.go +++ b/staging/src/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string_test.go @@ -98,6 +98,21 @@ func TestSetColonSeparatedMultimapStringString(t *testing.T) { &ColonSeparatedMultimapStringString{ initialized: true, Multimap: &map[string][]string{}}, ""}, + {"empty key no colon", []string{"foo"}, + NewColonSeparatedMultimapStringString(&nilMap), + &ColonSeparatedMultimapStringString{ + initialized: true, + Multimap: &map[string][]string{ + "": {"foo"}, + }}, "malformed pair, expect string:string"}, + {"empty key no colon allowed", []string{"foo"}, + NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(&nilMap), + &ColonSeparatedMultimapStringString{ + initialized: true, + allowDefaultEmptyKey: true, + Multimap: &map[string][]string{ + "": {"foo"}, + }}, ""}, {"empty key", []string{":foo"}, NewColonSeparatedMultimapStringString(&nilMap), &ColonSeparatedMultimapStringString{ diff --git a/staging/src/k8s.io/component-base/featuregate/feature_gate.go b/staging/src/k8s.io/component-base/featuregate/feature_gate.go index c6b4d7627f5..d164baf3878 100644 --- a/staging/src/k8s.io/component-base/featuregate/feature_gate.go +++ b/staging/src/k8s.io/component-base/featuregate/feature_gate.go @@ -115,7 +115,6 @@ type FeatureGate interface { // config against potential feature gate changes before committing those changes. DeepCopy() MutableVersionedFeatureGate // Validate checks if the flag gates are valid at the emulated version. - // Should always be called after Set when DeferErrorsToValidation is set to true. Validate() []error } @@ -125,7 +124,9 @@ type MutableFeatureGate interface { FeatureGate // AddFlag adds a flag for setting global feature gates to the specified FlagSet. - AddFlag(fs *pflag.FlagSet, prefix string) + AddFlag(fs *pflag.FlagSet) + // Close sets closed to true, and prevents subsequent calls to Add + Close() // Set parses and stores flag gates for known features // from a string like feature1=true,feature2=false,... Set(value string) error @@ -163,10 +164,6 @@ type MutableVersionedFeatureGate interface { // Otherwise, the emulationVersion will be the same as the binary version. // If set, the feature defaults and availability will be as if the binary is at the emulated version. SetEmulationVersion(emulationVersion *version.Version) error - // DeferErrorsToValidation defers the errors of Set function to the Validate() function if true. - // This is used when the user wants to set the feature gate flag before the emulationVersion is finalized. - // Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true. - DeferErrorsToValidation(val bool) // GetAll returns a copy of the map of known feature names to versioned feature specs. GetAllVersioned() map[Feature]VersionedSpecs // AddVersioned adds versioned feature specs to the featureGate. @@ -199,12 +196,8 @@ type featureGate struct { // while enabled keeps the values of all resolved features. enabledRaw atomic.Value // closed is set to true when AddFlag is called, and prevents subsequent calls to Add - closed bool - // deferErrorsToValidation could be set to true to defer checking flag setting error, - // because the emulationVersion may not be the final emulationVersion when the flag is set. - // Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true. - deferErrorsToValidation bool - emulationVersion atomic.Pointer[version.Version] + closed bool + emulationVersion atomic.Pointer[version.Version] } func setUnsetAlphaGates(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, cVer *version.Version) { @@ -288,17 +281,10 @@ func (f *featureGate) Set(value string) error { } m[k] = boolValue } - err := f.SetFromMap(m) - // ignores SetFromMap error, because the emulationVersion may not be the final emulationVersion when the flag is set. - // Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true. - if f.deferErrorsToValidation { - return nil - } - return err + return f.SetFromMap(m) } // Validate checks if the flag gates are valid at the emulated version. -// Should always be called after Set when DeferErrorsToValidation is set to true. func (f *featureGate) Validate() []error { m, ok := f.enabledRaw.Load().(map[string]bool) if !ok { @@ -502,15 +488,6 @@ func (f *featureGate) GetAllVersioned() map[Feature]VersionedSpecs { return retval } -// DeferErrorsToValidation could be used to defer checking flag setting error, -// because the emulationVersion may not be the final emulationVersion when the flag is set. -// Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true. -func (f *featureGate) DeferErrorsToValidation(val bool) { - f.lock.Lock() - defer f.lock.Unlock() - f.deferErrorsToValidation = val -} - func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) error { f.lock.Lock() defer f.lock.Unlock() @@ -577,21 +554,23 @@ func getCurrentVersion(v VersionedSpecs, emulationVersion *version.Version) *Fea } } -// AddFlag adds a flag for setting global feature gates to the specified FlagSet. -func (f *featureGate) AddFlag(fs *pflag.FlagSet, prefix string) { +// Close sets closed to true, and prevents subsequent calls to Add +func (f *featureGate) Close() { f.lock.Lock() + f.closed = true + f.lock.Unlock() +} + +// AddFlag adds a flag for setting global feature gates to the specified FlagSet. +func (f *featureGate) AddFlag(fs *pflag.FlagSet) { // TODO(mtaufen): Shouldn't we just close it on the first Set/SetFromMap instead? // Not all components expose a feature gates flag using this AddFlag method, and // in the future, all components will completely stop exposing a feature gates flag, // in favor of componentconfig. - f.closed = true - f.lock.Unlock() + f.Close() known := f.KnownFeatures() - if len(prefix) > 0 && !strings.HasSuffix(prefix, "-") { - prefix += "-" - } - fs.Var(f, prefix+flagName, ""+ + fs.Var(f, flagName, ""+ "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ "Options are:\n"+strings.Join(known, "\n")) } diff --git a/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go b/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go index b46d5483fb5..9156a024abe 100644 --- a/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go +++ b/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go @@ -264,7 +264,7 @@ func TestFeatureGateFlag(t *testing.T) { testLockedFalseGate: {Default: false, PreRelease: GA, LockToDefault: true}, }) require.NoError(t, err) - f.AddFlag(fs, "") + f.AddFlag(fs) err = fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)}) if test.parseError != "" { if !strings.Contains(err.Error(), test.parseError) { @@ -679,7 +679,7 @@ func TestFeatureGateOverrideDefault(t *testing.T) { t.Run("returns error if already added to flag set", func(t *testing.T) { f := NewFeatureGate() fs := pflag.NewFlagSet("test", pflag.ContinueOnError) - f.AddFlag(fs, "") + f.AddFlag(fs) if err := f.OverrideDefault("TestFeature", true); err == nil { t.Error("expected a non-nil error to be returned") @@ -986,7 +986,9 @@ func TestVersionedFeatureGateFlag(t *testing.T) { t.Run(test.arg, func(t *testing.T) { fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError) f := NewVersionedFeatureGate(version.MustParse("1.29")) - f.DeferErrorsToValidation(true) + if err := f.SetEmulationVersion(version.MustParse("1.28")); err != nil { + t.Fatalf("failed to SetEmulationVersion: %v", err) + } err := f.AddVersioned(map[Feature]VersionedSpecs{ testGAGate: { {Version: version.MustParse("1.29"), Default: true, PreRelease: GA}, @@ -1011,14 +1013,12 @@ func TestVersionedFeatureGateFlag(t *testing.T) { testBetaGateNoVersion: {Default: false, PreRelease: Beta}, }) require.NoError(t, err) - f.AddFlag(fs, "") + f.AddFlag(fs) var errs []error err = fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)}) if err != nil { errs = append(errs, err) - } else { - errs = append(errs, f.SetEmulationVersion(version.MustParse("1.28"))) } err = utilerrors.NewAggregate(errs) if test.parseError != "" { @@ -1425,7 +1425,7 @@ func TestVersionedFeatureGateOverrideDefault(t *testing.T) { f := NewVersionedFeatureGate(version.MustParse("1.29")) require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) fs := pflag.NewFlagSet("test", pflag.ContinueOnError) - f.AddFlag(fs, "") + f.AddFlag(fs) if err := f.OverrideDefault("TestFeature", true); err == nil { t.Error("expected a non-nil error to be returned") diff --git a/staging/src/k8s.io/component-base/logs/example/cmd/logger.go b/staging/src/k8s.io/component-base/logs/example/cmd/logger.go index af51c793e30..b0d4f9baaca 100644 --- a/staging/src/k8s.io/component-base/logs/example/cmd/logger.go +++ b/staging/src/k8s.io/component-base/logs/example/cmd/logger.go @@ -67,7 +67,7 @@ func NewLoggerCommand() *cobra.Command { }, } logsapi.AddFeatureGates(featureGate) - featureGate.AddFlag(cmd.Flags(), "") + featureGate.AddFlag(cmd.Flags()) logsapi.AddFlags(c, cmd.Flags()) return cmd } diff --git a/staging/src/k8s.io/component-base/logs/example/k8s2slog/k8s2slog.go b/staging/src/k8s.io/component-base/logs/example/k8s2slog/k8s2slog.go index 6c29c206a3b..c39ef9fe7a6 100644 --- a/staging/src/k8s.io/component-base/logs/example/k8s2slog/k8s2slog.go +++ b/staging/src/k8s.io/component-base/logs/example/k8s2slog/k8s2slog.go @@ -83,7 +83,7 @@ func NewLoggerCommand() *cobra.Command { // Shouldn't happen. panic(err) } - featureGate.AddFlag(cmd.Flags(), "") + featureGate.AddFlag(cmd.Flags()) logsapi.AddFlags(c, cmd.Flags()) return cmd } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go index a70c4126cf2..6e3c02bfef8 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go @@ -33,7 +33,6 @@ import ( genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" - "k8s.io/component-base/featuregate" "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" "k8s.io/kube-aggregator/pkg/apiserver" aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" @@ -61,15 +60,13 @@ type AggregatorOptions struct { // with a default AggregatorOptions. func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions) *cobra.Command { o := *defaults - featureGate := o.ServerRunOptions.FeatureGate.(featuregate.MutableVersionedFeatureGate) - effectiveVersion := o.ServerRunOptions.EffectiveVersion.(utilversion.MutableEffectiveVersion) cmd := &cobra.Command{ Short: "Launch a API aggregator and proxy server", Long: "Launch a API aggregator and proxy server", + PersistentPreRunE: func(*cobra.Command, []string) error { + return utilversion.DefaultComponentGlobalsRegistry.Set() + }, RunE: func(c *cobra.Command, args []string) error { - if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { - return err - } if err := o.Complete(); err != nil { return err } @@ -85,8 +82,7 @@ func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions) cmd.SetContext(ctx) fs := cmd.Flags() - featureGate.AddFlag(fs, "") - effectiveVersion.AddFlags(fs, "") + utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs) o.AddFlags(fs) return cmd @@ -107,7 +103,7 @@ func NewDefaultOptions(out, err io.Writer) *AggregatorOptions { // You can also have the flag setting the effectiveVersion of the aggregator apiserver, and // having a mapping from the aggregator apiserver version to generic apiserver version. effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( - utilversion.ComponentGenericAPIServer, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) + utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) o := &AggregatorOptions{ ServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion), RecommendedOptions: genericoptions.NewRecommendedOptions( diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go index a968d36177b..6558efda6f1 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go @@ -41,7 +41,7 @@ var ( // Codecs provides methods for retrieving codecs and serializers for specific // versions and content types. Codecs = serializer.NewCodecFactory(Scheme) - WardleComponentName = "wardle-server" + WardleComponentName = "wardle" ) func init() { diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go index 275c1a8718f..df497e450f2 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go @@ -18,7 +18,6 @@ package server import ( "context" - "errors" "fmt" "io" "net" @@ -36,6 +35,7 @@ import ( genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" + "k8s.io/component-base/featuregate" "k8s.io/sample-apiserver/pkg/admission/plugin/banflunder" "k8s.io/sample-apiserver/pkg/admission/wardleinitializer" "k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1" @@ -61,7 +61,7 @@ type WardleServerOptions struct { func mapWardleEffectiveVersionToKubeEffectiveVersion(registry utilversion.ComponentGlobalsRegistry) error { wardleVer := registry.EffectiveVersionFor(apiserver.WardleComponentName) - kubeVer := registry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer).(utilversion.MutableEffectiveVersion) + kubeVer := registry.EffectiveVersionFor(utilversion.DefaultKubeComponent).(utilversion.MutableEffectiveVersion) // map from wardle emulation version to kube emulation version. emulationVersionMap := map[string]string{ "1.2": kubeVer.BinaryVersion().AddMinor(1).String(), @@ -99,6 +99,13 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti cmd := &cobra.Command{ Short: "Launch a wardle API server", Long: "Launch a wardle API server", + PersistentPreRunE: func(*cobra.Command, []string) error { + if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil { + return err + } + // convert wardle effective version to kube effective version to be used in generic api server, and set the generic api server feature gate. + return mapWardleEffectiveVersionToKubeEffectiveVersion(utilversion.DefaultComponentGlobalsRegistry) + }, RunE: func(c *cobra.Command, args []string) error { if err := o.Complete(); err != nil { return err @@ -118,12 +125,19 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti o.RecommendedOptions.AddFlags(flags) wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2") - utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, nil, false)) - _, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( - utilversion.ComponentGenericAPIServer, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) + wardleFeatureGate := featuregate.NewVersionedFeatureGate(version.MustParse("1.2")) + utilruntime.Must(wardleFeatureGate.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ + "BanFlunder": { + {Version: version.MustParse("1.2"), Default: true, PreRelease: featuregate.GA}, + {Version: version.MustParse("1.1"), Default: false, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.0"), Default: false, PreRelease: featuregate.Alpha}, + }, + })) + utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate, false)) + _, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( + utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) - wardleEffectiveVersion.AddFlags(flags, "wardle-") - featureGate.AddFlag(flags, "") + utilversion.DefaultComponentGlobalsRegistry.AddFlags(flags) return cmd } @@ -132,29 +146,19 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti func (o WardleServerOptions) Validate(args []string) error { errors := []error{} errors = append(errors, o.RecommendedOptions.Validate()...) - errors = append(errors, utilversion.DefaultComponentGlobalsRegistry.ValidateAllComponents()...) + errors = append(errors, utilversion.DefaultComponentGlobalsRegistry.Validate()...) return utilerrors.NewAggregate(errors) } // Complete fills in fields required to have valid data func (o *WardleServerOptions) Complete() error { - // register admission plugins - banflunder.Register(o.RecommendedOptions.Admission.Plugins) + if utilversion.DefaultComponentGlobalsRegistry.FeatureGateFor(apiserver.WardleComponentName).Enabled("BanFlunder") { + // register admission plugins + banflunder.Register(o.RecommendedOptions.Admission.Plugins) - // add admission plugins to the RecommendedPluginOrder - o.RecommendedOptions.Admission.RecommendedPluginOrder = append(o.RecommendedOptions.Admission.RecommendedPluginOrder, "BanFlunder") - - // convert wardle effective version to kube effective version to be used in generic api server, and set the generic api server feature gate. - if err := mapWardleEffectiveVersionToKubeEffectiveVersion(utilversion.DefaultComponentGlobalsRegistry); err != nil { - return err + // add admission plugins to the RecommendedPluginOrder + o.RecommendedOptions.Admission.RecommendedPluginOrder = append(o.RecommendedOptions.Admission.RecommendedPluginOrder, "BanFlunder") } - if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { - return err - } - if errs := utilversion.DefaultComponentGlobalsRegistry.ValidateAllComponents(); len(errs) > 0 { - return errors.Join(errs...) - } - return nil } @@ -185,8 +189,8 @@ func (o *WardleServerOptions) Config() (*apiserver.Config, error) { serverConfig.OpenAPIV3Config.Info.Title = "Wardle" serverConfig.OpenAPIV3Config.Info.Version = "0.1" - serverConfig.FeatureGate = utilversion.DefaultComponentGlobalsRegistry.FeatureGateFor(utilversion.ComponentGenericAPIServer) - serverConfig.EffectiveVersion = utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer) + serverConfig.FeatureGate = utilversion.DefaultComponentGlobalsRegistry.FeatureGateFor(utilversion.DefaultKubeComponent) + serverConfig.EffectiveVersion = utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil { return nil, err diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go index 6ee274c65b6..7d7a0bc74e7 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go @@ -50,7 +50,7 @@ func TestMapBinaryEffectiveVersionToKubeEffectiveVersion(t *testing.T) { t.Run(tc.desc, func(t *testing.T) { registry := utilversion.NewComponentGlobalsRegistry() _ = registry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, nil, true) - _ = registry.Register(utilversion.ComponentGenericAPIServer, defaultKubeEffectiveVersion, nil, true) + _ = registry.Register(utilversion.DefaultKubeComponent, defaultKubeEffectiveVersion, nil, true) wardleEffectiveVersion.SetEmulationVersion(tc.wardleEmulationVer) err := mapWardleEffectiveVersionToKubeEffectiveVersion(registry) @@ -59,7 +59,7 @@ func TestMapBinaryEffectiveVersionToKubeEffectiveVersion(t *testing.T) { t.Fatal("expected error, no error found") } } else { - assert.True(t, registry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer).EmulationVersion().EqualTo(tc.expectedKubeEmulationVer)) + assert.True(t, registry.EffectiveVersionFor(utilversion.DefaultKubeComponent).EmulationVersion().EqualTo(tc.expectedKubeEmulationVer)) } }) } diff --git a/test/e2e/dra/test-driver/app/server.go b/test/e2e/dra/test-driver/app/server.go index 59e4ac42a99..3f55c541463 100644 --- a/test/e2e/dra/test-driver/app/server.go +++ b/test/e2e/dra/test-driver/app/server.go @@ -88,7 +88,7 @@ func NewCommand() *cobra.Command { fs = sharedFlagSets.FlagSet("other") featureGate := featuregate.NewFeatureGate() utilruntime.Must(logsapi.AddFeatureGates(featureGate)) - featureGate.AddFlag(fs, "") + featureGate.AddFlag(fs) fs = cmd.PersistentFlags() for _, f := range sharedFlagSets.FlagSets { diff --git a/test/integration/apiserver/apiserver_test.go b/test/integration/apiserver/apiserver_test.go index 81edfbbc0aa..eba370d6e91 100644 --- a/test/integration/apiserver/apiserver_test.go +++ b/test/integration/apiserver/apiserver_test.go @@ -3008,7 +3008,7 @@ func TestEmulatedStorageVersion(t *testing.T) { t.Run(emulatedVersion, func(t *testing.T) { server := kubeapiservertesting.StartTestServerOrDie( t, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: emulatedVersion}, - []string{"--emulated-version=" + emulatedVersion, `--storage-media-type=application/json`}, framework.SharedEtcd()) + []string{"--emulated-version=kube=" + emulatedVersion, `--storage-media-type=application/json`}, framework.SharedEtcd()) defer server.TearDownFn() client := clientset.NewForConfigOrDie(server.ClientConfig) @@ -3106,7 +3106,7 @@ func TestEmulatedStorageVersion(t *testing.T) { func TestEnableEmulationVersion(t *testing.T) { server := kubeapiservertesting.StartTestServerOrDie(t, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"}, - []string{"--emulated-version=1.31"}, framework.SharedEtcd()) + []string{"--emulated-version=kube=1.31"}, framework.SharedEtcd()) defer server.TearDownFn() rt, err := restclient.TransportFor(server.ClientConfig) diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index 2ed1f1c001c..8d4164ea9b5 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -292,7 +292,8 @@ func TestAggregatedAPIServer(t *testing.T) { "--etcd-servers", framework.GetEtcdURL(), "--cert-dir", wardleCertDir, "--kubeconfig", wardleToKASKubeConfigFile, - "--wardle-emulated-version", "1.1", + "--emulated-version", "wardle=1.1", + "--feature-gates", "wardle:BanFlunder=true", }) if err := wardleCmd.Execute(); err != nil { t.Error(err) @@ -387,6 +388,7 @@ func TestAggregatedAPIServer(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "panda", }, + DisallowedFlunders: []string{"badname"}, }, metav1.CreateOptions{}) if err != nil { t.Fatal(err) @@ -402,11 +404,22 @@ func TestAggregatedAPIServer(t *testing.T) { t.Error("expected non-empty resource version for fischer list") } + _, err = wardleClient.Flunders(metav1.NamespaceSystem).Create(ctx, &wardlev1alpha1.Flunder{ + ObjectMeta: metav1.ObjectMeta{ + Name: "badname", + }, + }, metav1.CreateOptions{}) + if err == nil { + t.Fatal("expect flunder:badname not admitted") + } _, err = wardleClient.Flunders(metav1.NamespaceSystem).Create(ctx, &wardlev1alpha1.Flunder{ ObjectMeta: metav1.ObjectMeta{ Name: "panda", }, }, metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } flunderList, err := wardleClient.Flunders(metav1.NamespaceSystem).List(ctx, metav1.ListOptions{}) if err != nil { t.Fatal(err) From 4352c4ad2762ce49ce30e62381f8ceb24723fbcc Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Fri, 31 May 2024 20:29:48 -0700 Subject: [PATCH 3/4] Add version mapping in ComponentGlobalsRegistry. Signed-off-by: Siyuan Zhang --- .../app/options/options_test.go | 2 +- cmd/kube-apiserver/app/testing/testserver.go | 9 +- pkg/controlplane/apiserver/config.go | 4 - .../apiserver/options/options_test.go | 2 +- pkg/controlplane/instance_test.go | 11 +- .../pkg/apiserver/apiserver.go | 7 - .../pkg/cmd/server/testing/testserver.go | 4 +- .../apimachinery/pkg/util/version/version.go | 53 ++- .../pkg/util/version/version_test.go | 19 +- .../src/k8s.io/apiserver/pkg/server/config.go | 14 +- .../apiserver/pkg/server/config_test.go | 3 + .../apiserver/pkg/server/deleted_kinds.go | 2 +- .../apiserver/pkg/server/genericapiserver.go | 3 - .../pkg/server/genericapiserver_test.go | 7 +- .../pkg/server/options/serving_test.go | 1 - .../storage/resource_encoding_config.go | 1 + .../apiserver/pkg/util/version/registry.go | 229 +++++++++++-- .../pkg/util/version/registry_test.go | 307 +++++++++++++----- .../apiserver/pkg/util/version/version.go | 107 +++--- .../pkg/util/version/version_test.go | 54 --- .../featuregate/feature_gate.go | 154 ++++++--- .../featuregate/feature_gate_test.go | 80 ++++- .../pkg/apiserver/apiserver.go | 3 - .../pkg/apiserver/apiserver.go | 9 - .../sample-apiserver/pkg/cmd/server/start.go | 36 +- .../pkg/cmd/server/start_test.go | 28 +- test/integration/apiserver/apiserver_test.go | 43 +++ test/integration/client/client_test.go | 8 +- .../etcd/crd_overlap_storage_test.go | 10 +- .../integration/etcd/etcd_cross_group_test.go | 2 +- test/integration/examples/apiserver_test.go | 48 ++- .../integration/servicecidr/allocator_test.go | 2 - 32 files changed, 853 insertions(+), 409 deletions(-) diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index 1e4d5a679f8..fa53175c9ad 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -53,7 +53,7 @@ func TestAddFlags(t *testing.T) { featureGate := featuregate.NewFeatureGate() componentRegistry := utilversion.NewComponentGlobalsRegistry() effectiveVersion := utilversion.NewEffectiveVersion("1.32") - _ = componentRegistry.Register("test", effectiveVersion, featureGate, true) + utilruntime.Must(componentRegistry.Register("test", effectiveVersion, featureGate)) s := NewServerRunOptions(featureGate, effectiveVersion) for _, f := range s.Flags().FlagSets { fs.AddFlagSet(f) diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go index f4268b23a6d..0e6411fe869 100644 --- a/cmd/kube-apiserver/app/testing/testserver.go +++ b/cmd/kube-apiserver/app/testing/testserver.go @@ -43,6 +43,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" serveroptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/storage/storagebackend" @@ -182,12 +183,12 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, fs := pflag.NewFlagSet("test", pflag.PanicOnError) featureGate := utilfeature.DefaultMutableFeatureGate - binaryVersion := utilversion.DefaultKubeEffectiveVersion().BinaryVersion().String() + effectiveVersion := utilversion.DefaultKubeEffectiveVersion() if instanceOptions.BinaryVersion != "" { - binaryVersion = instanceOptions.BinaryVersion + effectiveVersion = utilversion.NewEffectiveVersion(instanceOptions.BinaryVersion) } - effectiveVersion := utilversion.NewEffectiveVersion(binaryVersion) - _ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate, true) + utilversion.DefaultComponentGlobalsRegistry.Reset() + utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate)) s := options.NewServerRunOptions(featureGate, effectiveVersion) diff --git a/pkg/controlplane/apiserver/config.go b/pkg/controlplane/apiserver/config.go index 7933f076ec6..1fc862c34c9 100644 --- a/pkg/controlplane/apiserver/config.go +++ b/pkg/controlplane/apiserver/config.go @@ -46,7 +46,6 @@ import ( clientgoinformers "k8s.io/client-go/informers" clientgoclientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/keyutil" - "k8s.io/component-base/version" aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" openapicommon "k8s.io/kube-openapi/pkg/common" @@ -172,9 +171,6 @@ func BuildGenericConfig( sets.NewString("attach", "exec", "proxy", "log", "portforward"), ) - kubeVersion := version.Get() - genericConfig.Version = &kubeVersion - if genericConfig.EgressSelector != nil { s.Etcd.StorageConfig.Transport.EgressLookup = genericConfig.EgressSelector.Lookup } diff --git a/pkg/controlplane/apiserver/options/options_test.go b/pkg/controlplane/apiserver/options/options_test.go index a3f65ebde7f..557983c3679 100644 --- a/pkg/controlplane/apiserver/options/options_test.go +++ b/pkg/controlplane/apiserver/options/options_test.go @@ -48,7 +48,7 @@ func TestAddFlags(t *testing.T) { featureGate := featuregate.NewFeatureGate() effectiveVersion := utilversion.NewEffectiveVersion("1.32") componentRegistry := utilversion.NewComponentGlobalsRegistry() - _ = componentRegistry.Register("test", effectiveVersion, featureGate, true) + utilruntime.Must(componentRegistry.Register("test", effectiveVersion, featureGate)) s := NewOptions(featureGate, effectiveVersion) var fss cliflag.NamedFlagSets s.AddFlags(&fss) diff --git a/pkg/controlplane/instance_test.go b/pkg/controlplane/instance_test.go index 32d1811d82c..524a529520a 100644 --- a/pkg/controlplane/instance_test.go +++ b/pkg/controlplane/instance_test.go @@ -20,6 +20,7 @@ import ( "context" "crypto/tls" "encoding/json" + "fmt" "io" "net" "net/http" @@ -118,9 +119,7 @@ func setUp(t *testing.T) (*etcd3testing.EtcdTestServer, Config, *assert.Assertio t.Fatal(err) } - kubeVersion := kubeversion.Get() config.ControlPlane.Generic.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer() - config.ControlPlane.Generic.Version = &kubeVersion config.ControlPlane.StorageFactory = storageFactory config.ControlPlane.Generic.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}} config.ControlPlane.Generic.PublicAddress = netutils.ParseIPSloppy("192.168.10.4") @@ -243,9 +242,13 @@ func TestVersion(t *testing.T) { if err != nil { t.Errorf("unexpected error: %v", err) } + expectedInfo := kubeversion.Get() + kubeVersion := utilversion.DefaultKubeEffectiveVersion().BinaryVersion() + expectedInfo.Major = fmt.Sprintf("%d", kubeVersion.Major()) + expectedInfo.Minor = fmt.Sprintf("%d", kubeVersion.Minor()) - if !reflect.DeepEqual(kubeversion.Get(), info) { - t.Errorf("Expected %#v, Got %#v", kubeversion.Get(), info) + if !reflect.DeepEqual(expectedInfo, info) { + t.Errorf("Expected %#v, Got %#v", expectedInfo, info) } } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go index 517fc9e531a..a6d6213a08b 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go @@ -40,7 +40,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/endpoints/discovery" "k8s.io/apiserver/pkg/endpoints/discovery/aggregated" genericregistry "k8s.io/apiserver/pkg/registry/generic" @@ -118,12 +117,6 @@ func (cfg *Config) Complete() CompletedConfig { } c.GenericConfig.EnableDiscovery = false - if c.GenericConfig.Version == nil { - c.GenericConfig.Version = &version.Info{ - Major: "0", - Minor: "1", - } - } return CompletedConfig{&c} } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go index 547a0ba7712..ef5fce2fead 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go @@ -31,6 +31,7 @@ import ( extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" "k8s.io/apiextensions-apiserver/pkg/cmd/server/options" generatedopenapi "k8s.io/apiextensions-apiserver/pkg/generated/openapi" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" genericapiserver "k8s.io/apiserver/pkg/server" @@ -124,7 +125,8 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin featureGate := utilfeature.DefaultMutableFeatureGate effectiveVersion := utilversion.DefaultKubeEffectiveVersion() - _ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate, true) + utilversion.DefaultComponentGlobalsRegistry.Reset() + utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate)) s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr, featureGate, effectiveVersion) utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/version/version.go b/staging/src/k8s.io/apimachinery/pkg/util/version/version.go index d327f00b3be..b7812ff2d15 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/version/version.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/version/version.go @@ -23,6 +23,8 @@ import ( "regexp" "strconv" "strings" + + apimachineryversion "k8s.io/apimachinery/pkg/version" ) // Version is an opaque representation of a version number @@ -31,6 +33,7 @@ type Version struct { semver bool preRelease string buildMetadata string + info apimachineryversion.Info } var ( @@ -252,19 +255,30 @@ func (v *Version) WithMinor(minor uint) *Version { return &result } -// SubtractMinor returns the version diff minor versions back, with the same major and no patch. -// If diff >= current minor, the minor would be 0. -func (v *Version) SubtractMinor(diff uint) *Version { +// SubtractMinor returns the version with offset from the original minor, with the same major and no patch. +// If -offset >= current minor, the minor would be 0. +func (v *Version) OffsetMinor(offset int) *Version { var minor uint - if diff < v.Minor() { - minor = v.Minor() - diff + if offset >= 0 { + minor = v.Minor() + uint(offset) + } else { + diff := uint(-offset) + if diff < v.Minor() { + minor = v.Minor() - diff + } } return MajorMinor(v.Major(), minor) } +// SubtractMinor returns the version diff minor versions back, with the same major and no patch. +// If diff >= current minor, the minor would be 0. +func (v *Version) SubtractMinor(diff uint) *Version { + return v.OffsetMinor(-int(diff)) +} + // AddMinor returns the version diff minor versions forward, with the same major and no patch. func (v *Version) AddMinor(diff uint) *Version { - return MajorMinor(v.Major(), v.Minor()+diff) + return v.OffsetMinor(int(diff)) } // WithPatch returns copy of the version object with requested patch number @@ -441,3 +455,30 @@ func (v *Version) Compare(other string) (int, error) { } return v.compareInternal(ov), nil } + +// WithInfo returns copy of the version object with requested info +func (v *Version) WithInfo(info apimachineryversion.Info) *Version { + result := *v + result.info = info + return &result +} + +func (v *Version) Info() *apimachineryversion.Info { + if v == nil { + return nil + } + // in case info is empty, or the major and minor in info is different from the actual major and minor + v.info.Major = itoa(v.Major()) + v.info.Minor = itoa(v.Minor()) + if v.info.GitVersion == "" { + v.info.GitVersion = v.String() + } + return &v.info +} + +func itoa(i uint) string { + if i == 0 { + return "" + } + return strconv.Itoa(int(i)) +} diff --git a/staging/src/k8s.io/apimachinery/pkg/util/version/version_test.go b/staging/src/k8s.io/apimachinery/pkg/util/version/version_test.go index c8625fb8091..a3345ddfdde 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/version/version_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/version/version_test.go @@ -453,37 +453,42 @@ func TestHighestSupportedVersion(t *testing.T) { } } -func TestSubtractMinor(t *testing.T) { +func TestOffsetMinor(t *testing.T) { var tests = []struct { version string - diff uint + diff int expectedComponents []uint }{ { version: "1.0.2", - diff: 3, + diff: -3, expectedComponents: []uint{1, 0}, }, { version: "1.3.2-alpha+001", - diff: 2, + diff: -2, expectedComponents: []uint{1, 1}, }, { version: "1.3.2-alpha+001", - diff: 3, + diff: -3, expectedComponents: []uint{1, 0}, }, { version: "1.20", - diff: 5, + diff: -5, expectedComponents: []uint{1, 15}, }, + { + version: "1.20", + diff: 5, + expectedComponents: []uint{1, 25}, + }, } for _, test := range tests { version, _ := ParseGeneric(test.version) - if !reflect.DeepEqual(test.expectedComponents, version.SubtractMinor(test.diff).Components()) { + if !reflect.DeepEqual(test.expectedComponents, version.OffsetMinor(test.diff).Components()) { t.Error("parse returned un'expected components") } } diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index b266cb2e494..931226bb678 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -44,7 +44,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/version" utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup" - apimachineryversion "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/audit" "k8s.io/apiserver/pkg/authentication/authenticator" @@ -150,8 +149,6 @@ type Config struct { // done values in this values for this map are ignored. PostStartHooks map[string]PostStartHookConfigEntry - // Version will enable the /version endpoint if non-nil - Version *apimachineryversion.Info // EffectiveVersion determines which apis and features are available // based on when the api/feature lifecyle. EffectiveVersion utilversion.EffectiveVersion @@ -702,12 +699,8 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo } c.ExternalAddress = net.JoinHostPort(c.ExternalAddress, strconv.Itoa(port)) } - var ver *version.Version - if c.EffectiveVersion != nil { - ver = c.EffectiveVersion.EmulationVersion() - } - completeOpenAPI(c.OpenAPIConfig, ver) - completeOpenAPIV3(c.OpenAPIV3Config, ver) + completeOpenAPI(c.OpenAPIConfig, c.EffectiveVersion.EmulationVersion()) + completeOpenAPIV3(c.OpenAPIV3Config, c.EffectiveVersion.EmulationVersion()) if c.DiscoveryAddresses == nil { c.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: c.ExternalAddress} @@ -834,7 +827,6 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G StorageVersionManager: c.StorageVersionManager, EffectiveVersion: c.EffectiveVersion, - Version: c.Version, FeatureGate: c.FeatureGate, muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{}, @@ -1103,7 +1095,7 @@ func installAPI(s *GenericAPIServer, c *Config) { } } - routes.Version{Version: c.Version}.Install(s.Handler.GoRestfulContainer) + routes.Version{Version: c.EffectiveVersion.BinaryVersion().Info()}.Install(s.Handler.GoRestfulContainer) if c.EnableDiscovery { if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) { diff --git a/staging/src/k8s.io/apiserver/pkg/server/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/config_test.go index f58f3bf9c2b..6d25272b0a8 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config_test.go @@ -40,6 +40,7 @@ import ( "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/server/healthz" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/rest" @@ -90,6 +91,7 @@ func TestNewWithDelegate(t *testing.T) { delegateConfig.PublicAddress = netutils.ParseIPSloppy("192.168.10.4") delegateConfig.LegacyAPIGroupPrefixes = sets.NewString("/api") delegateConfig.LoopbackClientConfig = &rest.Config{} + delegateConfig.EffectiveVersion = utilversion.NewEffectiveVersion("") clientset := fake.NewSimpleClientset() if clientset == nil { t.Fatal("unable to create fake client set") @@ -122,6 +124,7 @@ func TestNewWithDelegate(t *testing.T) { wrappingConfig.PublicAddress = netutils.ParseIPSloppy("192.168.10.4") wrappingConfig.LegacyAPIGroupPrefixes = sets.NewString("/api") wrappingConfig.LoopbackClientConfig = &rest.Config{} + wrappingConfig.EffectiveVersion = utilversion.NewEffectiveVersion("") wrappingConfig.HealthzChecks = append(wrappingConfig.HealthzChecks, healthz.NamedCheck("wrapping-health", func(r *http.Request) error { return fmt.Errorf("wrapping failed healthcheck") diff --git a/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go index 841c1f5729c..564b5bc4352 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go +++ b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go @@ -100,7 +100,7 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio } introduced, ok := versionedPtr.(introducedInterface) - // skip the introduced check for test where currentVersion is 0.0 + // skip the introduced check for test when currentVersion is 0.0 to test all apis if ok && (e.currentVersion.Major() > 0 || e.currentVersion.Minor() > 0) { majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced() verIntroduced := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced)) diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go index c47fc5b23f0..7b8e13da256 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go @@ -40,7 +40,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup" - "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/audit" "k8s.io/apiserver/pkg/authorization/authorizer" @@ -238,8 +237,6 @@ type GenericAPIServer struct { // StorageVersionManager holds the storage versions of the API resources installed by this server. StorageVersionManager storageversion.Manager - // Version will enable the /version endpoint if non-nil - Version *version.Info // EffectiveVersion determines which apis and features are available // based on when the api/feature lifecyle. EffectiveVersion utilversion.EffectiveVersion diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go index c2bfef4dda4..c6a7cbdcff0 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go @@ -138,7 +138,7 @@ func setUp(t *testing.T) (Config, *assert.Assertions) { if clientset == nil { t.Fatal("unable to create fake client set") } - + config.EffectiveVersion = utilversion.NewEffectiveVersion("") config.OpenAPIConfig = DefaultOpenAPIConfig(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme())) config.OpenAPIConfig.Info.Version = "unversioned" config.OpenAPIV3Config = DefaultOpenAPIV3Config(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme())) @@ -460,8 +460,9 @@ func TestNotRestRoutesHaveAuth(t *testing.T) { config.EnableProfiling = true kubeVersion := fakeVersion() - config.Version = &kubeVersion - config.EffectiveVersion = utilversion.NewEffectiveVersion(kubeVersion.String()) + effectiveVersion := utilversion.NewEffectiveVersion(kubeVersion.String()) + effectiveVersion.Set(effectiveVersion.BinaryVersion().WithInfo(kubeVersion), effectiveVersion.EmulationVersion(), effectiveVersion.MinCompatibilityVersion()) + config.EffectiveVersion = effectiveVersion s, err := config.Complete(nil).New("test", NewEmptyDelegate()) if err != nil { diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go index 40feeaf62c1..dc87524efcf 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go @@ -278,7 +278,6 @@ func TestServerRunWithSNI(t *testing.T) { // launch server config := setUp(t) v := fakeVersion() - config.Version = &v config.EffectiveVersion = utilversion.NewEffectiveVersion(v.String()) config.EnableIndex = true diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go b/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go index 5d5cfccc7fd..7339d17df50 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go @@ -157,6 +157,7 @@ func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example } // If it was introduced after current compatibility version, don't use it + // skip the introduced check for test when currentVersion is 0.0 to test all apis if introduced, hasIntroduced := exampleOfGVK.(introducedInterface); hasIntroduced && (compatibilityVersion.Major() > 0 || compatibilityVersion.Minor() > 0) { // API resource lifecycles should be relative to k8s api version majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced() diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/registry.go b/staging/src/k8s.io/apiserver/pkg/util/version/registry.go index 589d42a1ade..170ac28045e 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/registry.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/registry.go @@ -59,12 +59,29 @@ var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGloba const ( DefaultKubeComponent = "kube" + + klogLevel = 2 ) +type VersionMapping func(from *version.Version) *version.Version + // ComponentGlobals stores the global variables for a component for easy access. type ComponentGlobals struct { effectiveVersion MutableEffectiveVersion featureGate featuregate.MutableVersionedFeatureGate + + // emulationVersionMapping contains the mapping from the emulation version of this component + // to the emulation version of another component. + emulationVersionMapping map[string]VersionMapping + // dependentEmulationVersion stores whether or not this component's EmulationVersion is dependent through mapping on another component. + // If true, the emulation version cannot be set from the flag, or version mapping from another component. + dependentEmulationVersion bool + // minCompatibilityVersionMapping contains the mapping from the min compatibility version of this component + // to the min compatibility version of another component. + minCompatibilityVersionMapping map[string]VersionMapping + // dependentMinCompatibilityVersion stores whether or not this component's MinCompatibilityVersion is dependent through mapping on another component + // If true, the min compatibility version cannot be set from the flag, or version mapping from another component. + dependentMinCompatibilityVersion bool } type ComponentGlobalsRegistry interface { @@ -75,9 +92,8 @@ type ComponentGlobalsRegistry interface { // Returns nil if the component is not registered. FeatureGateFor(component string) featuregate.FeatureGate // Register registers the EffectiveVersion and FeatureGate for a component. - // Overrides existing ComponentGlobals if it is already in the registry if override is true, - // otherwise returns error if the component is already registered. - Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error + // returns error if the component is already registered. + Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error // ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry. // Otherwise, the provided variables would be registered under the component, and the same variables would be returned. ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) @@ -85,25 +101,43 @@ type ComponentGlobalsRegistry interface { AddFlags(fs *pflag.FlagSet) // Set sets the flags for all global variables for all components registered. Set() error - // SetAllComponents calls the Validate() function for all the global variables for all components registered. + // Validate calls the Validate() function for all the global variables for all components registered. Validate() []error + // Reset removes all stored ComponentGlobals, configurations, and version mappings. + Reset() + // SetEmulationVersionMapping sets the mapping from the emulation version of one component + // to the emulation version of another component. + // Once set, the emulation version of the toComponent will be determined by the emulation version of the fromComponent, + // and cannot be set from cmd flags anymore. + // For a given component, its emulation version can only depend on one other component, no multiple dependency is allowed. + SetEmulationVersionMapping(fromComponent, toComponent string, f VersionMapping) error } type componentGlobalsRegistry struct { - componentGlobals map[string]ComponentGlobals + componentGlobals map[string]*ComponentGlobals mutex sync.RWMutex - // map of component name to emulation version set from the flag. - emulationVersionConfig cliflag.ConfigurationMap + // list of component name to emulation version set from the flag. + emulationVersionConfig []string // map of component name to the list of feature gates set from the flag. featureGatesConfig map[string][]string } -func NewComponentGlobalsRegistry() ComponentGlobalsRegistry { +func NewComponentGlobalsRegistry() *componentGlobalsRegistry { return &componentGlobalsRegistry{ - componentGlobals: make(map[string]ComponentGlobals), + componentGlobals: make(map[string]*ComponentGlobals), + emulationVersionConfig: nil, + featureGatesConfig: nil, } } +func (r *componentGlobalsRegistry) Reset() { + r.mutex.RLock() + defer r.mutex.RUnlock() + r.componentGlobals = make(map[string]*ComponentGlobals) + r.emulationVersionConfig = nil + r.featureGatesConfig = nil +} + func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion { r.mutex.RLock() defer r.mutex.RUnlock() @@ -124,8 +158,8 @@ func (r *componentGlobalsRegistry) FeatureGateFor(component string) featuregate. return globals.featureGate } -func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error { - if _, ok := r.componentGlobals[component]; ok && !override { +func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error { + if _, ok := r.componentGlobals[component]; ok { return fmt.Errorf("component globals of %s already registered", component) } if featureGate != nil { @@ -133,18 +167,23 @@ func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVer return err } } - c := ComponentGlobals{effectiveVersion: effectiveVersion, featureGate: featureGate} - r.componentGlobals[component] = c + c := ComponentGlobals{ + effectiveVersion: effectiveVersion, + featureGate: featureGate, + emulationVersionMapping: make(map[string]VersionMapping), + minCompatibilityVersionMapping: make(map[string]VersionMapping), + } + r.componentGlobals[component] = &c return nil } -func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error { +func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error { if effectiveVersion == nil { return fmt.Errorf("cannot register nil effectiveVersion") } r.mutex.Lock() defer r.mutex.Unlock() - return r.unsafeRegister(component, effectiveVersion, featureGate, override) + return r.unsafeRegister(component, effectiveVersion, featureGate) } func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) { @@ -154,13 +193,11 @@ func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, if ok { return globals.effectiveVersion, globals.featureGate } - utilruntime.Must(r.unsafeRegister(component, effectiveVersion, featureGate, false)) + utilruntime.Must(r.unsafeRegister(component, effectiveVersion, featureGate)) return effectiveVersion, featureGate } -func (r *componentGlobalsRegistry) knownFeatures() []string { - r.mutex.Lock() - defer r.mutex.Unlock() +func (r *componentGlobalsRegistry) unsafeKnownFeatures() []string { var known []string for component, globals := range r.componentGlobals { if globals.featureGate == nil { @@ -174,18 +211,22 @@ func (r *componentGlobalsRegistry) knownFeatures() []string { return known } -func (r *componentGlobalsRegistry) versionFlagOptions(isEmulation bool) []string { - r.mutex.Lock() - defer r.mutex.Unlock() +func (r *componentGlobalsRegistry) unsafeVersionFlagOptions(isEmulation bool) []string { var vs []string for component, globals := range r.componentGlobals { binaryVer := globals.effectiveVersion.BinaryVersion() if isEmulation { + if globals.dependentEmulationVersion { + continue + } // emulated version could be between binaryMajor.{binaryMinor} and binaryMajor.{binaryMinor} // TODO: change to binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} in 1.32 vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component, binaryVer.SubtractMinor(0).String(), binaryVer.String(), globals.effectiveVersion.EmulationVersion().String())) } else { + if globals.dependentMinCompatibilityVersion { + continue + } // min compatibility version could be between binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component, binaryVer.SubtractMinor(1).String(), binaryVer.String(), globals.effectiveVersion.MinCompatibilityVersion().String())) @@ -200,51 +241,133 @@ func (r *componentGlobalsRegistry) AddFlags(fs *pflag.FlagSet) { return } r.mutex.Lock() + defer r.mutex.Unlock() for _, globals := range r.componentGlobals { if globals.featureGate != nil { globals.featureGate.Close() } } - r.emulationVersionConfig = make(cliflag.ConfigurationMap) + if r.emulationVersionConfig != nil || r.featureGatesConfig != nil { + klog.Warning("calling componentGlobalsRegistry.AddFlags more than once, the registry will be set by the latest flags") + } + r.emulationVersionConfig = []string{} r.featureGatesConfig = make(map[string][]string) - r.mutex.Unlock() - fs.Var(&r.emulationVersionConfig, "emulated-version", ""+ + fs.StringSliceVar(&r.emulationVersionConfig, "emulated-version", r.emulationVersionConfig, ""+ "The versions different components emulate their capabilities (APIs, features, ...) of.\n"+ "If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+ - "Version format could only be major.minor, for example: '--emulated-version=wardle=1.2,kube=1.31'. Options are:\n"+strings.Join(r.versionFlagOptions(true), "\n")) + "Version format could only be major.minor, for example: '--emulated-version=wardle=1.2,kube=1.31'. Options are:\n"+strings.Join(r.unsafeVersionFlagOptions(true), "\n")+ + "If the component is not specified, defaults to \"kube\"") fs.Var(cliflag.NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(&r.featureGatesConfig), "feature-gates", "Comma-separated list of component:key=value pairs that describe feature gates for alpha/experimental features of different components.\n"+ "If the component is not specified, defaults to \"kube\". This flag can be repeatedly invoked. For example: --feature-gates 'wardle:featureA=true,wardle:featureB=false' --feature-gates 'kube:featureC=true'"+ - "Options are:\n"+strings.Join(r.knownFeatures(), "\n")) + "Options are:\n"+strings.Join(r.unsafeKnownFeatures(), "\n")) +} + +type componentVersion struct { + component string + ver *version.Version +} + +// getFullEmulationVersionConfig expands the given version config with version registered version mapping, +// and returns the map of component to Version. +func (r *componentGlobalsRegistry) getFullEmulationVersionConfig( + versionConfigMap map[string]*version.Version) (map[string]*version.Version, error) { + result := map[string]*version.Version{} + setQueue := []componentVersion{} + for comp, ver := range versionConfigMap { + if _, ok := r.componentGlobals[comp]; !ok { + return result, fmt.Errorf("component not registered: %s", comp) + } + klog.V(klogLevel).Infof("setting version %s=%s", comp, ver.String()) + setQueue = append(setQueue, componentVersion{comp, ver}) + } + for len(setQueue) > 0 { + cv := setQueue[0] + if _, visited := result[cv.component]; visited { + return result, fmt.Errorf("setting version of %s more than once, probably version mapping loop", cv.component) + } + setQueue = setQueue[1:] + result[cv.component] = cv.ver + for toComp, f := range r.componentGlobals[cv.component].emulationVersionMapping { + toVer := f(cv.ver) + if toVer == nil { + return result, fmt.Errorf("got nil version from mapping of %s=%s to component:%s", cv.component, cv.ver.String(), toComp) + } + klog.V(klogLevel).Infof("setting version %s=%s from version mapping of %s=%s", toComp, toVer.String(), cv.component, cv.ver.String()) + setQueue = append(setQueue, componentVersion{toComp, toVer}) + } + } + return result, nil +} + +func toVersionMap(versionConfig []string) (map[string]*version.Version, error) { + m := map[string]*version.Version{} + for _, compVer := range versionConfig { + // default to "kube" of component is not specified + k := "kube" + v := compVer + if strings.Contains(compVer, "=") { + arr := strings.SplitN(compVer, "=", 2) + if len(arr) != 2 { + return m, fmt.Errorf("malformed pair, expect string=string") + } + k = strings.TrimSpace(arr[0]) + v = strings.TrimSpace(arr[1]) + } + ver, err := version.Parse(v) + if err != nil { + return m, err + } + if ver.Patch() != 0 { + return m, fmt.Errorf("patch version not allowed, got: %s=%s", k, ver.String()) + } + if existingVer, ok := m[k]; ok { + return m, fmt.Errorf("duplicate version flag, %s=%s and %s=%s", k, existingVer.String(), k, ver.String()) + } + m[k] = ver + } + return m, nil } func (r *componentGlobalsRegistry) Set() error { r.mutex.Lock() defer r.mutex.Unlock() - for comp, emuVer := range r.emulationVersionConfig { + emulationVersionConfigMap, err := toVersionMap(r.emulationVersionConfig) + if err != nil { + return err + } + for comp := range emulationVersionConfigMap { if _, ok := r.componentGlobals[comp]; !ok { return fmt.Errorf("component not registered: %s", comp) } - klog.V(2).Infof("setting %s:emulation version to %s\n", comp, emuVer) - v, err := version.Parse(emuVer) - if err != nil { - return err + // only components without any dependencies can be set from the flag. + if r.componentGlobals[comp].dependentEmulationVersion { + return fmt.Errorf("EmulationVersion of %s is set by mapping, cannot set it by flag", comp) + } + } + if emulationVersions, err := r.getFullEmulationVersionConfig(emulationVersionConfigMap); err != nil { + return err + } else { + for comp, ver := range emulationVersions { + r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(ver) } - r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(v) } // Set feature gate emulation version before setting feature gate flag values. for comp, globals := range r.componentGlobals { if globals.featureGate == nil { continue } - klog.V(2).Infof("setting %s:feature gate emulation version to %s\n", comp, globals.effectiveVersion.EmulationVersion().String()) + klog.V(klogLevel).Infof("setting %s:feature gate emulation version to %s", comp, globals.effectiveVersion.EmulationVersion().String()) if err := globals.featureGate.SetEmulationVersion(globals.effectiveVersion.EmulationVersion()); err != nil { return err } } for comp, fg := range r.featureGatesConfig { if comp == "" { + if _, ok := r.featureGatesConfig[DefaultKubeComponent]; ok { + return fmt.Errorf("set kube feature gates with default empty prefix or kube: prefix consistently, do not mix use") + } comp = DefaultKubeComponent } if _, ok := r.componentGlobals[comp]; !ok { @@ -255,7 +378,7 @@ func (r *componentGlobalsRegistry) Set() error { return fmt.Errorf("component featureGate not registered: %s", comp) } flagVal := strings.Join(fg, ",") - klog.V(2).Infof("setting %s:feature-gates=%s\n", comp, flagVal) + klog.V(klogLevel).Infof("setting %s:feature-gates=%s", comp, flagVal) if err := featureGate.Set(flagVal); err != nil { return err } @@ -275,3 +398,39 @@ func (r *componentGlobalsRegistry) Validate() []error { } return errs } + +func (r *componentGlobalsRegistry) SetEmulationVersionMapping(fromComponent, toComponent string, f VersionMapping) error { + if f == nil { + return nil + } + klog.V(klogLevel).Infof("setting EmulationVersion mapping from %s to %s", fromComponent, toComponent) + r.mutex.Lock() + defer r.mutex.Unlock() + if _, ok := r.componentGlobals[fromComponent]; !ok { + return fmt.Errorf("component not registered: %s", fromComponent) + } + if _, ok := r.componentGlobals[toComponent]; !ok { + return fmt.Errorf("component not registered: %s", toComponent) + } + // check multiple dependency + if r.componentGlobals[toComponent].dependentEmulationVersion { + return fmt.Errorf("mapping of %s already exists from another component", toComponent) + } + r.componentGlobals[toComponent].dependentEmulationVersion = true + + versionMapping := r.componentGlobals[fromComponent].emulationVersionMapping + if _, ok := versionMapping[toComponent]; ok { + return fmt.Errorf("EmulationVersion from %s to %s already exists", fromComponent, toComponent) + } + versionMapping[toComponent] = f + klog.V(klogLevel).Infof("setting the default EmulationVersion of %s based on mapping from the default EmulationVersion of %s", fromComponent, toComponent) + defaultFromVersion := r.componentGlobals[fromComponent].effectiveVersion.EmulationVersion() + emulationVersions, err := r.getFullEmulationVersionConfig(map[string]*version.Version{fromComponent: defaultFromVersion}) + if err != nil { + return err + } + for comp, ver := range emulationVersions { + r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(ver) + } + return nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go b/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go index 68625e526b0..1badd5344d2 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go @@ -22,8 +22,8 @@ import ( "testing" "github.com/spf13/pflag" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/version" - cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/featuregate" ) @@ -39,30 +39,23 @@ func TestEffectiveVersionRegistry(t *testing.T) { if r.EffectiveVersionFor(testComponent) != nil { t.Fatalf("expected nil EffectiveVersion initially") } - if err := r.Register(testComponent, ver1, nil, false); err != nil { + if err := r.Register(testComponent, ver1, nil); err != nil { t.Fatalf("expected no error to register new component, but got err: %v", err) } if !r.EffectiveVersionFor(testComponent).EqualTo(ver1) { t.Fatalf("expected EffectiveVersionFor to return the version registered") } // overwrite - if err := r.Register(testComponent, ver2, nil, false); err == nil { + if err := r.Register(testComponent, ver2, nil); err == nil { t.Fatalf("expected error to register existing component when override is false") } - if err := r.Register(testComponent, ver2, nil, true); err != nil { - t.Fatalf("expected no error to overriding existing component, but got err: %v", err) - } - if !r.EffectiveVersionFor(testComponent).EqualTo(ver2) { + if !r.EffectiveVersionFor(testComponent).EqualTo(ver1) { t.Fatalf("expected EffectiveVersionFor to return the version overridden") } } func testRegistry(t *testing.T) *componentGlobalsRegistry { - r := componentGlobalsRegistry{ - componentGlobals: map[string]ComponentGlobals{}, - emulationVersionConfig: make(cliflag.ConfigurationMap), - featureGatesConfig: make(map[string][]string), - } + r := NewComponentGlobalsRegistry() verKube := NewEffectiveVersion("1.31") fgKube := featuregate.NewVersionedFeatureGate(version.MustParse("0.0")) err := fgKube.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ @@ -102,19 +95,35 @@ func testRegistry(t *testing.T) *componentGlobalsRegistry { if err != nil { t.Fatal(err) } - _ = r.Register(DefaultKubeComponent, verKube, fgKube, true) - _ = r.Register(testComponent, verTest, fgTest, true) - return &r + utilruntime.Must(r.Register(DefaultKubeComponent, verKube, fgKube)) + utilruntime.Must(r.Register(testComponent, verTest, fgTest)) + return r } func TestVersionFlagOptions(t *testing.T) { r := testRegistry(t) - emuVers := strings.Join(r.versionFlagOptions(true), "\n") + emuVers := strings.Join(r.unsafeVersionFlagOptions(true), "\n") expectedEmuVers := "kube=1.31..1.31 (default=1.31)\ntest=2.8..2.8 (default=2.8)" if emuVers != expectedEmuVers { t.Errorf("wanted emulation version flag options to be: %s, got %s", expectedEmuVers, emuVers) } - minCompVers := strings.Join(r.versionFlagOptions(false), "\n") + minCompVers := strings.Join(r.unsafeVersionFlagOptions(false), "\n") + expectedMinCompVers := "kube=1.30..1.31 (default=1.30)\ntest=2.7..2.8 (default=2.7)" + if minCompVers != expectedMinCompVers { + t.Errorf("wanted min compatibility version flag options to be: %s, got %s", expectedMinCompVers, minCompVers) + } +} + +func TestVersionFlagOptionsWithMapping(t *testing.T) { + r := testRegistry(t) + utilruntime.Must(r.SetEmulationVersionMapping(testComponent, DefaultKubeComponent, + func(from *version.Version) *version.Version { return from.OffsetMinor(3) })) + emuVers := strings.Join(r.unsafeVersionFlagOptions(true), "\n") + expectedEmuVers := "test=2.8..2.8 (default=2.8)" + if emuVers != expectedEmuVers { + t.Errorf("wanted emulation version flag options to be: %s, got %s", expectedEmuVers, emuVers) + } + minCompVers := strings.Join(r.unsafeVersionFlagOptions(false), "\n") expectedMinCompVers := "kube=1.30..1.31 (default=1.30)\ntest=2.7..2.8 (default=2.7)" if minCompVers != expectedMinCompVers { t.Errorf("wanted min compatibility version flag options to be: %s, got %s", expectedMinCompVers, minCompVers) @@ -123,7 +132,7 @@ func TestVersionFlagOptions(t *testing.T) { func TestVersionedFeatureGateFlag(t *testing.T) { r := testRegistry(t) - known := strings.Join(r.knownFeatures(), "\n") + known := strings.Join(r.unsafeKnownFeatures(), "\n") expectedKnown := "kube:AllAlpha=true|false (ALPHA - default=false)\n" + "kube:AllBeta=true|false (BETA - default=false)\n" + "kube:commonC=true|false (BETA - default=true)\n" + @@ -140,86 +149,127 @@ func TestVersionedFeatureGateFlag(t *testing.T) { func TestFlags(t *testing.T) { tests := []struct { name string - emulationVersionFlag string - featureGatesFlag string + flags []string parseError string - expectedKubeEmulationVersion *version.Version - expectedTestEmulationVersion *version.Version + expectedKubeEmulationVersion string + expectedTestEmulationVersion string expectedKubeFeatureValues map[featuregate.Feature]bool expectedTestFeatureValues map[featuregate.Feature]bool }{ { name: "setting kube emulation version", - emulationVersionFlag: "kube=1.30", - expectedKubeEmulationVersion: version.MajorMinor(1, 30), + flags: []string{"--emulated-version=kube=1.30"}, + expectedKubeEmulationVersion: "1.30", }, { - name: "setting kube emulation version, prefix v ok", - emulationVersionFlag: "kube=v1.30", - expectedKubeEmulationVersion: version.MajorMinor(1, 30), + name: "setting kube emulation version twice", + flags: []string{ + "--emulated-version=kube=1.30", + "--emulated-version=kube=1.32", + }, + parseError: "duplicate version flag, kube=1.30 and kube=1.32", + }, + { + name: "prefix v ok", + flags: []string{"--emulated-version=kube=v1.30"}, + expectedKubeEmulationVersion: "1.30", + }, + { + name: "patch version not ok", + flags: []string{"--emulated-version=kube=1.30.2"}, + parseError: "patch version not allowed, got: kube=1.30.2", }, { name: "setting test emulation version", - emulationVersionFlag: "test=2.7", - expectedKubeEmulationVersion: version.MajorMinor(1, 31), - expectedTestEmulationVersion: version.MajorMinor(2, 7), + flags: []string{"--emulated-version=test=2.7"}, + expectedKubeEmulationVersion: "1.31", + expectedTestEmulationVersion: "2.7", }, { - name: "version missing component", - emulationVersionFlag: "1.31", - parseError: "component not registered: 1.31", + name: "version missing component default to kube", + flags: []string{"--emulated-version=1.30"}, + expectedKubeEmulationVersion: "1.30", }, { - name: "version unregistered component", - emulationVersionFlag: "test3=1.31", - parseError: "component not registered: test3", + name: "version missing component default to kube with duplicate", + flags: []string{"--emulated-version=1.30", "--emulated-version=kube=1.30"}, + parseError: "duplicate version flag, kube=1.30 and kube=1.30", }, { - name: "invalid version", - emulationVersionFlag: "test=1.foo", - parseError: "illegal version string \"1.foo\"", + name: "version unregistered component", + flags: []string{"--emulated-version=test3=1.31"}, + parseError: "component not registered: test3", }, { - name: "setting test feature flag", - emulationVersionFlag: "test=2.7", - featureGatesFlag: "test:testA=true", - expectedKubeEmulationVersion: version.MajorMinor(1, 31), - expectedTestEmulationVersion: version.MajorMinor(2, 7), + name: "invalid version", + flags: []string{"--emulated-version=test=1.foo"}, + parseError: "illegal version string \"1.foo\"", + }, + { + name: "setting test feature flag", + flags: []string{ + "--emulated-version=test=2.7", + "--feature-gates=test:testA=true", + }, + expectedKubeEmulationVersion: "1.31", + expectedTestEmulationVersion: "2.7", expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": true, "kubeB": false, "commonC": true}, expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": true, "testB": false, "commonC": false}, }, { - name: "setting future test feature flag", - emulationVersionFlag: "test=2.7", - featureGatesFlag: "test:testA=true,test:testB=true", - parseError: "cannot set feature gate testB to true, feature is PreAlpha at emulated version 2.7", + name: "setting future test feature flag", + flags: []string{ + "--emulated-version=test=2.7", + "--feature-gates=test:testA=true,test:testB=true", + }, + parseError: "cannot set feature gate testB to true, feature is PreAlpha at emulated version 2.7", }, { - name: "setting kube feature flag", - emulationVersionFlag: "test=2.7,kube=1.30", - featureGatesFlag: "test:commonC=true,commonC=false,kube:kubeB=true", - expectedKubeEmulationVersion: version.MajorMinor(1, 30), - expectedTestEmulationVersion: version.MajorMinor(2, 7), + name: "setting kube feature flag", + flags: []string{ + "--emulated-version=test=2.7", + "--emulated-version=kube=1.30", + "--feature-gates=kubeB=false,test:commonC=true", + "--feature-gates=commonC=false,kubeB=true", + }, + expectedKubeEmulationVersion: "1.30", + expectedTestEmulationVersion: "2.7", expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": false, "kubeB": true, "commonC": false}, expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": false, "testB": false, "commonC": true}, }, { - name: "setting locked kube feature flag", - emulationVersionFlag: "test=2.7", - featureGatesFlag: "kubeA=false", - parseError: "cannot set feature gate kubeA to false, feature is locked to true", + name: "setting kube feature flag with different prefix", + flags: []string{ + "--emulated-version=test=2.7", + "--emulated-version=kube=1.30", + "--feature-gates=kube:kubeB=false,test:commonC=true", + "--feature-gates=commonC=false,kubeB=true", + }, + parseError: "set kube feature gates with default empty prefix or kube: prefix consistently, do not mix use", }, { - name: "setting unknown test feature flag", - emulationVersionFlag: "test=2.7", - featureGatesFlag: "test:testD=true", - parseError: "unrecognized feature gate: testD", + name: "setting locked kube feature flag", + flags: []string{ + "--emulated-version=test=2.7", + "--feature-gates=kubeA=false", + }, + parseError: "cannot set feature gate kubeA to false, feature is locked to true", }, { - name: "setting unknown component feature flag", - emulationVersionFlag: "test=2.7", - featureGatesFlag: "test3:commonC=true", - parseError: "component not registered: test3", + name: "setting unknown test feature flag", + flags: []string{ + "--emulated-version=test=2.7", + "--feature-gates=test:testD=true", + }, + parseError: "unrecognized feature gate: testD", + }, + { + name: "setting unknown component feature flag", + flags: []string{ + "--emulated-version=test=2.7", + "--feature-gates=test3:commonC=true", + }, + parseError: "component not registered: test3", }, } for i, test := range tests { @@ -227,9 +277,7 @@ func TestFlags(t *testing.T) { fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError) r := testRegistry(t) r.AddFlags(fs) - - err := fs.Parse([]string{fmt.Sprintf("--emulated-version=%s", test.emulationVersionFlag), - fmt.Sprintf("--feature-gates=%s", test.featureGatesFlag)}) + err := fs.Parse(test.flags) if err == nil { err = r.Set() } @@ -242,19 +290,11 @@ func TestFlags(t *testing.T) { if err != nil { t.Fatalf("%d: Parse() expected: nil, got: %v", i, err) } - if test.expectedKubeEmulationVersion != nil { - v := r.EffectiveVersionFor("kube").EmulationVersion() - if !v.EqualTo(test.expectedKubeEmulationVersion) { - t.Fatalf("%d: EmulationVersion expected: %s, got: %s", i, test.expectedKubeEmulationVersion.String(), v.String()) - return - } + if len(test.expectedKubeEmulationVersion) > 0 { + assertVersionEqualTo(t, r.EffectiveVersionFor(DefaultKubeComponent).EmulationVersion(), test.expectedKubeEmulationVersion) } - if test.expectedTestEmulationVersion != nil { - v := r.EffectiveVersionFor("test").EmulationVersion() - if !v.EqualTo(test.expectedTestEmulationVersion) { - t.Fatalf("%d: EmulationVersion expected: %s, got: %s", i, test.expectedTestEmulationVersion.String(), v.String()) - return - } + if len(test.expectedTestEmulationVersion) > 0 { + assertVersionEqualTo(t, r.EffectiveVersionFor(testComponent).EmulationVersion(), test.expectedTestEmulationVersion) } for f, v := range test.expectedKubeFeatureValues { if r.FeatureGateFor(DefaultKubeComponent).Enabled(f) != v { @@ -269,3 +309,110 @@ func TestFlags(t *testing.T) { }) } } + +func TestVersionMapping(t *testing.T) { + r := NewComponentGlobalsRegistry() + ver1 := NewEffectiveVersion("0.58") + ver2 := NewEffectiveVersion("1.28") + ver3 := NewEffectiveVersion("2.10") + + utilruntime.Must(r.Register("test1", ver1, nil)) + utilruntime.Must(r.Register("test2", ver2, nil)) + utilruntime.Must(r.Register("test3", ver3, nil)) + + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.10") + + utilruntime.Must(r.SetEmulationVersionMapping("test2", "test3", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()+1, from.Minor()-19) + })) + utilruntime.Must(r.SetEmulationVersionMapping("test1", "test2", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()+1, from.Minor()-28) + })) + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.30") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.11") + + fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError) + r.AddFlags(fs) + + if err := fs.Parse([]string{fmt.Sprintf("--emulated-version=%s", "test1=0.56")}); err != nil { + t.Fatal(err) + return + } + if err := r.Set(); err != nil { + t.Fatal(err) + return + } + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.56") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.09") +} + +func TestVersionMappingWithMultipleDependency(t *testing.T) { + r := NewComponentGlobalsRegistry() + ver1 := NewEffectiveVersion("0.58") + ver2 := NewEffectiveVersion("1.28") + ver3 := NewEffectiveVersion("2.10") + + utilruntime.Must(r.Register("test1", ver1, nil)) + utilruntime.Must(r.Register("test2", ver2, nil)) + utilruntime.Must(r.Register("test3", ver3, nil)) + + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.10") + + utilruntime.Must(r.SetEmulationVersionMapping("test1", "test2", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()+1, from.Minor()-28) + })) + err := r.SetEmulationVersionMapping("test3", "test2", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()-1, from.Minor()+19) + }) + if err == nil { + t.Errorf("expect error when setting 2nd mapping to test2") + } +} + +func TestVersionMappingWithCyclicDependency(t *testing.T) { + r := NewComponentGlobalsRegistry() + ver1 := NewEffectiveVersion("0.58") + ver2 := NewEffectiveVersion("1.28") + ver3 := NewEffectiveVersion("2.10") + + utilruntime.Must(r.Register("test1", ver1, nil)) + utilruntime.Must(r.Register("test2", ver2, nil)) + utilruntime.Must(r.Register("test3", ver3, nil)) + + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.10") + + utilruntime.Must(r.SetEmulationVersionMapping("test1", "test2", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()+1, from.Minor()-28) + })) + utilruntime.Must(r.SetEmulationVersionMapping("test2", "test3", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()+1, from.Minor()-19) + })) + err := r.SetEmulationVersionMapping("test3", "test1", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()-2, from.Minor()+48) + }) + if err == nil { + t.Errorf("expect cyclic version mapping error") + } +} + +func assertVersionEqualTo(t *testing.T, ver *version.Version, expectedVer string) { + if ver.EqualTo(version.MustParse(expectedVer)) { + return + } + t.Errorf("expected: %s, got %s", expectedVer, ver.String()) +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/version.go b/staging/src/k8s.io/apiserver/pkg/util/version/version.go index fce927c6c61..1596aef389b 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/version.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/version.go @@ -18,10 +18,8 @@ package version import ( "fmt" - "strings" "sync/atomic" - "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/version" baseversion "k8s.io/component-base/version" ) @@ -40,37 +38,6 @@ type MutableEffectiveVersion interface { Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) SetEmulationVersion(emulationVersion *version.Version) SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) - // AddFlags adds the "{prefix}-emulated-version" to the flagset. - AddFlags(fs *pflag.FlagSet, prefix string) -} - -type VersionVar struct { - val atomic.Pointer[version.Version] -} - -// Set sets the flag value -func (v *VersionVar) Set(s string) error { - components := strings.Split(s, ".") - if len(components) != 2 { - return fmt.Errorf("version %s is not in the format of major.minor", s) - } - ver, err := version.ParseGeneric(s) - if err != nil { - return err - } - v.val.Store(ver) - return nil -} - -// String returns the flag value -func (v *VersionVar) String() string { - ver := v.val.Load() - return ver.String() -} - -// Type gets the flag type -func (v *VersionVar) Type() string { - return "version" } type effectiveVersion struct { @@ -78,9 +45,9 @@ type effectiveVersion struct { // If the emulationVersion is set by the users, it could only contain major and minor versions. // In tests, emulationVersion could be the same as the binary version, or set directly, // which can have "alpha" as pre-release to continue serving expired apis while we clean up the test. - emulationVersion VersionVar + emulationVersion atomic.Pointer[version.Version] // minCompatibilityVersion could only contain major and minor versions. - minCompatibilityVersion VersionVar + minCompatibilityVersion atomic.Pointer[version.Version] } func (m *effectiveVersion) BinaryVersion() *version.Version { @@ -88,13 +55,17 @@ func (m *effectiveVersion) BinaryVersion() *version.Version { } func (m *effectiveVersion) EmulationVersion() *version.Version { - // Emulation version can have "alpha" as pre-release to continue serving expired apis while we clean up the test. - // The pre-release should not be accessible to the users. - return m.emulationVersion.val.Load().WithPreRelease(m.BinaryVersion().PreRelease()) + ver := m.emulationVersion.Load() + if ver != nil { + // Emulation version can have "alpha" as pre-release to continue serving expired apis while we clean up the test. + // The pre-release should not be accessible to the users. + return ver.WithPreRelease(m.BinaryVersion().PreRelease()) + } + return ver } func (m *effectiveVersion) MinCompatibilityVersion() *version.Version { - return m.minCompatibilityVersion.val.Load() + return m.minCompatibilityVersion.Load() } func (m *effectiveVersion) EqualTo(other EffectiveVersion) bool { @@ -109,26 +80,33 @@ func (m *effectiveVersion) String() string { m.BinaryVersion().String(), m.EmulationVersion().String(), m.MinCompatibilityVersion().String()) } +func majorMinor(ver *version.Version) *version.Version { + if ver == nil { + return ver + } + return version.MajorMinor(ver.Major(), ver.Minor()) +} + func (m *effectiveVersion) Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) { m.binaryVersion.Store(binaryVersion) - m.emulationVersion.val.Store(version.MajorMinor(emulationVersion.Major(), emulationVersion.Minor())) - m.minCompatibilityVersion.val.Store(version.MajorMinor(minCompatibilityVersion.Major(), minCompatibilityVersion.Minor())) + m.emulationVersion.Store(majorMinor(emulationVersion)) + m.minCompatibilityVersion.Store(majorMinor(minCompatibilityVersion)) } func (m *effectiveVersion) SetEmulationVersion(emulationVersion *version.Version) { - m.emulationVersion.val.Store(version.MajorMinor(emulationVersion.Major(), emulationVersion.Minor())) + m.emulationVersion.Store(majorMinor(emulationVersion)) } func (m *effectiveVersion) SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) { - m.minCompatibilityVersion.val.Store(version.MajorMinor(minCompatibilityVersion.Major(), minCompatibilityVersion.Minor())) + m.minCompatibilityVersion.Store(majorMinor(minCompatibilityVersion)) } func (m *effectiveVersion) Validate() []error { var errs []error // Validate only checks the major and minor versions. binaryVersion := m.binaryVersion.Load().WithPatch(0) - emulationVersion := m.emulationVersion.val.Load() - minCompatibilityVersion := m.minCompatibilityVersion.val.Load() + emulationVersion := m.emulationVersion.Load() + minCompatibilityVersion := m.minCompatibilityVersion.Load() // emulationVersion can only be 1.{binaryMinor-1}...1.{binaryMinor}. maxEmuVer := binaryVersion @@ -151,45 +129,36 @@ func (m *effectiveVersion) Validate() []error { return errs } -// AddFlags adds the "{prefix}-emulated-version" to the flagset. -func (m *effectiveVersion) AddFlags(fs *pflag.FlagSet, prefix string) { - if m == nil { - return - } - if len(prefix) > 0 && !strings.HasSuffix(prefix, "-") { - prefix += "-" - } - fs.Var(&m.emulationVersion, prefix+"emulated-version", ""+ - "The version the K8s component emulates its capabilities (APIs, features, ...) of.\n"+ - "If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+ - "Any capabilities present in the binary version that were introduced after the emulated version will be unavailable and any capabilities removed after the emulated version will be available.\n"+ - "This flag applies only to component capabilities, and does not disable bug fixes and performance improvements present in the binary version.\n"+ - "Defaults to the binary version. The value should be between 1.{binaryMinorVersion-1} and 1.{binaryMinorVersion}.\n"+ - "Format could only be major.minor") -} - -func NewEffectiveVersion(binaryVer string) MutableEffectiveVersion { +func newEffectiveVersion(binaryVersion *version.Version) MutableEffectiveVersion { effective := &effectiveVersion{} - binaryVersion := version.MustParse(binaryVer) compatVersion := binaryVersion.SubtractMinor(1) effective.Set(binaryVersion, binaryVersion, compatVersion) return effective } +func NewEffectiveVersion(binaryVer string) MutableEffectiveVersion { + if binaryVer == "" { + return &effectiveVersion{} + } + binaryVersion := version.MustParse(binaryVer) + return newEffectiveVersion(binaryVersion) +} + // DefaultBuildEffectiveVersion returns the MutableEffectiveVersion based on the // current build information. func DefaultBuildEffectiveVersion() MutableEffectiveVersion { verInfo := baseversion.Get() - ver := NewEffectiveVersion(verInfo.String()) - if ver.BinaryVersion().Major() == 0 && ver.BinaryVersion().Minor() == 0 { - ver = DefaultKubeEffectiveVersion() + binaryVersion := version.MustParse(verInfo.String()).WithInfo(verInfo) + if binaryVersion.Major() == 0 && binaryVersion.Minor() == 0 { + return DefaultKubeEffectiveVersion() } - return ver + return newEffectiveVersion(binaryVersion) } // DefaultKubeEffectiveVersion returns the MutableEffectiveVersion based on the // latest K8s release. // Should update for each minor release! func DefaultKubeEffectiveVersion() MutableEffectiveVersion { - return NewEffectiveVersion("1.31") + binaryVersion := version.MustParse("1.31").WithInfo(baseversion.Get()) + return newEffectiveVersion(binaryVersion) } diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go b/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go index 24db0318f25..784e3e4e186 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go @@ -17,11 +17,8 @@ limitations under the License. package version import ( - "fmt" - "strings" "testing" - "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/version" ) @@ -127,54 +124,3 @@ func TestValidate(t *testing.T) { }) } } - -func TestEffectiveVersionsFlag(t *testing.T) { - tests := []struct { - name string - emulationVersion string - expectedEmulationVersion *version.Version - parseError string - }{ - { - name: "major.minor ok", - emulationVersion: "1.30", - expectedEmulationVersion: version.MajorMinor(1, 30), - }, - { - name: "v prefix ok", - emulationVersion: "v1.30", - expectedEmulationVersion: version.MajorMinor(1, 30), - }, - { - name: "semantic version not ok", - emulationVersion: "1.30.1", - parseError: "version 1.30.1 is not in the format of major.minor", - }, - { - name: "invalid version", - emulationVersion: "1.foo", - parseError: "illegal version string", - }, - } - for i, test := range tests { - t.Run(test.name, func(t *testing.T) { - fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError) - effective := NewEffectiveVersion("1.30") - effective.AddFlags(fs, "test") - - err := fs.Parse([]string{fmt.Sprintf("--test-emulated-version=%s", test.emulationVersion)}) - if test.parseError != "" { - if !strings.Contains(err.Error(), test.parseError) { - t.Fatalf("%d: Parse() Expected %v, Got %v", i, test.parseError, err) - } - return - } - if err != nil { - t.Fatalf("%d: Parse() Expected nil, Got %v", i, err) - } - if !effective.EmulationVersion().EqualTo(test.expectedEmulationVersion) { - t.Errorf("%d: EmulationVersion Expected %s, Got %s", i, test.expectedEmulationVersion.String(), effective.EmulationVersion().String()) - } - }) - } -} diff --git a/staging/src/k8s.io/component-base/featuregate/feature_gate.go b/staging/src/k8s.io/component-base/featuregate/feature_gate.go index d164baf3878..680d92d3d83 100644 --- a/staging/src/k8s.io/component-base/featuregate/feature_gate.go +++ b/staging/src/k8s.io/component-base/featuregate/feature_gate.go @@ -127,6 +127,9 @@ type MutableFeatureGate interface { AddFlag(fs *pflag.FlagSet) // Close sets closed to true, and prevents subsequent calls to Add Close() + // OpenForModification sets closedForModification to false, and allows subsequent calls to SetEmulationVersion to change enabled features + // before the next Enabled is called. + OpenForModification() // Set parses and stores flag gates for known features // from a string like feature1=true,feature2=false,... Set(value string) error @@ -163,11 +166,25 @@ type MutableVersionedFeatureGate interface { // SetEmulationVersion overrides the emulationVersion of the feature gate. // Otherwise, the emulationVersion will be the same as the binary version. // If set, the feature defaults and availability will be as if the binary is at the emulated version. + // Returns error if the new emulationVersion will change the enablement state of a feature that has already been queried. + // If you have to use featureGate.Enabled before parsing the flags, call featureGate.OpenForModification following featureGate.Enabled. SetEmulationVersion(emulationVersion *version.Version) error // GetAll returns a copy of the map of known feature names to versioned feature specs. GetAllVersioned() map[Feature]VersionedSpecs // AddVersioned adds versioned feature specs to the featureGate. AddVersioned(features map[Feature]VersionedSpecs) error + // OverrideDefaultAtVersion sets a local override for the registered default value of a named + // feature for the prerelease lifecycle the given version is at. + // If the feature has not been previously registered (e.g. by a call to Add), + // has a locked default, or if the gate has already registered itself with a FlagSet, a non-nil + // error is returned. + // + // When two or more components consume a common feature, one component can override its + // default at runtime in order to adopt new defaults before or after the other + // components. For example, a new feature can be evaluated with a limited blast radius by + // overriding its default to true for a limited number of components without simultaneously + // changing its default for all consuming components. + OverrideDefaultAtVersion(name Feature, override bool, ver *version.Version) error } // MutableVersionedFeatureGateForTests is a feature gate interface that should only be used in tests. @@ -196,7 +213,14 @@ type featureGate struct { // while enabled keeps the values of all resolved features. enabledRaw atomic.Value // closed is set to true when AddFlag is called, and prevents subsequent calls to Add - closed bool + closed bool + // closedForModification is set to true when Enabled is called, and prevents subsequent calls to SetEmulationVersion to change the enabled features. + // TODO: after all feature gates have migrated to versioned feature gates, + // closedForModification should also prevents subsequent calls to Set and SetFromMap to change the enabled features + closedForModification atomic.Bool + // queriedFeatures stores all the features that have been queried through the Enabled interface. + // It is reset when closedForModification is reset. + queriedFeatures atomic.Value emulationVersion atomic.Pointer[version.Version] } @@ -205,8 +229,8 @@ func setUnsetAlphaGates(known map[Feature]VersionedSpecs, enabled map[Feature]bo if k == "AllAlpha" || k == "AllBeta" { continue } - currentVersion := getCurrentVersion(v, cVer) - if currentVersion.PreRelease == Alpha { + featureSpec := featureSpecAtEmulationVersion(v, cVer) + if featureSpec.PreRelease == Alpha { if _, found := enabled[k]; !found { enabled[k] = val } @@ -219,8 +243,8 @@ func setUnsetBetaGates(known map[Feature]VersionedSpecs, enabled map[Feature]boo if k == "AllAlpha" || k == "AllBeta" { continue } - currentVersion := getCurrentVersion(v, cVer) - if currentVersion.PreRelease == Beta { + featureSpec := featureSpecAtEmulationVersion(v, cVer) + if featureSpec.PreRelease == Beta { if _, found := enabled[k]; !found { enabled[k] = val } @@ -251,6 +275,7 @@ func NewVersionedFeatureGate(emulationVersion *version.Version) *featureGate { f.enabled.Store(map[Feature]bool{}) f.enabledRaw.Store(map[string]bool{}) f.emulationVersion.Store(emulationVersion) + f.queriedFeatures.Store(map[Feature]struct{}{}) klog.V(1).Infof("new feature gate with emulationVersion=%s", f.emulationVersion.Load().String()) return f } @@ -291,11 +316,11 @@ func (f *featureGate) Validate() []error { return []error{fmt.Errorf("cannot cast enabledRaw to map[string]bool")} } enabled := map[Feature]bool{} - return f.unsafeSetFromMap(enabled, m) + return f.unsafeSetFromMap(enabled, m, f.EmulationVersion()) } // unsafeSetFromMap stores flag gates for known features from a map[string]bool into an enabled map. -func (f *featureGate) unsafeSetFromMap(enabled map[Feature]bool, m map[string]bool) []error { +func (f *featureGate) unsafeSetFromMap(enabled map[Feature]bool, m map[string]bool, emulationVersion *version.Version) []error { var errs []error // Copy existing state known := map[Feature]VersionedSpecs{} @@ -312,26 +337,26 @@ func (f *featureGate) unsafeSetFromMap(enabled map[Feature]bool, m map[string]bo errs = append(errs, fmt.Errorf("unrecognized feature gate: %s", k)) return errs } - currentVersion := f.getCurrentVersion(versionedSpecs) - if currentVersion.LockToDefault && currentVersion.Default != v { - errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is locked to %v", k, v, currentVersion.Default)) + featureSpec := featureSpecAtEmulationVersion(versionedSpecs, emulationVersion) + if featureSpec.LockToDefault && featureSpec.Default != v { + errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is locked to %v", k, v, featureSpec.Default)) continue } // Handle "special" features like "all alpha gates" if fn, found := f.special[key]; found { - fn(known, enabled, v, f.emulationVersion.Load()) + fn(known, enabled, v, emulationVersion) enabled[key] = v continue } - if currentVersion.PreRelease == PreAlpha { - errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is PreAlpha at emulated version %s", k, v, f.EmulationVersion().String())) + if featureSpec.PreRelease == PreAlpha { + errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is PreAlpha at emulated version %s", k, v, emulationVersion.String())) continue } enabled[key] = v - if currentVersion.PreRelease == Deprecated { + if featureSpec.PreRelease == Deprecated { klog.Warningf("Setting deprecated feature gate %s=%t. It will be removed in a future release.", k, v) - } else if currentVersion.PreRelease == GA { + } else if featureSpec.PreRelease == GA { klog.Warningf("Setting GA feature gate %s=%t. It will be removed in a future release.", k, v) } } @@ -361,7 +386,7 @@ func (f *featureGate) SetFromMap(m map[string]bool) error { } f.enabledRaw.Store(enabledRaw) - errs := f.unsafeSetFromMap(enabled, enabledRaw) + errs := f.unsafeSetFromMap(enabled, enabledRaw, f.EmulationVersion()) if len(errs) == 0 { // Persist changes f.enabled.Store(enabled) @@ -429,6 +454,10 @@ func (f *featureGate) AddVersioned(features map[Feature]VersionedSpecs) error { } func (f *featureGate) OverrideDefault(name Feature, override bool) error { + return f.OverrideDefaultAtVersion(name, override, f.EmulationVersion()) +} + +func (f *featureGate) OverrideDefaultAtVersion(name Feature, override bool, ver *version.Version) error { f.lock.Lock() defer f.lock.Unlock() @@ -446,12 +475,12 @@ func (f *featureGate) OverrideDefault(name Feature, override bool) error { if !ok { return fmt.Errorf("cannot override default: feature %q is not registered", name) } - spec := f.getCurrentVersion(specs) + spec := featureSpecAtEmulationVersion(specs, ver) switch { case spec.LockToDefault: return fmt.Errorf("cannot override default: feature %q default is locked to %t", name, spec.Default) case spec.PreRelease == PreAlpha: - return fmt.Errorf("cannot override default: feature %q is not available before emulation version %s", name, f.EmulationVersion().String()) + return fmt.Errorf("cannot override default: feature %q is not available before version %s", name, ver.String()) case spec.PreRelease == Deprecated: klog.Warningf("Overriding default of deprecated feature gate %s=%t. It will be removed in a future release.", name, override) case spec.PreRelease == GA: @@ -469,12 +498,12 @@ func (f *featureGate) OverrideDefault(name Feature, override bool) error { func (f *featureGate) GetAll() map[Feature]FeatureSpec { retval := map[Feature]FeatureSpec{} for k, v := range f.GetAllVersioned() { - spec := f.getCurrentVersion(v) + spec := f.featureSpecAtEmulationVersion(v) if spec.PreRelease == PreAlpha { // The feature is not available at the emulation version. continue } - retval[k] = *f.getCurrentVersion(v) + retval[k] = *f.featureSpecAtEmulationVersion(v) } return retval } @@ -492,7 +521,6 @@ func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) err f.lock.Lock() defer f.lock.Unlock() klog.V(1).Infof("set feature gate emulationVersion to %s", emulationVersion.String()) - f.emulationVersion.Store(emulationVersion) // Copy existing state enabledRaw := map[string]bool{} @@ -501,10 +529,24 @@ func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) err } // enabled map should be reset whenever emulationVersion is changed. enabled := map[Feature]bool{} - errs := f.unsafeSetFromMap(enabled, enabledRaw) + errs := f.unsafeSetFromMap(enabled, enabledRaw, emulationVersion) + + if f.closedForModification.Load() { + queriedFeatures := f.queriedFeatures.Load().(map[Feature]struct{}) + known := f.known.Load().(map[Feature]VersionedSpecs) + for feature := range queriedFeatures { + newVal := featureEnabled(feature, enabled, known, emulationVersion) + oldVal := f.Enabled(feature) + // it is ok to modify emulation version if it does not result in feature enablemennt change for features that have already been queried. + if newVal != oldVal { + errs = append(errs, fmt.Errorf("SetEmulationVersion will change already queried feature:%s from %v to %v\ncall featureGate.OpenForModification() first to override", feature, oldVal, newVal)) + } + } + } if len(errs) == 0 { // Persist changes f.enabled.Store(enabled) + f.emulationVersion.Store(emulationVersion) } return utilerrors.NewAggregate(errs) } @@ -514,32 +556,49 @@ func (f *featureGate) EmulationVersion() *version.Version { } // FeatureSpec returns the FeatureSpec at the EmulationVersion if the key exists, an error otherwise. +// This is useful to keep multiple implementations of a feature based on the PreRelease or Version info. func (f *featureGate) FeatureSpec(key Feature) (FeatureSpec, error) { if v, ok := f.known.Load().(map[Feature]VersionedSpecs)[key]; ok { - currentVersion := f.getCurrentVersion(v) - return *currentVersion, nil + featureSpec := f.featureSpecAtEmulationVersion(v) + return *featureSpec, nil } return FeatureSpec{}, fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName) } -// Enabled returns true if the key is enabled. If the key is not known, this call will panic. -func (f *featureGate) Enabled(key Feature) bool { - // fallback to default behavior, since we don't have emulation version set - if v, ok := f.enabled.Load().(map[Feature]bool)[key]; ok { +func (f *featureGate) recordQueried(key Feature) { + queriedFeatures := map[Feature]struct{}{} + for k := range f.queriedFeatures.Load().(map[Feature]struct{}) { + queriedFeatures[k] = struct{}{} + } + queriedFeatures[key] = struct{}{} + f.queriedFeatures.Store(queriedFeatures) + f.closedForModification.Store(true) +} + +func featureEnabled(key Feature, enabled map[Feature]bool, known map[Feature]VersionedSpecs, emulationVersion *version.Version) bool { + // check explicitly set enabled list + if v, ok := enabled[key]; ok { return v } - if v, ok := f.known.Load().(map[Feature]VersionedSpecs)[key]; ok { - return f.getCurrentVersion(v).Default + if v, ok := known[key]; ok { + return featureSpecAtEmulationVersion(v, emulationVersion).Default } - panic(fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName)) + panic(fmt.Errorf("feature %q is not registered in FeatureGate", key)) } -func (f *featureGate) getCurrentVersion(v VersionedSpecs) *FeatureSpec { - return getCurrentVersion(v, f.EmulationVersion()) +// Enabled returns true if the key is enabled. If the key is not known, this call will panic. +func (f *featureGate) Enabled(key Feature) bool { + v := featureEnabled(key, f.enabled.Load().(map[Feature]bool), f.known.Load().(map[Feature]VersionedSpecs), f.EmulationVersion()) + f.recordQueried(key) + return v } -func getCurrentVersion(v VersionedSpecs, emulationVersion *version.Version) *FeatureSpec { +func (f *featureGate) featureSpecAtEmulationVersion(v VersionedSpecs) *FeatureSpec { + return featureSpecAtEmulationVersion(v, f.EmulationVersion()) +} + +func featureSpecAtEmulationVersion(v VersionedSpecs, emulationVersion *version.Version) *FeatureSpec { i := len(v) - 1 for ; i >= 0; i-- { if v[i].Version.GreaterThan(emulationVersion) { @@ -561,6 +620,18 @@ func (f *featureGate) Close() { f.lock.Unlock() } +// OpenForModification sets closedForModification to false, and allows subsequent calls to SetEmulationVersion to change enabled features +// before the next Enabled is called. +func (f *featureGate) OpenForModification() { + queriedFeatures := []Feature{} + for feature := range f.queriedFeatures.Load().(map[Feature]struct{}) { + queriedFeatures = append(queriedFeatures, feature) + } + klog.Warningf("open feature gate for modification after querying features: %v.", queriedFeatures) + f.closedForModification.Store(false) + f.queriedFeatures.Store(map[Feature]struct{}{}) +} + // AddFlag adds a flag for setting global feature gates to the specified FlagSet. func (f *featureGate) AddFlag(fs *pflag.FlagSet) { // TODO(mtaufen): Shouldn't we just close it on the first Set/SetFromMap instead? @@ -590,11 +661,11 @@ func (f *featureGate) KnownFeatures() []string { known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, v[0].PreRelease, v[0].Default)) continue } - currentV := f.getCurrentVersion(v) - if currentV.PreRelease == GA || currentV.PreRelease == Deprecated || currentV.PreRelease == PreAlpha { + featureSpec := f.featureSpecAtEmulationVersion(v) + if featureSpec.PreRelease == GA || featureSpec.PreRelease == Deprecated || featureSpec.PreRelease == PreAlpha { continue } - known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, currentV.PreRelease, currentV.Default)) + known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, featureSpec.PreRelease, featureSpec.Default)) } sort.Strings(known) return known @@ -620,7 +691,7 @@ func (f *featureGate) DeepCopy() MutableVersionedFeatureGate { // Construct a new featureGate around the copied state. // Note that specialFeatures is treated as immutable by convention, - // and we maintain the value of f.closed across the copy. + // and we maintain the value of f.closed across the copy, but resets closedForModification. fg := &featureGate{ special: specialFeatures, closed: f.closed, @@ -629,6 +700,7 @@ func (f *featureGate) DeepCopy() MutableVersionedFeatureGate { fg.known.Store(known) fg.enabled.Store(enabled) fg.enabledRaw.Store(enabledRaw) + fg.queriedFeatures.Store(map[Feature]struct{}{}) return fg } @@ -636,9 +708,15 @@ func (f *featureGate) DeepCopy() MutableVersionedFeatureGate { func (f *featureGate) Reset(m map[string]bool) { enabled := map[Feature]bool{} enabledRaw := map[string]bool{} + queriedFeatures := map[Feature]struct{}{} f.enabled.Store(enabled) f.enabledRaw.Store(enabledRaw) _ = f.SetFromMap(m) + f.closedForModification.Store(false) + f.queriedFeatures.Store(queriedFeatures) + f.lock.Lock() + defer f.lock.Unlock() + f.closed = false } func (f *featureGate) EnabledRawMap() map[string]bool { diff --git a/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go b/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go index 9156a024abe..85f99d25787 100644 --- a/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go +++ b/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go @@ -1280,6 +1280,48 @@ func TestVersionedFeatureGateOverrideDefault(t *testing.T) { } }) + t.Run("overrides at specific version take effect", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.AddVersioned(map[Feature]VersionedSpecs{ + "TestFeature1": { + {Version: version.MustParse("1.28"), Default: true}, + }, + "TestFeature2": { + {Version: version.MustParse("1.26"), Default: false}, + {Version: version.MustParse("1.29"), Default: false}, + }, + }); err != nil { + t.Fatal(err) + } + if f.OverrideDefaultAtVersion("TestFeature1", false, version.MustParse("1.27")) == nil { + t.Error("expected error when attempting to override the default for a feature not available at given version") + } + require.NoError(t, f.OverrideDefaultAtVersion("TestFeature2", true, version.MustParse("1.27"))) + if !f.Enabled("TestFeature1") { + t.Error("expected TestFeature1 to have effective default of true") + } + if !f.Enabled("TestFeature2") { + t.Error("expected TestFeature2 to have effective default of true") + } + f.OpenForModification() + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.29"))) + if !f.Enabled("TestFeature1") { + t.Error("expected TestFeature1 to have effective default of true") + } + if f.Enabled("TestFeature2") { + t.Error("expected TestFeature2 to have effective default of false") + } + f.OpenForModification() + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.26"))) + if f.Enabled("TestFeature1") { + t.Error("expected TestFeature1 to have effective default of false") + } + if !f.Enabled("TestFeature2") { + t.Error("expected TestFeature2 to have effective default of true") + } + }) + t.Run("overrides are preserved across deep copies", func(t *testing.T) { f := NewVersionedFeatureGate(version.MustParse("1.29")) require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) @@ -1433,7 +1475,7 @@ func TestVersionedFeatureGateOverrideDefault(t *testing.T) { }) } -func TestGetCurrentVersion(t *testing.T) { +func TestFeatureSpecAtEmulationVersion(t *testing.T) { specs := VersionedSpecs{{Version: version.MustParse("1.29"), Default: true, PreRelease: GA}, {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, {Version: version.MustParse("1.25"), Default: false, PreRelease: Alpha}, @@ -1469,11 +1511,41 @@ func TestGetCurrentVersion(t *testing.T) { }, } for i, test := range tests { - t.Run(fmt.Sprintf("getCurrentVersion for emulationVersion %s", test.cVersion), func(t *testing.T) { - result := getCurrentVersion(specs, version.MustParse(test.cVersion)) + t.Run(fmt.Sprintf("featureSpecAtEmulationVersion for emulationVersion %s", test.cVersion), func(t *testing.T) { + result := featureSpecAtEmulationVersion(specs, version.MustParse(test.cVersion)) if !reflect.DeepEqual(*result, test.expect) { - t.Errorf("%d: getCurrentVersion(, %s) Expected %v, Got %v", i, test.cVersion, test.expect, result) + t.Errorf("%d: featureSpecAtEmulationVersion(, %s) Expected %v, Got %v", i, test.cVersion, test.expect, result) } }) } } + +func TestOpenForModification(t *testing.T) { + const testBetaGate Feature = "testBetaGate" + f := NewVersionedFeatureGate(version.MustParse("1.29")) + + err := f.AddVersioned(map[Feature]VersionedSpecs{ + testBetaGate: { + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.26"), Default: false, PreRelease: Alpha}, + }, + }) + require.NoError(t, err) + + if f.Enabled(testBetaGate) != true { + t.Errorf("Expected true") + } + err = f.SetEmulationVersion(version.MustParse("1.28")) + if err == nil { + t.Fatalf("Expected error when SetEmulationVersion after querying features") + } + if f.Enabled(testBetaGate) != true { + t.Errorf("Expected true") + } + f.OpenForModification() + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if f.Enabled(testBetaGate) != false { + t.Errorf("Expected false at 1.28") + } +} diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go index 93ac2b14b31..8b2499a8b6c 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go @@ -38,7 +38,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/transport" "k8s.io/component-base/tracing" - "k8s.io/component-base/version" v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" v1helper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper" "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" @@ -185,8 +184,6 @@ func (cfg *Config) Complete() CompletedConfig { // the kube aggregator wires its own discovery mechanism // TODO eventually collapse this by extracting all of the discovery out c.GenericConfig.EnableDiscovery = false - version := version.Get() - c.GenericConfig.Version = &version return CompletedConfig{&c} } diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go index 6558efda6f1..e8307e72f04 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go @@ -17,17 +17,13 @@ limitations under the License. package apiserver import ( - "strconv" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" - utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/sample-apiserver/pkg/apis/wardle" "k8s.io/sample-apiserver/pkg/apis/wardle/install" wardleregistry "k8s.io/sample-apiserver/pkg/registry" @@ -94,11 +90,6 @@ func (cfg *Config) Complete() CompletedConfig { cfg.GenericConfig.Complete(), &cfg.ExtraConfig, } - wardleEffectiveVersion := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(WardleComponentName) - c.GenericConfig.Version = &version.Info{ - Major: strconv.Itoa(int(wardleEffectiveVersion.BinaryVersion().Major())), - Minor: strconv.Itoa(int(wardleEffectiveVersion.BinaryVersion().Minor())), - } return CompletedConfig{&c} } diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go index df497e450f2..6152d0b84ca 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go @@ -59,22 +59,14 @@ type WardleServerOptions struct { AlternateDNS []string } -func mapWardleEffectiveVersionToKubeEffectiveVersion(registry utilversion.ComponentGlobalsRegistry) error { - wardleVer := registry.EffectiveVersionFor(apiserver.WardleComponentName) - kubeVer := registry.EffectiveVersionFor(utilversion.DefaultKubeComponent).(utilversion.MutableEffectiveVersion) - // map from wardle emulation version to kube emulation version. - emulationVersionMap := map[string]string{ - "1.2": kubeVer.BinaryVersion().AddMinor(1).String(), - "1.1": kubeVer.BinaryVersion().String(), - "1.0": kubeVer.BinaryVersion().SubtractMinor(1).String(), +func wardleEmulationVersionToKubeEmulationVersion(ver *version.Version) *version.Version { + if ver.Major() != 1 { + return nil } - wardleEmulationVer := wardleVer.EmulationVersion() - if kubeEmulationVer, ok := emulationVersionMap[wardleEmulationVer.String()]; ok { - kubeVer.SetEmulationVersion(version.MustParse(kubeEmulationVer)) - } else { - return fmt.Errorf("cannot find mapping from wardle emulation version: %s to kube version", wardleVer.EmulationVersion().String()) - } - return nil + kubeVer := utilversion.DefaultKubeEffectiveVersion().BinaryVersion() + // "1.1" maps to kubeVer + offset := int(ver.Minor()) - 1 + return kubeVer.OffsetMinor(offset) } // NewWardleServerOptions returns a new WardleServerOptions @@ -100,11 +92,7 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti Short: "Launch a wardle API server", Long: "Launch a wardle API server", PersistentPreRunE: func(*cobra.Command, []string) error { - if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil { - return err - } - // convert wardle effective version to kube effective version to be used in generic api server, and set the generic api server feature gate. - return mapWardleEffectiveVersionToKubeEffectiveVersion(utilversion.DefaultComponentGlobalsRegistry) + return utilversion.DefaultComponentGlobalsRegistry.Set() }, RunE: func(c *cobra.Command, args []string) error { if err := o.Complete(); err != nil { @@ -133,10 +121,10 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti {Version: version.MustParse("1.0"), Default: false, PreRelease: featuregate.Alpha}, }, })) - utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate, false)) + utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate)) _, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) - + utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.SetEmulationVersionMapping(apiserver.WardleComponentName, utilversion.DefaultKubeComponent, wardleEmulationVersionToKubeEmulationVersion)) utilversion.DefaultComponentGlobalsRegistry.AddFlags(flags) return cmd @@ -189,8 +177,8 @@ func (o *WardleServerOptions) Config() (*apiserver.Config, error) { serverConfig.OpenAPIV3Config.Info.Title = "Wardle" serverConfig.OpenAPIV3Config.Info.Version = "0.1" - serverConfig.FeatureGate = utilversion.DefaultComponentGlobalsRegistry.FeatureGateFor(utilversion.DefaultKubeComponent) - serverConfig.EffectiveVersion = utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) + serverConfig.FeatureGate = utilversion.DefaultComponentGlobalsRegistry.FeatureGateFor(apiserver.WardleComponentName) + serverConfig.EffectiveVersion = utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(apiserver.WardleComponentName) if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil { return nil, err diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go index 7d7a0bc74e7..5a803318c12 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go @@ -21,13 +21,11 @@ import ( "k8s.io/apimachinery/pkg/util/version" utilversion "k8s.io/apiserver/pkg/util/version" - "k8s.io/sample-apiserver/pkg/apiserver" "github.com/stretchr/testify/assert" ) -func TestMapBinaryEffectiveVersionToKubeEffectiveVersion(t *testing.T) { - wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2") +func TestWardleEmulationVersionToKubeEmulationVersion(t *testing.T) { defaultKubeEffectiveVersion := utilversion.DefaultKubeEffectiveVersion() testCases := []struct { @@ -35,32 +33,26 @@ func TestMapBinaryEffectiveVersionToKubeEffectiveVersion(t *testing.T) { wardleEmulationVer *version.Version expectedKubeEmulationVer *version.Version }{ + { + desc: "same version as than kube binary", + wardleEmulationVer: version.MajorMinor(1, 1), + expectedKubeEmulationVer: defaultKubeEffectiveVersion.BinaryVersion(), + }, { desc: "1 version higher than kube binary", wardleEmulationVer: version.MajorMinor(1, 2), - expectedKubeEmulationVer: defaultKubeEffectiveVersion.BinaryVersion().AddMinor(1), + expectedKubeEmulationVer: defaultKubeEffectiveVersion.BinaryVersion().OffsetMinor(1), }, { desc: "no mapping", - wardleEmulationVer: version.MajorMinor(1, 10), + wardleEmulationVer: version.MajorMinor(2, 10), }, } for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - registry := utilversion.NewComponentGlobalsRegistry() - _ = registry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, nil, true) - _ = registry.Register(utilversion.DefaultKubeComponent, defaultKubeEffectiveVersion, nil, true) - - wardleEffectiveVersion.SetEmulationVersion(tc.wardleEmulationVer) - err := mapWardleEffectiveVersionToKubeEffectiveVersion(registry) - if tc.expectedKubeEmulationVer == nil { - if err == nil { - t.Fatal("expected error, no error found") - } - } else { - assert.True(t, registry.EffectiveVersionFor(utilversion.DefaultKubeComponent).EmulationVersion().EqualTo(tc.expectedKubeEmulationVer)) - } + mappedKubeEmulationVer := wardleEmulationVersionToKubeEmulationVersion(tc.wardleEmulationVer) + assert.True(t, mappedKubeEmulationVer.EqualTo(tc.expectedKubeEmulationVer)) }) } } diff --git a/test/integration/apiserver/apiserver_test.go b/test/integration/apiserver/apiserver_test.go index eba370d6e91..5c4152915a5 100644 --- a/test/integration/apiserver/apiserver_test.go +++ b/test/integration/apiserver/apiserver_test.go @@ -56,6 +56,7 @@ import ( "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/endpoints/handlers" "k8s.io/apiserver/pkg/storage/storagebackend" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" @@ -3103,6 +3104,48 @@ func TestEmulatedStorageVersion(t *testing.T) { } } +// TestAllowedEmulationVersions tests the TestServer can start without problem for all allowed emulation versions. +func TestAllowedEmulationVersions(t *testing.T) { + tcs := []struct { + name string + emulationVersion string + }{ + { + name: "default", + emulationVersion: utilversion.DefaultKubeEffectiveVersion().EmulationVersion().String(), + }, + } + + for _, tc := range tcs { + t.Run(tc.emulationVersion, func(t *testing.T) { + server := kubeapiservertesting.StartTestServerOrDie(t, nil, + []string{fmt.Sprintf("--emulated-version=kube=%s", tc.emulationVersion)}, framework.SharedEtcd()) + defer server.TearDownFn() + + rt, err := restclient.TransportFor(server.ClientConfig) + if err != nil { + t.Fatal(err) + } + + req, err := http.NewRequest("GET", server.ClientConfig.Host+"/", nil) + if err != nil { + t.Fatal(err) + } + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatal(err) + } + expectedStatusCode := 200 + if resp.StatusCode != expectedStatusCode { + t.Errorf("expect status code: %d, got : %d\n", expectedStatusCode, resp.StatusCode) + } + defer func() { + _ = resp.Body.Close() + }() + }) + } +} + func TestEnableEmulationVersion(t *testing.T) { server := kubeapiservertesting.StartTestServerOrDie(t, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"}, diff --git a/test/integration/client/client_test.go b/test/integration/client/client_test.go index 4802ca1e750..02b56c6a64f 100644 --- a/test/integration/client/client_test.go +++ b/test/integration/client/client_test.go @@ -48,6 +48,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/utils/pointer" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/component-base/version" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" @@ -65,7 +66,12 @@ func TestClient(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if e, a := version.Get(), *info; !reflect.DeepEqual(e, a) { + expectedInfo := version.Get() + kubeVersion := utilversion.DefaultKubeEffectiveVersion().BinaryVersion() + expectedInfo.Major = fmt.Sprintf("%d", kubeVersion.Major()) + expectedInfo.Minor = fmt.Sprintf("%d", kubeVersion.Minor()) + + if e, a := expectedInfo, *info; !reflect.DeepEqual(e, a) { t.Errorf("expected %#v, got %#v", e, a) } diff --git a/test/integration/etcd/crd_overlap_storage_test.go b/test/integration/etcd/crd_overlap_storage_test.go index ebe1e042dc3..7aa898a0bf2 100644 --- a/test/integration/etcd/crd_overlap_storage_test.go +++ b/test/integration/etcd/crd_overlap_storage_test.go @@ -32,11 +32,9 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" - utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/dynamic" apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1" - "k8s.io/kubernetes/cmd/kube-apiserver/app/options" ) // TestOverlappingBuiltInResources ensures the list of group-resources the custom resource finalizer should skip is up to date @@ -71,9 +69,7 @@ func TestOverlappingBuiltInResources(t *testing.T) { // TestOverlappingCustomResourceAPIService ensures creating and deleting a custom resource overlapping with APIServices does not destroy APIService data func TestOverlappingCustomResourceAPIService(t *testing.T) { - apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { - opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.30") - }) + apiServer := StartRealAPIServerOrDie(t) defer apiServer.Cleanup() apiServiceClient, err := apiregistrationclient.NewForConfig(apiServer.Config) @@ -235,9 +231,7 @@ func TestOverlappingCustomResourceAPIService(t *testing.T) { // TestOverlappingCustomResourceCustomResourceDefinition ensures creating and deleting a custom resource overlapping with CustomResourceDefinition does not destroy CustomResourceDefinition data func TestOverlappingCustomResourceCustomResourceDefinition(t *testing.T) { - apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { - opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.30") - }) + apiServer := StartRealAPIServerOrDie(t) defer apiServer.Cleanup() crdClient, err := crdclient.NewForConfig(apiServer.Config) diff --git a/test/integration/etcd/etcd_cross_group_test.go b/test/integration/etcd/etcd_cross_group_test.go index f24f0ee39b4..c2592fc0a49 100644 --- a/test/integration/etcd/etcd_cross_group_test.go +++ b/test/integration/etcd/etcd_cross_group_test.go @@ -39,7 +39,7 @@ import ( func TestCrossGroupStorage(t *testing.T) { apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { // force enable all resources so we can check storage. - opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.30") + opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("0.0") }) defer apiServer.Cleanup() diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index 8d4164ea9b5..55cc7f61d2e 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -226,6 +226,18 @@ func TestAPIServiceWaitOnStart(t *testing.T) { } func TestAggregatedAPIServer(t *testing.T) { + t.Run("WithoutWardleFeatureGateAtV1.2", func(t *testing.T) { + testAggregatedAPIServer(t, false, "1.2") + }) + t.Run("WithoutWardleFeatureGateAtV1.1", func(t *testing.T) { + testAggregatedAPIServer(t, false, "1.1") + }) + t.Run("WithWardleFeatureGateAtV1.1", func(t *testing.T) { + testAggregatedAPIServer(t, true, "1.1") + }) +} + +func testAggregatedAPIServer(t *testing.T, enableWardleFeatureGate bool, emulationVersion string) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) t.Cleanup(cancel) @@ -240,7 +252,7 @@ func TestAggregatedAPIServer(t *testing.T) { // endpoints cannot have loopback IPs so we need to override the resolver itself t.Cleanup(app.SetServiceResolverForTests(staticURLServiceResolver(fmt.Sprintf("https://127.0.0.1:%d", wardlePort)))) - testServer := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true}, nil, framework.SharedEtcd()) + testServer := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, BinaryVersion: "1.32"}, nil, framework.SharedEtcd()) defer testServer.TearDownFn() kubeClientConfig := rest.CopyConfig(testServer.ClientConfig) // force json because everything speaks it @@ -286,15 +298,18 @@ func TestAggregatedAPIServer(t *testing.T) { o.RecommendedOptions.SecureServing.Listener = listener o.RecommendedOptions.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1") wardleCmd := sampleserver.NewCommandStartWardleServer(ctx, o) - wardleCmd.SetArgs([]string{ + args := []string{ "--authentication-kubeconfig", wardleToKASKubeConfigFile, "--authorization-kubeconfig", wardleToKASKubeConfigFile, "--etcd-servers", framework.GetEtcdURL(), "--cert-dir", wardleCertDir, "--kubeconfig", wardleToKASKubeConfigFile, - "--emulated-version", "wardle=1.1", - "--feature-gates", "wardle:BanFlunder=true", - }) + "--emulated-version", fmt.Sprintf("wardle=%s", emulationVersion), + } + if enableWardleFeatureGate { + args = append(args, "--feature-gates", "wardle:BanFlunder=true") + } + wardleCmd.SetArgs(args) if err := wardleCmd.Execute(); err != nil { t.Error(err) } @@ -393,6 +408,8 @@ func TestAggregatedAPIServer(t *testing.T) { if err != nil { t.Fatal(err) } + // clean up data after test is done + defer wardleClient.Fischers().Delete(ctx, "panda", metav1.DeleteOptions{}) fischersList, err := wardleClient.Fischers().List(ctx, metav1.ListOptions{}) if err != nil { t.Fatal(err) @@ -409,8 +426,16 @@ func TestAggregatedAPIServer(t *testing.T) { Name: "badname", }, }, metav1.CreateOptions{}) - if err == nil { - t.Fatal("expect flunder:badname not admitted") + banFlunder := enableWardleFeatureGate || emulationVersion == "1.2" + if banFlunder && err == nil { + t.Fatal("expect flunder:badname not admitted when wardle feature gates are specified") + } + if !banFlunder { + if err != nil { + t.Fatal("expect flunder:badname admitted when wardle feature gates are not specified") + } else { + defer wardleClient.Flunders(metav1.NamespaceSystem).Delete(ctx, "badname", metav1.DeleteOptions{}) + } } _, err = wardleClient.Flunders(metav1.NamespaceSystem).Create(ctx, &wardlev1alpha1.Flunder{ ObjectMeta: metav1.ObjectMeta{ @@ -420,12 +445,17 @@ func TestAggregatedAPIServer(t *testing.T) { if err != nil { t.Fatal(err) } + defer wardleClient.Flunders(metav1.NamespaceSystem).Delete(ctx, "panda", metav1.DeleteOptions{}) flunderList, err := wardleClient.Flunders(metav1.NamespaceSystem).List(ctx, metav1.ListOptions{}) if err != nil { t.Fatal(err) } - if len(flunderList.Items) != 1 { - t.Errorf("expected one flunder: %#v", flunderList.Items) + expectedFlunderCount := 2 + if banFlunder { + expectedFlunderCount = 1 + } + if len(flunderList.Items) != expectedFlunderCount { + t.Errorf("expected %d flunder: %#v", expectedFlunderCount, flunderList.Items) } if len(flunderList.ResourceVersion) == 0 { t.Error("expected non-empty resource version for flunder list") diff --git a/test/integration/servicecidr/allocator_test.go b/test/integration/servicecidr/allocator_test.go index ad6d6280dc2..dd4b45cc521 100644 --- a/test/integration/servicecidr/allocator_test.go +++ b/test/integration/servicecidr/allocator_test.go @@ -29,7 +29,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" - utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/kubernetes" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" @@ -123,7 +122,6 @@ func TestServiceAllocIPAddress(t *testing.T) { ModifyServerRunOptions: func(opts *options.ServerRunOptions) { opts.ServiceClusterIPRanges = serviceCIDR opts.GenericServerRunOptions.AdvertiseAddress = netutils.ParseIPSloppy("2001:db8::10") - opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.31") opts.APIEnablement.RuntimeConfig.Set("networking.k8s.io/v1alpha1=true") }, }) From 379676c4bef48e5d2add28851302b55b41fcabcf Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Mon, 10 Jun 2024 17:50:22 +0000 Subject: [PATCH 4/4] add DefaultComponentGlobalsRegistry flags in ServerRunOptions Signed-off-by: Siyuan Zhang --- cmd/kube-apiserver/app/options/options.go | 6 +- .../app/options/options_test.go | 24 +- cmd/kube-apiserver/app/options/validation.go | 10 + cmd/kube-apiserver/app/server.go | 6 +- cmd/kube-apiserver/app/testing/testserver.go | 6 +- pkg/controlplane/apiserver/config_test.go | 6 +- pkg/controlplane/apiserver/options/options.go | 6 +- .../apiserver/options/options_test.go | 23 +- .../apiserver/options/validation_test.go | 4 +- .../pkg/cmd/server/options/options.go | 6 +- .../pkg/cmd/server/server.go | 9 +- .../pkg/cmd/server/testing/testserver.go | 4 +- .../src/k8s.io/apiserver/pkg/server/config.go | 8 +- .../apiserver/pkg/server/config_test.go | 2 + .../apiserver/pkg/server/genericapiserver.go | 9 +- .../pkg/server/options/api_enablement_test.go | 1 - .../pkg/server/options/server_run_options.go | 50 ++-- .../server/options/server_run_options_test.go | 66 +++--- .../storage/resource_encoding_config.go | 6 +- .../apiserver/pkg/util/version/registry.go | 18 ++ .../apiserver/pkg/util/version/version.go | 9 +- .../pkg/util/version/version_test.go | 19 -- .../featuregate/feature_gate.go | 151 ++++++------ .../featuregate/feature_gate_test.go | 200 ++++++++++++++-- .../featuregate/testing/feature_gate.go | 92 +++++++- .../featuregate/testing/feature_gate_test.go | 220 +++++++++++++++++- .../src/k8s.io/component-base/version/base.go | 7 + .../kube-aggregator/pkg/cmd/server/start.go | 13 +- staging/src/k8s.io/sample-apiserver/go.mod | 2 + staging/src/k8s.io/sample-apiserver/go.sum | 1 + .../sample-apiserver/pkg/cmd/server/start.go | 2 +- test/e2e_node/services/apiserver.go | 6 +- .../integration/etcd/etcd_cross_group_test.go | 8 +- .../etcd/etcd_storage_path_test.go | 7 +- test/integration/etcd/server.go | 6 +- test/integration/framework/test_server.go | 6 +- 36 files changed, 734 insertions(+), 285 deletions(-) diff --git a/cmd/kube-apiserver/app/options/options.go b/cmd/kube-apiserver/app/options/options.go index 53a5b3c06ee..ebed12af1d6 100644 --- a/cmd/kube-apiserver/app/options/options.go +++ b/cmd/kube-apiserver/app/options/options.go @@ -26,8 +26,6 @@ import ( utilnet "k8s.io/apimachinery/pkg/util/net" cliflag "k8s.io/component-base/cli/flag" - utilversion "k8s.io/apiserver/pkg/util/version" - "k8s.io/component-base/featuregate" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/cluster/ports" controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" @@ -66,9 +64,9 @@ type Extra struct { } // NewServerRunOptions creates and returns ServerRunOptions according to the given featureGate and effectiveVersion of the server binary to run. -func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions { +func NewServerRunOptions() *ServerRunOptions { s := ServerRunOptions{ - Options: controlplaneapiserver.NewOptions(featureGate, effectiveVersion), + Options: controlplaneapiserver.NewOptions(), CloudProvider: kubeoptions.NewCloudProviderOptions(), Extra: Extra{ diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index fa53175c9ad..6668fd19054 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -48,17 +48,17 @@ import ( ) func TestAddFlags(t *testing.T) { + componentGlobalsRegistry := utilversion.DefaultComponentGlobalsRegistry + t.Cleanup(func() { + componentGlobalsRegistry.Reset() + }) fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError) - featureGate := featuregate.NewFeatureGate() - componentRegistry := utilversion.NewComponentGlobalsRegistry() - effectiveVersion := utilversion.NewEffectiveVersion("1.32") - utilruntime.Must(componentRegistry.Register("test", effectiveVersion, featureGate)) - s := NewServerRunOptions(featureGate, effectiveVersion) + utilruntime.Must(componentGlobalsRegistry.Register("test", utilversion.NewEffectiveVersion("1.32"), featuregate.NewFeatureGate())) + s := NewServerRunOptions() for _, f := range s.Flags().FlagSets { fs.AddFlagSet(f) } - componentRegistry.AddFlags(fs) args := []string{ "--enable-admission-plugins=AlwaysDeny", @@ -133,7 +133,7 @@ func TestAddFlags(t *testing.T) { "--emulated-version=test=1.31", } fs.Parse(args) - utilruntime.Must(componentRegistry.Set()) + utilruntime.Must(componentGlobalsRegistry.Set()) // This is a snapshot of expected options parsed by args. expected := &ServerRunOptions{ @@ -147,8 +147,8 @@ func TestAddFlags(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024), MaxRequestBodyBytes: int64(3 * 1024 * 1024), - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: componentGlobalsRegistry, + ComponentName: utilversion.DefaultKubeComponent, }, Admission: &kubeoptions.AdmissionOptions{ GenericAdmission: &apiserveroptions.AdmissionOptions{ @@ -350,8 +350,8 @@ func TestAddFlags(t *testing.T) { if !reflect.DeepEqual(expected, s) { t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{}))) } - - if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" { - t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31") + testEffectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor("test") + if testEffectiveVersion.EmulationVersion().String() != "1.31" { + t.Errorf("Got emulation version %s, wanted %s", testEffectiveVersion.EmulationVersion().String(), "1.31") } } diff --git a/cmd/kube-apiserver/app/options/validation.go b/cmd/kube-apiserver/app/options/validation.go index cc64a288635..4c36f2bce17 100644 --- a/cmd/kube-apiserver/app/options/validation.go +++ b/cmd/kube-apiserver/app/options/validation.go @@ -26,6 +26,7 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" netutils "k8s.io/utils/net" + "k8s.io/apimachinery/pkg/util/version" controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" "k8s.io/kubernetes/pkg/controlplane/reconcilers" "k8s.io/kubernetes/pkg/features" @@ -139,5 +140,14 @@ func (s CompletedOptions) Validate() []error { errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount)) } + // TODO: remove in 1.32 + // emulationVersion is introduced in 1.31, so it is only allowed to be equal to the binary version at 1.31. + effectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor(s.GenericServerRunOptions.ComponentName) + binaryVersion := version.MajorMinor(effectiveVersion.BinaryVersion().Major(), effectiveVersion.BinaryVersion().Minor()) + if binaryVersion.EqualTo(version.MajorMinor(1, 31)) && !effectiveVersion.EmulationVersion().EqualTo(binaryVersion) { + errs = append(errs, fmt.Errorf("emulation version needs to be equal to binary version(%s) in compatibility-version alpha, got %s", + binaryVersion.String(), effectiveVersion.EmulationVersion().String())) + } + return errs } diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 9d32d738f84..fad0f1e9579 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -64,9 +64,9 @@ func init() { // NewAPIServerCommand creates a *cobra.Command object with default parameters func NewAPIServerCommand() *cobra.Command { - effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( + _, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( utilversion.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) - s := options.NewServerRunOptions(featureGate, effectiveVersion) + s := options.NewServerRunOptions() cmd := &cobra.Command{ Use: "kube-apiserver", @@ -124,8 +124,6 @@ cluster's shared state through which all other components interact.`, fs := cmd.Flags() namedFlagSets := s.Flags() verflag.AddFlags(namedFlagSets.FlagSet("global")) - utilversion.DefaultComponentGlobalsRegistry.AddFlags(namedFlagSets.FlagSet("global")) - globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags()) options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic")) for _, f := range namedFlagSets.FlagSets { diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go index 0e6411fe869..785724e7448 100644 --- a/cmd/kube-apiserver/app/testing/testserver.go +++ b/cmd/kube-apiserver/app/testing/testserver.go @@ -55,6 +55,7 @@ import ( clientgotransport "k8s.io/client-go/transport" "k8s.io/client-go/util/cert" "k8s.io/client-go/util/keyutil" + featuregatetesting "k8s.io/component-base/featuregate/testing" logsapi "k8s.io/component-base/logs/api/v1" "k8s.io/klog/v2" "k8s.io/kube-aggregator/pkg/apiserver" @@ -187,15 +188,16 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, if instanceOptions.BinaryVersion != "" { effectiveVersion = utilversion.NewEffectiveVersion(instanceOptions.BinaryVersion) } + // need to call SetFeatureGateEmulationVersionDuringTest to reset the feature gate emulation version at the end of the test. + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, featureGate, effectiveVersion.EmulationVersion()) utilversion.DefaultComponentGlobalsRegistry.Reset() utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate)) - s := options.NewServerRunOptions(featureGate, effectiveVersion) + s := options.NewServerRunOptions() for _, f := range s.Flags().FlagSets { fs.AddFlagSet(f) } - utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs) s.SecureServing.Listener, s.SecureServing.BindPort, err = createLocalhostListenerOnFreePort() if err != nil { diff --git a/pkg/controlplane/apiserver/config_test.go b/pkg/controlplane/apiserver/config_test.go index c5ff6a79b83..0e6516e62ca 100644 --- a/pkg/controlplane/apiserver/config_test.go +++ b/pkg/controlplane/apiserver/config_test.go @@ -24,8 +24,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" apiserveroptions "k8s.io/apiserver/pkg/server/options" - utilversion "k8s.io/apiserver/pkg/util/version" - "k8s.io/component-base/featuregate" aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/controlplane/apiserver/options" @@ -34,9 +32,7 @@ import ( ) func TestBuildGenericConfig(t *testing.T) { - featureGate := featuregate.NewFeatureGate() - effectiveVersion := utilversion.DefaultKubeEffectiveVersion() - opts := options.NewOptions(featureGate, effectiveVersion) + opts := options.NewOptions() s := (&apiserveroptions.SecureServingOptions{ BindAddress: netutils.ParseIPSloppy("127.0.0.1"), }).WithLoopback() diff --git a/pkg/controlplane/apiserver/options/options.go b/pkg/controlplane/apiserver/options/options.go index bdaa56ffede..2da89f9555d 100644 --- a/pkg/controlplane/apiserver/options/options.go +++ b/pkg/controlplane/apiserver/options/options.go @@ -28,10 +28,8 @@ import ( peerreconcilers "k8s.io/apiserver/pkg/reconcilers" genericoptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/storage/storagebackend" - utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/util/keyutil" cliflag "k8s.io/component-base/cli/flag" - "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" logsapi "k8s.io/component-base/logs/api/v1" "k8s.io/component-base/metrics" @@ -100,9 +98,9 @@ type CompletedOptions struct { } // NewOptions creates a new ServerRunOptions object with default parameters -func NewOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *Options { +func NewOptions() *Options { s := Options{ - GenericServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion), + GenericServerRunOptions: genericoptions.NewServerRunOptions(), Etcd: genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)), SecureServing: kubeoptions.NewSecureServingOptions(), Audit: genericoptions.NewAuditOptions(), diff --git a/pkg/controlplane/apiserver/options/options_test.go b/pkg/controlplane/apiserver/options/options_test.go index 557983c3679..6e3ea1a52d6 100644 --- a/pkg/controlplane/apiserver/options/options_test.go +++ b/pkg/controlplane/apiserver/options/options_test.go @@ -44,18 +44,18 @@ import ( ) func TestAddFlags(t *testing.T) { + componentGlobalsRegistry := utilversion.DefaultComponentGlobalsRegistry + t.Cleanup(func() { + componentGlobalsRegistry.Reset() + }) fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError) - featureGate := featuregate.NewFeatureGate() - effectiveVersion := utilversion.NewEffectiveVersion("1.32") - componentRegistry := utilversion.NewComponentGlobalsRegistry() - utilruntime.Must(componentRegistry.Register("test", effectiveVersion, featureGate)) - s := NewOptions(featureGate, effectiveVersion) + utilruntime.Must(componentGlobalsRegistry.Register("test", utilversion.NewEffectiveVersion("1.32"), featuregate.NewFeatureGate())) + s := NewOptions() var fss cliflag.NamedFlagSets s.AddFlags(&fss) for _, f := range fss.FlagSets { fs.AddFlagSet(f) } - componentRegistry.AddFlags(fs) args := []string{ "--enable-admission-plugins=AlwaysDeny", @@ -119,7 +119,7 @@ func TestAddFlags(t *testing.T) { "--emulated-version=test=1.31", } fs.Parse(args) - utilruntime.Must(componentRegistry.Set()) + utilruntime.Must(componentGlobalsRegistry.Set()) // This is a snapshot of expected options parsed by args. expected := &Options{ @@ -132,8 +132,8 @@ func TestAddFlags(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024), MaxRequestBodyBytes: int64(3 * 1024 * 1024), - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: componentGlobalsRegistry, + ComponentName: utilversion.DefaultKubeComponent, }, Admission: &kubeoptions.AdmissionOptions{ GenericAdmission: &apiserveroptions.AdmissionOptions{ @@ -305,7 +305,8 @@ func TestAddFlags(t *testing.T) { t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{}))) } - if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" { - t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31") + testEffectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor("test") + if testEffectiveVersion.EmulationVersion().String() != "1.31" { + t.Errorf("Got emulation version %s, wanted %s", testEffectiveVersion.EmulationVersion().String(), "1.31") } } diff --git a/pkg/controlplane/apiserver/options/validation_test.go b/pkg/controlplane/apiserver/options/validation_test.go index 9dddc971a42..518d8a547d2 100644 --- a/pkg/controlplane/apiserver/options/validation_test.go +++ b/pkg/controlplane/apiserver/options/validation_test.go @@ -201,7 +201,7 @@ func TestValidateOptions(t *testing.T) { name: "validate master count equal 0", expectErrors: true, options: &Options{ - GenericServerRunOptions: &genericoptions.ServerRunOptions{FeatureGate: utilfeature.DefaultFeatureGate.DeepCopy(), EffectiveVersion: utilversion.NewEffectiveVersion("1.32")}, + GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry}, Etcd: &genericoptions.EtcdOptions{}, SecureServing: &genericoptions.SecureServingOptionsWithLoopback{}, Audit: &genericoptions.AuditOptions{}, @@ -228,7 +228,7 @@ func TestValidateOptions(t *testing.T) { name: "validate token request enable not attempted", expectErrors: true, options: &Options{ - GenericServerRunOptions: &genericoptions.ServerRunOptions{FeatureGate: utilfeature.DefaultFeatureGate.DeepCopy(), EffectiveVersion: utilversion.NewEffectiveVersion("1.32")}, + GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry}, Etcd: &genericoptions.EtcdOptions{}, SecureServing: &genericoptions.SecureServingOptionsWithLoopback{}, Audit: &genericoptions.AuditOptions{}, diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go index 07fdcb940f5..8ceab525415 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go @@ -39,11 +39,9 @@ import ( flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request" "k8s.io/apiserver/pkg/util/openapi" "k8s.io/apiserver/pkg/util/proxy" - utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/apiserver/pkg/util/webhook" scheme "k8s.io/client-go/kubernetes/scheme" corev1 "k8s.io/client-go/listers/core/v1" - "k8s.io/component-base/featuregate" netutils "k8s.io/utils/net" ) @@ -60,9 +58,9 @@ type CustomResourceDefinitionsServerOptions struct { } // NewCustomResourceDefinitionsServerOptions creates default options of an apiextensions-apiserver. -func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer, featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *CustomResourceDefinitionsServerOptions { +func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer) *CustomResourceDefinitionsServerOptions { o := &CustomResourceDefinitionsServerOptions{ - ServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion), + ServerRunOptions: genericoptions.NewServerRunOptions(), RecommendedOptions: genericoptions.NewRecommendedOptions( defaultEtcdPathPrefix, apiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion), diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go index 6aa64f0f071..39be3d19a0e 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go @@ -24,17 +24,11 @@ import ( "k8s.io/apiextensions-apiserver/pkg/cmd/server/options" genericapiserver "k8s.io/apiserver/pkg/server" - utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" ) func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command { - // effectiveVersion is used to set what apis and feature gates the generic api server is compatible with. - // You can also have the flag setting the effectiveVersion of the apiextensions apiserver, and - // having a mapping from the apiextensions apiserver version to generic apiserver version. - effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( - utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) - o := options.NewCustomResourceDefinitionsServerOptions(out, errOut, featureGate, effectiveVersion) + o := options.NewCustomResourceDefinitionsServerOptions(out, errOut) cmd := &cobra.Command{ Short: "Launch an API extensions API server", @@ -58,7 +52,6 @@ func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command cmd.SetContext(ctx) fs := cmd.Flags() - utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs) o.AddFlags(fs) return cmd } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go index ef5fce2fead..4c4156d14be 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go @@ -127,9 +127,7 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin effectiveVersion := utilversion.DefaultKubeEffectiveVersion() utilversion.DefaultComponentGlobalsRegistry.Reset() utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate)) - s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr, featureGate, effectiveVersion) - - utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs) + s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr) s.AddFlags(fs) s.RecommendedOptions.SecureServing.Listener, s.RecommendedOptions.SecureServing.BindPort, err = createLocalhostListenerOnFreePort() diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index 931226bb678..6f0ca1bcac4 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -832,7 +832,7 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{}, } - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) { + if c.FeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) { manager := c.AggregatedDiscoveryGroupManager if manager == nil { manager = discoveryendpoint.NewResourceManager("apis") @@ -1047,14 +1047,14 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler { handler = genericfilters.WithRetryAfter(handler, c.lifecycleSignals.NotAcceptingNewRequest.Signaled()) } handler = genericfilters.WithHTTPLogging(handler) - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) { + if c.FeatureGate.Enabled(genericfeatures.APIServerTracing) { handler = genericapifilters.WithTracing(handler, c.TracerProvider) } handler = genericapifilters.WithLatencyTrackers(handler) // WithRoutine will execute future handlers in a separate goroutine and serving // handler in current goroutine to minimize the stack memory usage. It must be // after WithPanicRecover() to be protected from panics. - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServingWithRoutine) { + if c.FeatureGate.Enabled(genericfeatures.APIServingWithRoutine) { handler = genericfilters.WithRoutine(handler, c.LongRunningFunc) } handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver) @@ -1098,7 +1098,7 @@ func installAPI(s *GenericAPIServer, c *Config) { routes.Version{Version: c.EffectiveVersion.BinaryVersion().Info()}.Install(s.Handler.GoRestfulContainer) if c.EnableDiscovery { - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) { + if c.FeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) { wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(s.DiscoveryGroupManager, s.AggregatedDiscoveryGroupManager) s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/apis", metav1.APIGroupList{})) } else { diff --git a/staging/src/k8s.io/apiserver/pkg/server/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/config_test.go index 6d25272b0a8..a7f76d0142e 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config_test.go @@ -40,6 +40,7 @@ import ( "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/server/healthz" + utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" @@ -308,6 +309,7 @@ func TestAuthenticationAuditAnnotationsDefaultChain(t *testing.T) { LongRunningFunc: func(_ *http.Request, _ *request.RequestInfo) bool { return false }, lifecycleSignals: newLifecycleSignals(), TracerProvider: tracing.NewNoopTracerProvider(), + FeatureGate: utilfeature.DefaultFeatureGate, } h := DefaultBuildHandlerChain(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go index 7b8e13da256..fed94bf4abd 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go @@ -51,7 +51,6 @@ import ( "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/server/routes" "k8s.io/apiserver/pkg/storageversion" - utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" restclient "k8s.io/client-go/rest" "k8s.io/component-base/featuregate" @@ -780,7 +779,7 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A } resourceInfos = append(resourceInfos, r...) - if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) { + if s.FeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) { // Aggregated discovery only aggregates resources under /apis if apiPrefix == APIGroupPrefix { s.AggregatedDiscoveryGroupManager.AddGroupVersion( @@ -808,8 +807,8 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A s.RegisterDestroyFunc(apiGroupInfo.destroyStorage) - if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionAPI) && - utilfeature.DefaultFeatureGate.Enabled(features.APIServerIdentity) { + if s.FeatureGate.Enabled(features.StorageVersionAPI) && + s.FeatureGate.Enabled(features.APIServerIdentity) { // API installation happens before we start listening on the handlers, // therefore it is safe to register ResourceInfos here. The handler will block // write requests until the storage versions of the targeting resources are updated. @@ -839,7 +838,7 @@ func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo // Install the version handler. // Add a handler at / to enumerate the supported api versions. legacyRootAPIHandler := discovery.NewLegacyRootAPIHandler(s.discoveryAddresses, s.Serializer, apiPrefix) - if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) { + if s.FeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) { wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(legacyRootAPIHandler, s.AggregatedLegacyDiscoveryGroupManager) s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/api", metav1.APIVersions{})) } else { diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement_test.go index 7d60244004d..a14319e5373 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/api_enablement_test.go @@ -33,7 +33,6 @@ func (f fakeGroupRegistry) IsGroupRegistered(group string) bool { func TestAPIEnablementOptionsValidate(t *testing.T) { testCases := []struct { name string - testOptions *APIEnablementOptions runtimeConfig cliflag.ConfigurationMap expectErr string }{ diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go index 88dc3802f7f..593d591638e 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go @@ -25,9 +25,10 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/server" + utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" - "k8s.io/component-base/featuregate" "github.com/spf13/pflag" ) @@ -91,12 +92,23 @@ type ServerRunOptions struct { // it is not overridden by any other grace period. ShutdownWatchTerminationGracePeriod time.Duration - // FeatureGate are the featuregate to install on the CLI - FeatureGate featuregate.FeatureGate - EffectiveVersion utilversion.EffectiveVersion + // ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored. + ComponentGlobalsRegistry utilversion.ComponentGlobalsRegistry + // ComponentName is name under which the server's global variabled are registered in the ComponentGlobalsRegistry. + ComponentName string } -func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions { +func NewServerRunOptions() *ServerRunOptions { + if utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) == nil { + featureGate := utilfeature.DefaultMutableFeatureGate + effectiveVersion := utilversion.DefaultKubeEffectiveVersion() + utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate)) + } + + return NewServerRunOptionsForComponent(utilversion.DefaultKubeComponent, utilversion.DefaultComponentGlobalsRegistry) +} + +func NewServerRunOptionsForComponent(componentName string, componentGlobalsRegistry utilversion.ComponentGlobalsRegistry) *ServerRunOptions { defaults := server.NewConfig(serializer.CodecFactory{}) return &ServerRunOptions{ MaxRequestsInFlight: defaults.MaxRequestsInFlight, @@ -109,13 +121,16 @@ func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion u JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes, MaxRequestBodyBytes: defaults.MaxRequestBodyBytes, ShutdownSendRetryAfter: false, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentName: componentName, + ComponentGlobalsRegistry: componentGlobalsRegistry, } } // ApplyTo applies the run options to the method receiver and returns self func (s *ServerRunOptions) ApplyTo(c *server.Config) error { + if err := s.ComponentGlobalsRegistry.SetFallback(); err != nil { + return err + } c.CorsAllowedOriginList = s.CorsAllowedOriginList c.HSTSDirectives = s.HSTSDirectives c.ExternalAddress = s.ExternalHost @@ -131,8 +146,8 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error { c.PublicAddress = s.AdvertiseAddress c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter c.ShutdownWatchTerminationGracePeriod = s.ShutdownWatchTerminationGracePeriod - c.EffectiveVersion = s.EffectiveVersion - c.FeatureGate = s.FeatureGate + c.EffectiveVersion = s.ComponentGlobalsRegistry.EffectiveVersionFor(s.ComponentName) + c.FeatureGate = s.ComponentGlobalsRegistry.FeatureGateFor(s.ComponentName) return nil } @@ -205,12 +220,7 @@ func (s *ServerRunOptions) Validate() []error { if err := validateCorsAllowedOriginList(s.CorsAllowedOriginList); err != nil { errors = append(errors, err) } - if s.FeatureGate != nil { - if errs := s.FeatureGate.Validate(); len(errs) != 0 { - errors = append(errors, errs...) - } - } - if errs := s.EffectiveVersion.Validate(); len(errs) != 0 { + if errs := s.ComponentGlobalsRegistry.Validate(); len(errs) != 0 { errors = append(errors, errs...) } return errors @@ -353,15 +363,11 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { fs.DurationVar(&s.ShutdownWatchTerminationGracePeriod, "shutdown-watch-termination-grace-period", s.ShutdownWatchTerminationGracePeriod, ""+ "This option, if set, represents the maximum amount of grace period the apiserver will wait "+ "for active watch request(s) to drain during the graceful server shutdown window.") + + s.ComponentGlobalsRegistry.AddFlags(fs) } // Complete fills missing fields with defaults. func (s *ServerRunOptions) Complete() error { - if s.FeatureGate == nil { - return fmt.Errorf("nil FeatureGate in ServerRunOptions") - } - if s.EffectiveVersion == nil { - return fmt.Errorf("nil EffectiveVersion in ServerRunOptions") - } - return nil + return s.ComponentGlobalsRegistry.SetFallback() } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go index 0c62a4d6893..7bfcf0245f1 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go @@ -23,14 +23,21 @@ import ( "time" utilerrors "k8s.io/apimachinery/pkg/util/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/version" utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" netutils "k8s.io/utils/net" ) func TestServerRunOptionsValidate(t *testing.T) { + testRegistry := utilversion.NewComponentGlobalsRegistry() featureGate := utilfeature.DefaultFeatureGate.DeepCopy() effectiveVersion := utilversion.NewEffectiveVersion("1.30") + effectiveVersion.SetEmulationVersion(version.MajorMinor(1, 32)) + testComponent := "test" + utilruntime.Must(testRegistry.Register(testComponent, effectiveVersion, featureGate)) + testCases := []struct { name string testOptions *ServerRunOptions @@ -47,8 +54,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--max-requests-inflight can not be negative value", }, @@ -63,8 +69,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--max-mutating-requests-inflight can not be negative value", }, @@ -79,8 +84,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--request-timeout can not be negative value", }, @@ -95,8 +99,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: -1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--min-request-timeout can not be negative value", }, @@ -111,8 +114,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: -10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value", }, @@ -127,8 +129,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: -10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "ServerRunOptions.MaxRequestBodyBytes can not be negative value", }, @@ -144,8 +145,7 @@ func TestServerRunOptionsValidate(t *testing.T) { JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, LivezGracePeriod: -time.Second, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--livez-grace-period can not be a negative value", }, @@ -161,8 +161,7 @@ func TestServerRunOptionsValidate(t *testing.T) { JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, ShutdownDelayDuration: -time.Second, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--shutdown-delay-duration can not be negative value", }, @@ -178,11 +177,27 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectErr: "--strict-transport-security-directives invalid, allowed values: max-age=expireTime, includeSubDomains, preload. see https://tools.ietf.org/html/rfc6797#section-6.1 for more information", }, + { + name: "Test when emulation version is invalid", + testOptions: &ServerRunOptions{ + AdvertiseAddress: netutils.ParseIPSloppy("192.168.10.10"), + CorsAllowedOriginList: []string{"^10.10.10.100$", "^10.10.10.200$"}, + HSTSDirectives: []string{"max-age=31536000", "includeSubDomains", "preload"}, + MaxRequestsInFlight: 400, + MaxMutatingRequestsInFlight: 200, + RequestTimeout: time.Duration(2) * time.Minute, + MinRequestTimeout: 1800, + JSONPatchMaxCopyBytes: 10 * 1024 * 1024, + MaxRequestBodyBytes: 10 * 1024 * 1024, + ComponentName: testComponent, + ComponentGlobalsRegistry: testRegistry, + }, + expectErr: "emulation version 1.32 is not between [1.29, 1.30.0]", + }, { name: "Test when ServerRunOptions is valid", testOptions: &ServerRunOptions{ @@ -195,8 +210,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - FeatureGate: featureGate, - EffectiveVersion: effectiveVersion, + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, }, } @@ -216,8 +230,6 @@ func TestServerRunOptionsValidate(t *testing.T) { } func TestValidateCorsAllowedOriginList(t *testing.T) { - featureGate := utilfeature.DefaultFeatureGate.DeepCopy() - effectiveVersion := utilversion.NewEffectiveVersion("1.30") tests := []struct { regexp [][]string errShouldContain string @@ -265,7 +277,7 @@ func TestValidateCorsAllowedOriginList(t *testing.T) { for _, test := range tests { for _, regexp := range test.regexp { t.Run(fmt.Sprintf("regexp/%s", regexp), func(t *testing.T) { - options := NewServerRunOptions(featureGate, effectiveVersion) + options := NewServerRunOptions() if errs := options.Validate(); len(errs) != 0 { t.Fatalf("wrong test setup: %#v", errs) } @@ -289,8 +301,6 @@ func TestValidateCorsAllowedOriginList(t *testing.T) { } func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) { - featureGate := utilfeature.DefaultFeatureGate.DeepCopy() - effectiveVersion := utilversion.NewEffectiveVersion("1.30") tests := []struct { name string optionsFn func() *ServerRunOptions @@ -299,13 +309,13 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) { { name: "default should be valid", optionsFn: func() *ServerRunOptions { - return NewServerRunOptions(featureGate, effectiveVersion) + return NewServerRunOptions() }, }, { name: "negative not allowed", optionsFn: func() *ServerRunOptions { - o := NewServerRunOptions(featureGate, effectiveVersion) + o := NewServerRunOptions() o.ShutdownWatchTerminationGracePeriod = -time.Second return o }, @@ -332,7 +342,7 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) { } t.Run("default should be zero", func(t *testing.T) { - options := NewServerRunOptions(featureGate, effectiveVersion) + options := NewServerRunOptions() if options.ShutdownWatchTerminationGracePeriod != time.Duration(0) { t.Errorf("expected default of ShutdownWatchTerminationGracePeriod to be zero, but got: %s", options.ShutdownWatchTerminationGracePeriod) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go b/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go index 7339d17df50..d73c8e62c61 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go @@ -181,9 +181,9 @@ func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example // was being served which didn't exist at all in min-compatibility-version. // // In the alpha case - we do not support compatibility versioning of - // alpha types and recommend users do not mix the two. + // alpha types and recommend users do not mix the two. // In the skip-level case - The version of apiserver we are retaining - // compatibility with has no knowledge of the type, - // so storing it in another type is no issue. + // compatibility with has no knowledge of the type, + // so storing it in another type is no issue. return binaryVersionOfResource, nil } diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/registry.go b/staging/src/k8s.io/apiserver/pkg/util/version/registry.go index 170ac28045e..e6655277c0e 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/registry.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/registry.go @@ -101,6 +101,8 @@ type ComponentGlobalsRegistry interface { AddFlags(fs *pflag.FlagSet) // Set sets the flags for all global variables for all components registered. Set() error + // SetFallback calls Set() if it has never been called. + SetFallback() error // Validate calls the Validate() function for all the global variables for all components registered. Validate() []error // Reset removes all stored ComponentGlobals, configurations, and version mappings. @@ -120,6 +122,8 @@ type componentGlobalsRegistry struct { emulationVersionConfig []string // map of component name to the list of feature gates set from the flag. featureGatesConfig map[string][]string + // set stores if the Set() function for the registry is already called. + set bool } func NewComponentGlobalsRegistry() *componentGlobalsRegistry { @@ -136,6 +140,7 @@ func (r *componentGlobalsRegistry) Reset() { r.componentGlobals = make(map[string]*ComponentGlobals) r.emulationVersionConfig = nil r.featureGatesConfig = nil + r.set = false } func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion { @@ -330,9 +335,22 @@ func toVersionMap(versionConfig []string) (map[string]*version.Version, error) { return m, nil } +func (r *componentGlobalsRegistry) SetFallback() error { + r.mutex.Lock() + set := r.set + r.mutex.Unlock() + if set { + return nil + } + klog.Warning("setting componentGlobalsRegistry in SetFallback. We recommend calling componentGlobalsRegistry.Set()" + + " right after parsing flags to avoid using feature gates before their final values are set by the flags.") + return r.Set() +} + func (r *componentGlobalsRegistry) Set() error { r.mutex.Lock() defer r.mutex.Unlock() + r.set = true emulationVersionConfigMap, err := toVersionMap(r.emulationVersionConfig) if err != nil { return err diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/version.go b/staging/src/k8s.io/apiserver/pkg/util/version/version.go index 1596aef389b..a7a5fda87c7 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/version.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/version.go @@ -111,12 +111,6 @@ func (m *effectiveVersion) Validate() []error { // emulationVersion can only be 1.{binaryMinor-1}...1.{binaryMinor}. maxEmuVer := binaryVersion minEmuVer := binaryVersion.SubtractMinor(1) - // TODO: remove in 1.32 - // emulationVersion is introduced in 1.31, so it cannot be lower than that. - // binaryVersion could be lower than 1.31 in tests. So we are only checking 1.31. - if binaryVersion.EqualTo(version.MajorMinor(1, 31)) { - minEmuVer = version.MajorMinor(1, 31) - } if emulationVersion.GreaterThan(maxEmuVer) || emulationVersion.LessThan(minEmuVer) { errs = append(errs, fmt.Errorf("emulation version %s is not between [%s, %s]", emulationVersion.String(), minEmuVer.String(), maxEmuVer.String())) } @@ -157,8 +151,7 @@ func DefaultBuildEffectiveVersion() MutableEffectiveVersion { // DefaultKubeEffectiveVersion returns the MutableEffectiveVersion based on the // latest K8s release. -// Should update for each minor release! func DefaultKubeEffectiveVersion() MutableEffectiveVersion { - binaryVersion := version.MustParse("1.31").WithInfo(baseversion.Get()) + binaryVersion := version.MustParse(baseversion.DefaultKubeBinaryVersion).WithInfo(baseversion.Get()) return newEffectiveVersion(binaryVersion) } diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go b/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go index 784e3e4e186..aff8a8e4a0b 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go @@ -42,25 +42,6 @@ func TestValidate(t *testing.T) { emulationVersion: "v1.31.0", minCompatibilityVersion: "v1.31.0", }, - { - name: "binary version 1.31, emulation version lower than 1.31", - binaryVersion: "v1.31.2", - emulationVersion: "v1.30.0", - minCompatibilityVersion: "v1.30.0", - expectErrors: true, - }, - { - name: "binary version 1.31, emulation version 1.31", - binaryVersion: "v1.31.2", - emulationVersion: "v1.31.0", - minCompatibilityVersion: "v1.30.0", - }, - { - name: "binary version lower than 1.31", - binaryVersion: "v1.30.2", - emulationVersion: "v1.29.0", - minCompatibilityVersion: "v1.29.0", - }, { name: "emulation version two minor lower than binary not ok", binaryVersion: "v1.33.2", diff --git a/staging/src/k8s.io/component-base/featuregate/feature_gate.go b/staging/src/k8s.io/component-base/featuregate/feature_gate.go index 680d92d3d83..e3ca5e91dae 100644 --- a/staging/src/k8s.io/component-base/featuregate/feature_gate.go +++ b/staging/src/k8s.io/component-base/featuregate/feature_gate.go @@ -114,6 +114,9 @@ type FeatureGate interface { // set on the copy without mutating the original. This is useful for validating // config against potential feature gate changes before committing those changes. DeepCopy() MutableVersionedFeatureGate + // CopyKnownFeatures returns a partial copy of the FeatureGate object, with all the known features and overrides. + // This is useful for creating a new instance of feature gate without inheriting all the enabled configurations of the base feature gate. + CopyKnownFeatures() MutableVersionedFeatureGate // Validate checks if the flag gates are valid at the emulated version. Validate() []error } @@ -127,9 +130,6 @@ type MutableFeatureGate interface { AddFlag(fs *pflag.FlagSet) // Close sets closed to true, and prevents subsequent calls to Add Close() - // OpenForModification sets closedForModification to false, and allows subsequent calls to SetEmulationVersion to change enabled features - // before the next Enabled is called. - OpenForModification() // Set parses and stores flag gates for known features // from a string like feature1=true,feature2=false,... Set(value string) error @@ -166,8 +166,6 @@ type MutableVersionedFeatureGate interface { // SetEmulationVersion overrides the emulationVersion of the feature gate. // Otherwise, the emulationVersion will be the same as the binary version. // If set, the feature defaults and availability will be as if the binary is at the emulated version. - // Returns error if the new emulationVersion will change the enablement state of a feature that has already been queried. - // If you have to use featureGate.Enabled before parsing the flags, call featureGate.OpenForModification following featureGate.Enabled. SetEmulationVersion(emulationVersion *version.Version) error // GetAll returns a copy of the map of known feature names to versioned feature specs. GetAllVersioned() map[Feature]VersionedSpecs @@ -185,15 +183,11 @@ type MutableVersionedFeatureGate interface { // overriding its default to true for a limited number of components without simultaneously // changing its default for all consuming components. OverrideDefaultAtVersion(name Feature, override bool, ver *version.Version) error -} - -// MutableVersionedFeatureGateForTests is a feature gate interface that should only be used in tests. -type MutableVersionedFeatureGateForTests interface { - MutableVersionedFeatureGate - // Reset sets the enabled and enabledRaw to the input map. - Reset(m map[string]bool) - // EnabledRawMap returns the raw enable map from the feature gate. - EnabledRawMap() map[string]bool + // ExplicitlySet returns true if the feature value is explicitly set instead of + // being derived from the default values or special features. + ExplicitlySet(name Feature) bool + // ResetFeatureValueToDefault resets the value of the feature back to the default value. + ResetFeatureValueToDefault(name Feature) error } // featureGate implements FeatureGate as well as pflag.Value for flag parsing. @@ -214,12 +208,8 @@ type featureGate struct { enabledRaw atomic.Value // closed is set to true when AddFlag is called, and prevents subsequent calls to Add closed bool - // closedForModification is set to true when Enabled is called, and prevents subsequent calls to SetEmulationVersion to change the enabled features. - // TODO: after all feature gates have migrated to versioned feature gates, - // closedForModification should also prevents subsequent calls to Set and SetFromMap to change the enabled features - closedForModification atomic.Bool // queriedFeatures stores all the features that have been queried through the Enabled interface. - // It is reset when closedForModification is reset. + // It is reset when SetEmulationVersion is called. queriedFeatures atomic.Value emulationVersion atomic.Pointer[version.Version] } @@ -282,7 +272,7 @@ func NewVersionedFeatureGate(emulationVersion *version.Version) *featureGate { // NewFeatureGate creates a feature gate with the current binary version. func NewFeatureGate() *featureGate { - binaryVersison := version.MustParse(baseversion.Get().String()) + binaryVersison := version.MustParse(baseversion.DefaultKubeBinaryVersion) return NewVersionedFeatureGate(binaryVersison) } @@ -311,6 +301,8 @@ func (f *featureGate) Set(value string) error { // Validate checks if the flag gates are valid at the emulated version. func (f *featureGate) Validate() []error { + f.lock.Lock() + defer f.lock.Unlock() m, ok := f.enabledRaw.Load().(map[string]bool) if !ok { return []error{fmt.Errorf("cannot cast enabledRaw to map[string]bool")} @@ -497,13 +489,17 @@ func (f *featureGate) OverrideDefaultAtVersion(name Feature, override bool, ver // GetAll returns a copy of the map of known feature names to feature specs for the current emulationVersion. func (f *featureGate) GetAll() map[Feature]FeatureSpec { retval := map[Feature]FeatureSpec{} - for k, v := range f.GetAllVersioned() { - spec := f.featureSpecAtEmulationVersion(v) + f.lock.Lock() + versionedSpecs := f.GetAllVersioned() + emuVer := f.EmulationVersion() + f.lock.Unlock() + for k, v := range versionedSpecs { + spec := featureSpecAtEmulationVersion(v, emuVer) if spec.PreRelease == PreAlpha { // The feature is not available at the emulation version. continue } - retval[k] = *f.featureSpecAtEmulationVersion(v) + retval[k] = *spec } return retval } @@ -518,6 +514,9 @@ func (f *featureGate) GetAllVersioned() map[Feature]VersionedSpecs { } func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) error { + if emulationVersion.EqualTo(f.EmulationVersion()) { + return nil + } f.lock.Lock() defer f.lock.Unlock() klog.V(1).Infof("set feature gate emulationVersion to %s", emulationVersion.String()) @@ -531,22 +530,21 @@ func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) err enabled := map[Feature]bool{} errs := f.unsafeSetFromMap(enabled, enabledRaw, emulationVersion) - if f.closedForModification.Load() { - queriedFeatures := f.queriedFeatures.Load().(map[Feature]struct{}) - known := f.known.Load().(map[Feature]VersionedSpecs) - for feature := range queriedFeatures { - newVal := featureEnabled(feature, enabled, known, emulationVersion) - oldVal := f.Enabled(feature) - // it is ok to modify emulation version if it does not result in feature enablemennt change for features that have already been queried. - if newVal != oldVal { - errs = append(errs, fmt.Errorf("SetEmulationVersion will change already queried feature:%s from %v to %v\ncall featureGate.OpenForModification() first to override", feature, oldVal, newVal)) - } + queriedFeatures := f.queriedFeatures.Load().(map[Feature]struct{}) + known := f.known.Load().(map[Feature]VersionedSpecs) + for feature := range queriedFeatures { + newVal := featureEnabled(feature, enabled, known, emulationVersion) + oldVal := featureEnabled(feature, f.enabled.Load().(map[Feature]bool), known, f.EmulationVersion()) + if newVal != oldVal { + klog.Warningf("SetEmulationVersion will change already queried feature:%s from %v to %v", feature, oldVal, newVal) } } + if len(errs) == 0 { // Persist changes f.enabled.Store(enabled) f.emulationVersion.Store(emulationVersion) + f.queriedFeatures.Store(map[Feature]struct{}{}) } return utilerrors.NewAggregate(errs) } @@ -555,9 +553,9 @@ func (f *featureGate) EmulationVersion() *version.Version { return f.emulationVersion.Load() } -// FeatureSpec returns the FeatureSpec at the EmulationVersion if the key exists, an error otherwise. +// featureSpec returns the featureSpec at the EmulationVersion if the key exists, an error otherwise. // This is useful to keep multiple implementations of a feature based on the PreRelease or Version info. -func (f *featureGate) FeatureSpec(key Feature) (FeatureSpec, error) { +func (f *featureGate) featureSpec(key Feature) (FeatureSpec, error) { if v, ok := f.known.Load().(map[Feature]VersionedSpecs)[key]; ok { featureSpec := f.featureSpecAtEmulationVersion(v) return *featureSpec, nil @@ -565,14 +563,16 @@ func (f *featureGate) FeatureSpec(key Feature) (FeatureSpec, error) { return FeatureSpec{}, fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName) } -func (f *featureGate) recordQueried(key Feature) { +func (f *featureGate) unsafeRecordQueried(key Feature) { queriedFeatures := map[Feature]struct{}{} for k := range f.queriedFeatures.Load().(map[Feature]struct{}) { queriedFeatures[k] = struct{}{} } + if _, ok := queriedFeatures[key]; ok { + return + } queriedFeatures[key] = struct{}{} f.queriedFeatures.Store(queriedFeatures) - f.closedForModification.Store(true) } func featureEnabled(key Feature, enabled map[Feature]bool, known map[Feature]VersionedSpecs, emulationVersion *version.Version) bool { @@ -589,8 +589,9 @@ func featureEnabled(key Feature, enabled map[Feature]bool, known map[Feature]Ver // Enabled returns true if the key is enabled. If the key is not known, this call will panic. func (f *featureGate) Enabled(key Feature) bool { + // TODO: ideally we should lock the feature gate in this call to be safe, need to evaluate how much performance impact locking would have. v := featureEnabled(key, f.enabled.Load().(map[Feature]bool), f.known.Load().(map[Feature]VersionedSpecs), f.EmulationVersion()) - f.recordQueried(key) + f.unsafeRecordQueried(key) return v } @@ -620,18 +621,6 @@ func (f *featureGate) Close() { f.lock.Unlock() } -// OpenForModification sets closedForModification to false, and allows subsequent calls to SetEmulationVersion to change enabled features -// before the next Enabled is called. -func (f *featureGate) OpenForModification() { - queriedFeatures := []Feature{} - for feature := range f.queriedFeatures.Load().(map[Feature]struct{}) { - queriedFeatures = append(queriedFeatures, feature) - } - klog.Warningf("open feature gate for modification after querying features: %v.", queriedFeatures) - f.closedForModification.Store(false) - f.queriedFeatures.Store(map[Feature]struct{}{}) -} - // AddFlag adds a flag for setting global feature gates to the specified FlagSet. func (f *featureGate) AddFlag(fs *pflag.FlagSet) { // TODO(mtaufen): Shouldn't we just close it on the first Set/SetFromMap instead? @@ -671,10 +660,21 @@ func (f *featureGate) KnownFeatures() []string { return known } +// CopyKnownFeatures returns a partial copy of the FeatureGate object, with all the known features and overrides. +// This is useful for creating a new instance of feature gate without inheriting all the enabled configurations of the base feature gate. +func (f *featureGate) CopyKnownFeatures() MutableVersionedFeatureGate { + fg := NewVersionedFeatureGate(f.EmulationVersion()) + known := f.GetAllVersioned() + fg.known.Store(known) + return fg +} + // DeepCopy returns a deep copy of the FeatureGate object, such that gates can be // set on the copy without mutating the original. This is useful for validating // config against potential feature gate changes before committing those changes. func (f *featureGate) DeepCopy() MutableVersionedFeatureGate { + f.lock.Lock() + defer f.lock.Unlock() // Copy existing state. known := map[Feature]VersionedSpecs{} for k, v := range f.known.Load().(map[Feature]VersionedSpecs) { @@ -691,7 +691,7 @@ func (f *featureGate) DeepCopy() MutableVersionedFeatureGate { // Construct a new featureGate around the copied state. // Note that specialFeatures is treated as immutable by convention, - // and we maintain the value of f.closed across the copy, but resets closedForModification. + // and we maintain the value of f.closed across the copy. fg := &featureGate{ special: specialFeatures, closed: f.closed, @@ -704,21 +704,40 @@ func (f *featureGate) DeepCopy() MutableVersionedFeatureGate { return fg } -// Reset sets the enabled and enabledRaw to the input map. -func (f *featureGate) Reset(m map[string]bool) { - enabled := map[Feature]bool{} - enabledRaw := map[string]bool{} - queriedFeatures := map[Feature]struct{}{} - f.enabled.Store(enabled) - f.enabledRaw.Store(enabledRaw) - _ = f.SetFromMap(m) - f.closedForModification.Store(false) - f.queriedFeatures.Store(queriedFeatures) - f.lock.Lock() - defer f.lock.Unlock() - f.closed = false +// ExplicitlySet returns true if the feature value is explicitly set instead of +// being derived from the default values or special features. +func (f *featureGate) ExplicitlySet(name Feature) bool { + enabledRaw := f.enabledRaw.Load().(map[string]bool) + _, ok := enabledRaw[string(name)] + return ok } -func (f *featureGate) EnabledRawMap() map[string]bool { - return f.enabledRaw.Load().(map[string]bool) +// ResetFeatureValueToDefault resets the value of the feature back to the default value. +func (f *featureGate) ResetFeatureValueToDefault(name Feature) error { + f.lock.Lock() + defer f.lock.Unlock() + enabled := map[Feature]bool{} + for k, v := range f.enabled.Load().(map[Feature]bool) { + enabled[k] = v + } + enabledRaw := map[string]bool{} + for k, v := range f.enabledRaw.Load().(map[string]bool) { + enabledRaw[k] = v + } + _, inEnabled := enabled[name] + if inEnabled { + delete(enabled, name) + } + _, inEnabledRaw := enabledRaw[string(name)] + if inEnabledRaw { + delete(enabledRaw, string(name)) + } + // some features could be in enabled map but not enabledRaw map, + // for example some Alpha feature when AllAlpha is set. + if inEnabledRaw && !inEnabled { + return fmt.Errorf("feature:%s was explicitly set, but not in enabled map", name) + } + f.enabled.Store(enabled) + f.enabledRaw.Store(enabledRaw) + return nil } diff --git a/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go b/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go index 85f99d25787..cd6eeb83ecb 100644 --- a/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go +++ b/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go @@ -596,6 +596,19 @@ func TestFeatureGateOverrideDefault(t *testing.T) { } }) + t.Run("overrides are preserved across CopyKnownFeatures", func(t *testing.T) { + f := NewFeatureGate() + require.NoError(t, f.Add(map[Feature]FeatureSpec{"TestFeature": {Default: false}})) + require.NoError(t, f.OverrideDefault("TestFeature", true)) + fcopy := f.CopyKnownFeatures() + if !f.Enabled("TestFeature") { + t.Error("TestFeature should be enabled by override") + } + if !fcopy.Enabled("TestFeature") { + t.Error("default override was not preserved by CopyKnownFeatures") + } + }) + t.Run("reflected in known features", func(t *testing.T) { f := NewFeatureGate() if err := f.Add(map[Feature]FeatureSpec{"TestFeature": { @@ -1119,22 +1132,22 @@ func TestVersionedFeatureGateFlagDefaults(t *testing.T) { if f.Enabled(testAlphaGate) != false { t.Errorf("Expected false") } - if fs, _ := f.FeatureSpec(testAlphaGate); fs.PreRelease != PreAlpha || fs.Version.String() != "0.0" { + if fs, _ := f.featureSpec(testAlphaGate); fs.PreRelease != PreAlpha || fs.Version.String() != "0.0" { t.Errorf("Expected (PreAlpha, 0.0)") } if f.Enabled(testBetaGate) != false { t.Errorf("Expected false") } - if fs, _ := f.FeatureSpec(testBetaGate); fs.PreRelease != Beta || fs.Version.String() != "1.28" { + if fs, _ := f.featureSpec(testBetaGate); fs.PreRelease != Beta || fs.Version.String() != "1.28" { t.Errorf("Expected (Beta, 1.28)") } if f.Enabled(testGAGate) != true { t.Errorf("Expected true") } - if fs, _ := f.FeatureSpec(testGAGate); fs.PreRelease != Beta || fs.Version.String() != "1.27" { + if fs, _ := f.featureSpec(testGAGate); fs.PreRelease != Beta || fs.Version.String() != "1.27" { t.Errorf("Expected (Beta, 1.27)") } - if _, err := f.FeatureSpec("NonExist"); err == nil { + if _, err := f.featureSpec("NonExist"); err == nil { t.Errorf("Expected Error") } allFeatures := f.GetAll() @@ -1304,7 +1317,6 @@ func TestVersionedFeatureGateOverrideDefault(t *testing.T) { if !f.Enabled("TestFeature2") { t.Error("expected TestFeature2 to have effective default of true") } - f.OpenForModification() require.NoError(t, f.SetEmulationVersion(version.MustParse("1.29"))) if !f.Enabled("TestFeature1") { t.Error("expected TestFeature1 to have effective default of true") @@ -1312,7 +1324,6 @@ func TestVersionedFeatureGateOverrideDefault(t *testing.T) { if f.Enabled("TestFeature2") { t.Error("expected TestFeature2 to have effective default of false") } - f.OpenForModification() require.NoError(t, f.SetEmulationVersion(version.MustParse("1.26"))) if f.Enabled("TestFeature1") { t.Error("expected TestFeature1 to have effective default of false") @@ -1520,32 +1531,173 @@ func TestFeatureSpecAtEmulationVersion(t *testing.T) { } } -func TestOpenForModification(t *testing.T) { - const testBetaGate Feature = "testBetaGate" - f := NewVersionedFeatureGate(version.MustParse("1.29")) +func TestCopyKnownFeatures(t *testing.T) { + f := NewFeatureGate() + require.NoError(t, f.Add(map[Feature]FeatureSpec{"FeatureA": {Default: false}, "FeatureB": {Default: false}})) + require.NoError(t, f.Set("FeatureA=true")) + require.NoError(t, f.OverrideDefault("FeatureB", true)) + fcopy := f.CopyKnownFeatures() + require.NoError(t, f.Add(map[Feature]FeatureSpec{"FeatureC": {Default: false}})) + assert.True(t, f.Enabled("FeatureA")) + assert.True(t, f.Enabled("FeatureB")) + assert.False(t, f.Enabled("FeatureC")) + + assert.False(t, fcopy.Enabled("FeatureA")) + assert.True(t, fcopy.Enabled("FeatureB")) + + require.NoError(t, fcopy.Set("FeatureB=false")) + assert.True(t, f.Enabled("FeatureB")) + assert.False(t, fcopy.Enabled("FeatureB")) + if err := fcopy.Set("FeatureC=true"); err == nil { + t.Error("expected FeatureC not registered in the copied feature gate") + } +} + +func TestExplicitlySet(t *testing.T) { + // gates for testing + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" + + tests := []struct { + arg string + expectedFeatureValue map[Feature]bool + expectedExplicitlySet map[Feature]bool + }{ + { + arg: "", + expectedFeatureValue: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + }, + expectedExplicitlySet: map[Feature]bool{ + allAlphaGate: false, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: false, + }, + }, + { + arg: "AllAlpha=true,TestBeta=false", + expectedFeatureValue: map[Feature]bool{ + allAlphaGate: true, + allBetaGate: false, + testAlphaGate: true, + testBetaGate: false, + }, + expectedExplicitlySet: map[Feature]bool{ + allAlphaGate: true, + allBetaGate: false, + testAlphaGate: false, + testBetaGate: true, + }, + }, + { + arg: "AllAlpha=true,AllBeta=false", + expectedFeatureValue: map[Feature]bool{ + allAlphaGate: true, + allBetaGate: false, + testAlphaGate: true, + testBetaGate: false, + }, + expectedExplicitlySet: map[Feature]bool{ + allAlphaGate: true, + allBetaGate: true, + testAlphaGate: false, + testBetaGate: false, + }, + }, + } + for i, test := range tests { + t.Run(test.arg, func(t *testing.T) { + fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError) + f := NewVersionedFeatureGate(version.MustParse("1.29")) + err := f.AddVersioned(map[Feature]VersionedSpecs{ + testAlphaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + }, + testBetaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha}, + }, + }) + require.NoError(t, err) + f.AddFlag(fs) + + var errs []error + err = fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)}) + if err != nil { + errs = append(errs, err) + } + err = utilerrors.NewAggregate(errs) + require.NoError(t, err) + for k, v := range test.expectedFeatureValue { + if actual := f.Enabled(k); actual != v { + t.Errorf("%d: expected %s=%v, Got %v", i, k, v, actual) + } + } + for k, v := range test.expectedExplicitlySet { + if actual := f.ExplicitlySet(k); actual != v { + t.Errorf("%d: expected ExplicitlySet(%s)=%v, Got %v", i, k, v, actual) + } + } + }) + } +} + +func TestResetFeatureValueToDefault(t *testing.T) { + // gates for testing + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" + + f := NewVersionedFeatureGate(version.MustParse("1.29")) err := f.AddVersioned(map[Feature]VersionedSpecs{ + testAlphaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + }, testBetaGate: { {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta}, - {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, - {Version: version.MustParse("1.26"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha}, }, }) require.NoError(t, err) - if f.Enabled(testBetaGate) != true { - t.Errorf("Expected true") + fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError) + assert.False(t, f.Enabled("AllAlpha")) + assert.False(t, f.Enabled("AllBeta")) + assert.False(t, f.Enabled("TestAlpha")) + assert.True(t, f.Enabled("TestBeta")) + + f.AddFlag(fs) + var errs []error + err = fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, "AllAlpha=true,TestBeta=false")}) + if err != nil { + errs = append(errs, err) } - err = f.SetEmulationVersion(version.MustParse("1.28")) - if err == nil { - t.Fatalf("Expected error when SetEmulationVersion after querying features") - } - if f.Enabled(testBetaGate) != true { - t.Errorf("Expected true") - } - f.OpenForModification() + err = utilerrors.NewAggregate(errs) + require.NoError(t, err) + assert.True(t, f.Enabled("AllAlpha")) + assert.False(t, f.Enabled("AllBeta")) + assert.True(t, f.Enabled("TestAlpha")) + assert.False(t, f.Enabled("TestBeta")) + + require.NoError(t, f.ResetFeatureValueToDefault("AllAlpha")) + assert.False(t, f.Enabled("AllAlpha")) + assert.False(t, f.Enabled("AllBeta")) + assert.True(t, f.Enabled("TestAlpha")) + assert.False(t, f.Enabled("TestBeta")) + + require.NoError(t, f.ResetFeatureValueToDefault("TestBeta")) + assert.False(t, f.Enabled("AllAlpha")) + assert.False(t, f.Enabled("AllBeta")) + assert.True(t, f.Enabled("TestAlpha")) + assert.True(t, f.Enabled("TestBeta")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) - if f.Enabled(testBetaGate) != false { - t.Errorf("Expected false at 1.28") - } + assert.False(t, f.Enabled("AllAlpha")) + assert.False(t, f.Enabled("AllBeta")) + assert.False(t, f.Enabled("TestAlpha")) + assert.False(t, f.Enabled("TestBeta")) } diff --git a/staging/src/k8s.io/component-base/featuregate/testing/feature_gate.go b/staging/src/k8s.io/component-base/featuregate/testing/feature_gate.go index 47c0f927738..eb28217aff5 100644 --- a/staging/src/k8s.io/component-base/featuregate/testing/feature_gate.go +++ b/staging/src/k8s.io/component-base/featuregate/testing/feature_gate.go @@ -20,14 +20,16 @@ import ( "fmt" "strings" "sync" - "testing" + "k8s.io/apimachinery/pkg/util/version" "k8s.io/component-base/featuregate" ) var ( - overrideLock sync.Mutex - featureFlagOverride map[featuregate.Feature]string + overrideLock sync.Mutex + featureFlagOverride map[featuregate.Feature]string + emulationVersionOverride string + emulationVersionOverrideValue *version.Version ) func init() { @@ -42,10 +44,12 @@ func init() { // Example use: // // featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features., true) -func SetFeatureGateDuringTest(tb testing.TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) { +func SetFeatureGateDuringTest(tb TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) { tb.Helper() detectParallelOverrideCleanup := detectParallelOverride(tb, f) - originalEnabled := gate.(featuregate.MutableVersionedFeatureGateForTests).EnabledRawMap() + originalValue := gate.Enabled(f) + originalEmuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion() + originalExplicitlySet := gate.(featuregate.MutableVersionedFeatureGate).ExplicitlySet(f) // Specially handle AllAlpha and AllBeta if f == "AllAlpha" || f == "AllBeta" { @@ -67,11 +71,46 @@ func SetFeatureGateDuringTest(tb testing.TB, gate featuregate.FeatureGate, f fea tb.Cleanup(func() { tb.Helper() detectParallelOverrideCleanup() - gate.(featuregate.MutableVersionedFeatureGateForTests).Reset(originalEnabled) + emuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion() + if !emuVer.EqualTo(originalEmuVer) { + tb.Fatalf("change of feature gate emulation version from %s to %s in the chain of SetFeatureGateDuringTest is not allowed\nuse SetFeatureGateEmulationVersionDuringTest to change emulation version in tests", + originalEmuVer.String(), emuVer.String()) + } + if originalExplicitlySet { + if err := gate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, originalValue)); err != nil { + tb.Errorf("error restoring %s=%v: %v", f, originalValue, err) + } + } else { + if err := gate.(featuregate.MutableVersionedFeatureGate).ResetFeatureValueToDefault(f); err != nil { + tb.Errorf("error restoring %s=%v: %v", f, originalValue, err) + } + } }) } -func detectParallelOverride(tb testing.TB, f featuregate.Feature) func() { +// SetFeatureGateEmulationVersionDuringTest sets the specified gate to the specified emulation version for duration of the test. +// Fails when it detects second call to set a different emulation version or is unable to set or restore emulation version. +// WARNING: Can leak set variable when called in test calling t.Parallel(), however second attempt to set a different emulation version will cause fatal. +// Example use: + +// featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) +func SetFeatureGateEmulationVersionDuringTest(tb TB, gate featuregate.FeatureGate, ver *version.Version) { + tb.Helper() + detectParallelOverrideCleanup := detectParallelOverrideEmulationVersion(tb, ver) + originalEmuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion() + if err := gate.(featuregate.MutableVersionedFeatureGate).SetEmulationVersion(ver); err != nil { + tb.Fatalf("failed to set emulation version to %s during test", ver.String()) + } + tb.Cleanup(func() { + tb.Helper() + detectParallelOverrideCleanup() + if err := gate.(featuregate.MutableVersionedFeatureGate).SetEmulationVersion(originalEmuVer); err != nil { + tb.Fatalf("failed to restore emulation version to %s during test", originalEmuVer.String()) + } + }) +} + +func detectParallelOverride(tb TB, f featuregate.Feature) func() { tb.Helper() overrideLock.Lock() defer overrideLock.Unlock() @@ -92,7 +131,44 @@ func detectParallelOverride(tb testing.TB, f featuregate.Feature) func() { } } -func sameTestOrSubtest(tb testing.TB, testName string) bool { +func detectParallelOverrideEmulationVersion(tb TB, ver *version.Version) func() { + tb.Helper() + overrideLock.Lock() + defer overrideLock.Unlock() + beforeOverrideTestName := emulationVersionOverride + beforeOverrideValue := emulationVersionOverrideValue + if ver.EqualTo(beforeOverrideValue) { + return func() {} + } + if beforeOverrideTestName != "" && !sameTestOrSubtest(tb, beforeOverrideTestName) { + tb.Fatalf("Detected parallel setting of a feature gate emulation version by both %q and %q", beforeOverrideTestName, tb.Name()) + } + emulationVersionOverride = tb.Name() + emulationVersionOverrideValue = ver + + return func() { + tb.Helper() + overrideLock.Lock() + defer overrideLock.Unlock() + if afterOverrideTestName := emulationVersionOverride; afterOverrideTestName != tb.Name() { + tb.Fatalf("Detected parallel setting of a feature gate emulation version between both %q and %q", afterOverrideTestName, tb.Name()) + } + emulationVersionOverride = beforeOverrideTestName + emulationVersionOverrideValue = beforeOverrideValue + } +} + +func sameTestOrSubtest(tb TB, testName string) bool { // Assumes that "/" is not used in test names. return tb.Name() == testName || strings.HasPrefix(tb.Name(), testName+"/") } + +type TB interface { + Cleanup(func()) + Error(args ...any) + Errorf(format string, args ...any) + Fatal(args ...any) + Fatalf(format string, args ...any) + Helper() + Name() string +} diff --git a/staging/src/k8s.io/component-base/featuregate/testing/feature_gate_test.go b/staging/src/k8s.io/component-base/featuregate/testing/feature_gate_test.go index cec7309eb2e..189efcb11f0 100644 --- a/staging/src/k8s.io/component-base/featuregate/testing/feature_gate_test.go +++ b/staging/src/k8s.io/component-base/featuregate/testing/feature_gate_test.go @@ -21,6 +21,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/util/version" "k8s.io/component-base/featuregate" ) @@ -71,6 +72,7 @@ func TestSpecialGates(t *gotest.T) { expect(t, gate, before) t.Cleanup(func() { expect(t, gate, before) + cleanup() }) SetFeatureGateDuringTest(t, gate, "AllAlpha", true) @@ -156,10 +158,107 @@ func TestSetFeatureGateInTest(t *gotest.T) { assert.True(t, gate.Enabled("feature")) } -func TestDetectLeakToMainTest(t *gotest.T) { - t.Cleanup(func() { - featureFlagOverride = map[featuregate.Feature]string{} +func TestSpecialGatesVersioned(t *gotest.T) { + originalEmulationVersion := version.MustParse("1.31") + gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion) + + err := gate.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ + "alpha_default_on": { + {Version: version.MustParse("1.27"), Default: true, PreRelease: featuregate.Alpha}, + }, + "alpha_default_off": { + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, + }, + "beta_default_on": { + {Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta}, + }, + "beta_default_on_set_off": { + {Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, + }, + "beta_default_off": { + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Beta}, + }, + "beta_default_off_set_on": { + {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, + }, }) + require.NoError(t, err) + + require.NoError(t, gate.Set("beta_default_on_set_off=false")) + require.NoError(t, gate.Set("beta_default_off_set_on=true")) + + before := map[featuregate.Feature]bool{ + "AllAlpha": false, + "AllBeta": false, + + "alpha_default_on": true, + "alpha_default_off": false, + + "beta_default_on": true, + "beta_default_on_set_off": false, + "beta_default_off": false, + "beta_default_off_set_on": true, + } + expect(t, gate, before) + t.Cleanup(func() { + expect(t, gate, before) + cleanup() + }) + + t.Run("OverwriteInSubtest", func(t *gotest.T) { + SetFeatureGateDuringTest(t, gate, "AllAlpha", true) + expect(t, gate, map[featuregate.Feature]bool{ + "AllAlpha": true, + "AllBeta": false, + + "alpha_default_on": true, + "alpha_default_off": true, + + "beta_default_on": true, + "beta_default_on_set_off": false, + "beta_default_off": false, + "beta_default_off_set_on": true, + }) + + SetFeatureGateDuringTest(t, gate, "AllBeta", true) + expect(t, gate, map[featuregate.Feature]bool{ + "AllAlpha": true, + "AllBeta": true, + + "alpha_default_on": true, + "alpha_default_off": true, + + "beta_default_on": true, + "beta_default_on_set_off": true, + "beta_default_off": true, + "beta_default_off_set_on": true, + }) + }) + expect(t, gate, before) + + t.Run("OverwriteInSubtestAtDifferentVersion", func(t *gotest.T) { + SetFeatureGateEmulationVersionDuringTest(t, gate, version.MustParse("1.28")) + SetFeatureGateDuringTest(t, gate, "AllAlpha", true) + expect(t, gate, map[featuregate.Feature]bool{ + "AllAlpha": true, + "AllBeta": false, + + "alpha_default_on": true, + "alpha_default_off": true, + + "beta_default_on": false, + "beta_default_on_set_off": true, + "beta_default_off": false, + "beta_default_off_set_on": true, + }) + }) + +} + +func TestDetectLeakToMainTest(t *gotest.T) { + t.Cleanup(cleanup) gate := featuregate.NewFeatureGate() err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{ "feature": {PreRelease: featuregate.Alpha, Default: false}, @@ -183,9 +282,7 @@ func TestDetectLeakToMainTest(t *gotest.T) { } func TestDetectLeakToOtherSubtest(t *gotest.T) { - t.Cleanup(func() { - featureFlagOverride = map[featuregate.Feature]string{} - }) + t.Cleanup(cleanup) gate := featuregate.NewFeatureGate() err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{ "feature": {PreRelease: featuregate.Alpha, Default: false}, @@ -211,9 +308,7 @@ func TestDetectLeakToOtherSubtest(t *gotest.T) { } func TestCannotDetectLeakFromSubtest(t *gotest.T) { - t.Cleanup(func() { - featureFlagOverride = map[featuregate.Feature]string{} - }) + t.Cleanup(cleanup) gate := featuregate.NewFeatureGate() err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{ "feature": {PreRelease: featuregate.Alpha, Default: false}, @@ -230,6 +325,107 @@ func TestCannotDetectLeakFromSubtest(t *gotest.T) { assert.True(t, gate.Enabled("feature")) } +func TestCannotDetectLeakFromTwoSubtestsWithDifferentFeatures(t *gotest.T) { + t.Cleanup(cleanup) + gate := featuregate.NewFeatureGate() + err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{ + "feature1": {PreRelease: featuregate.Alpha, Default: false}, + "feature2": {PreRelease: featuregate.Alpha, Default: false}, + }) + require.NoError(t, err) + + assert.False(t, gate.Enabled("feature1")) + assert.False(t, gate.Enabled("feature2")) + subtestName := "Subtest" + // Subtest setting feature gate and calling parallel will leak it out + t.Run(subtestName, func(t *gotest.T) { + SetFeatureGateDuringTest(t, gate, "feature1", true) + t.Parallel() + assert.True(t, gate.Enabled("feature1")) + assert.False(t, gate.Enabled("feature2")) + }) + // Leaked true + assert.True(t, gate.Enabled("feature1")) + assert.False(t, gate.Enabled("feature2")) + // Add suffix to name to prevent tests with the same prefix. + t.Run(subtestName+"Suffix", func(t *gotest.T) { + // Leaked true + assert.True(t, gate.Enabled("feature1")) + assert.False(t, gate.Enabled("feature2")) + SetFeatureGateDuringTest(t, gate, "feature2", true) + assert.True(t, gate.Enabled("feature1")) + assert.True(t, gate.Enabled("feature2")) + }) +} + +func TestDetectEmulationVersionLeakToMainTest(t *gotest.T) { + t.Cleanup(cleanup) + originalEmulationVersion := version.MustParse("1.31") + newEmulationVersion := version.MustParse("1.30") + gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion) + assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion)) + + // Subtest setting feature gate and calling parallel will leak it out + t.Run("LeakingSubtest", func(t *gotest.T) { + fakeT := &ignoreFatalT{T: t} + SetFeatureGateEmulationVersionDuringTest(fakeT, gate, newEmulationVersion) + // Calling t.Parallel in subtest will resume the main test body + t.Parallel() + // Leaked from main test + assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion)) + }) + // Leaked from subtest + assert.True(t, gate.EmulationVersion().EqualTo(newEmulationVersion)) + fakeT := &ignoreFatalT{T: t} + SetFeatureGateEmulationVersionDuringTest(fakeT, gate, originalEmulationVersion) + assert.True(t, fakeT.fatalRecorded) +} + +func TestNoLeakFromSameEmulationVersionToMainTest(t *gotest.T) { + t.Cleanup(cleanup) + originalEmulationVersion := version.MustParse("1.31") + newEmulationVersion := version.MustParse("1.31") + gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion) + assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion)) + + // Subtest setting feature gate and calling parallel will leak it out + t.Run("LeakingSubtest", func(t *gotest.T) { + SetFeatureGateEmulationVersionDuringTest(t, gate, newEmulationVersion) + // Calling t.Parallel in subtest will resume the main test body + t.Parallel() + // Leaked from main test + assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion)) + }) + // Leaked from subtest + assert.True(t, gate.EmulationVersion().EqualTo(newEmulationVersion)) + SetFeatureGateEmulationVersionDuringTest(t, gate, originalEmulationVersion) +} + +func TestDetectEmulationVersionLeakToOtherSubtest(t *gotest.T) { + t.Cleanup(cleanup) + originalEmulationVersion := version.MustParse("1.31") + newEmulationVersion := version.MustParse("1.30") + gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion) + assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion)) + + subtestName := "Subtest" + // Subtest setting feature gate and calling parallel will leak it out + t.Run(subtestName, func(t *gotest.T) { + fakeT := &ignoreFatalT{T: t} + SetFeatureGateEmulationVersionDuringTest(fakeT, gate, newEmulationVersion) + t.Parallel() + }) + // Add suffix to name to prevent tests with the same prefix. + t.Run(subtestName+"Suffix", func(t *gotest.T) { + // Leaked newEmulationVersion + assert.True(t, gate.EmulationVersion().EqualTo(newEmulationVersion)) + + fakeT := &ignoreFatalT{T: t} + SetFeatureGateEmulationVersionDuringTest(fakeT, gate, originalEmulationVersion) + assert.True(t, fakeT.fatalRecorded) + }) +} + type ignoreFatalT struct { *gotest.T fatalRecorded bool @@ -248,3 +444,9 @@ func (f *ignoreFatalT) Fatalf(format string, args ...any) { f.fatalRecorded = true f.T.Logf("[IGNORED] "+format, args...) } + +func cleanup() { + featureFlagOverride = map[featuregate.Feature]string{} + emulationVersionOverride = "" + emulationVersionOverrideValue = nil +} diff --git a/staging/src/k8s.io/component-base/version/base.go b/staging/src/k8s.io/component-base/version/base.go index b753b7d191f..60154678249 100644 --- a/staging/src/k8s.io/component-base/version/base.go +++ b/staging/src/k8s.io/component-base/version/base.go @@ -61,3 +61,10 @@ var ( buildDate = "1970-01-01T00:00:00Z" // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ') ) + +const ( + // DefaultKubeBinaryVersion is the hard coded k8 binary version based on the latest K8s release. + // It is supposed to be consistent with gitMajor and gitMinor, except for local tests, where gitMajor and gitMinor are "". + // Should update for each minor release! + DefaultKubeBinaryVersion = "1.31" +) diff --git a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go index 6e3c02bfef8..d081702d246 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go @@ -31,7 +31,6 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/filters" genericoptions "k8s.io/apiserver/pkg/server/options" - utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" "k8s.io/kube-aggregator/pkg/apiserver" @@ -81,10 +80,7 @@ func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions) } cmd.SetContext(ctx) - fs := cmd.Flags() - utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs) - - o.AddFlags(fs) + o.AddFlags(cmd.Flags()) return cmd } @@ -99,13 +95,8 @@ func (o *AggregatorOptions) AddFlags(fs *pflag.FlagSet) { // NewDefaultOptions builds a "normal" set of options. You wouldn't normally expose this, but hyperkube isn't cobra compatible func NewDefaultOptions(out, err io.Writer) *AggregatorOptions { - // effectiveVersion is used to set what apis and feature gates the generic api server is compatible with. - // You can also have the flag setting the effectiveVersion of the aggregator apiserver, and - // having a mapping from the aggregator apiserver version to generic apiserver version. - effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( - utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) o := &AggregatorOptions{ - ServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion), + ServerRunOptions: genericoptions.NewServerRunOptions(), RecommendedOptions: genericoptions.NewRecommendedOptions( defaultEtcdPathPrefix, aggregatorscheme.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion), diff --git a/staging/src/k8s.io/sample-apiserver/go.mod b/staging/src/k8s.io/sample-apiserver/go.mod index 32f73176861..8f2dee1a0a2 100644 --- a/staging/src/k8s.io/sample-apiserver/go.mod +++ b/staging/src/k8s.io/sample-apiserver/go.mod @@ -7,6 +7,7 @@ go 1.22.0 require ( github.com/google/gofuzz v1.2.0 github.com/spf13/cobra v1.8.1 + github.com/stretchr/testify v1.9.0 k8s.io/apimachinery v0.0.0 k8s.io/apiserver v0.0.0 k8s.io/client-go v0.0.0 @@ -55,6 +56,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.19.0 // indirect github.com/prometheus/client_model v0.6.0 // indirect github.com/prometheus/common v0.48.0 // indirect diff --git a/staging/src/k8s.io/sample-apiserver/go.sum b/staging/src/k8s.io/sample-apiserver/go.sum index dc64c5fca30..bfb8e18b984 100644 --- a/staging/src/k8s.io/sample-apiserver/go.sum +++ b/staging/src/k8s.io/sample-apiserver/go.sum @@ -170,6 +170,7 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go index 6152d0b84ca..5e30e2bb319 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go @@ -113,7 +113,7 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti o.RecommendedOptions.AddFlags(flags) wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2") - wardleFeatureGate := featuregate.NewVersionedFeatureGate(version.MustParse("1.2")) + wardleFeatureGate := utilfeature.DefaultFeatureGate.CopyKnownFeatures() utilruntime.Must(wardleFeatureGate.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ "BanFlunder": { {Version: version.MustParse("1.2"), Default: true, PreRelease: featuregate.GA}, diff --git a/test/e2e_node/services/apiserver.go b/test/e2e_node/services/apiserver.go index c092d9d69b6..4488715b3bd 100644 --- a/test/e2e_node/services/apiserver.go +++ b/test/e2e_node/services/apiserver.go @@ -29,8 +29,6 @@ import ( netutils "k8s.io/utils/net" utilerrors "k8s.io/apimachinery/pkg/util/errors" - utilfeature "k8s.io/apiserver/pkg/util/feature" - utilversion "k8s.io/apiserver/pkg/util/version" apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/test/e2e/framework" @@ -65,9 +63,7 @@ func NewAPIServer(storageConfig storagebackend.Config) *APIServer { func (a *APIServer) Start(ctx context.Context) error { const tokenFilePath = "known_tokens.csv" - featureGate := utilfeature.DefaultFeatureGate - effectiveVersion := utilversion.DefaultKubeEffectiveVersion() - o := options.NewServerRunOptions(featureGate, effectiveVersion) + o := options.NewServerRunOptions() o.Etcd.StorageConfig = a.storageConfig _, ipnet, err := netutils.ParseCIDRSloppy(clusterIPRange) if err != nil { diff --git a/test/integration/etcd/etcd_cross_group_test.go b/test/integration/etcd/etcd_cross_group_test.go index c2592fc0a49..efc4fcc4e6c 100644 --- a/test/integration/etcd/etcd_cross_group_test.go +++ b/test/integration/etcd/etcd_cross_group_test.go @@ -27,9 +27,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/storage" + utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/dynamic" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" @@ -37,9 +39,13 @@ import ( // TestCrossGroupStorage tests to make sure that all objects stored in an expected location in etcd can be converted/read. func TestCrossGroupStorage(t *testing.T) { + testRegistry := utilversion.NewComponentGlobalsRegistry() + utilruntime.Must(testRegistry.Register("test", utilversion.NewEffectiveVersion("0.0"), utilfeature.DefaultFeatureGate.DeepCopy())) + apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { // force enable all resources so we can check storage. - opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("0.0") + opts.GenericServerRunOptions.ComponentName = "test" + opts.GenericServerRunOptions.ComponentGlobalsRegistry = testRegistry }) defer apiServer.Cleanup() diff --git a/test/integration/etcd/etcd_storage_path_test.go b/test/integration/etcd/etcd_storage_path_test.go index eda94aee93f..43cbedbbfdc 100644 --- a/test/integration/etcd/etcd_storage_path_test.go +++ b/test/integration/etcd/etcd_storage_path_test.go @@ -34,6 +34,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" @@ -75,8 +76,12 @@ var allowMissingTestdataFixtures = map[schema.GroupVersionKind]bool{ func TestEtcdStoragePath(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true) featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true) + testRegistry := utilversion.NewComponentGlobalsRegistry() + utilruntime.Must(testRegistry.Register("test", utilversion.NewEffectiveVersion("0.0"), feature.DefaultFeatureGate.DeepCopy())) + apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { - opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("0.0") + opts.GenericServerRunOptions.ComponentName = "test" + opts.GenericServerRunOptions.ComponentGlobalsRegistry = testRegistry }) defer apiServer.Cleanup() defer dumpEtcdKVOnFailure(t, apiServer.KV) diff --git a/test/integration/etcd/server.go b/test/integration/etcd/server.go index fb57211ea43..ed69e3d532f 100644 --- a/test/integration/etcd/server.go +++ b/test/integration/etcd/server.go @@ -40,8 +40,6 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" genericapiserveroptions "k8s.io/apiserver/pkg/server/options" - utilfeature "k8s.io/apiserver/pkg/util/feature" - utilversion "k8s.io/apiserver/pkg/util/version" cacheddiscovery "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" @@ -93,9 +91,7 @@ func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRu t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err) } - featureGate := utilfeature.DefaultFeatureGate - effectiveVersion := utilversion.DefaultBuildEffectiveVersion() - opts := options.NewServerRunOptions(featureGate, effectiveVersion) + opts := options.NewServerRunOptions() opts.Options.SecureServing.Listener = listener opts.Options.SecureServing.ServerCert.CertDirectory = certDir opts.Options.ServiceAccountSigningKeyFile = saSigningKeyFile.Name() diff --git a/test/integration/framework/test_server.go b/test/integration/framework/test_server.go index 5d514b360d4..bb43850466d 100644 --- a/test/integration/framework/test_server.go +++ b/test/integration/framework/test_server.go @@ -35,8 +35,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" genericapiserver "k8s.io/apiserver/pkg/server" genericapiserveroptions "k8s.io/apiserver/pkg/server/options" - utilfeature "k8s.io/apiserver/pkg/util/feature" - utilversion "k8s.io/apiserver/pkg/util/version" client "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/util/cert" @@ -137,9 +135,7 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) ( t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err) } - featureGate := utilfeature.DefaultFeatureGate - effectiveVersion := utilversion.DefaultKubeEffectiveVersion() - opts := options.NewServerRunOptions(featureGate, effectiveVersion) + opts := options.NewServerRunOptions() opts.SecureServing.Listener = listener opts.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1") opts.SecureServing.ServerCert.CertDirectory = certDir