add DefaultComponentGlobalsRegistry flags in ServerRunOptions

Signed-off-by: Siyuan Zhang <sizhang@google.com>
This commit is contained in:
Siyuan Zhang
2024-06-10 17:50:22 +00:00
parent 4352c4ad27
commit 379676c4be
36 changed files with 734 additions and 285 deletions

View File

@@ -26,8 +26,6 @@ import (
utilnet "k8s.io/apimachinery/pkg/util/net"
cliflag "k8s.io/component-base/cli/flag"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/component-base/featuregate"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/cluster/ports"
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
@@ -66,9 +64,9 @@ type Extra struct {
}
// NewServerRunOptions creates and returns ServerRunOptions according to the given featureGate and effectiveVersion of the server binary to run.
func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions {
func NewServerRunOptions() *ServerRunOptions {
s := ServerRunOptions{
Options: controlplaneapiserver.NewOptions(featureGate, effectiveVersion),
Options: controlplaneapiserver.NewOptions(),
CloudProvider: kubeoptions.NewCloudProviderOptions(),
Extra: Extra{

View File

@@ -48,17 +48,17 @@ import (
)
func TestAddFlags(t *testing.T) {
componentGlobalsRegistry := utilversion.DefaultComponentGlobalsRegistry
t.Cleanup(func() {
componentGlobalsRegistry.Reset()
})
fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
featureGate := featuregate.NewFeatureGate()
componentRegistry := utilversion.NewComponentGlobalsRegistry()
effectiveVersion := utilversion.NewEffectiveVersion("1.32")
utilruntime.Must(componentRegistry.Register("test", effectiveVersion, featureGate))
s := NewServerRunOptions(featureGate, effectiveVersion)
utilruntime.Must(componentGlobalsRegistry.Register("test", utilversion.NewEffectiveVersion("1.32"), featuregate.NewFeatureGate()))
s := NewServerRunOptions()
for _, f := range s.Flags().FlagSets {
fs.AddFlagSet(f)
}
componentRegistry.AddFlags(fs)
args := []string{
"--enable-admission-plugins=AlwaysDeny",
@@ -133,7 +133,7 @@ func TestAddFlags(t *testing.T) {
"--emulated-version=test=1.31",
}
fs.Parse(args)
utilruntime.Must(componentRegistry.Set())
utilruntime.Must(componentGlobalsRegistry.Set())
// This is a snapshot of expected options parsed by args.
expected := &ServerRunOptions{
@@ -147,8 +147,8 @@ func TestAddFlags(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
ComponentGlobalsRegistry: componentGlobalsRegistry,
ComponentName: utilversion.DefaultKubeComponent,
},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &apiserveroptions.AdmissionOptions{
@@ -350,8 +350,8 @@ func TestAddFlags(t *testing.T) {
if !reflect.DeepEqual(expected, s) {
t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{})))
}
if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" {
t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31")
testEffectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor("test")
if testEffectiveVersion.EmulationVersion().String() != "1.31" {
t.Errorf("Got emulation version %s, wanted %s", testEffectiveVersion.EmulationVersion().String(), "1.31")
}
}

View File

@@ -26,6 +26,7 @@ import (
utilfeature "k8s.io/apiserver/pkg/util/feature"
netutils "k8s.io/utils/net"
"k8s.io/apimachinery/pkg/util/version"
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
"k8s.io/kubernetes/pkg/controlplane/reconcilers"
"k8s.io/kubernetes/pkg/features"
@@ -139,5 +140,14 @@ func (s CompletedOptions) Validate() []error {
errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount))
}
// TODO: remove in 1.32
// emulationVersion is introduced in 1.31, so it is only allowed to be equal to the binary version at 1.31.
effectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor(s.GenericServerRunOptions.ComponentName)
binaryVersion := version.MajorMinor(effectiveVersion.BinaryVersion().Major(), effectiveVersion.BinaryVersion().Minor())
if binaryVersion.EqualTo(version.MajorMinor(1, 31)) && !effectiveVersion.EmulationVersion().EqualTo(binaryVersion) {
errs = append(errs, fmt.Errorf("emulation version needs to be equal to binary version(%s) in compatibility-version alpha, got %s",
binaryVersion.String(), effectiveVersion.EmulationVersion().String()))
}
return errs
}

View File

@@ -64,9 +64,9 @@ func init() {
// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
_, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
utilversion.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
s := options.NewServerRunOptions(featureGate, effectiveVersion)
s := options.NewServerRunOptions()
cmd := &cobra.Command{
Use: "kube-apiserver",
@@ -124,8 +124,6 @@ cluster's shared state through which all other components interact.`,
fs := cmd.Flags()
namedFlagSets := s.Flags()
verflag.AddFlags(namedFlagSets.FlagSet("global"))
utilversion.DefaultComponentGlobalsRegistry.AddFlags(namedFlagSets.FlagSet("global"))
globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags())
options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic"))
for _, f := range namedFlagSets.FlagSets {

View File

@@ -55,6 +55,7 @@ import (
clientgotransport "k8s.io/client-go/transport"
"k8s.io/client-go/util/cert"
"k8s.io/client-go/util/keyutil"
featuregatetesting "k8s.io/component-base/featuregate/testing"
logsapi "k8s.io/component-base/logs/api/v1"
"k8s.io/klog/v2"
"k8s.io/kube-aggregator/pkg/apiserver"
@@ -187,15 +188,16 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
if instanceOptions.BinaryVersion != "" {
effectiveVersion = utilversion.NewEffectiveVersion(instanceOptions.BinaryVersion)
}
// need to call SetFeatureGateEmulationVersionDuringTest to reset the feature gate emulation version at the end of the test.
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, featureGate, effectiveVersion.EmulationVersion())
utilversion.DefaultComponentGlobalsRegistry.Reset()
utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate))
s := options.NewServerRunOptions(featureGate, effectiveVersion)
s := options.NewServerRunOptions()
for _, f := range s.Flags().FlagSets {
fs.AddFlagSet(f)
}
utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs)
s.SecureServing.Listener, s.SecureServing.BindPort, err = createLocalhostListenerOnFreePort()
if err != nil {

View File

@@ -24,8 +24,6 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
apiserveroptions "k8s.io/apiserver/pkg/server/options"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/component-base/featuregate"
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
@@ -34,9 +32,7 @@ import (
)
func TestBuildGenericConfig(t *testing.T) {
featureGate := featuregate.NewFeatureGate()
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
opts := options.NewOptions(featureGate, effectiveVersion)
opts := options.NewOptions()
s := (&apiserveroptions.SecureServingOptions{
BindAddress: netutils.ParseIPSloppy("127.0.0.1"),
}).WithLoopback()

View File

@@ -28,10 +28,8 @@ import (
peerreconcilers "k8s.io/apiserver/pkg/reconcilers"
genericoptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/storagebackend"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/util/keyutil"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
logsapi "k8s.io/component-base/logs/api/v1"
"k8s.io/component-base/metrics"
@@ -100,9 +98,9 @@ type CompletedOptions struct {
}
// NewOptions creates a new ServerRunOptions object with default parameters
func NewOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *Options {
func NewOptions() *Options {
s := Options{
GenericServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion),
GenericServerRunOptions: genericoptions.NewServerRunOptions(),
Etcd: genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)),
SecureServing: kubeoptions.NewSecureServingOptions(),
Audit: genericoptions.NewAuditOptions(),

View File

@@ -44,18 +44,18 @@ import (
)
func TestAddFlags(t *testing.T) {
componentGlobalsRegistry := utilversion.DefaultComponentGlobalsRegistry
t.Cleanup(func() {
componentGlobalsRegistry.Reset()
})
fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
featureGate := featuregate.NewFeatureGate()
effectiveVersion := utilversion.NewEffectiveVersion("1.32")
componentRegistry := utilversion.NewComponentGlobalsRegistry()
utilruntime.Must(componentRegistry.Register("test", effectiveVersion, featureGate))
s := NewOptions(featureGate, effectiveVersion)
utilruntime.Must(componentGlobalsRegistry.Register("test", utilversion.NewEffectiveVersion("1.32"), featuregate.NewFeatureGate()))
s := NewOptions()
var fss cliflag.NamedFlagSets
s.AddFlags(&fss)
for _, f := range fss.FlagSets {
fs.AddFlagSet(f)
}
componentRegistry.AddFlags(fs)
args := []string{
"--enable-admission-plugins=AlwaysDeny",
@@ -119,7 +119,7 @@ func TestAddFlags(t *testing.T) {
"--emulated-version=test=1.31",
}
fs.Parse(args)
utilruntime.Must(componentRegistry.Set())
utilruntime.Must(componentGlobalsRegistry.Set())
// This is a snapshot of expected options parsed by args.
expected := &Options{
@@ -132,8 +132,8 @@ func TestAddFlags(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
ComponentGlobalsRegistry: componentGlobalsRegistry,
ComponentName: utilversion.DefaultKubeComponent,
},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &apiserveroptions.AdmissionOptions{
@@ -305,7 +305,8 @@ func TestAddFlags(t *testing.T) {
t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{})))
}
if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" {
t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31")
testEffectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor("test")
if testEffectiveVersion.EmulationVersion().String() != "1.31" {
t.Errorf("Got emulation version %s, wanted %s", testEffectiveVersion.EmulationVersion().String(), "1.31")
}
}

View File

@@ -201,7 +201,7 @@ func TestValidateOptions(t *testing.T) {
name: "validate master count equal 0",
expectErrors: true,
options: &Options{
GenericServerRunOptions: &genericoptions.ServerRunOptions{FeatureGate: utilfeature.DefaultFeatureGate.DeepCopy(), EffectiveVersion: utilversion.NewEffectiveVersion("1.32")},
GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry},
Etcd: &genericoptions.EtcdOptions{},
SecureServing: &genericoptions.SecureServingOptionsWithLoopback{},
Audit: &genericoptions.AuditOptions{},
@@ -228,7 +228,7 @@ func TestValidateOptions(t *testing.T) {
name: "validate token request enable not attempted",
expectErrors: true,
options: &Options{
GenericServerRunOptions: &genericoptions.ServerRunOptions{FeatureGate: utilfeature.DefaultFeatureGate.DeepCopy(), EffectiveVersion: utilversion.NewEffectiveVersion("1.32")},
GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry},
Etcd: &genericoptions.EtcdOptions{},
SecureServing: &genericoptions.SecureServingOptionsWithLoopback{},
Audit: &genericoptions.AuditOptions{},

View File

@@ -39,11 +39,9 @@ import (
flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
"k8s.io/apiserver/pkg/util/openapi"
"k8s.io/apiserver/pkg/util/proxy"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/apiserver/pkg/util/webhook"
scheme "k8s.io/client-go/kubernetes/scheme"
corev1 "k8s.io/client-go/listers/core/v1"
"k8s.io/component-base/featuregate"
netutils "k8s.io/utils/net"
)
@@ -60,9 +58,9 @@ type CustomResourceDefinitionsServerOptions struct {
}
// NewCustomResourceDefinitionsServerOptions creates default options of an apiextensions-apiserver.
func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer, featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *CustomResourceDefinitionsServerOptions {
func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer) *CustomResourceDefinitionsServerOptions {
o := &CustomResourceDefinitionsServerOptions{
ServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion),
ServerRunOptions: genericoptions.NewServerRunOptions(),
RecommendedOptions: genericoptions.NewRecommendedOptions(
defaultEtcdPathPrefix,
apiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion),

View File

@@ -24,17 +24,11 @@ import (
"k8s.io/apiextensions-apiserver/pkg/cmd/server/options"
genericapiserver "k8s.io/apiserver/pkg/server"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
)
func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command {
// effectiveVersion is used to set what apis and feature gates the generic api server is compatible with.
// You can also have the flag setting the effectiveVersion of the apiextensions apiserver, and
// having a mapping from the apiextensions apiserver version to generic apiserver version.
effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
o := options.NewCustomResourceDefinitionsServerOptions(out, errOut, featureGate, effectiveVersion)
o := options.NewCustomResourceDefinitionsServerOptions(out, errOut)
cmd := &cobra.Command{
Short: "Launch an API extensions API server",
@@ -58,7 +52,6 @@ func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command
cmd.SetContext(ctx)
fs := cmd.Flags()
utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs)
o.AddFlags(fs)
return cmd
}

View File

@@ -127,9 +127,7 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
utilversion.DefaultComponentGlobalsRegistry.Reset()
utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate))
s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr, featureGate, effectiveVersion)
utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs)
s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr)
s.AddFlags(fs)
s.RecommendedOptions.SecureServing.Listener, s.RecommendedOptions.SecureServing.BindPort, err = createLocalhostListenerOnFreePort()

View File

@@ -832,7 +832,7 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{},
}
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
if c.FeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
manager := c.AggregatedDiscoveryGroupManager
if manager == nil {
manager = discoveryendpoint.NewResourceManager("apis")
@@ -1047,14 +1047,14 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
handler = genericfilters.WithRetryAfter(handler, c.lifecycleSignals.NotAcceptingNewRequest.Signaled())
}
handler = genericfilters.WithHTTPLogging(handler)
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) {
if c.FeatureGate.Enabled(genericfeatures.APIServerTracing) {
handler = genericapifilters.WithTracing(handler, c.TracerProvider)
}
handler = genericapifilters.WithLatencyTrackers(handler)
// WithRoutine will execute future handlers in a separate goroutine and serving
// handler in current goroutine to minimize the stack memory usage. It must be
// after WithPanicRecover() to be protected from panics.
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServingWithRoutine) {
if c.FeatureGate.Enabled(genericfeatures.APIServingWithRoutine) {
handler = genericfilters.WithRoutine(handler, c.LongRunningFunc)
}
handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver)
@@ -1098,7 +1098,7 @@ func installAPI(s *GenericAPIServer, c *Config) {
routes.Version{Version: c.EffectiveVersion.BinaryVersion().Info()}.Install(s.Handler.GoRestfulContainer)
if c.EnableDiscovery {
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
if c.FeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(s.DiscoveryGroupManager, s.AggregatedDiscoveryGroupManager)
s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/apis", metav1.APIGroupList{}))
} else {

View File

@@ -40,6 +40,7 @@ import (
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/server/healthz"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake"
@@ -308,6 +309,7 @@ func TestAuthenticationAuditAnnotationsDefaultChain(t *testing.T) {
LongRunningFunc: func(_ *http.Request, _ *request.RequestInfo) bool { return false },
lifecycleSignals: newLifecycleSignals(),
TracerProvider: tracing.NewNoopTracerProvider(),
FeatureGate: utilfeature.DefaultFeatureGate,
}
h := DefaultBuildHandlerChain(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

View File

@@ -51,7 +51,6 @@ import (
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/server/routes"
"k8s.io/apiserver/pkg/storageversion"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
restclient "k8s.io/client-go/rest"
"k8s.io/component-base/featuregate"
@@ -780,7 +779,7 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A
}
resourceInfos = append(resourceInfos, r...)
if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
if s.FeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
// Aggregated discovery only aggregates resources under /apis
if apiPrefix == APIGroupPrefix {
s.AggregatedDiscoveryGroupManager.AddGroupVersion(
@@ -808,8 +807,8 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A
s.RegisterDestroyFunc(apiGroupInfo.destroyStorage)
if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionAPI) &&
utilfeature.DefaultFeatureGate.Enabled(features.APIServerIdentity) {
if s.FeatureGate.Enabled(features.StorageVersionAPI) &&
s.FeatureGate.Enabled(features.APIServerIdentity) {
// API installation happens before we start listening on the handlers,
// therefore it is safe to register ResourceInfos here. The handler will block
// write requests until the storage versions of the targeting resources are updated.
@@ -839,7 +838,7 @@ func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo
// Install the version handler.
// Add a handler at /<apiPrefix> to enumerate the supported api versions.
legacyRootAPIHandler := discovery.NewLegacyRootAPIHandler(s.discoveryAddresses, s.Serializer, apiPrefix)
if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
if s.FeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(legacyRootAPIHandler, s.AggregatedLegacyDiscoveryGroupManager)
s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/api", metav1.APIVersions{}))
} else {

View File

@@ -33,7 +33,6 @@ func (f fakeGroupRegistry) IsGroupRegistered(group string) bool {
func TestAPIEnablementOptionsValidate(t *testing.T) {
testCases := []struct {
name string
testOptions *APIEnablementOptions
runtimeConfig cliflag.ConfigurationMap
expectErr string
}{

View File

@@ -25,9 +25,10 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/server"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/component-base/featuregate"
"github.com/spf13/pflag"
)
@@ -91,12 +92,23 @@ type ServerRunOptions struct {
// it is not overridden by any other grace period.
ShutdownWatchTerminationGracePeriod time.Duration
// FeatureGate are the featuregate to install on the CLI
FeatureGate featuregate.FeatureGate
EffectiveVersion utilversion.EffectiveVersion
// ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored.
ComponentGlobalsRegistry utilversion.ComponentGlobalsRegistry
// ComponentName is name under which the server's global variabled are registered in the ComponentGlobalsRegistry.
ComponentName string
}
func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions {
func NewServerRunOptions() *ServerRunOptions {
if utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) == nil {
featureGate := utilfeature.DefaultMutableFeatureGate
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate))
}
return NewServerRunOptionsForComponent(utilversion.DefaultKubeComponent, utilversion.DefaultComponentGlobalsRegistry)
}
func NewServerRunOptionsForComponent(componentName string, componentGlobalsRegistry utilversion.ComponentGlobalsRegistry) *ServerRunOptions {
defaults := server.NewConfig(serializer.CodecFactory{})
return &ServerRunOptions{
MaxRequestsInFlight: defaults.MaxRequestsInFlight,
@@ -109,13 +121,16 @@ func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion u
JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes,
MaxRequestBodyBytes: defaults.MaxRequestBodyBytes,
ShutdownSendRetryAfter: false,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
ComponentName: componentName,
ComponentGlobalsRegistry: componentGlobalsRegistry,
}
}
// ApplyTo applies the run options to the method receiver and returns self
func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
if err := s.ComponentGlobalsRegistry.SetFallback(); err != nil {
return err
}
c.CorsAllowedOriginList = s.CorsAllowedOriginList
c.HSTSDirectives = s.HSTSDirectives
c.ExternalAddress = s.ExternalHost
@@ -131,8 +146,8 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
c.PublicAddress = s.AdvertiseAddress
c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter
c.ShutdownWatchTerminationGracePeriod = s.ShutdownWatchTerminationGracePeriod
c.EffectiveVersion = s.EffectiveVersion
c.FeatureGate = s.FeatureGate
c.EffectiveVersion = s.ComponentGlobalsRegistry.EffectiveVersionFor(s.ComponentName)
c.FeatureGate = s.ComponentGlobalsRegistry.FeatureGateFor(s.ComponentName)
return nil
}
@@ -205,12 +220,7 @@ func (s *ServerRunOptions) Validate() []error {
if err := validateCorsAllowedOriginList(s.CorsAllowedOriginList); err != nil {
errors = append(errors, err)
}
if s.FeatureGate != nil {
if errs := s.FeatureGate.Validate(); len(errs) != 0 {
errors = append(errors, errs...)
}
}
if errs := s.EffectiveVersion.Validate(); len(errs) != 0 {
if errs := s.ComponentGlobalsRegistry.Validate(); len(errs) != 0 {
errors = append(errors, errs...)
}
return errors
@@ -353,15 +363,11 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
fs.DurationVar(&s.ShutdownWatchTerminationGracePeriod, "shutdown-watch-termination-grace-period", s.ShutdownWatchTerminationGracePeriod, ""+
"This option, if set, represents the maximum amount of grace period the apiserver will wait "+
"for active watch request(s) to drain during the graceful server shutdown window.")
s.ComponentGlobalsRegistry.AddFlags(fs)
}
// Complete fills missing fields with defaults.
func (s *ServerRunOptions) Complete() error {
if s.FeatureGate == nil {
return fmt.Errorf("nil FeatureGate in ServerRunOptions")
}
if s.EffectiveVersion == nil {
return fmt.Errorf("nil EffectiveVersion in ServerRunOptions")
}
return nil
return s.ComponentGlobalsRegistry.SetFallback()
}

View File

@@ -23,14 +23,21 @@ import (
"time"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/version"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
netutils "k8s.io/utils/net"
)
func TestServerRunOptionsValidate(t *testing.T) {
testRegistry := utilversion.NewComponentGlobalsRegistry()
featureGate := utilfeature.DefaultFeatureGate.DeepCopy()
effectiveVersion := utilversion.NewEffectiveVersion("1.30")
effectiveVersion.SetEmulationVersion(version.MajorMinor(1, 32))
testComponent := "test"
utilruntime.Must(testRegistry.Register(testComponent, effectiveVersion, featureGate))
testCases := []struct {
name string
testOptions *ServerRunOptions
@@ -47,8 +54,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--max-requests-inflight can not be negative value",
},
@@ -63,8 +69,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--max-mutating-requests-inflight can not be negative value",
},
@@ -79,8 +84,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--request-timeout can not be negative value",
},
@@ -95,8 +99,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: -1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--min-request-timeout can not be negative value",
},
@@ -111,8 +114,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: -10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value",
},
@@ -127,8 +129,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: -10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "ServerRunOptions.MaxRequestBodyBytes can not be negative value",
},
@@ -144,8 +145,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
LivezGracePeriod: -time.Second,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--livez-grace-period can not be a negative value",
},
@@ -161,8 +161,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ShutdownDelayDuration: -time.Second,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--shutdown-delay-duration can not be negative value",
},
@@ -178,11 +177,27 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--strict-transport-security-directives invalid, allowed values: max-age=expireTime, includeSubDomains, preload. see https://tools.ietf.org/html/rfc6797#section-6.1 for more information",
},
{
name: "Test when emulation version is invalid",
testOptions: &ServerRunOptions{
AdvertiseAddress: netutils.ParseIPSloppy("192.168.10.10"),
CorsAllowedOriginList: []string{"^10.10.10.100$", "^10.10.10.200$"},
HSTSDirectives: []string{"max-age=31536000", "includeSubDomains", "preload"},
MaxRequestsInFlight: 400,
MaxMutatingRequestsInFlight: 200,
RequestTimeout: time.Duration(2) * time.Minute,
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentName: testComponent,
ComponentGlobalsRegistry: testRegistry,
},
expectErr: "emulation version 1.32 is not between [1.29, 1.30.0]",
},
{
name: "Test when ServerRunOptions is valid",
testOptions: &ServerRunOptions{
@@ -195,8 +210,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
},
}
@@ -216,8 +230,6 @@ func TestServerRunOptionsValidate(t *testing.T) {
}
func TestValidateCorsAllowedOriginList(t *testing.T) {
featureGate := utilfeature.DefaultFeatureGate.DeepCopy()
effectiveVersion := utilversion.NewEffectiveVersion("1.30")
tests := []struct {
regexp [][]string
errShouldContain string
@@ -265,7 +277,7 @@ func TestValidateCorsAllowedOriginList(t *testing.T) {
for _, test := range tests {
for _, regexp := range test.regexp {
t.Run(fmt.Sprintf("regexp/%s", regexp), func(t *testing.T) {
options := NewServerRunOptions(featureGate, effectiveVersion)
options := NewServerRunOptions()
if errs := options.Validate(); len(errs) != 0 {
t.Fatalf("wrong test setup: %#v", errs)
}
@@ -289,8 +301,6 @@ func TestValidateCorsAllowedOriginList(t *testing.T) {
}
func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) {
featureGate := utilfeature.DefaultFeatureGate.DeepCopy()
effectiveVersion := utilversion.NewEffectiveVersion("1.30")
tests := []struct {
name string
optionsFn func() *ServerRunOptions
@@ -299,13 +309,13 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) {
{
name: "default should be valid",
optionsFn: func() *ServerRunOptions {
return NewServerRunOptions(featureGate, effectiveVersion)
return NewServerRunOptions()
},
},
{
name: "negative not allowed",
optionsFn: func() *ServerRunOptions {
o := NewServerRunOptions(featureGate, effectiveVersion)
o := NewServerRunOptions()
o.ShutdownWatchTerminationGracePeriod = -time.Second
return o
},
@@ -332,7 +342,7 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) {
}
t.Run("default should be zero", func(t *testing.T) {
options := NewServerRunOptions(featureGate, effectiveVersion)
options := NewServerRunOptions()
if options.ShutdownWatchTerminationGracePeriod != time.Duration(0) {
t.Errorf("expected default of ShutdownWatchTerminationGracePeriod to be zero, but got: %s", options.ShutdownWatchTerminationGracePeriod)
}

View File

@@ -181,9 +181,9 @@ func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example
// was being served which didn't exist at all in min-compatibility-version.
//
// In the alpha case - we do not support compatibility versioning of
// alpha types and recommend users do not mix the two.
// alpha types and recommend users do not mix the two.
// In the skip-level case - The version of apiserver we are retaining
// compatibility with has no knowledge of the type,
// so storing it in another type is no issue.
// compatibility with has no knowledge of the type,
// so storing it in another type is no issue.
return binaryVersionOfResource, nil
}

View File

@@ -101,6 +101,8 @@ type ComponentGlobalsRegistry interface {
AddFlags(fs *pflag.FlagSet)
// Set sets the flags for all global variables for all components registered.
Set() error
// SetFallback calls Set() if it has never been called.
SetFallback() error
// Validate calls the Validate() function for all the global variables for all components registered.
Validate() []error
// Reset removes all stored ComponentGlobals, configurations, and version mappings.
@@ -120,6 +122,8 @@ type componentGlobalsRegistry struct {
emulationVersionConfig []string
// map of component name to the list of feature gates set from the flag.
featureGatesConfig map[string][]string
// set stores if the Set() function for the registry is already called.
set bool
}
func NewComponentGlobalsRegistry() *componentGlobalsRegistry {
@@ -136,6 +140,7 @@ func (r *componentGlobalsRegistry) Reset() {
r.componentGlobals = make(map[string]*ComponentGlobals)
r.emulationVersionConfig = nil
r.featureGatesConfig = nil
r.set = false
}
func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion {
@@ -330,9 +335,22 @@ func toVersionMap(versionConfig []string) (map[string]*version.Version, error) {
return m, nil
}
func (r *componentGlobalsRegistry) SetFallback() error {
r.mutex.Lock()
set := r.set
r.mutex.Unlock()
if set {
return nil
}
klog.Warning("setting componentGlobalsRegistry in SetFallback. We recommend calling componentGlobalsRegistry.Set()" +
" right after parsing flags to avoid using feature gates before their final values are set by the flags.")
return r.Set()
}
func (r *componentGlobalsRegistry) Set() error {
r.mutex.Lock()
defer r.mutex.Unlock()
r.set = true
emulationVersionConfigMap, err := toVersionMap(r.emulationVersionConfig)
if err != nil {
return err

View File

@@ -111,12 +111,6 @@ func (m *effectiveVersion) Validate() []error {
// emulationVersion can only be 1.{binaryMinor-1}...1.{binaryMinor}.
maxEmuVer := binaryVersion
minEmuVer := binaryVersion.SubtractMinor(1)
// TODO: remove in 1.32
// emulationVersion is introduced in 1.31, so it cannot be lower than that.
// binaryVersion could be lower than 1.31 in tests. So we are only checking 1.31.
if binaryVersion.EqualTo(version.MajorMinor(1, 31)) {
minEmuVer = version.MajorMinor(1, 31)
}
if emulationVersion.GreaterThan(maxEmuVer) || emulationVersion.LessThan(minEmuVer) {
errs = append(errs, fmt.Errorf("emulation version %s is not between [%s, %s]", emulationVersion.String(), minEmuVer.String(), maxEmuVer.String()))
}
@@ -157,8 +151,7 @@ func DefaultBuildEffectiveVersion() MutableEffectiveVersion {
// DefaultKubeEffectiveVersion returns the MutableEffectiveVersion based on the
// latest K8s release.
// Should update for each minor release!
func DefaultKubeEffectiveVersion() MutableEffectiveVersion {
binaryVersion := version.MustParse("1.31").WithInfo(baseversion.Get())
binaryVersion := version.MustParse(baseversion.DefaultKubeBinaryVersion).WithInfo(baseversion.Get())
return newEffectiveVersion(binaryVersion)
}

View File

@@ -42,25 +42,6 @@ func TestValidate(t *testing.T) {
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.31.0",
},
{
name: "binary version 1.31, emulation version lower than 1.31",
binaryVersion: "v1.31.2",
emulationVersion: "v1.30.0",
minCompatibilityVersion: "v1.30.0",
expectErrors: true,
},
{
name: "binary version 1.31, emulation version 1.31",
binaryVersion: "v1.31.2",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.30.0",
},
{
name: "binary version lower than 1.31",
binaryVersion: "v1.30.2",
emulationVersion: "v1.29.0",
minCompatibilityVersion: "v1.29.0",
},
{
name: "emulation version two minor lower than binary not ok",
binaryVersion: "v1.33.2",

View File

@@ -114,6 +114,9 @@ type FeatureGate interface {
// set on the copy without mutating the original. This is useful for validating
// config against potential feature gate changes before committing those changes.
DeepCopy() MutableVersionedFeatureGate
// CopyKnownFeatures returns a partial copy of the FeatureGate object, with all the known features and overrides.
// This is useful for creating a new instance of feature gate without inheriting all the enabled configurations of the base feature gate.
CopyKnownFeatures() MutableVersionedFeatureGate
// Validate checks if the flag gates are valid at the emulated version.
Validate() []error
}
@@ -127,9 +130,6 @@ type MutableFeatureGate interface {
AddFlag(fs *pflag.FlagSet)
// Close sets closed to true, and prevents subsequent calls to Add
Close()
// OpenForModification sets closedForModification to false, and allows subsequent calls to SetEmulationVersion to change enabled features
// before the next Enabled is called.
OpenForModification()
// Set parses and stores flag gates for known features
// from a string like feature1=true,feature2=false,...
Set(value string) error
@@ -166,8 +166,6 @@ type MutableVersionedFeatureGate interface {
// SetEmulationVersion overrides the emulationVersion of the feature gate.
// Otherwise, the emulationVersion will be the same as the binary version.
// If set, the feature defaults and availability will be as if the binary is at the emulated version.
// Returns error if the new emulationVersion will change the enablement state of a feature that has already been queried.
// If you have to use featureGate.Enabled before parsing the flags, call featureGate.OpenForModification following featureGate.Enabled.
SetEmulationVersion(emulationVersion *version.Version) error
// GetAll returns a copy of the map of known feature names to versioned feature specs.
GetAllVersioned() map[Feature]VersionedSpecs
@@ -185,15 +183,11 @@ type MutableVersionedFeatureGate interface {
// overriding its default to true for a limited number of components without simultaneously
// changing its default for all consuming components.
OverrideDefaultAtVersion(name Feature, override bool, ver *version.Version) error
}
// MutableVersionedFeatureGateForTests is a feature gate interface that should only be used in tests.
type MutableVersionedFeatureGateForTests interface {
MutableVersionedFeatureGate
// Reset sets the enabled and enabledRaw to the input map.
Reset(m map[string]bool)
// EnabledRawMap returns the raw enable map from the feature gate.
EnabledRawMap() map[string]bool
// ExplicitlySet returns true if the feature value is explicitly set instead of
// being derived from the default values or special features.
ExplicitlySet(name Feature) bool
// ResetFeatureValueToDefault resets the value of the feature back to the default value.
ResetFeatureValueToDefault(name Feature) error
}
// featureGate implements FeatureGate as well as pflag.Value for flag parsing.
@@ -214,12 +208,8 @@ type featureGate struct {
enabledRaw atomic.Value
// closed is set to true when AddFlag is called, and prevents subsequent calls to Add
closed bool
// closedForModification is set to true when Enabled is called, and prevents subsequent calls to SetEmulationVersion to change the enabled features.
// TODO: after all feature gates have migrated to versioned feature gates,
// closedForModification should also prevents subsequent calls to Set and SetFromMap to change the enabled features
closedForModification atomic.Bool
// queriedFeatures stores all the features that have been queried through the Enabled interface.
// It is reset when closedForModification is reset.
// It is reset when SetEmulationVersion is called.
queriedFeatures atomic.Value
emulationVersion atomic.Pointer[version.Version]
}
@@ -282,7 +272,7 @@ func NewVersionedFeatureGate(emulationVersion *version.Version) *featureGate {
// NewFeatureGate creates a feature gate with the current binary version.
func NewFeatureGate() *featureGate {
binaryVersison := version.MustParse(baseversion.Get().String())
binaryVersison := version.MustParse(baseversion.DefaultKubeBinaryVersion)
return NewVersionedFeatureGate(binaryVersison)
}
@@ -311,6 +301,8 @@ func (f *featureGate) Set(value string) error {
// Validate checks if the flag gates are valid at the emulated version.
func (f *featureGate) Validate() []error {
f.lock.Lock()
defer f.lock.Unlock()
m, ok := f.enabledRaw.Load().(map[string]bool)
if !ok {
return []error{fmt.Errorf("cannot cast enabledRaw to map[string]bool")}
@@ -497,13 +489,17 @@ func (f *featureGate) OverrideDefaultAtVersion(name Feature, override bool, ver
// GetAll returns a copy of the map of known feature names to feature specs for the current emulationVersion.
func (f *featureGate) GetAll() map[Feature]FeatureSpec {
retval := map[Feature]FeatureSpec{}
for k, v := range f.GetAllVersioned() {
spec := f.featureSpecAtEmulationVersion(v)
f.lock.Lock()
versionedSpecs := f.GetAllVersioned()
emuVer := f.EmulationVersion()
f.lock.Unlock()
for k, v := range versionedSpecs {
spec := featureSpecAtEmulationVersion(v, emuVer)
if spec.PreRelease == PreAlpha {
// The feature is not available at the emulation version.
continue
}
retval[k] = *f.featureSpecAtEmulationVersion(v)
retval[k] = *spec
}
return retval
}
@@ -518,6 +514,9 @@ func (f *featureGate) GetAllVersioned() map[Feature]VersionedSpecs {
}
func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) error {
if emulationVersion.EqualTo(f.EmulationVersion()) {
return nil
}
f.lock.Lock()
defer f.lock.Unlock()
klog.V(1).Infof("set feature gate emulationVersion to %s", emulationVersion.String())
@@ -531,22 +530,21 @@ func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) err
enabled := map[Feature]bool{}
errs := f.unsafeSetFromMap(enabled, enabledRaw, emulationVersion)
if f.closedForModification.Load() {
queriedFeatures := f.queriedFeatures.Load().(map[Feature]struct{})
known := f.known.Load().(map[Feature]VersionedSpecs)
for feature := range queriedFeatures {
newVal := featureEnabled(feature, enabled, known, emulationVersion)
oldVal := f.Enabled(feature)
// it is ok to modify emulation version if it does not result in feature enablemennt change for features that have already been queried.
if newVal != oldVal {
errs = append(errs, fmt.Errorf("SetEmulationVersion will change already queried feature:%s from %v to %v\ncall featureGate.OpenForModification() first to override", feature, oldVal, newVal))
}
queriedFeatures := f.queriedFeatures.Load().(map[Feature]struct{})
known := f.known.Load().(map[Feature]VersionedSpecs)
for feature := range queriedFeatures {
newVal := featureEnabled(feature, enabled, known, emulationVersion)
oldVal := featureEnabled(feature, f.enabled.Load().(map[Feature]bool), known, f.EmulationVersion())
if newVal != oldVal {
klog.Warningf("SetEmulationVersion will change already queried feature:%s from %v to %v", feature, oldVal, newVal)
}
}
if len(errs) == 0 {
// Persist changes
f.enabled.Store(enabled)
f.emulationVersion.Store(emulationVersion)
f.queriedFeatures.Store(map[Feature]struct{}{})
}
return utilerrors.NewAggregate(errs)
}
@@ -555,9 +553,9 @@ func (f *featureGate) EmulationVersion() *version.Version {
return f.emulationVersion.Load()
}
// FeatureSpec returns the FeatureSpec at the EmulationVersion if the key exists, an error otherwise.
// featureSpec returns the featureSpec at the EmulationVersion if the key exists, an error otherwise.
// This is useful to keep multiple implementations of a feature based on the PreRelease or Version info.
func (f *featureGate) FeatureSpec(key Feature) (FeatureSpec, error) {
func (f *featureGate) featureSpec(key Feature) (FeatureSpec, error) {
if v, ok := f.known.Load().(map[Feature]VersionedSpecs)[key]; ok {
featureSpec := f.featureSpecAtEmulationVersion(v)
return *featureSpec, nil
@@ -565,14 +563,16 @@ func (f *featureGate) FeatureSpec(key Feature) (FeatureSpec, error) {
return FeatureSpec{}, fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName)
}
func (f *featureGate) recordQueried(key Feature) {
func (f *featureGate) unsafeRecordQueried(key Feature) {
queriedFeatures := map[Feature]struct{}{}
for k := range f.queriedFeatures.Load().(map[Feature]struct{}) {
queriedFeatures[k] = struct{}{}
}
if _, ok := queriedFeatures[key]; ok {
return
}
queriedFeatures[key] = struct{}{}
f.queriedFeatures.Store(queriedFeatures)
f.closedForModification.Store(true)
}
func featureEnabled(key Feature, enabled map[Feature]bool, known map[Feature]VersionedSpecs, emulationVersion *version.Version) bool {
@@ -589,8 +589,9 @@ func featureEnabled(key Feature, enabled map[Feature]bool, known map[Feature]Ver
// Enabled returns true if the key is enabled. If the key is not known, this call will panic.
func (f *featureGate) Enabled(key Feature) bool {
// TODO: ideally we should lock the feature gate in this call to be safe, need to evaluate how much performance impact locking would have.
v := featureEnabled(key, f.enabled.Load().(map[Feature]bool), f.known.Load().(map[Feature]VersionedSpecs), f.EmulationVersion())
f.recordQueried(key)
f.unsafeRecordQueried(key)
return v
}
@@ -620,18 +621,6 @@ func (f *featureGate) Close() {
f.lock.Unlock()
}
// OpenForModification sets closedForModification to false, and allows subsequent calls to SetEmulationVersion to change enabled features
// before the next Enabled is called.
func (f *featureGate) OpenForModification() {
queriedFeatures := []Feature{}
for feature := range f.queriedFeatures.Load().(map[Feature]struct{}) {
queriedFeatures = append(queriedFeatures, feature)
}
klog.Warningf("open feature gate for modification after querying features: %v.", queriedFeatures)
f.closedForModification.Store(false)
f.queriedFeatures.Store(map[Feature]struct{}{})
}
// AddFlag adds a flag for setting global feature gates to the specified FlagSet.
func (f *featureGate) AddFlag(fs *pflag.FlagSet) {
// TODO(mtaufen): Shouldn't we just close it on the first Set/SetFromMap instead?
@@ -671,10 +660,21 @@ func (f *featureGate) KnownFeatures() []string {
return known
}
// CopyKnownFeatures returns a partial copy of the FeatureGate object, with all the known features and overrides.
// This is useful for creating a new instance of feature gate without inheriting all the enabled configurations of the base feature gate.
func (f *featureGate) CopyKnownFeatures() MutableVersionedFeatureGate {
fg := NewVersionedFeatureGate(f.EmulationVersion())
known := f.GetAllVersioned()
fg.known.Store(known)
return fg
}
// DeepCopy returns a deep copy of the FeatureGate object, such that gates can be
// set on the copy without mutating the original. This is useful for validating
// config against potential feature gate changes before committing those changes.
func (f *featureGate) DeepCopy() MutableVersionedFeatureGate {
f.lock.Lock()
defer f.lock.Unlock()
// Copy existing state.
known := map[Feature]VersionedSpecs{}
for k, v := range f.known.Load().(map[Feature]VersionedSpecs) {
@@ -691,7 +691,7 @@ func (f *featureGate) DeepCopy() MutableVersionedFeatureGate {
// Construct a new featureGate around the copied state.
// Note that specialFeatures is treated as immutable by convention,
// and we maintain the value of f.closed across the copy, but resets closedForModification.
// and we maintain the value of f.closed across the copy.
fg := &featureGate{
special: specialFeatures,
closed: f.closed,
@@ -704,21 +704,40 @@ func (f *featureGate) DeepCopy() MutableVersionedFeatureGate {
return fg
}
// Reset sets the enabled and enabledRaw to the input map.
func (f *featureGate) Reset(m map[string]bool) {
enabled := map[Feature]bool{}
enabledRaw := map[string]bool{}
queriedFeatures := map[Feature]struct{}{}
f.enabled.Store(enabled)
f.enabledRaw.Store(enabledRaw)
_ = f.SetFromMap(m)
f.closedForModification.Store(false)
f.queriedFeatures.Store(queriedFeatures)
f.lock.Lock()
defer f.lock.Unlock()
f.closed = false
// ExplicitlySet returns true if the feature value is explicitly set instead of
// being derived from the default values or special features.
func (f *featureGate) ExplicitlySet(name Feature) bool {
enabledRaw := f.enabledRaw.Load().(map[string]bool)
_, ok := enabledRaw[string(name)]
return ok
}
func (f *featureGate) EnabledRawMap() map[string]bool {
return f.enabledRaw.Load().(map[string]bool)
// ResetFeatureValueToDefault resets the value of the feature back to the default value.
func (f *featureGate) ResetFeatureValueToDefault(name Feature) error {
f.lock.Lock()
defer f.lock.Unlock()
enabled := map[Feature]bool{}
for k, v := range f.enabled.Load().(map[Feature]bool) {
enabled[k] = v
}
enabledRaw := map[string]bool{}
for k, v := range f.enabledRaw.Load().(map[string]bool) {
enabledRaw[k] = v
}
_, inEnabled := enabled[name]
if inEnabled {
delete(enabled, name)
}
_, inEnabledRaw := enabledRaw[string(name)]
if inEnabledRaw {
delete(enabledRaw, string(name))
}
// some features could be in enabled map but not enabledRaw map,
// for example some Alpha feature when AllAlpha is set.
if inEnabledRaw && !inEnabled {
return fmt.Errorf("feature:%s was explicitly set, but not in enabled map", name)
}
f.enabled.Store(enabled)
f.enabledRaw.Store(enabledRaw)
return nil
}

View File

@@ -596,6 +596,19 @@ func TestFeatureGateOverrideDefault(t *testing.T) {
}
})
t.Run("overrides are preserved across CopyKnownFeatures", func(t *testing.T) {
f := NewFeatureGate()
require.NoError(t, f.Add(map[Feature]FeatureSpec{"TestFeature": {Default: false}}))
require.NoError(t, f.OverrideDefault("TestFeature", true))
fcopy := f.CopyKnownFeatures()
if !f.Enabled("TestFeature") {
t.Error("TestFeature should be enabled by override")
}
if !fcopy.Enabled("TestFeature") {
t.Error("default override was not preserved by CopyKnownFeatures")
}
})
t.Run("reflected in known features", func(t *testing.T) {
f := NewFeatureGate()
if err := f.Add(map[Feature]FeatureSpec{"TestFeature": {
@@ -1119,22 +1132,22 @@ func TestVersionedFeatureGateFlagDefaults(t *testing.T) {
if f.Enabled(testAlphaGate) != false {
t.Errorf("Expected false")
}
if fs, _ := f.FeatureSpec(testAlphaGate); fs.PreRelease != PreAlpha || fs.Version.String() != "0.0" {
if fs, _ := f.featureSpec(testAlphaGate); fs.PreRelease != PreAlpha || fs.Version.String() != "0.0" {
t.Errorf("Expected (PreAlpha, 0.0)")
}
if f.Enabled(testBetaGate) != false {
t.Errorf("Expected false")
}
if fs, _ := f.FeatureSpec(testBetaGate); fs.PreRelease != Beta || fs.Version.String() != "1.28" {
if fs, _ := f.featureSpec(testBetaGate); fs.PreRelease != Beta || fs.Version.String() != "1.28" {
t.Errorf("Expected (Beta, 1.28)")
}
if f.Enabled(testGAGate) != true {
t.Errorf("Expected true")
}
if fs, _ := f.FeatureSpec(testGAGate); fs.PreRelease != Beta || fs.Version.String() != "1.27" {
if fs, _ := f.featureSpec(testGAGate); fs.PreRelease != Beta || fs.Version.String() != "1.27" {
t.Errorf("Expected (Beta, 1.27)")
}
if _, err := f.FeatureSpec("NonExist"); err == nil {
if _, err := f.featureSpec("NonExist"); err == nil {
t.Errorf("Expected Error")
}
allFeatures := f.GetAll()
@@ -1304,7 +1317,6 @@ func TestVersionedFeatureGateOverrideDefault(t *testing.T) {
if !f.Enabled("TestFeature2") {
t.Error("expected TestFeature2 to have effective default of true")
}
f.OpenForModification()
require.NoError(t, f.SetEmulationVersion(version.MustParse("1.29")))
if !f.Enabled("TestFeature1") {
t.Error("expected TestFeature1 to have effective default of true")
@@ -1312,7 +1324,6 @@ func TestVersionedFeatureGateOverrideDefault(t *testing.T) {
if f.Enabled("TestFeature2") {
t.Error("expected TestFeature2 to have effective default of false")
}
f.OpenForModification()
require.NoError(t, f.SetEmulationVersion(version.MustParse("1.26")))
if f.Enabled("TestFeature1") {
t.Error("expected TestFeature1 to have effective default of false")
@@ -1520,32 +1531,173 @@ func TestFeatureSpecAtEmulationVersion(t *testing.T) {
}
}
func TestOpenForModification(t *testing.T) {
const testBetaGate Feature = "testBetaGate"
f := NewVersionedFeatureGate(version.MustParse("1.29"))
func TestCopyKnownFeatures(t *testing.T) {
f := NewFeatureGate()
require.NoError(t, f.Add(map[Feature]FeatureSpec{"FeatureA": {Default: false}, "FeatureB": {Default: false}}))
require.NoError(t, f.Set("FeatureA=true"))
require.NoError(t, f.OverrideDefault("FeatureB", true))
fcopy := f.CopyKnownFeatures()
require.NoError(t, f.Add(map[Feature]FeatureSpec{"FeatureC": {Default: false}}))
assert.True(t, f.Enabled("FeatureA"))
assert.True(t, f.Enabled("FeatureB"))
assert.False(t, f.Enabled("FeatureC"))
assert.False(t, fcopy.Enabled("FeatureA"))
assert.True(t, fcopy.Enabled("FeatureB"))
require.NoError(t, fcopy.Set("FeatureB=false"))
assert.True(t, f.Enabled("FeatureB"))
assert.False(t, fcopy.Enabled("FeatureB"))
if err := fcopy.Set("FeatureC=true"); err == nil {
t.Error("expected FeatureC not registered in the copied feature gate")
}
}
func TestExplicitlySet(t *testing.T) {
// gates for testing
const testAlphaGate Feature = "TestAlpha"
const testBetaGate Feature = "TestBeta"
tests := []struct {
arg string
expectedFeatureValue map[Feature]bool
expectedExplicitlySet map[Feature]bool
}{
{
arg: "",
expectedFeatureValue: map[Feature]bool{
allAlphaGate: false,
allBetaGate: false,
testAlphaGate: false,
testBetaGate: false,
},
expectedExplicitlySet: map[Feature]bool{
allAlphaGate: false,
allBetaGate: false,
testAlphaGate: false,
testBetaGate: false,
},
},
{
arg: "AllAlpha=true,TestBeta=false",
expectedFeatureValue: map[Feature]bool{
allAlphaGate: true,
allBetaGate: false,
testAlphaGate: true,
testBetaGate: false,
},
expectedExplicitlySet: map[Feature]bool{
allAlphaGate: true,
allBetaGate: false,
testAlphaGate: false,
testBetaGate: true,
},
},
{
arg: "AllAlpha=true,AllBeta=false",
expectedFeatureValue: map[Feature]bool{
allAlphaGate: true,
allBetaGate: false,
testAlphaGate: true,
testBetaGate: false,
},
expectedExplicitlySet: map[Feature]bool{
allAlphaGate: true,
allBetaGate: true,
testAlphaGate: false,
testBetaGate: false,
},
},
}
for i, test := range tests {
t.Run(test.arg, func(t *testing.T) {
fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError)
f := NewVersionedFeatureGate(version.MustParse("1.29"))
err := f.AddVersioned(map[Feature]VersionedSpecs{
testAlphaGate: {
{Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha},
},
testBetaGate: {
{Version: version.MustParse("1.29"), Default: false, PreRelease: Beta},
{Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha},
},
})
require.NoError(t, err)
f.AddFlag(fs)
var errs []error
err = fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)})
if err != nil {
errs = append(errs, err)
}
err = utilerrors.NewAggregate(errs)
require.NoError(t, err)
for k, v := range test.expectedFeatureValue {
if actual := f.Enabled(k); actual != v {
t.Errorf("%d: expected %s=%v, Got %v", i, k, v, actual)
}
}
for k, v := range test.expectedExplicitlySet {
if actual := f.ExplicitlySet(k); actual != v {
t.Errorf("%d: expected ExplicitlySet(%s)=%v, Got %v", i, k, v, actual)
}
}
})
}
}
func TestResetFeatureValueToDefault(t *testing.T) {
// gates for testing
const testAlphaGate Feature = "TestAlpha"
const testBetaGate Feature = "TestBeta"
f := NewVersionedFeatureGate(version.MustParse("1.29"))
err := f.AddVersioned(map[Feature]VersionedSpecs{
testAlphaGate: {
{Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha},
},
testBetaGate: {
{Version: version.MustParse("1.29"), Default: true, PreRelease: Beta},
{Version: version.MustParse("1.28"), Default: false, PreRelease: Beta},
{Version: version.MustParse("1.26"), Default: false, PreRelease: Alpha},
{Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha},
},
})
require.NoError(t, err)
if f.Enabled(testBetaGate) != true {
t.Errorf("Expected true")
fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError)
assert.False(t, f.Enabled("AllAlpha"))
assert.False(t, f.Enabled("AllBeta"))
assert.False(t, f.Enabled("TestAlpha"))
assert.True(t, f.Enabled("TestBeta"))
f.AddFlag(fs)
var errs []error
err = fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, "AllAlpha=true,TestBeta=false")})
if err != nil {
errs = append(errs, err)
}
err = f.SetEmulationVersion(version.MustParse("1.28"))
if err == nil {
t.Fatalf("Expected error when SetEmulationVersion after querying features")
}
if f.Enabled(testBetaGate) != true {
t.Errorf("Expected true")
}
f.OpenForModification()
err = utilerrors.NewAggregate(errs)
require.NoError(t, err)
assert.True(t, f.Enabled("AllAlpha"))
assert.False(t, f.Enabled("AllBeta"))
assert.True(t, f.Enabled("TestAlpha"))
assert.False(t, f.Enabled("TestBeta"))
require.NoError(t, f.ResetFeatureValueToDefault("AllAlpha"))
assert.False(t, f.Enabled("AllAlpha"))
assert.False(t, f.Enabled("AllBeta"))
assert.True(t, f.Enabled("TestAlpha"))
assert.False(t, f.Enabled("TestBeta"))
require.NoError(t, f.ResetFeatureValueToDefault("TestBeta"))
assert.False(t, f.Enabled("AllAlpha"))
assert.False(t, f.Enabled("AllBeta"))
assert.True(t, f.Enabled("TestAlpha"))
assert.True(t, f.Enabled("TestBeta"))
require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28")))
if f.Enabled(testBetaGate) != false {
t.Errorf("Expected false at 1.28")
}
assert.False(t, f.Enabled("AllAlpha"))
assert.False(t, f.Enabled("AllBeta"))
assert.False(t, f.Enabled("TestAlpha"))
assert.False(t, f.Enabled("TestBeta"))
}

View File

@@ -20,14 +20,16 @@ import (
"fmt"
"strings"
"sync"
"testing"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/component-base/featuregate"
)
var (
overrideLock sync.Mutex
featureFlagOverride map[featuregate.Feature]string
overrideLock sync.Mutex
featureFlagOverride map[featuregate.Feature]string
emulationVersionOverride string
emulationVersionOverrideValue *version.Version
)
func init() {
@@ -42,10 +44,12 @@ func init() {
// Example use:
//
// featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.<FeatureName>, true)
func SetFeatureGateDuringTest(tb testing.TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) {
func SetFeatureGateDuringTest(tb TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) {
tb.Helper()
detectParallelOverrideCleanup := detectParallelOverride(tb, f)
originalEnabled := gate.(featuregate.MutableVersionedFeatureGateForTests).EnabledRawMap()
originalValue := gate.Enabled(f)
originalEmuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion()
originalExplicitlySet := gate.(featuregate.MutableVersionedFeatureGate).ExplicitlySet(f)
// Specially handle AllAlpha and AllBeta
if f == "AllAlpha" || f == "AllBeta" {
@@ -67,11 +71,46 @@ func SetFeatureGateDuringTest(tb testing.TB, gate featuregate.FeatureGate, f fea
tb.Cleanup(func() {
tb.Helper()
detectParallelOverrideCleanup()
gate.(featuregate.MutableVersionedFeatureGateForTests).Reset(originalEnabled)
emuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion()
if !emuVer.EqualTo(originalEmuVer) {
tb.Fatalf("change of feature gate emulation version from %s to %s in the chain of SetFeatureGateDuringTest is not allowed\nuse SetFeatureGateEmulationVersionDuringTest to change emulation version in tests",
originalEmuVer.String(), emuVer.String())
}
if originalExplicitlySet {
if err := gate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, originalValue)); err != nil {
tb.Errorf("error restoring %s=%v: %v", f, originalValue, err)
}
} else {
if err := gate.(featuregate.MutableVersionedFeatureGate).ResetFeatureValueToDefault(f); err != nil {
tb.Errorf("error restoring %s=%v: %v", f, originalValue, err)
}
}
})
}
func detectParallelOverride(tb testing.TB, f featuregate.Feature) func() {
// SetFeatureGateEmulationVersionDuringTest sets the specified gate to the specified emulation version for duration of the test.
// Fails when it detects second call to set a different emulation version or is unable to set or restore emulation version.
// WARNING: Can leak set variable when called in test calling t.Parallel(), however second attempt to set a different emulation version will cause fatal.
// Example use:
// featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31"))
func SetFeatureGateEmulationVersionDuringTest(tb TB, gate featuregate.FeatureGate, ver *version.Version) {
tb.Helper()
detectParallelOverrideCleanup := detectParallelOverrideEmulationVersion(tb, ver)
originalEmuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion()
if err := gate.(featuregate.MutableVersionedFeatureGate).SetEmulationVersion(ver); err != nil {
tb.Fatalf("failed to set emulation version to %s during test", ver.String())
}
tb.Cleanup(func() {
tb.Helper()
detectParallelOverrideCleanup()
if err := gate.(featuregate.MutableVersionedFeatureGate).SetEmulationVersion(originalEmuVer); err != nil {
tb.Fatalf("failed to restore emulation version to %s during test", originalEmuVer.String())
}
})
}
func detectParallelOverride(tb TB, f featuregate.Feature) func() {
tb.Helper()
overrideLock.Lock()
defer overrideLock.Unlock()
@@ -92,7 +131,44 @@ func detectParallelOverride(tb testing.TB, f featuregate.Feature) func() {
}
}
func sameTestOrSubtest(tb testing.TB, testName string) bool {
func detectParallelOverrideEmulationVersion(tb TB, ver *version.Version) func() {
tb.Helper()
overrideLock.Lock()
defer overrideLock.Unlock()
beforeOverrideTestName := emulationVersionOverride
beforeOverrideValue := emulationVersionOverrideValue
if ver.EqualTo(beforeOverrideValue) {
return func() {}
}
if beforeOverrideTestName != "" && !sameTestOrSubtest(tb, beforeOverrideTestName) {
tb.Fatalf("Detected parallel setting of a feature gate emulation version by both %q and %q", beforeOverrideTestName, tb.Name())
}
emulationVersionOverride = tb.Name()
emulationVersionOverrideValue = ver
return func() {
tb.Helper()
overrideLock.Lock()
defer overrideLock.Unlock()
if afterOverrideTestName := emulationVersionOverride; afterOverrideTestName != tb.Name() {
tb.Fatalf("Detected parallel setting of a feature gate emulation version between both %q and %q", afterOverrideTestName, tb.Name())
}
emulationVersionOverride = beforeOverrideTestName
emulationVersionOverrideValue = beforeOverrideValue
}
}
func sameTestOrSubtest(tb TB, testName string) bool {
// Assumes that "/" is not used in test names.
return tb.Name() == testName || strings.HasPrefix(tb.Name(), testName+"/")
}
type TB interface {
Cleanup(func())
Error(args ...any)
Errorf(format string, args ...any)
Fatal(args ...any)
Fatalf(format string, args ...any)
Helper()
Name() string
}

View File

@@ -21,6 +21,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/component-base/featuregate"
)
@@ -71,6 +72,7 @@ func TestSpecialGates(t *gotest.T) {
expect(t, gate, before)
t.Cleanup(func() {
expect(t, gate, before)
cleanup()
})
SetFeatureGateDuringTest(t, gate, "AllAlpha", true)
@@ -156,10 +158,107 @@ func TestSetFeatureGateInTest(t *gotest.T) {
assert.True(t, gate.Enabled("feature"))
}
func TestDetectLeakToMainTest(t *gotest.T) {
t.Cleanup(func() {
featureFlagOverride = map[featuregate.Feature]string{}
func TestSpecialGatesVersioned(t *gotest.T) {
originalEmulationVersion := version.MustParse("1.31")
gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion)
err := gate.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
"alpha_default_on": {
{Version: version.MustParse("1.27"), Default: true, PreRelease: featuregate.Alpha},
},
"alpha_default_off": {
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
},
"beta_default_on": {
{Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta},
},
"beta_default_on_set_off": {
{Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta},
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
},
"beta_default_off": {
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Beta},
},
"beta_default_off_set_on": {
{Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Beta},
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
},
})
require.NoError(t, err)
require.NoError(t, gate.Set("beta_default_on_set_off=false"))
require.NoError(t, gate.Set("beta_default_off_set_on=true"))
before := map[featuregate.Feature]bool{
"AllAlpha": false,
"AllBeta": false,
"alpha_default_on": true,
"alpha_default_off": false,
"beta_default_on": true,
"beta_default_on_set_off": false,
"beta_default_off": false,
"beta_default_off_set_on": true,
}
expect(t, gate, before)
t.Cleanup(func() {
expect(t, gate, before)
cleanup()
})
t.Run("OverwriteInSubtest", func(t *gotest.T) {
SetFeatureGateDuringTest(t, gate, "AllAlpha", true)
expect(t, gate, map[featuregate.Feature]bool{
"AllAlpha": true,
"AllBeta": false,
"alpha_default_on": true,
"alpha_default_off": true,
"beta_default_on": true,
"beta_default_on_set_off": false,
"beta_default_off": false,
"beta_default_off_set_on": true,
})
SetFeatureGateDuringTest(t, gate, "AllBeta", true)
expect(t, gate, map[featuregate.Feature]bool{
"AllAlpha": true,
"AllBeta": true,
"alpha_default_on": true,
"alpha_default_off": true,
"beta_default_on": true,
"beta_default_on_set_off": true,
"beta_default_off": true,
"beta_default_off_set_on": true,
})
})
expect(t, gate, before)
t.Run("OverwriteInSubtestAtDifferentVersion", func(t *gotest.T) {
SetFeatureGateEmulationVersionDuringTest(t, gate, version.MustParse("1.28"))
SetFeatureGateDuringTest(t, gate, "AllAlpha", true)
expect(t, gate, map[featuregate.Feature]bool{
"AllAlpha": true,
"AllBeta": false,
"alpha_default_on": true,
"alpha_default_off": true,
"beta_default_on": false,
"beta_default_on_set_off": true,
"beta_default_off": false,
"beta_default_off_set_on": true,
})
})
}
func TestDetectLeakToMainTest(t *gotest.T) {
t.Cleanup(cleanup)
gate := featuregate.NewFeatureGate()
err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{
"feature": {PreRelease: featuregate.Alpha, Default: false},
@@ -183,9 +282,7 @@ func TestDetectLeakToMainTest(t *gotest.T) {
}
func TestDetectLeakToOtherSubtest(t *gotest.T) {
t.Cleanup(func() {
featureFlagOverride = map[featuregate.Feature]string{}
})
t.Cleanup(cleanup)
gate := featuregate.NewFeatureGate()
err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{
"feature": {PreRelease: featuregate.Alpha, Default: false},
@@ -211,9 +308,7 @@ func TestDetectLeakToOtherSubtest(t *gotest.T) {
}
func TestCannotDetectLeakFromSubtest(t *gotest.T) {
t.Cleanup(func() {
featureFlagOverride = map[featuregate.Feature]string{}
})
t.Cleanup(cleanup)
gate := featuregate.NewFeatureGate()
err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{
"feature": {PreRelease: featuregate.Alpha, Default: false},
@@ -230,6 +325,107 @@ func TestCannotDetectLeakFromSubtest(t *gotest.T) {
assert.True(t, gate.Enabled("feature"))
}
func TestCannotDetectLeakFromTwoSubtestsWithDifferentFeatures(t *gotest.T) {
t.Cleanup(cleanup)
gate := featuregate.NewFeatureGate()
err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{
"feature1": {PreRelease: featuregate.Alpha, Default: false},
"feature2": {PreRelease: featuregate.Alpha, Default: false},
})
require.NoError(t, err)
assert.False(t, gate.Enabled("feature1"))
assert.False(t, gate.Enabled("feature2"))
subtestName := "Subtest"
// Subtest setting feature gate and calling parallel will leak it out
t.Run(subtestName, func(t *gotest.T) {
SetFeatureGateDuringTest(t, gate, "feature1", true)
t.Parallel()
assert.True(t, gate.Enabled("feature1"))
assert.False(t, gate.Enabled("feature2"))
})
// Leaked true
assert.True(t, gate.Enabled("feature1"))
assert.False(t, gate.Enabled("feature2"))
// Add suffix to name to prevent tests with the same prefix.
t.Run(subtestName+"Suffix", func(t *gotest.T) {
// Leaked true
assert.True(t, gate.Enabled("feature1"))
assert.False(t, gate.Enabled("feature2"))
SetFeatureGateDuringTest(t, gate, "feature2", true)
assert.True(t, gate.Enabled("feature1"))
assert.True(t, gate.Enabled("feature2"))
})
}
func TestDetectEmulationVersionLeakToMainTest(t *gotest.T) {
t.Cleanup(cleanup)
originalEmulationVersion := version.MustParse("1.31")
newEmulationVersion := version.MustParse("1.30")
gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion)
assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion))
// Subtest setting feature gate and calling parallel will leak it out
t.Run("LeakingSubtest", func(t *gotest.T) {
fakeT := &ignoreFatalT{T: t}
SetFeatureGateEmulationVersionDuringTest(fakeT, gate, newEmulationVersion)
// Calling t.Parallel in subtest will resume the main test body
t.Parallel()
// Leaked from main test
assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion))
})
// Leaked from subtest
assert.True(t, gate.EmulationVersion().EqualTo(newEmulationVersion))
fakeT := &ignoreFatalT{T: t}
SetFeatureGateEmulationVersionDuringTest(fakeT, gate, originalEmulationVersion)
assert.True(t, fakeT.fatalRecorded)
}
func TestNoLeakFromSameEmulationVersionToMainTest(t *gotest.T) {
t.Cleanup(cleanup)
originalEmulationVersion := version.MustParse("1.31")
newEmulationVersion := version.MustParse("1.31")
gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion)
assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion))
// Subtest setting feature gate and calling parallel will leak it out
t.Run("LeakingSubtest", func(t *gotest.T) {
SetFeatureGateEmulationVersionDuringTest(t, gate, newEmulationVersion)
// Calling t.Parallel in subtest will resume the main test body
t.Parallel()
// Leaked from main test
assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion))
})
// Leaked from subtest
assert.True(t, gate.EmulationVersion().EqualTo(newEmulationVersion))
SetFeatureGateEmulationVersionDuringTest(t, gate, originalEmulationVersion)
}
func TestDetectEmulationVersionLeakToOtherSubtest(t *gotest.T) {
t.Cleanup(cleanup)
originalEmulationVersion := version.MustParse("1.31")
newEmulationVersion := version.MustParse("1.30")
gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion)
assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion))
subtestName := "Subtest"
// Subtest setting feature gate and calling parallel will leak it out
t.Run(subtestName, func(t *gotest.T) {
fakeT := &ignoreFatalT{T: t}
SetFeatureGateEmulationVersionDuringTest(fakeT, gate, newEmulationVersion)
t.Parallel()
})
// Add suffix to name to prevent tests with the same prefix.
t.Run(subtestName+"Suffix", func(t *gotest.T) {
// Leaked newEmulationVersion
assert.True(t, gate.EmulationVersion().EqualTo(newEmulationVersion))
fakeT := &ignoreFatalT{T: t}
SetFeatureGateEmulationVersionDuringTest(fakeT, gate, originalEmulationVersion)
assert.True(t, fakeT.fatalRecorded)
})
}
type ignoreFatalT struct {
*gotest.T
fatalRecorded bool
@@ -248,3 +444,9 @@ func (f *ignoreFatalT) Fatalf(format string, args ...any) {
f.fatalRecorded = true
f.T.Logf("[IGNORED] "+format, args...)
}
func cleanup() {
featureFlagOverride = map[featuregate.Feature]string{}
emulationVersionOverride = ""
emulationVersionOverrideValue = nil
}

View File

@@ -61,3 +61,10 @@ var (
buildDate = "1970-01-01T00:00:00Z" // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ')
)
const (
// DefaultKubeBinaryVersion is the hard coded k8 binary version based on the latest K8s release.
// It is supposed to be consistent with gitMajor and gitMinor, except for local tests, where gitMajor and gitMinor are "".
// Should update for each minor release!
DefaultKubeBinaryVersion = "1.31"
)

View File

@@ -31,7 +31,6 @@ import (
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/filters"
genericoptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
"k8s.io/kube-aggregator/pkg/apiserver"
@@ -81,10 +80,7 @@ func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions)
}
cmd.SetContext(ctx)
fs := cmd.Flags()
utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs)
o.AddFlags(fs)
o.AddFlags(cmd.Flags())
return cmd
}
@@ -99,13 +95,8 @@ func (o *AggregatorOptions) AddFlags(fs *pflag.FlagSet) {
// NewDefaultOptions builds a "normal" set of options. You wouldn't normally expose this, but hyperkube isn't cobra compatible
func NewDefaultOptions(out, err io.Writer) *AggregatorOptions {
// effectiveVersion is used to set what apis and feature gates the generic api server is compatible with.
// You can also have the flag setting the effectiveVersion of the aggregator apiserver, and
// having a mapping from the aggregator apiserver version to generic apiserver version.
effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
o := &AggregatorOptions{
ServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion),
ServerRunOptions: genericoptions.NewServerRunOptions(),
RecommendedOptions: genericoptions.NewRecommendedOptions(
defaultEtcdPathPrefix,
aggregatorscheme.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion),

View File

@@ -7,6 +7,7 @@ go 1.22.0
require (
github.com/google/gofuzz v1.2.0
github.com/spf13/cobra v1.8.1
github.com/stretchr/testify v1.9.0
k8s.io/apimachinery v0.0.0
k8s.io/apiserver v0.0.0
k8s.io/client-go v0.0.0
@@ -55,6 +56,7 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.19.0 // indirect
github.com/prometheus/client_model v0.6.0 // indirect
github.com/prometheus/common v0.48.0 // indirect

View File

@@ -170,6 +170,7 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

View File

@@ -113,7 +113,7 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti
o.RecommendedOptions.AddFlags(flags)
wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2")
wardleFeatureGate := featuregate.NewVersionedFeatureGate(version.MustParse("1.2"))
wardleFeatureGate := utilfeature.DefaultFeatureGate.CopyKnownFeatures()
utilruntime.Must(wardleFeatureGate.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
"BanFlunder": {
{Version: version.MustParse("1.2"), Default: true, PreRelease: featuregate.GA},

View File

@@ -29,8 +29,6 @@ import (
netutils "k8s.io/utils/net"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
"k8s.io/kubernetes/test/e2e/framework"
@@ -65,9 +63,7 @@ func NewAPIServer(storageConfig storagebackend.Config) *APIServer {
func (a *APIServer) Start(ctx context.Context) error {
const tokenFilePath = "known_tokens.csv"
featureGate := utilfeature.DefaultFeatureGate
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
o := options.NewServerRunOptions(featureGate, effectiveVersion)
o := options.NewServerRunOptions()
o.Etcd.StorageConfig = a.storageConfig
_, ipnet, err := netutils.ParseCIDRSloppy(clusterIPRange)
if err != nil {

View File

@@ -27,9 +27,11 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/apiserver/pkg/storage"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/dynamic"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
@@ -37,9 +39,13 @@ import (
// TestCrossGroupStorage tests to make sure that all objects stored in an expected location in etcd can be converted/read.
func TestCrossGroupStorage(t *testing.T) {
testRegistry := utilversion.NewComponentGlobalsRegistry()
utilruntime.Must(testRegistry.Register("test", utilversion.NewEffectiveVersion("0.0"), utilfeature.DefaultFeatureGate.DeepCopy()))
apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) {
// force enable all resources so we can check storage.
opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("0.0")
opts.GenericServerRunOptions.ComponentName = "test"
opts.GenericServerRunOptions.ComponentGlobalsRegistry = testRegistry
})
defer apiServer.Cleanup()

View File

@@ -34,6 +34,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
@@ -75,8 +76,12 @@ var allowMissingTestdataFixtures = map[schema.GroupVersionKind]bool{
func TestEtcdStoragePath(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true)
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)
testRegistry := utilversion.NewComponentGlobalsRegistry()
utilruntime.Must(testRegistry.Register("test", utilversion.NewEffectiveVersion("0.0"), feature.DefaultFeatureGate.DeepCopy()))
apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) {
opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("0.0")
opts.GenericServerRunOptions.ComponentName = "test"
opts.GenericServerRunOptions.ComponentGlobalsRegistry = testRegistry
})
defer apiServer.Cleanup()
defer dumpEtcdKVOnFailure(t, apiServer.KV)

View File

@@ -40,8 +40,6 @@ import (
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/wait"
genericapiserveroptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
cacheddiscovery "k8s.io/client-go/discovery/cached/memory"
"k8s.io/client-go/dynamic"
clientset "k8s.io/client-go/kubernetes"
@@ -93,9 +91,7 @@ func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRu
t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err)
}
featureGate := utilfeature.DefaultFeatureGate
effectiveVersion := utilversion.DefaultBuildEffectiveVersion()
opts := options.NewServerRunOptions(featureGate, effectiveVersion)
opts := options.NewServerRunOptions()
opts.Options.SecureServing.Listener = listener
opts.Options.SecureServing.ServerCert.CertDirectory = certDir
opts.Options.ServiceAccountSigningKeyFile = saSigningKeyFile.Name()

View File

@@ -35,8 +35,6 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
genericapiserver "k8s.io/apiserver/pkg/server"
genericapiserveroptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
client "k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/util/cert"
@@ -137,9 +135,7 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) (
t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err)
}
featureGate := utilfeature.DefaultFeatureGate
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
opts := options.NewServerRunOptions(featureGate, effectiveVersion)
opts := options.NewServerRunOptions()
opts.SecureServing.Listener = listener
opts.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1")
opts.SecureServing.ServerCert.CertDirectory = certDir