add DefaultComponentGlobalsRegistry flags in ServerRunOptions
Signed-off-by: Siyuan Zhang <sizhang@google.com>
This commit is contained in:
@@ -26,8 +26,6 @@ import (
|
||||
utilnet "k8s.io/apimachinery/pkg/util/net"
|
||||
cliflag "k8s.io/component-base/cli/flag"
|
||||
|
||||
utilversion "k8s.io/apiserver/pkg/util/version"
|
||||
"k8s.io/component-base/featuregate"
|
||||
api "k8s.io/kubernetes/pkg/apis/core"
|
||||
"k8s.io/kubernetes/pkg/cluster/ports"
|
||||
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
|
||||
@@ -66,9 +64,9 @@ type Extra struct {
|
||||
}
|
||||
|
||||
// NewServerRunOptions creates and returns ServerRunOptions according to the given featureGate and effectiveVersion of the server binary to run.
|
||||
func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions {
|
||||
func NewServerRunOptions() *ServerRunOptions {
|
||||
s := ServerRunOptions{
|
||||
Options: controlplaneapiserver.NewOptions(featureGate, effectiveVersion),
|
||||
Options: controlplaneapiserver.NewOptions(),
|
||||
CloudProvider: kubeoptions.NewCloudProviderOptions(),
|
||||
|
||||
Extra: Extra{
|
||||
|
||||
@@ -48,17 +48,17 @@ import (
|
||||
)
|
||||
|
||||
func TestAddFlags(t *testing.T) {
|
||||
componentGlobalsRegistry := utilversion.DefaultComponentGlobalsRegistry
|
||||
t.Cleanup(func() {
|
||||
componentGlobalsRegistry.Reset()
|
||||
})
|
||||
fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
|
||||
|
||||
featureGate := featuregate.NewFeatureGate()
|
||||
componentRegistry := utilversion.NewComponentGlobalsRegistry()
|
||||
effectiveVersion := utilversion.NewEffectiveVersion("1.32")
|
||||
utilruntime.Must(componentRegistry.Register("test", effectiveVersion, featureGate))
|
||||
s := NewServerRunOptions(featureGate, effectiveVersion)
|
||||
utilruntime.Must(componentGlobalsRegistry.Register("test", utilversion.NewEffectiveVersion("1.32"), featuregate.NewFeatureGate()))
|
||||
s := NewServerRunOptions()
|
||||
for _, f := range s.Flags().FlagSets {
|
||||
fs.AddFlagSet(f)
|
||||
}
|
||||
componentRegistry.AddFlags(fs)
|
||||
|
||||
args := []string{
|
||||
"--enable-admission-plugins=AlwaysDeny",
|
||||
@@ -133,7 +133,7 @@ func TestAddFlags(t *testing.T) {
|
||||
"--emulated-version=test=1.31",
|
||||
}
|
||||
fs.Parse(args)
|
||||
utilruntime.Must(componentRegistry.Set())
|
||||
utilruntime.Must(componentGlobalsRegistry.Set())
|
||||
|
||||
// This is a snapshot of expected options parsed by args.
|
||||
expected := &ServerRunOptions{
|
||||
@@ -147,8 +147,8 @@ func TestAddFlags(t *testing.T) {
|
||||
MinRequestTimeout: 1800,
|
||||
JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),
|
||||
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
|
||||
FeatureGate: featureGate,
|
||||
EffectiveVersion: effectiveVersion,
|
||||
ComponentGlobalsRegistry: componentGlobalsRegistry,
|
||||
ComponentName: utilversion.DefaultKubeComponent,
|
||||
},
|
||||
Admission: &kubeoptions.AdmissionOptions{
|
||||
GenericAdmission: &apiserveroptions.AdmissionOptions{
|
||||
@@ -350,8 +350,8 @@ func TestAddFlags(t *testing.T) {
|
||||
if !reflect.DeepEqual(expected, s) {
|
||||
t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{})))
|
||||
}
|
||||
|
||||
if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" {
|
||||
t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31")
|
||||
testEffectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor("test")
|
||||
if testEffectiveVersion.EmulationVersion().String() != "1.31" {
|
||||
t.Errorf("Got emulation version %s, wanted %s", testEffectiveVersion.EmulationVersion().String(), "1.31")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
netutils "k8s.io/utils/net"
|
||||
|
||||
"k8s.io/apimachinery/pkg/util/version"
|
||||
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
|
||||
"k8s.io/kubernetes/pkg/controlplane/reconcilers"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
@@ -139,5 +140,14 @@ func (s CompletedOptions) Validate() []error {
|
||||
errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount))
|
||||
}
|
||||
|
||||
// TODO: remove in 1.32
|
||||
// emulationVersion is introduced in 1.31, so it is only allowed to be equal to the binary version at 1.31.
|
||||
effectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor(s.GenericServerRunOptions.ComponentName)
|
||||
binaryVersion := version.MajorMinor(effectiveVersion.BinaryVersion().Major(), effectiveVersion.BinaryVersion().Minor())
|
||||
if binaryVersion.EqualTo(version.MajorMinor(1, 31)) && !effectiveVersion.EmulationVersion().EqualTo(binaryVersion) {
|
||||
errs = append(errs, fmt.Errorf("emulation version needs to be equal to binary version(%s) in compatibility-version alpha, got %s",
|
||||
binaryVersion.String(), effectiveVersion.EmulationVersion().String()))
|
||||
}
|
||||
|
||||
return errs
|
||||
}
|
||||
|
||||
@@ -64,9 +64,9 @@ func init() {
|
||||
|
||||
// NewAPIServerCommand creates a *cobra.Command object with default parameters
|
||||
func NewAPIServerCommand() *cobra.Command {
|
||||
effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
|
||||
_, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
|
||||
utilversion.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
|
||||
s := options.NewServerRunOptions(featureGate, effectiveVersion)
|
||||
s := options.NewServerRunOptions()
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "kube-apiserver",
|
||||
@@ -124,8 +124,6 @@ cluster's shared state through which all other components interact.`,
|
||||
fs := cmd.Flags()
|
||||
namedFlagSets := s.Flags()
|
||||
verflag.AddFlags(namedFlagSets.FlagSet("global"))
|
||||
utilversion.DefaultComponentGlobalsRegistry.AddFlags(namedFlagSets.FlagSet("global"))
|
||||
|
||||
globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags())
|
||||
options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic"))
|
||||
for _, f := range namedFlagSets.FlagSets {
|
||||
|
||||
@@ -55,6 +55,7 @@ import (
|
||||
clientgotransport "k8s.io/client-go/transport"
|
||||
"k8s.io/client-go/util/cert"
|
||||
"k8s.io/client-go/util/keyutil"
|
||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||
logsapi "k8s.io/component-base/logs/api/v1"
|
||||
"k8s.io/klog/v2"
|
||||
"k8s.io/kube-aggregator/pkg/apiserver"
|
||||
@@ -187,15 +188,16 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
|
||||
if instanceOptions.BinaryVersion != "" {
|
||||
effectiveVersion = utilversion.NewEffectiveVersion(instanceOptions.BinaryVersion)
|
||||
}
|
||||
// need to call SetFeatureGateEmulationVersionDuringTest to reset the feature gate emulation version at the end of the test.
|
||||
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, featureGate, effectiveVersion.EmulationVersion())
|
||||
utilversion.DefaultComponentGlobalsRegistry.Reset()
|
||||
utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate))
|
||||
|
||||
s := options.NewServerRunOptions(featureGate, effectiveVersion)
|
||||
s := options.NewServerRunOptions()
|
||||
|
||||
for _, f := range s.Flags().FlagSets {
|
||||
fs.AddFlagSet(f)
|
||||
}
|
||||
utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs)
|
||||
|
||||
s.SecureServing.Listener, s.SecureServing.BindPort, err = createLocalhostListenerOnFreePort()
|
||||
if err != nil {
|
||||
|
||||
@@ -24,8 +24,6 @@ import (
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
apiserveroptions "k8s.io/apiserver/pkg/server/options"
|
||||
utilversion "k8s.io/apiserver/pkg/util/version"
|
||||
"k8s.io/component-base/featuregate"
|
||||
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
|
||||
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
||||
"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
|
||||
@@ -34,9 +32,7 @@ import (
|
||||
)
|
||||
|
||||
func TestBuildGenericConfig(t *testing.T) {
|
||||
featureGate := featuregate.NewFeatureGate()
|
||||
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
|
||||
opts := options.NewOptions(featureGate, effectiveVersion)
|
||||
opts := options.NewOptions()
|
||||
s := (&apiserveroptions.SecureServingOptions{
|
||||
BindAddress: netutils.ParseIPSloppy("127.0.0.1"),
|
||||
}).WithLoopback()
|
||||
|
||||
@@ -28,10 +28,8 @@ import (
|
||||
peerreconcilers "k8s.io/apiserver/pkg/reconcilers"
|
||||
genericoptions "k8s.io/apiserver/pkg/server/options"
|
||||
"k8s.io/apiserver/pkg/storage/storagebackend"
|
||||
utilversion "k8s.io/apiserver/pkg/util/version"
|
||||
"k8s.io/client-go/util/keyutil"
|
||||
cliflag "k8s.io/component-base/cli/flag"
|
||||
"k8s.io/component-base/featuregate"
|
||||
"k8s.io/component-base/logs"
|
||||
logsapi "k8s.io/component-base/logs/api/v1"
|
||||
"k8s.io/component-base/metrics"
|
||||
@@ -100,9 +98,9 @@ type CompletedOptions struct {
|
||||
}
|
||||
|
||||
// NewOptions creates a new ServerRunOptions object with default parameters
|
||||
func NewOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *Options {
|
||||
func NewOptions() *Options {
|
||||
s := Options{
|
||||
GenericServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion),
|
||||
GenericServerRunOptions: genericoptions.NewServerRunOptions(),
|
||||
Etcd: genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)),
|
||||
SecureServing: kubeoptions.NewSecureServingOptions(),
|
||||
Audit: genericoptions.NewAuditOptions(),
|
||||
|
||||
@@ -44,18 +44,18 @@ import (
|
||||
)
|
||||
|
||||
func TestAddFlags(t *testing.T) {
|
||||
componentGlobalsRegistry := utilversion.DefaultComponentGlobalsRegistry
|
||||
t.Cleanup(func() {
|
||||
componentGlobalsRegistry.Reset()
|
||||
})
|
||||
fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
|
||||
featureGate := featuregate.NewFeatureGate()
|
||||
effectiveVersion := utilversion.NewEffectiveVersion("1.32")
|
||||
componentRegistry := utilversion.NewComponentGlobalsRegistry()
|
||||
utilruntime.Must(componentRegistry.Register("test", effectiveVersion, featureGate))
|
||||
s := NewOptions(featureGate, effectiveVersion)
|
||||
utilruntime.Must(componentGlobalsRegistry.Register("test", utilversion.NewEffectiveVersion("1.32"), featuregate.NewFeatureGate()))
|
||||
s := NewOptions()
|
||||
var fss cliflag.NamedFlagSets
|
||||
s.AddFlags(&fss)
|
||||
for _, f := range fss.FlagSets {
|
||||
fs.AddFlagSet(f)
|
||||
}
|
||||
componentRegistry.AddFlags(fs)
|
||||
|
||||
args := []string{
|
||||
"--enable-admission-plugins=AlwaysDeny",
|
||||
@@ -119,7 +119,7 @@ func TestAddFlags(t *testing.T) {
|
||||
"--emulated-version=test=1.31",
|
||||
}
|
||||
fs.Parse(args)
|
||||
utilruntime.Must(componentRegistry.Set())
|
||||
utilruntime.Must(componentGlobalsRegistry.Set())
|
||||
|
||||
// This is a snapshot of expected options parsed by args.
|
||||
expected := &Options{
|
||||
@@ -132,8 +132,8 @@ func TestAddFlags(t *testing.T) {
|
||||
MinRequestTimeout: 1800,
|
||||
JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),
|
||||
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
|
||||
FeatureGate: featureGate,
|
||||
EffectiveVersion: effectiveVersion,
|
||||
ComponentGlobalsRegistry: componentGlobalsRegistry,
|
||||
ComponentName: utilversion.DefaultKubeComponent,
|
||||
},
|
||||
Admission: &kubeoptions.AdmissionOptions{
|
||||
GenericAdmission: &apiserveroptions.AdmissionOptions{
|
||||
@@ -305,7 +305,8 @@ func TestAddFlags(t *testing.T) {
|
||||
t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{})))
|
||||
}
|
||||
|
||||
if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" {
|
||||
t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31")
|
||||
testEffectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor("test")
|
||||
if testEffectiveVersion.EmulationVersion().String() != "1.31" {
|
||||
t.Errorf("Got emulation version %s, wanted %s", testEffectiveVersion.EmulationVersion().String(), "1.31")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -201,7 +201,7 @@ func TestValidateOptions(t *testing.T) {
|
||||
name: "validate master count equal 0",
|
||||
expectErrors: true,
|
||||
options: &Options{
|
||||
GenericServerRunOptions: &genericoptions.ServerRunOptions{FeatureGate: utilfeature.DefaultFeatureGate.DeepCopy(), EffectiveVersion: utilversion.NewEffectiveVersion("1.32")},
|
||||
GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry},
|
||||
Etcd: &genericoptions.EtcdOptions{},
|
||||
SecureServing: &genericoptions.SecureServingOptionsWithLoopback{},
|
||||
Audit: &genericoptions.AuditOptions{},
|
||||
@@ -228,7 +228,7 @@ func TestValidateOptions(t *testing.T) {
|
||||
name: "validate token request enable not attempted",
|
||||
expectErrors: true,
|
||||
options: &Options{
|
||||
GenericServerRunOptions: &genericoptions.ServerRunOptions{FeatureGate: utilfeature.DefaultFeatureGate.DeepCopy(), EffectiveVersion: utilversion.NewEffectiveVersion("1.32")},
|
||||
GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry},
|
||||
Etcd: &genericoptions.EtcdOptions{},
|
||||
SecureServing: &genericoptions.SecureServingOptionsWithLoopback{},
|
||||
Audit: &genericoptions.AuditOptions{},
|
||||
|
||||
@@ -39,11 +39,9 @@ import (
|
||||
flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
|
||||
"k8s.io/apiserver/pkg/util/openapi"
|
||||
"k8s.io/apiserver/pkg/util/proxy"
|
||||
utilversion "k8s.io/apiserver/pkg/util/version"
|
||||
"k8s.io/apiserver/pkg/util/webhook"
|
||||
scheme "k8s.io/client-go/kubernetes/scheme"
|
||||
corev1 "k8s.io/client-go/listers/core/v1"
|
||||
"k8s.io/component-base/featuregate"
|
||||
netutils "k8s.io/utils/net"
|
||||
)
|
||||
|
||||
@@ -60,9 +58,9 @@ type CustomResourceDefinitionsServerOptions struct {
|
||||
}
|
||||
|
||||
// NewCustomResourceDefinitionsServerOptions creates default options of an apiextensions-apiserver.
|
||||
func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer, featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *CustomResourceDefinitionsServerOptions {
|
||||
func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer) *CustomResourceDefinitionsServerOptions {
|
||||
o := &CustomResourceDefinitionsServerOptions{
|
||||
ServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion),
|
||||
ServerRunOptions: genericoptions.NewServerRunOptions(),
|
||||
RecommendedOptions: genericoptions.NewRecommendedOptions(
|
||||
defaultEtcdPathPrefix,
|
||||
apiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion),
|
||||
|
||||
@@ -24,17 +24,11 @@ import (
|
||||
|
||||
"k8s.io/apiextensions-apiserver/pkg/cmd/server/options"
|
||||
genericapiserver "k8s.io/apiserver/pkg/server"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
utilversion "k8s.io/apiserver/pkg/util/version"
|
||||
)
|
||||
|
||||
func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command {
|
||||
// effectiveVersion is used to set what apis and feature gates the generic api server is compatible with.
|
||||
// You can also have the flag setting the effectiveVersion of the apiextensions apiserver, and
|
||||
// having a mapping from the apiextensions apiserver version to generic apiserver version.
|
||||
effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
|
||||
utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
|
||||
o := options.NewCustomResourceDefinitionsServerOptions(out, errOut, featureGate, effectiveVersion)
|
||||
o := options.NewCustomResourceDefinitionsServerOptions(out, errOut)
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Short: "Launch an API extensions API server",
|
||||
@@ -58,7 +52,6 @@ func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command
|
||||
cmd.SetContext(ctx)
|
||||
|
||||
fs := cmd.Flags()
|
||||
utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs)
|
||||
o.AddFlags(fs)
|
||||
return cmd
|
||||
}
|
||||
|
||||
@@ -127,9 +127,7 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin
|
||||
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
|
||||
utilversion.DefaultComponentGlobalsRegistry.Reset()
|
||||
utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate))
|
||||
s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr, featureGate, effectiveVersion)
|
||||
|
||||
utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs)
|
||||
s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr)
|
||||
s.AddFlags(fs)
|
||||
|
||||
s.RecommendedOptions.SecureServing.Listener, s.RecommendedOptions.SecureServing.BindPort, err = createLocalhostListenerOnFreePort()
|
||||
|
||||
@@ -832,7 +832,7 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
|
||||
muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{},
|
||||
}
|
||||
|
||||
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
|
||||
if c.FeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
|
||||
manager := c.AggregatedDiscoveryGroupManager
|
||||
if manager == nil {
|
||||
manager = discoveryendpoint.NewResourceManager("apis")
|
||||
@@ -1047,14 +1047,14 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
|
||||
handler = genericfilters.WithRetryAfter(handler, c.lifecycleSignals.NotAcceptingNewRequest.Signaled())
|
||||
}
|
||||
handler = genericfilters.WithHTTPLogging(handler)
|
||||
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) {
|
||||
if c.FeatureGate.Enabled(genericfeatures.APIServerTracing) {
|
||||
handler = genericapifilters.WithTracing(handler, c.TracerProvider)
|
||||
}
|
||||
handler = genericapifilters.WithLatencyTrackers(handler)
|
||||
// WithRoutine will execute future handlers in a separate goroutine and serving
|
||||
// handler in current goroutine to minimize the stack memory usage. It must be
|
||||
// after WithPanicRecover() to be protected from panics.
|
||||
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServingWithRoutine) {
|
||||
if c.FeatureGate.Enabled(genericfeatures.APIServingWithRoutine) {
|
||||
handler = genericfilters.WithRoutine(handler, c.LongRunningFunc)
|
||||
}
|
||||
handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver)
|
||||
@@ -1098,7 +1098,7 @@ func installAPI(s *GenericAPIServer, c *Config) {
|
||||
routes.Version{Version: c.EffectiveVersion.BinaryVersion().Info()}.Install(s.Handler.GoRestfulContainer)
|
||||
|
||||
if c.EnableDiscovery {
|
||||
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
|
||||
if c.FeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
|
||||
wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(s.DiscoveryGroupManager, s.AggregatedDiscoveryGroupManager)
|
||||
s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/apis", metav1.APIGroupList{}))
|
||||
} else {
|
||||
|
||||
@@ -40,6 +40,7 @@ import (
|
||||
"k8s.io/apiserver/pkg/authentication/user"
|
||||
"k8s.io/apiserver/pkg/endpoints/request"
|
||||
"k8s.io/apiserver/pkg/server/healthz"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
utilversion "k8s.io/apiserver/pkg/util/version"
|
||||
"k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
@@ -308,6 +309,7 @@ func TestAuthenticationAuditAnnotationsDefaultChain(t *testing.T) {
|
||||
LongRunningFunc: func(_ *http.Request, _ *request.RequestInfo) bool { return false },
|
||||
lifecycleSignals: newLifecycleSignals(),
|
||||
TracerProvider: tracing.NewNoopTracerProvider(),
|
||||
FeatureGate: utilfeature.DefaultFeatureGate,
|
||||
}
|
||||
|
||||
h := DefaultBuildHandlerChain(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -51,7 +51,6 @@ import (
|
||||
"k8s.io/apiserver/pkg/server/healthz"
|
||||
"k8s.io/apiserver/pkg/server/routes"
|
||||
"k8s.io/apiserver/pkg/storageversion"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
utilversion "k8s.io/apiserver/pkg/util/version"
|
||||
restclient "k8s.io/client-go/rest"
|
||||
"k8s.io/component-base/featuregate"
|
||||
@@ -780,7 +779,7 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A
|
||||
}
|
||||
resourceInfos = append(resourceInfos, r...)
|
||||
|
||||
if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
|
||||
if s.FeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
|
||||
// Aggregated discovery only aggregates resources under /apis
|
||||
if apiPrefix == APIGroupPrefix {
|
||||
s.AggregatedDiscoveryGroupManager.AddGroupVersion(
|
||||
@@ -808,8 +807,8 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A
|
||||
|
||||
s.RegisterDestroyFunc(apiGroupInfo.destroyStorage)
|
||||
|
||||
if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionAPI) &&
|
||||
utilfeature.DefaultFeatureGate.Enabled(features.APIServerIdentity) {
|
||||
if s.FeatureGate.Enabled(features.StorageVersionAPI) &&
|
||||
s.FeatureGate.Enabled(features.APIServerIdentity) {
|
||||
// API installation happens before we start listening on the handlers,
|
||||
// therefore it is safe to register ResourceInfos here. The handler will block
|
||||
// write requests until the storage versions of the targeting resources are updated.
|
||||
@@ -839,7 +838,7 @@ func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo
|
||||
// Install the version handler.
|
||||
// Add a handler at /<apiPrefix> to enumerate the supported api versions.
|
||||
legacyRootAPIHandler := discovery.NewLegacyRootAPIHandler(s.discoveryAddresses, s.Serializer, apiPrefix)
|
||||
if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
|
||||
if s.FeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
|
||||
wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(legacyRootAPIHandler, s.AggregatedLegacyDiscoveryGroupManager)
|
||||
s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/api", metav1.APIVersions{}))
|
||||
} else {
|
||||
|
||||
@@ -33,7 +33,6 @@ func (f fakeGroupRegistry) IsGroupRegistered(group string) bool {
|
||||
func TestAPIEnablementOptionsValidate(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
testOptions *APIEnablementOptions
|
||||
runtimeConfig cliflag.ConfigurationMap
|
||||
expectErr string
|
||||
}{
|
||||
|
||||
@@ -25,9 +25,10 @@ import (
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer"
|
||||
"k8s.io/apimachinery/pkg/util/errors"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
"k8s.io/apiserver/pkg/server"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
utilversion "k8s.io/apiserver/pkg/util/version"
|
||||
"k8s.io/component-base/featuregate"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
@@ -91,12 +92,23 @@ type ServerRunOptions struct {
|
||||
// it is not overridden by any other grace period.
|
||||
ShutdownWatchTerminationGracePeriod time.Duration
|
||||
|
||||
// FeatureGate are the featuregate to install on the CLI
|
||||
FeatureGate featuregate.FeatureGate
|
||||
EffectiveVersion utilversion.EffectiveVersion
|
||||
// ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored.
|
||||
ComponentGlobalsRegistry utilversion.ComponentGlobalsRegistry
|
||||
// ComponentName is name under which the server's global variabled are registered in the ComponentGlobalsRegistry.
|
||||
ComponentName string
|
||||
}
|
||||
|
||||
func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions {
|
||||
func NewServerRunOptions() *ServerRunOptions {
|
||||
if utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) == nil {
|
||||
featureGate := utilfeature.DefaultMutableFeatureGate
|
||||
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
|
||||
utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate))
|
||||
}
|
||||
|
||||
return NewServerRunOptionsForComponent(utilversion.DefaultKubeComponent, utilversion.DefaultComponentGlobalsRegistry)
|
||||
}
|
||||
|
||||
func NewServerRunOptionsForComponent(componentName string, componentGlobalsRegistry utilversion.ComponentGlobalsRegistry) *ServerRunOptions {
|
||||
defaults := server.NewConfig(serializer.CodecFactory{})
|
||||
return &ServerRunOptions{
|
||||
MaxRequestsInFlight: defaults.MaxRequestsInFlight,
|
||||
@@ -109,13 +121,16 @@ func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion u
|
||||
JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes,
|
||||
MaxRequestBodyBytes: defaults.MaxRequestBodyBytes,
|
||||
ShutdownSendRetryAfter: false,
|
||||
FeatureGate: featureGate,
|
||||
EffectiveVersion: effectiveVersion,
|
||||
ComponentName: componentName,
|
||||
ComponentGlobalsRegistry: componentGlobalsRegistry,
|
||||
}
|
||||
}
|
||||
|
||||
// ApplyTo applies the run options to the method receiver and returns self
|
||||
func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
|
||||
if err := s.ComponentGlobalsRegistry.SetFallback(); err != nil {
|
||||
return err
|
||||
}
|
||||
c.CorsAllowedOriginList = s.CorsAllowedOriginList
|
||||
c.HSTSDirectives = s.HSTSDirectives
|
||||
c.ExternalAddress = s.ExternalHost
|
||||
@@ -131,8 +146,8 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
|
||||
c.PublicAddress = s.AdvertiseAddress
|
||||
c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter
|
||||
c.ShutdownWatchTerminationGracePeriod = s.ShutdownWatchTerminationGracePeriod
|
||||
c.EffectiveVersion = s.EffectiveVersion
|
||||
c.FeatureGate = s.FeatureGate
|
||||
c.EffectiveVersion = s.ComponentGlobalsRegistry.EffectiveVersionFor(s.ComponentName)
|
||||
c.FeatureGate = s.ComponentGlobalsRegistry.FeatureGateFor(s.ComponentName)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -205,12 +220,7 @@ func (s *ServerRunOptions) Validate() []error {
|
||||
if err := validateCorsAllowedOriginList(s.CorsAllowedOriginList); err != nil {
|
||||
errors = append(errors, err)
|
||||
}
|
||||
if s.FeatureGate != nil {
|
||||
if errs := s.FeatureGate.Validate(); len(errs) != 0 {
|
||||
errors = append(errors, errs...)
|
||||
}
|
||||
}
|
||||
if errs := s.EffectiveVersion.Validate(); len(errs) != 0 {
|
||||
if errs := s.ComponentGlobalsRegistry.Validate(); len(errs) != 0 {
|
||||
errors = append(errors, errs...)
|
||||
}
|
||||
return errors
|
||||
@@ -353,15 +363,11 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
|
||||
fs.DurationVar(&s.ShutdownWatchTerminationGracePeriod, "shutdown-watch-termination-grace-period", s.ShutdownWatchTerminationGracePeriod, ""+
|
||||
"This option, if set, represents the maximum amount of grace period the apiserver will wait "+
|
||||
"for active watch request(s) to drain during the graceful server shutdown window.")
|
||||
|
||||
s.ComponentGlobalsRegistry.AddFlags(fs)
|
||||
}
|
||||
|
||||
// Complete fills missing fields with defaults.
|
||||
func (s *ServerRunOptions) Complete() error {
|
||||
if s.FeatureGate == nil {
|
||||
return fmt.Errorf("nil FeatureGate in ServerRunOptions")
|
||||
}
|
||||
if s.EffectiveVersion == nil {
|
||||
return fmt.Errorf("nil EffectiveVersion in ServerRunOptions")
|
||||
}
|
||||
return nil
|
||||
return s.ComponentGlobalsRegistry.SetFallback()
|
||||
}
|
||||
|
||||
@@ -23,14 +23,21 @@ import (
|
||||
"time"
|
||||
|
||||
utilerrors "k8s.io/apimachinery/pkg/util/errors"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/version"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
utilversion "k8s.io/apiserver/pkg/util/version"
|
||||
netutils "k8s.io/utils/net"
|
||||
)
|
||||
|
||||
func TestServerRunOptionsValidate(t *testing.T) {
|
||||
testRegistry := utilversion.NewComponentGlobalsRegistry()
|
||||
featureGate := utilfeature.DefaultFeatureGate.DeepCopy()
|
||||
effectiveVersion := utilversion.NewEffectiveVersion("1.30")
|
||||
effectiveVersion.SetEmulationVersion(version.MajorMinor(1, 32))
|
||||
testComponent := "test"
|
||||
utilruntime.Must(testRegistry.Register(testComponent, effectiveVersion, featureGate))
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
testOptions *ServerRunOptions
|
||||
@@ -47,8 +54,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
|
||||
MinRequestTimeout: 1800,
|
||||
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
|
||||
MaxRequestBodyBytes: 10 * 1024 * 1024,
|
||||
FeatureGate: featureGate,
|
||||
EffectiveVersion: effectiveVersion,
|
||||
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
|
||||
},
|
||||
expectErr: "--max-requests-inflight can not be negative value",
|
||||
},
|
||||
@@ -63,8 +69,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
|
||||
MinRequestTimeout: 1800,
|
||||
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
|
||||
MaxRequestBodyBytes: 10 * 1024 * 1024,
|
||||
FeatureGate: featureGate,
|
||||
EffectiveVersion: effectiveVersion,
|
||||
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
|
||||
},
|
||||
expectErr: "--max-mutating-requests-inflight can not be negative value",
|
||||
},
|
||||
@@ -79,8 +84,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
|
||||
MinRequestTimeout: 1800,
|
||||
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
|
||||
MaxRequestBodyBytes: 10 * 1024 * 1024,
|
||||
FeatureGate: featureGate,
|
||||
EffectiveVersion: effectiveVersion,
|
||||
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
|
||||
},
|
||||
expectErr: "--request-timeout can not be negative value",
|
||||
},
|
||||
@@ -95,8 +99,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
|
||||
MinRequestTimeout: -1800,
|
||||
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
|
||||
MaxRequestBodyBytes: 10 * 1024 * 1024,
|
||||
FeatureGate: featureGate,
|
||||
EffectiveVersion: effectiveVersion,
|
||||
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
|
||||
},
|
||||
expectErr: "--min-request-timeout can not be negative value",
|
||||
},
|
||||
@@ -111,8 +114,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
|
||||
MinRequestTimeout: 1800,
|
||||
JSONPatchMaxCopyBytes: -10 * 1024 * 1024,
|
||||
MaxRequestBodyBytes: 10 * 1024 * 1024,
|
||||
FeatureGate: featureGate,
|
||||
EffectiveVersion: effectiveVersion,
|
||||
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
|
||||
},
|
||||
expectErr: "ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value",
|
||||
},
|
||||
@@ -127,8 +129,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
|
||||
MinRequestTimeout: 1800,
|
||||
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
|
||||
MaxRequestBodyBytes: -10 * 1024 * 1024,
|
||||
FeatureGate: featureGate,
|
||||
EffectiveVersion: effectiveVersion,
|
||||
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
|
||||
},
|
||||
expectErr: "ServerRunOptions.MaxRequestBodyBytes can not be negative value",
|
||||
},
|
||||
@@ -144,8 +145,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
|
||||
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
|
||||
MaxRequestBodyBytes: 10 * 1024 * 1024,
|
||||
LivezGracePeriod: -time.Second,
|
||||
FeatureGate: featureGate,
|
||||
EffectiveVersion: effectiveVersion,
|
||||
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
|
||||
},
|
||||
expectErr: "--livez-grace-period can not be a negative value",
|
||||
},
|
||||
@@ -161,8 +161,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
|
||||
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
|
||||
MaxRequestBodyBytes: 10 * 1024 * 1024,
|
||||
ShutdownDelayDuration: -time.Second,
|
||||
FeatureGate: featureGate,
|
||||
EffectiveVersion: effectiveVersion,
|
||||
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
|
||||
},
|
||||
expectErr: "--shutdown-delay-duration can not be negative value",
|
||||
},
|
||||
@@ -178,11 +177,27 @@ func TestServerRunOptionsValidate(t *testing.T) {
|
||||
MinRequestTimeout: 1800,
|
||||
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
|
||||
MaxRequestBodyBytes: 10 * 1024 * 1024,
|
||||
FeatureGate: featureGate,
|
||||
EffectiveVersion: effectiveVersion,
|
||||
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
|
||||
},
|
||||
expectErr: "--strict-transport-security-directives invalid, allowed values: max-age=expireTime, includeSubDomains, preload. see https://tools.ietf.org/html/rfc6797#section-6.1 for more information",
|
||||
},
|
||||
{
|
||||
name: "Test when emulation version is invalid",
|
||||
testOptions: &ServerRunOptions{
|
||||
AdvertiseAddress: netutils.ParseIPSloppy("192.168.10.10"),
|
||||
CorsAllowedOriginList: []string{"^10.10.10.100$", "^10.10.10.200$"},
|
||||
HSTSDirectives: []string{"max-age=31536000", "includeSubDomains", "preload"},
|
||||
MaxRequestsInFlight: 400,
|
||||
MaxMutatingRequestsInFlight: 200,
|
||||
RequestTimeout: time.Duration(2) * time.Minute,
|
||||
MinRequestTimeout: 1800,
|
||||
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
|
||||
MaxRequestBodyBytes: 10 * 1024 * 1024,
|
||||
ComponentName: testComponent,
|
||||
ComponentGlobalsRegistry: testRegistry,
|
||||
},
|
||||
expectErr: "emulation version 1.32 is not between [1.29, 1.30.0]",
|
||||
},
|
||||
{
|
||||
name: "Test when ServerRunOptions is valid",
|
||||
testOptions: &ServerRunOptions{
|
||||
@@ -195,8 +210,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
|
||||
MinRequestTimeout: 1800,
|
||||
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
|
||||
MaxRequestBodyBytes: 10 * 1024 * 1024,
|
||||
FeatureGate: featureGate,
|
||||
EffectiveVersion: effectiveVersion,
|
||||
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -216,8 +230,6 @@ func TestServerRunOptionsValidate(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestValidateCorsAllowedOriginList(t *testing.T) {
|
||||
featureGate := utilfeature.DefaultFeatureGate.DeepCopy()
|
||||
effectiveVersion := utilversion.NewEffectiveVersion("1.30")
|
||||
tests := []struct {
|
||||
regexp [][]string
|
||||
errShouldContain string
|
||||
@@ -265,7 +277,7 @@ func TestValidateCorsAllowedOriginList(t *testing.T) {
|
||||
for _, test := range tests {
|
||||
for _, regexp := range test.regexp {
|
||||
t.Run(fmt.Sprintf("regexp/%s", regexp), func(t *testing.T) {
|
||||
options := NewServerRunOptions(featureGate, effectiveVersion)
|
||||
options := NewServerRunOptions()
|
||||
if errs := options.Validate(); len(errs) != 0 {
|
||||
t.Fatalf("wrong test setup: %#v", errs)
|
||||
}
|
||||
@@ -289,8 +301,6 @@ func TestValidateCorsAllowedOriginList(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) {
|
||||
featureGate := utilfeature.DefaultFeatureGate.DeepCopy()
|
||||
effectiveVersion := utilversion.NewEffectiveVersion("1.30")
|
||||
tests := []struct {
|
||||
name string
|
||||
optionsFn func() *ServerRunOptions
|
||||
@@ -299,13 +309,13 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) {
|
||||
{
|
||||
name: "default should be valid",
|
||||
optionsFn: func() *ServerRunOptions {
|
||||
return NewServerRunOptions(featureGate, effectiveVersion)
|
||||
return NewServerRunOptions()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "negative not allowed",
|
||||
optionsFn: func() *ServerRunOptions {
|
||||
o := NewServerRunOptions(featureGate, effectiveVersion)
|
||||
o := NewServerRunOptions()
|
||||
o.ShutdownWatchTerminationGracePeriod = -time.Second
|
||||
return o
|
||||
},
|
||||
@@ -332,7 +342,7 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) {
|
||||
}
|
||||
|
||||
t.Run("default should be zero", func(t *testing.T) {
|
||||
options := NewServerRunOptions(featureGate, effectiveVersion)
|
||||
options := NewServerRunOptions()
|
||||
if options.ShutdownWatchTerminationGracePeriod != time.Duration(0) {
|
||||
t.Errorf("expected default of ShutdownWatchTerminationGracePeriod to be zero, but got: %s", options.ShutdownWatchTerminationGracePeriod)
|
||||
}
|
||||
|
||||
@@ -181,9 +181,9 @@ func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example
|
||||
// was being served which didn't exist at all in min-compatibility-version.
|
||||
//
|
||||
// In the alpha case - we do not support compatibility versioning of
|
||||
// alpha types and recommend users do not mix the two.
|
||||
// alpha types and recommend users do not mix the two.
|
||||
// In the skip-level case - The version of apiserver we are retaining
|
||||
// compatibility with has no knowledge of the type,
|
||||
// so storing it in another type is no issue.
|
||||
// compatibility with has no knowledge of the type,
|
||||
// so storing it in another type is no issue.
|
||||
return binaryVersionOfResource, nil
|
||||
}
|
||||
|
||||
@@ -101,6 +101,8 @@ type ComponentGlobalsRegistry interface {
|
||||
AddFlags(fs *pflag.FlagSet)
|
||||
// Set sets the flags for all global variables for all components registered.
|
||||
Set() error
|
||||
// SetFallback calls Set() if it has never been called.
|
||||
SetFallback() error
|
||||
// Validate calls the Validate() function for all the global variables for all components registered.
|
||||
Validate() []error
|
||||
// Reset removes all stored ComponentGlobals, configurations, and version mappings.
|
||||
@@ -120,6 +122,8 @@ type componentGlobalsRegistry struct {
|
||||
emulationVersionConfig []string
|
||||
// map of component name to the list of feature gates set from the flag.
|
||||
featureGatesConfig map[string][]string
|
||||
// set stores if the Set() function for the registry is already called.
|
||||
set bool
|
||||
}
|
||||
|
||||
func NewComponentGlobalsRegistry() *componentGlobalsRegistry {
|
||||
@@ -136,6 +140,7 @@ func (r *componentGlobalsRegistry) Reset() {
|
||||
r.componentGlobals = make(map[string]*ComponentGlobals)
|
||||
r.emulationVersionConfig = nil
|
||||
r.featureGatesConfig = nil
|
||||
r.set = false
|
||||
}
|
||||
|
||||
func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion {
|
||||
@@ -330,9 +335,22 @@ func toVersionMap(versionConfig []string) (map[string]*version.Version, error) {
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (r *componentGlobalsRegistry) SetFallback() error {
|
||||
r.mutex.Lock()
|
||||
set := r.set
|
||||
r.mutex.Unlock()
|
||||
if set {
|
||||
return nil
|
||||
}
|
||||
klog.Warning("setting componentGlobalsRegistry in SetFallback. We recommend calling componentGlobalsRegistry.Set()" +
|
||||
" right after parsing flags to avoid using feature gates before their final values are set by the flags.")
|
||||
return r.Set()
|
||||
}
|
||||
|
||||
func (r *componentGlobalsRegistry) Set() error {
|
||||
r.mutex.Lock()
|
||||
defer r.mutex.Unlock()
|
||||
r.set = true
|
||||
emulationVersionConfigMap, err := toVersionMap(r.emulationVersionConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -111,12 +111,6 @@ func (m *effectiveVersion) Validate() []error {
|
||||
// emulationVersion can only be 1.{binaryMinor-1}...1.{binaryMinor}.
|
||||
maxEmuVer := binaryVersion
|
||||
minEmuVer := binaryVersion.SubtractMinor(1)
|
||||
// TODO: remove in 1.32
|
||||
// emulationVersion is introduced in 1.31, so it cannot be lower than that.
|
||||
// binaryVersion could be lower than 1.31 in tests. So we are only checking 1.31.
|
||||
if binaryVersion.EqualTo(version.MajorMinor(1, 31)) {
|
||||
minEmuVer = version.MajorMinor(1, 31)
|
||||
}
|
||||
if emulationVersion.GreaterThan(maxEmuVer) || emulationVersion.LessThan(minEmuVer) {
|
||||
errs = append(errs, fmt.Errorf("emulation version %s is not between [%s, %s]", emulationVersion.String(), minEmuVer.String(), maxEmuVer.String()))
|
||||
}
|
||||
@@ -157,8 +151,7 @@ func DefaultBuildEffectiveVersion() MutableEffectiveVersion {
|
||||
|
||||
// DefaultKubeEffectiveVersion returns the MutableEffectiveVersion based on the
|
||||
// latest K8s release.
|
||||
// Should update for each minor release!
|
||||
func DefaultKubeEffectiveVersion() MutableEffectiveVersion {
|
||||
binaryVersion := version.MustParse("1.31").WithInfo(baseversion.Get())
|
||||
binaryVersion := version.MustParse(baseversion.DefaultKubeBinaryVersion).WithInfo(baseversion.Get())
|
||||
return newEffectiveVersion(binaryVersion)
|
||||
}
|
||||
|
||||
@@ -42,25 +42,6 @@ func TestValidate(t *testing.T) {
|
||||
emulationVersion: "v1.31.0",
|
||||
minCompatibilityVersion: "v1.31.0",
|
||||
},
|
||||
{
|
||||
name: "binary version 1.31, emulation version lower than 1.31",
|
||||
binaryVersion: "v1.31.2",
|
||||
emulationVersion: "v1.30.0",
|
||||
minCompatibilityVersion: "v1.30.0",
|
||||
expectErrors: true,
|
||||
},
|
||||
{
|
||||
name: "binary version 1.31, emulation version 1.31",
|
||||
binaryVersion: "v1.31.2",
|
||||
emulationVersion: "v1.31.0",
|
||||
minCompatibilityVersion: "v1.30.0",
|
||||
},
|
||||
{
|
||||
name: "binary version lower than 1.31",
|
||||
binaryVersion: "v1.30.2",
|
||||
emulationVersion: "v1.29.0",
|
||||
minCompatibilityVersion: "v1.29.0",
|
||||
},
|
||||
{
|
||||
name: "emulation version two minor lower than binary not ok",
|
||||
binaryVersion: "v1.33.2",
|
||||
|
||||
@@ -114,6 +114,9 @@ type FeatureGate interface {
|
||||
// set on the copy without mutating the original. This is useful for validating
|
||||
// config against potential feature gate changes before committing those changes.
|
||||
DeepCopy() MutableVersionedFeatureGate
|
||||
// CopyKnownFeatures returns a partial copy of the FeatureGate object, with all the known features and overrides.
|
||||
// This is useful for creating a new instance of feature gate without inheriting all the enabled configurations of the base feature gate.
|
||||
CopyKnownFeatures() MutableVersionedFeatureGate
|
||||
// Validate checks if the flag gates are valid at the emulated version.
|
||||
Validate() []error
|
||||
}
|
||||
@@ -127,9 +130,6 @@ type MutableFeatureGate interface {
|
||||
AddFlag(fs *pflag.FlagSet)
|
||||
// Close sets closed to true, and prevents subsequent calls to Add
|
||||
Close()
|
||||
// OpenForModification sets closedForModification to false, and allows subsequent calls to SetEmulationVersion to change enabled features
|
||||
// before the next Enabled is called.
|
||||
OpenForModification()
|
||||
// Set parses and stores flag gates for known features
|
||||
// from a string like feature1=true,feature2=false,...
|
||||
Set(value string) error
|
||||
@@ -166,8 +166,6 @@ type MutableVersionedFeatureGate interface {
|
||||
// SetEmulationVersion overrides the emulationVersion of the feature gate.
|
||||
// Otherwise, the emulationVersion will be the same as the binary version.
|
||||
// If set, the feature defaults and availability will be as if the binary is at the emulated version.
|
||||
// Returns error if the new emulationVersion will change the enablement state of a feature that has already been queried.
|
||||
// If you have to use featureGate.Enabled before parsing the flags, call featureGate.OpenForModification following featureGate.Enabled.
|
||||
SetEmulationVersion(emulationVersion *version.Version) error
|
||||
// GetAll returns a copy of the map of known feature names to versioned feature specs.
|
||||
GetAllVersioned() map[Feature]VersionedSpecs
|
||||
@@ -185,15 +183,11 @@ type MutableVersionedFeatureGate interface {
|
||||
// overriding its default to true for a limited number of components without simultaneously
|
||||
// changing its default for all consuming components.
|
||||
OverrideDefaultAtVersion(name Feature, override bool, ver *version.Version) error
|
||||
}
|
||||
|
||||
// MutableVersionedFeatureGateForTests is a feature gate interface that should only be used in tests.
|
||||
type MutableVersionedFeatureGateForTests interface {
|
||||
MutableVersionedFeatureGate
|
||||
// Reset sets the enabled and enabledRaw to the input map.
|
||||
Reset(m map[string]bool)
|
||||
// EnabledRawMap returns the raw enable map from the feature gate.
|
||||
EnabledRawMap() map[string]bool
|
||||
// ExplicitlySet returns true if the feature value is explicitly set instead of
|
||||
// being derived from the default values or special features.
|
||||
ExplicitlySet(name Feature) bool
|
||||
// ResetFeatureValueToDefault resets the value of the feature back to the default value.
|
||||
ResetFeatureValueToDefault(name Feature) error
|
||||
}
|
||||
|
||||
// featureGate implements FeatureGate as well as pflag.Value for flag parsing.
|
||||
@@ -214,12 +208,8 @@ type featureGate struct {
|
||||
enabledRaw atomic.Value
|
||||
// closed is set to true when AddFlag is called, and prevents subsequent calls to Add
|
||||
closed bool
|
||||
// closedForModification is set to true when Enabled is called, and prevents subsequent calls to SetEmulationVersion to change the enabled features.
|
||||
// TODO: after all feature gates have migrated to versioned feature gates,
|
||||
// closedForModification should also prevents subsequent calls to Set and SetFromMap to change the enabled features
|
||||
closedForModification atomic.Bool
|
||||
// queriedFeatures stores all the features that have been queried through the Enabled interface.
|
||||
// It is reset when closedForModification is reset.
|
||||
// It is reset when SetEmulationVersion is called.
|
||||
queriedFeatures atomic.Value
|
||||
emulationVersion atomic.Pointer[version.Version]
|
||||
}
|
||||
@@ -282,7 +272,7 @@ func NewVersionedFeatureGate(emulationVersion *version.Version) *featureGate {
|
||||
|
||||
// NewFeatureGate creates a feature gate with the current binary version.
|
||||
func NewFeatureGate() *featureGate {
|
||||
binaryVersison := version.MustParse(baseversion.Get().String())
|
||||
binaryVersison := version.MustParse(baseversion.DefaultKubeBinaryVersion)
|
||||
return NewVersionedFeatureGate(binaryVersison)
|
||||
}
|
||||
|
||||
@@ -311,6 +301,8 @@ func (f *featureGate) Set(value string) error {
|
||||
|
||||
// Validate checks if the flag gates are valid at the emulated version.
|
||||
func (f *featureGate) Validate() []error {
|
||||
f.lock.Lock()
|
||||
defer f.lock.Unlock()
|
||||
m, ok := f.enabledRaw.Load().(map[string]bool)
|
||||
if !ok {
|
||||
return []error{fmt.Errorf("cannot cast enabledRaw to map[string]bool")}
|
||||
@@ -497,13 +489,17 @@ func (f *featureGate) OverrideDefaultAtVersion(name Feature, override bool, ver
|
||||
// GetAll returns a copy of the map of known feature names to feature specs for the current emulationVersion.
|
||||
func (f *featureGate) GetAll() map[Feature]FeatureSpec {
|
||||
retval := map[Feature]FeatureSpec{}
|
||||
for k, v := range f.GetAllVersioned() {
|
||||
spec := f.featureSpecAtEmulationVersion(v)
|
||||
f.lock.Lock()
|
||||
versionedSpecs := f.GetAllVersioned()
|
||||
emuVer := f.EmulationVersion()
|
||||
f.lock.Unlock()
|
||||
for k, v := range versionedSpecs {
|
||||
spec := featureSpecAtEmulationVersion(v, emuVer)
|
||||
if spec.PreRelease == PreAlpha {
|
||||
// The feature is not available at the emulation version.
|
||||
continue
|
||||
}
|
||||
retval[k] = *f.featureSpecAtEmulationVersion(v)
|
||||
retval[k] = *spec
|
||||
}
|
||||
return retval
|
||||
}
|
||||
@@ -518,6 +514,9 @@ func (f *featureGate) GetAllVersioned() map[Feature]VersionedSpecs {
|
||||
}
|
||||
|
||||
func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) error {
|
||||
if emulationVersion.EqualTo(f.EmulationVersion()) {
|
||||
return nil
|
||||
}
|
||||
f.lock.Lock()
|
||||
defer f.lock.Unlock()
|
||||
klog.V(1).Infof("set feature gate emulationVersion to %s", emulationVersion.String())
|
||||
@@ -531,22 +530,21 @@ func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) err
|
||||
enabled := map[Feature]bool{}
|
||||
errs := f.unsafeSetFromMap(enabled, enabledRaw, emulationVersion)
|
||||
|
||||
if f.closedForModification.Load() {
|
||||
queriedFeatures := f.queriedFeatures.Load().(map[Feature]struct{})
|
||||
known := f.known.Load().(map[Feature]VersionedSpecs)
|
||||
for feature := range queriedFeatures {
|
||||
newVal := featureEnabled(feature, enabled, known, emulationVersion)
|
||||
oldVal := f.Enabled(feature)
|
||||
// it is ok to modify emulation version if it does not result in feature enablemennt change for features that have already been queried.
|
||||
if newVal != oldVal {
|
||||
errs = append(errs, fmt.Errorf("SetEmulationVersion will change already queried feature:%s from %v to %v\ncall featureGate.OpenForModification() first to override", feature, oldVal, newVal))
|
||||
}
|
||||
queriedFeatures := f.queriedFeatures.Load().(map[Feature]struct{})
|
||||
known := f.known.Load().(map[Feature]VersionedSpecs)
|
||||
for feature := range queriedFeatures {
|
||||
newVal := featureEnabled(feature, enabled, known, emulationVersion)
|
||||
oldVal := featureEnabled(feature, f.enabled.Load().(map[Feature]bool), known, f.EmulationVersion())
|
||||
if newVal != oldVal {
|
||||
klog.Warningf("SetEmulationVersion will change already queried feature:%s from %v to %v", feature, oldVal, newVal)
|
||||
}
|
||||
}
|
||||
|
||||
if len(errs) == 0 {
|
||||
// Persist changes
|
||||
f.enabled.Store(enabled)
|
||||
f.emulationVersion.Store(emulationVersion)
|
||||
f.queriedFeatures.Store(map[Feature]struct{}{})
|
||||
}
|
||||
return utilerrors.NewAggregate(errs)
|
||||
}
|
||||
@@ -555,9 +553,9 @@ func (f *featureGate) EmulationVersion() *version.Version {
|
||||
return f.emulationVersion.Load()
|
||||
}
|
||||
|
||||
// FeatureSpec returns the FeatureSpec at the EmulationVersion if the key exists, an error otherwise.
|
||||
// featureSpec returns the featureSpec at the EmulationVersion if the key exists, an error otherwise.
|
||||
// This is useful to keep multiple implementations of a feature based on the PreRelease or Version info.
|
||||
func (f *featureGate) FeatureSpec(key Feature) (FeatureSpec, error) {
|
||||
func (f *featureGate) featureSpec(key Feature) (FeatureSpec, error) {
|
||||
if v, ok := f.known.Load().(map[Feature]VersionedSpecs)[key]; ok {
|
||||
featureSpec := f.featureSpecAtEmulationVersion(v)
|
||||
return *featureSpec, nil
|
||||
@@ -565,14 +563,16 @@ func (f *featureGate) FeatureSpec(key Feature) (FeatureSpec, error) {
|
||||
return FeatureSpec{}, fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName)
|
||||
}
|
||||
|
||||
func (f *featureGate) recordQueried(key Feature) {
|
||||
func (f *featureGate) unsafeRecordQueried(key Feature) {
|
||||
queriedFeatures := map[Feature]struct{}{}
|
||||
for k := range f.queriedFeatures.Load().(map[Feature]struct{}) {
|
||||
queriedFeatures[k] = struct{}{}
|
||||
}
|
||||
if _, ok := queriedFeatures[key]; ok {
|
||||
return
|
||||
}
|
||||
queriedFeatures[key] = struct{}{}
|
||||
f.queriedFeatures.Store(queriedFeatures)
|
||||
f.closedForModification.Store(true)
|
||||
}
|
||||
|
||||
func featureEnabled(key Feature, enabled map[Feature]bool, known map[Feature]VersionedSpecs, emulationVersion *version.Version) bool {
|
||||
@@ -589,8 +589,9 @@ func featureEnabled(key Feature, enabled map[Feature]bool, known map[Feature]Ver
|
||||
|
||||
// Enabled returns true if the key is enabled. If the key is not known, this call will panic.
|
||||
func (f *featureGate) Enabled(key Feature) bool {
|
||||
// TODO: ideally we should lock the feature gate in this call to be safe, need to evaluate how much performance impact locking would have.
|
||||
v := featureEnabled(key, f.enabled.Load().(map[Feature]bool), f.known.Load().(map[Feature]VersionedSpecs), f.EmulationVersion())
|
||||
f.recordQueried(key)
|
||||
f.unsafeRecordQueried(key)
|
||||
return v
|
||||
}
|
||||
|
||||
@@ -620,18 +621,6 @@ func (f *featureGate) Close() {
|
||||
f.lock.Unlock()
|
||||
}
|
||||
|
||||
// OpenForModification sets closedForModification to false, and allows subsequent calls to SetEmulationVersion to change enabled features
|
||||
// before the next Enabled is called.
|
||||
func (f *featureGate) OpenForModification() {
|
||||
queriedFeatures := []Feature{}
|
||||
for feature := range f.queriedFeatures.Load().(map[Feature]struct{}) {
|
||||
queriedFeatures = append(queriedFeatures, feature)
|
||||
}
|
||||
klog.Warningf("open feature gate for modification after querying features: %v.", queriedFeatures)
|
||||
f.closedForModification.Store(false)
|
||||
f.queriedFeatures.Store(map[Feature]struct{}{})
|
||||
}
|
||||
|
||||
// AddFlag adds a flag for setting global feature gates to the specified FlagSet.
|
||||
func (f *featureGate) AddFlag(fs *pflag.FlagSet) {
|
||||
// TODO(mtaufen): Shouldn't we just close it on the first Set/SetFromMap instead?
|
||||
@@ -671,10 +660,21 @@ func (f *featureGate) KnownFeatures() []string {
|
||||
return known
|
||||
}
|
||||
|
||||
// CopyKnownFeatures returns a partial copy of the FeatureGate object, with all the known features and overrides.
|
||||
// This is useful for creating a new instance of feature gate without inheriting all the enabled configurations of the base feature gate.
|
||||
func (f *featureGate) CopyKnownFeatures() MutableVersionedFeatureGate {
|
||||
fg := NewVersionedFeatureGate(f.EmulationVersion())
|
||||
known := f.GetAllVersioned()
|
||||
fg.known.Store(known)
|
||||
return fg
|
||||
}
|
||||
|
||||
// DeepCopy returns a deep copy of the FeatureGate object, such that gates can be
|
||||
// set on the copy without mutating the original. This is useful for validating
|
||||
// config against potential feature gate changes before committing those changes.
|
||||
func (f *featureGate) DeepCopy() MutableVersionedFeatureGate {
|
||||
f.lock.Lock()
|
||||
defer f.lock.Unlock()
|
||||
// Copy existing state.
|
||||
known := map[Feature]VersionedSpecs{}
|
||||
for k, v := range f.known.Load().(map[Feature]VersionedSpecs) {
|
||||
@@ -691,7 +691,7 @@ func (f *featureGate) DeepCopy() MutableVersionedFeatureGate {
|
||||
|
||||
// Construct a new featureGate around the copied state.
|
||||
// Note that specialFeatures is treated as immutable by convention,
|
||||
// and we maintain the value of f.closed across the copy, but resets closedForModification.
|
||||
// and we maintain the value of f.closed across the copy.
|
||||
fg := &featureGate{
|
||||
special: specialFeatures,
|
||||
closed: f.closed,
|
||||
@@ -704,21 +704,40 @@ func (f *featureGate) DeepCopy() MutableVersionedFeatureGate {
|
||||
return fg
|
||||
}
|
||||
|
||||
// Reset sets the enabled and enabledRaw to the input map.
|
||||
func (f *featureGate) Reset(m map[string]bool) {
|
||||
enabled := map[Feature]bool{}
|
||||
enabledRaw := map[string]bool{}
|
||||
queriedFeatures := map[Feature]struct{}{}
|
||||
f.enabled.Store(enabled)
|
||||
f.enabledRaw.Store(enabledRaw)
|
||||
_ = f.SetFromMap(m)
|
||||
f.closedForModification.Store(false)
|
||||
f.queriedFeatures.Store(queriedFeatures)
|
||||
f.lock.Lock()
|
||||
defer f.lock.Unlock()
|
||||
f.closed = false
|
||||
// ExplicitlySet returns true if the feature value is explicitly set instead of
|
||||
// being derived from the default values or special features.
|
||||
func (f *featureGate) ExplicitlySet(name Feature) bool {
|
||||
enabledRaw := f.enabledRaw.Load().(map[string]bool)
|
||||
_, ok := enabledRaw[string(name)]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (f *featureGate) EnabledRawMap() map[string]bool {
|
||||
return f.enabledRaw.Load().(map[string]bool)
|
||||
// ResetFeatureValueToDefault resets the value of the feature back to the default value.
|
||||
func (f *featureGate) ResetFeatureValueToDefault(name Feature) error {
|
||||
f.lock.Lock()
|
||||
defer f.lock.Unlock()
|
||||
enabled := map[Feature]bool{}
|
||||
for k, v := range f.enabled.Load().(map[Feature]bool) {
|
||||
enabled[k] = v
|
||||
}
|
||||
enabledRaw := map[string]bool{}
|
||||
for k, v := range f.enabledRaw.Load().(map[string]bool) {
|
||||
enabledRaw[k] = v
|
||||
}
|
||||
_, inEnabled := enabled[name]
|
||||
if inEnabled {
|
||||
delete(enabled, name)
|
||||
}
|
||||
_, inEnabledRaw := enabledRaw[string(name)]
|
||||
if inEnabledRaw {
|
||||
delete(enabledRaw, string(name))
|
||||
}
|
||||
// some features could be in enabled map but not enabledRaw map,
|
||||
// for example some Alpha feature when AllAlpha is set.
|
||||
if inEnabledRaw && !inEnabled {
|
||||
return fmt.Errorf("feature:%s was explicitly set, but not in enabled map", name)
|
||||
}
|
||||
f.enabled.Store(enabled)
|
||||
f.enabledRaw.Store(enabledRaw)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -596,6 +596,19 @@ func TestFeatureGateOverrideDefault(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("overrides are preserved across CopyKnownFeatures", func(t *testing.T) {
|
||||
f := NewFeatureGate()
|
||||
require.NoError(t, f.Add(map[Feature]FeatureSpec{"TestFeature": {Default: false}}))
|
||||
require.NoError(t, f.OverrideDefault("TestFeature", true))
|
||||
fcopy := f.CopyKnownFeatures()
|
||||
if !f.Enabled("TestFeature") {
|
||||
t.Error("TestFeature should be enabled by override")
|
||||
}
|
||||
if !fcopy.Enabled("TestFeature") {
|
||||
t.Error("default override was not preserved by CopyKnownFeatures")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("reflected in known features", func(t *testing.T) {
|
||||
f := NewFeatureGate()
|
||||
if err := f.Add(map[Feature]FeatureSpec{"TestFeature": {
|
||||
@@ -1119,22 +1132,22 @@ func TestVersionedFeatureGateFlagDefaults(t *testing.T) {
|
||||
if f.Enabled(testAlphaGate) != false {
|
||||
t.Errorf("Expected false")
|
||||
}
|
||||
if fs, _ := f.FeatureSpec(testAlphaGate); fs.PreRelease != PreAlpha || fs.Version.String() != "0.0" {
|
||||
if fs, _ := f.featureSpec(testAlphaGate); fs.PreRelease != PreAlpha || fs.Version.String() != "0.0" {
|
||||
t.Errorf("Expected (PreAlpha, 0.0)")
|
||||
}
|
||||
if f.Enabled(testBetaGate) != false {
|
||||
t.Errorf("Expected false")
|
||||
}
|
||||
if fs, _ := f.FeatureSpec(testBetaGate); fs.PreRelease != Beta || fs.Version.String() != "1.28" {
|
||||
if fs, _ := f.featureSpec(testBetaGate); fs.PreRelease != Beta || fs.Version.String() != "1.28" {
|
||||
t.Errorf("Expected (Beta, 1.28)")
|
||||
}
|
||||
if f.Enabled(testGAGate) != true {
|
||||
t.Errorf("Expected true")
|
||||
}
|
||||
if fs, _ := f.FeatureSpec(testGAGate); fs.PreRelease != Beta || fs.Version.String() != "1.27" {
|
||||
if fs, _ := f.featureSpec(testGAGate); fs.PreRelease != Beta || fs.Version.String() != "1.27" {
|
||||
t.Errorf("Expected (Beta, 1.27)")
|
||||
}
|
||||
if _, err := f.FeatureSpec("NonExist"); err == nil {
|
||||
if _, err := f.featureSpec("NonExist"); err == nil {
|
||||
t.Errorf("Expected Error")
|
||||
}
|
||||
allFeatures := f.GetAll()
|
||||
@@ -1304,7 +1317,6 @@ func TestVersionedFeatureGateOverrideDefault(t *testing.T) {
|
||||
if !f.Enabled("TestFeature2") {
|
||||
t.Error("expected TestFeature2 to have effective default of true")
|
||||
}
|
||||
f.OpenForModification()
|
||||
require.NoError(t, f.SetEmulationVersion(version.MustParse("1.29")))
|
||||
if !f.Enabled("TestFeature1") {
|
||||
t.Error("expected TestFeature1 to have effective default of true")
|
||||
@@ -1312,7 +1324,6 @@ func TestVersionedFeatureGateOverrideDefault(t *testing.T) {
|
||||
if f.Enabled("TestFeature2") {
|
||||
t.Error("expected TestFeature2 to have effective default of false")
|
||||
}
|
||||
f.OpenForModification()
|
||||
require.NoError(t, f.SetEmulationVersion(version.MustParse("1.26")))
|
||||
if f.Enabled("TestFeature1") {
|
||||
t.Error("expected TestFeature1 to have effective default of false")
|
||||
@@ -1520,32 +1531,173 @@ func TestFeatureSpecAtEmulationVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenForModification(t *testing.T) {
|
||||
const testBetaGate Feature = "testBetaGate"
|
||||
f := NewVersionedFeatureGate(version.MustParse("1.29"))
|
||||
func TestCopyKnownFeatures(t *testing.T) {
|
||||
f := NewFeatureGate()
|
||||
require.NoError(t, f.Add(map[Feature]FeatureSpec{"FeatureA": {Default: false}, "FeatureB": {Default: false}}))
|
||||
require.NoError(t, f.Set("FeatureA=true"))
|
||||
require.NoError(t, f.OverrideDefault("FeatureB", true))
|
||||
fcopy := f.CopyKnownFeatures()
|
||||
require.NoError(t, f.Add(map[Feature]FeatureSpec{"FeatureC": {Default: false}}))
|
||||
|
||||
assert.True(t, f.Enabled("FeatureA"))
|
||||
assert.True(t, f.Enabled("FeatureB"))
|
||||
assert.False(t, f.Enabled("FeatureC"))
|
||||
|
||||
assert.False(t, fcopy.Enabled("FeatureA"))
|
||||
assert.True(t, fcopy.Enabled("FeatureB"))
|
||||
|
||||
require.NoError(t, fcopy.Set("FeatureB=false"))
|
||||
assert.True(t, f.Enabled("FeatureB"))
|
||||
assert.False(t, fcopy.Enabled("FeatureB"))
|
||||
if err := fcopy.Set("FeatureC=true"); err == nil {
|
||||
t.Error("expected FeatureC not registered in the copied feature gate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplicitlySet(t *testing.T) {
|
||||
// gates for testing
|
||||
const testAlphaGate Feature = "TestAlpha"
|
||||
const testBetaGate Feature = "TestBeta"
|
||||
|
||||
tests := []struct {
|
||||
arg string
|
||||
expectedFeatureValue map[Feature]bool
|
||||
expectedExplicitlySet map[Feature]bool
|
||||
}{
|
||||
{
|
||||
arg: "",
|
||||
expectedFeatureValue: map[Feature]bool{
|
||||
allAlphaGate: false,
|
||||
allBetaGate: false,
|
||||
testAlphaGate: false,
|
||||
testBetaGate: false,
|
||||
},
|
||||
expectedExplicitlySet: map[Feature]bool{
|
||||
allAlphaGate: false,
|
||||
allBetaGate: false,
|
||||
testAlphaGate: false,
|
||||
testBetaGate: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
arg: "AllAlpha=true,TestBeta=false",
|
||||
expectedFeatureValue: map[Feature]bool{
|
||||
allAlphaGate: true,
|
||||
allBetaGate: false,
|
||||
testAlphaGate: true,
|
||||
testBetaGate: false,
|
||||
},
|
||||
expectedExplicitlySet: map[Feature]bool{
|
||||
allAlphaGate: true,
|
||||
allBetaGate: false,
|
||||
testAlphaGate: false,
|
||||
testBetaGate: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
arg: "AllAlpha=true,AllBeta=false",
|
||||
expectedFeatureValue: map[Feature]bool{
|
||||
allAlphaGate: true,
|
||||
allBetaGate: false,
|
||||
testAlphaGate: true,
|
||||
testBetaGate: false,
|
||||
},
|
||||
expectedExplicitlySet: map[Feature]bool{
|
||||
allAlphaGate: true,
|
||||
allBetaGate: true,
|
||||
testAlphaGate: false,
|
||||
testBetaGate: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
for i, test := range tests {
|
||||
t.Run(test.arg, func(t *testing.T) {
|
||||
fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError)
|
||||
f := NewVersionedFeatureGate(version.MustParse("1.29"))
|
||||
err := f.AddVersioned(map[Feature]VersionedSpecs{
|
||||
testAlphaGate: {
|
||||
{Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha},
|
||||
},
|
||||
testBetaGate: {
|
||||
{Version: version.MustParse("1.29"), Default: false, PreRelease: Beta},
|
||||
{Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
f.AddFlag(fs)
|
||||
|
||||
var errs []error
|
||||
err = fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)})
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
err = utilerrors.NewAggregate(errs)
|
||||
require.NoError(t, err)
|
||||
for k, v := range test.expectedFeatureValue {
|
||||
if actual := f.Enabled(k); actual != v {
|
||||
t.Errorf("%d: expected %s=%v, Got %v", i, k, v, actual)
|
||||
}
|
||||
}
|
||||
for k, v := range test.expectedExplicitlySet {
|
||||
if actual := f.ExplicitlySet(k); actual != v {
|
||||
t.Errorf("%d: expected ExplicitlySet(%s)=%v, Got %v", i, k, v, actual)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetFeatureValueToDefault(t *testing.T) {
|
||||
// gates for testing
|
||||
const testAlphaGate Feature = "TestAlpha"
|
||||
const testBetaGate Feature = "TestBeta"
|
||||
|
||||
f := NewVersionedFeatureGate(version.MustParse("1.29"))
|
||||
err := f.AddVersioned(map[Feature]VersionedSpecs{
|
||||
testAlphaGate: {
|
||||
{Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha},
|
||||
},
|
||||
testBetaGate: {
|
||||
{Version: version.MustParse("1.29"), Default: true, PreRelease: Beta},
|
||||
{Version: version.MustParse("1.28"), Default: false, PreRelease: Beta},
|
||||
{Version: version.MustParse("1.26"), Default: false, PreRelease: Alpha},
|
||||
{Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
if f.Enabled(testBetaGate) != true {
|
||||
t.Errorf("Expected true")
|
||||
fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError)
|
||||
assert.False(t, f.Enabled("AllAlpha"))
|
||||
assert.False(t, f.Enabled("AllBeta"))
|
||||
assert.False(t, f.Enabled("TestAlpha"))
|
||||
assert.True(t, f.Enabled("TestBeta"))
|
||||
|
||||
f.AddFlag(fs)
|
||||
var errs []error
|
||||
err = fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, "AllAlpha=true,TestBeta=false")})
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
err = f.SetEmulationVersion(version.MustParse("1.28"))
|
||||
if err == nil {
|
||||
t.Fatalf("Expected error when SetEmulationVersion after querying features")
|
||||
}
|
||||
if f.Enabled(testBetaGate) != true {
|
||||
t.Errorf("Expected true")
|
||||
}
|
||||
f.OpenForModification()
|
||||
err = utilerrors.NewAggregate(errs)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, f.Enabled("AllAlpha"))
|
||||
assert.False(t, f.Enabled("AllBeta"))
|
||||
assert.True(t, f.Enabled("TestAlpha"))
|
||||
assert.False(t, f.Enabled("TestBeta"))
|
||||
|
||||
require.NoError(t, f.ResetFeatureValueToDefault("AllAlpha"))
|
||||
assert.False(t, f.Enabled("AllAlpha"))
|
||||
assert.False(t, f.Enabled("AllBeta"))
|
||||
assert.True(t, f.Enabled("TestAlpha"))
|
||||
assert.False(t, f.Enabled("TestBeta"))
|
||||
|
||||
require.NoError(t, f.ResetFeatureValueToDefault("TestBeta"))
|
||||
assert.False(t, f.Enabled("AllAlpha"))
|
||||
assert.False(t, f.Enabled("AllBeta"))
|
||||
assert.True(t, f.Enabled("TestAlpha"))
|
||||
assert.True(t, f.Enabled("TestBeta"))
|
||||
|
||||
require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28")))
|
||||
if f.Enabled(testBetaGate) != false {
|
||||
t.Errorf("Expected false at 1.28")
|
||||
}
|
||||
assert.False(t, f.Enabled("AllAlpha"))
|
||||
assert.False(t, f.Enabled("AllBeta"))
|
||||
assert.False(t, f.Enabled("TestAlpha"))
|
||||
assert.False(t, f.Enabled("TestBeta"))
|
||||
}
|
||||
|
||||
@@ -20,14 +20,16 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"k8s.io/apimachinery/pkg/util/version"
|
||||
"k8s.io/component-base/featuregate"
|
||||
)
|
||||
|
||||
var (
|
||||
overrideLock sync.Mutex
|
||||
featureFlagOverride map[featuregate.Feature]string
|
||||
overrideLock sync.Mutex
|
||||
featureFlagOverride map[featuregate.Feature]string
|
||||
emulationVersionOverride string
|
||||
emulationVersionOverrideValue *version.Version
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -42,10 +44,12 @@ func init() {
|
||||
// Example use:
|
||||
//
|
||||
// featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.<FeatureName>, true)
|
||||
func SetFeatureGateDuringTest(tb testing.TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) {
|
||||
func SetFeatureGateDuringTest(tb TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) {
|
||||
tb.Helper()
|
||||
detectParallelOverrideCleanup := detectParallelOverride(tb, f)
|
||||
originalEnabled := gate.(featuregate.MutableVersionedFeatureGateForTests).EnabledRawMap()
|
||||
originalValue := gate.Enabled(f)
|
||||
originalEmuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion()
|
||||
originalExplicitlySet := gate.(featuregate.MutableVersionedFeatureGate).ExplicitlySet(f)
|
||||
|
||||
// Specially handle AllAlpha and AllBeta
|
||||
if f == "AllAlpha" || f == "AllBeta" {
|
||||
@@ -67,11 +71,46 @@ func SetFeatureGateDuringTest(tb testing.TB, gate featuregate.FeatureGate, f fea
|
||||
tb.Cleanup(func() {
|
||||
tb.Helper()
|
||||
detectParallelOverrideCleanup()
|
||||
gate.(featuregate.MutableVersionedFeatureGateForTests).Reset(originalEnabled)
|
||||
emuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion()
|
||||
if !emuVer.EqualTo(originalEmuVer) {
|
||||
tb.Fatalf("change of feature gate emulation version from %s to %s in the chain of SetFeatureGateDuringTest is not allowed\nuse SetFeatureGateEmulationVersionDuringTest to change emulation version in tests",
|
||||
originalEmuVer.String(), emuVer.String())
|
||||
}
|
||||
if originalExplicitlySet {
|
||||
if err := gate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, originalValue)); err != nil {
|
||||
tb.Errorf("error restoring %s=%v: %v", f, originalValue, err)
|
||||
}
|
||||
} else {
|
||||
if err := gate.(featuregate.MutableVersionedFeatureGate).ResetFeatureValueToDefault(f); err != nil {
|
||||
tb.Errorf("error restoring %s=%v: %v", f, originalValue, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func detectParallelOverride(tb testing.TB, f featuregate.Feature) func() {
|
||||
// SetFeatureGateEmulationVersionDuringTest sets the specified gate to the specified emulation version for duration of the test.
|
||||
// Fails when it detects second call to set a different emulation version or is unable to set or restore emulation version.
|
||||
// WARNING: Can leak set variable when called in test calling t.Parallel(), however second attempt to set a different emulation version will cause fatal.
|
||||
// Example use:
|
||||
|
||||
// featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31"))
|
||||
func SetFeatureGateEmulationVersionDuringTest(tb TB, gate featuregate.FeatureGate, ver *version.Version) {
|
||||
tb.Helper()
|
||||
detectParallelOverrideCleanup := detectParallelOverrideEmulationVersion(tb, ver)
|
||||
originalEmuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion()
|
||||
if err := gate.(featuregate.MutableVersionedFeatureGate).SetEmulationVersion(ver); err != nil {
|
||||
tb.Fatalf("failed to set emulation version to %s during test", ver.String())
|
||||
}
|
||||
tb.Cleanup(func() {
|
||||
tb.Helper()
|
||||
detectParallelOverrideCleanup()
|
||||
if err := gate.(featuregate.MutableVersionedFeatureGate).SetEmulationVersion(originalEmuVer); err != nil {
|
||||
tb.Fatalf("failed to restore emulation version to %s during test", originalEmuVer.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func detectParallelOverride(tb TB, f featuregate.Feature) func() {
|
||||
tb.Helper()
|
||||
overrideLock.Lock()
|
||||
defer overrideLock.Unlock()
|
||||
@@ -92,7 +131,44 @@ func detectParallelOverride(tb testing.TB, f featuregate.Feature) func() {
|
||||
}
|
||||
}
|
||||
|
||||
func sameTestOrSubtest(tb testing.TB, testName string) bool {
|
||||
func detectParallelOverrideEmulationVersion(tb TB, ver *version.Version) func() {
|
||||
tb.Helper()
|
||||
overrideLock.Lock()
|
||||
defer overrideLock.Unlock()
|
||||
beforeOverrideTestName := emulationVersionOverride
|
||||
beforeOverrideValue := emulationVersionOverrideValue
|
||||
if ver.EqualTo(beforeOverrideValue) {
|
||||
return func() {}
|
||||
}
|
||||
if beforeOverrideTestName != "" && !sameTestOrSubtest(tb, beforeOverrideTestName) {
|
||||
tb.Fatalf("Detected parallel setting of a feature gate emulation version by both %q and %q", beforeOverrideTestName, tb.Name())
|
||||
}
|
||||
emulationVersionOverride = tb.Name()
|
||||
emulationVersionOverrideValue = ver
|
||||
|
||||
return func() {
|
||||
tb.Helper()
|
||||
overrideLock.Lock()
|
||||
defer overrideLock.Unlock()
|
||||
if afterOverrideTestName := emulationVersionOverride; afterOverrideTestName != tb.Name() {
|
||||
tb.Fatalf("Detected parallel setting of a feature gate emulation version between both %q and %q", afterOverrideTestName, tb.Name())
|
||||
}
|
||||
emulationVersionOverride = beforeOverrideTestName
|
||||
emulationVersionOverrideValue = beforeOverrideValue
|
||||
}
|
||||
}
|
||||
|
||||
func sameTestOrSubtest(tb TB, testName string) bool {
|
||||
// Assumes that "/" is not used in test names.
|
||||
return tb.Name() == testName || strings.HasPrefix(tb.Name(), testName+"/")
|
||||
}
|
||||
|
||||
type TB interface {
|
||||
Cleanup(func())
|
||||
Error(args ...any)
|
||||
Errorf(format string, args ...any)
|
||||
Fatal(args ...any)
|
||||
Fatalf(format string, args ...any)
|
||||
Helper()
|
||||
Name() string
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"k8s.io/apimachinery/pkg/util/version"
|
||||
"k8s.io/component-base/featuregate"
|
||||
)
|
||||
|
||||
@@ -71,6 +72,7 @@ func TestSpecialGates(t *gotest.T) {
|
||||
expect(t, gate, before)
|
||||
t.Cleanup(func() {
|
||||
expect(t, gate, before)
|
||||
cleanup()
|
||||
})
|
||||
|
||||
SetFeatureGateDuringTest(t, gate, "AllAlpha", true)
|
||||
@@ -156,10 +158,107 @@ func TestSetFeatureGateInTest(t *gotest.T) {
|
||||
assert.True(t, gate.Enabled("feature"))
|
||||
}
|
||||
|
||||
func TestDetectLeakToMainTest(t *gotest.T) {
|
||||
t.Cleanup(func() {
|
||||
featureFlagOverride = map[featuregate.Feature]string{}
|
||||
func TestSpecialGatesVersioned(t *gotest.T) {
|
||||
originalEmulationVersion := version.MustParse("1.31")
|
||||
gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion)
|
||||
|
||||
err := gate.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
|
||||
"alpha_default_on": {
|
||||
{Version: version.MustParse("1.27"), Default: true, PreRelease: featuregate.Alpha},
|
||||
},
|
||||
"alpha_default_off": {
|
||||
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
|
||||
},
|
||||
"beta_default_on": {
|
||||
{Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta},
|
||||
},
|
||||
"beta_default_on_set_off": {
|
||||
{Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta},
|
||||
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
|
||||
},
|
||||
"beta_default_off": {
|
||||
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Beta},
|
||||
},
|
||||
"beta_default_off_set_on": {
|
||||
{Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Beta},
|
||||
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, gate.Set("beta_default_on_set_off=false"))
|
||||
require.NoError(t, gate.Set("beta_default_off_set_on=true"))
|
||||
|
||||
before := map[featuregate.Feature]bool{
|
||||
"AllAlpha": false,
|
||||
"AllBeta": false,
|
||||
|
||||
"alpha_default_on": true,
|
||||
"alpha_default_off": false,
|
||||
|
||||
"beta_default_on": true,
|
||||
"beta_default_on_set_off": false,
|
||||
"beta_default_off": false,
|
||||
"beta_default_off_set_on": true,
|
||||
}
|
||||
expect(t, gate, before)
|
||||
t.Cleanup(func() {
|
||||
expect(t, gate, before)
|
||||
cleanup()
|
||||
})
|
||||
|
||||
t.Run("OverwriteInSubtest", func(t *gotest.T) {
|
||||
SetFeatureGateDuringTest(t, gate, "AllAlpha", true)
|
||||
expect(t, gate, map[featuregate.Feature]bool{
|
||||
"AllAlpha": true,
|
||||
"AllBeta": false,
|
||||
|
||||
"alpha_default_on": true,
|
||||
"alpha_default_off": true,
|
||||
|
||||
"beta_default_on": true,
|
||||
"beta_default_on_set_off": false,
|
||||
"beta_default_off": false,
|
||||
"beta_default_off_set_on": true,
|
||||
})
|
||||
|
||||
SetFeatureGateDuringTest(t, gate, "AllBeta", true)
|
||||
expect(t, gate, map[featuregate.Feature]bool{
|
||||
"AllAlpha": true,
|
||||
"AllBeta": true,
|
||||
|
||||
"alpha_default_on": true,
|
||||
"alpha_default_off": true,
|
||||
|
||||
"beta_default_on": true,
|
||||
"beta_default_on_set_off": true,
|
||||
"beta_default_off": true,
|
||||
"beta_default_off_set_on": true,
|
||||
})
|
||||
})
|
||||
expect(t, gate, before)
|
||||
|
||||
t.Run("OverwriteInSubtestAtDifferentVersion", func(t *gotest.T) {
|
||||
SetFeatureGateEmulationVersionDuringTest(t, gate, version.MustParse("1.28"))
|
||||
SetFeatureGateDuringTest(t, gate, "AllAlpha", true)
|
||||
expect(t, gate, map[featuregate.Feature]bool{
|
||||
"AllAlpha": true,
|
||||
"AllBeta": false,
|
||||
|
||||
"alpha_default_on": true,
|
||||
"alpha_default_off": true,
|
||||
|
||||
"beta_default_on": false,
|
||||
"beta_default_on_set_off": true,
|
||||
"beta_default_off": false,
|
||||
"beta_default_off_set_on": true,
|
||||
})
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
func TestDetectLeakToMainTest(t *gotest.T) {
|
||||
t.Cleanup(cleanup)
|
||||
gate := featuregate.NewFeatureGate()
|
||||
err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{
|
||||
"feature": {PreRelease: featuregate.Alpha, Default: false},
|
||||
@@ -183,9 +282,7 @@ func TestDetectLeakToMainTest(t *gotest.T) {
|
||||
}
|
||||
|
||||
func TestDetectLeakToOtherSubtest(t *gotest.T) {
|
||||
t.Cleanup(func() {
|
||||
featureFlagOverride = map[featuregate.Feature]string{}
|
||||
})
|
||||
t.Cleanup(cleanup)
|
||||
gate := featuregate.NewFeatureGate()
|
||||
err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{
|
||||
"feature": {PreRelease: featuregate.Alpha, Default: false},
|
||||
@@ -211,9 +308,7 @@ func TestDetectLeakToOtherSubtest(t *gotest.T) {
|
||||
}
|
||||
|
||||
func TestCannotDetectLeakFromSubtest(t *gotest.T) {
|
||||
t.Cleanup(func() {
|
||||
featureFlagOverride = map[featuregate.Feature]string{}
|
||||
})
|
||||
t.Cleanup(cleanup)
|
||||
gate := featuregate.NewFeatureGate()
|
||||
err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{
|
||||
"feature": {PreRelease: featuregate.Alpha, Default: false},
|
||||
@@ -230,6 +325,107 @@ func TestCannotDetectLeakFromSubtest(t *gotest.T) {
|
||||
assert.True(t, gate.Enabled("feature"))
|
||||
}
|
||||
|
||||
func TestCannotDetectLeakFromTwoSubtestsWithDifferentFeatures(t *gotest.T) {
|
||||
t.Cleanup(cleanup)
|
||||
gate := featuregate.NewFeatureGate()
|
||||
err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{
|
||||
"feature1": {PreRelease: featuregate.Alpha, Default: false},
|
||||
"feature2": {PreRelease: featuregate.Alpha, Default: false},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.False(t, gate.Enabled("feature1"))
|
||||
assert.False(t, gate.Enabled("feature2"))
|
||||
subtestName := "Subtest"
|
||||
// Subtest setting feature gate and calling parallel will leak it out
|
||||
t.Run(subtestName, func(t *gotest.T) {
|
||||
SetFeatureGateDuringTest(t, gate, "feature1", true)
|
||||
t.Parallel()
|
||||
assert.True(t, gate.Enabled("feature1"))
|
||||
assert.False(t, gate.Enabled("feature2"))
|
||||
})
|
||||
// Leaked true
|
||||
assert.True(t, gate.Enabled("feature1"))
|
||||
assert.False(t, gate.Enabled("feature2"))
|
||||
// Add suffix to name to prevent tests with the same prefix.
|
||||
t.Run(subtestName+"Suffix", func(t *gotest.T) {
|
||||
// Leaked true
|
||||
assert.True(t, gate.Enabled("feature1"))
|
||||
assert.False(t, gate.Enabled("feature2"))
|
||||
SetFeatureGateDuringTest(t, gate, "feature2", true)
|
||||
assert.True(t, gate.Enabled("feature1"))
|
||||
assert.True(t, gate.Enabled("feature2"))
|
||||
})
|
||||
}
|
||||
|
||||
func TestDetectEmulationVersionLeakToMainTest(t *gotest.T) {
|
||||
t.Cleanup(cleanup)
|
||||
originalEmulationVersion := version.MustParse("1.31")
|
||||
newEmulationVersion := version.MustParse("1.30")
|
||||
gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion)
|
||||
assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion))
|
||||
|
||||
// Subtest setting feature gate and calling parallel will leak it out
|
||||
t.Run("LeakingSubtest", func(t *gotest.T) {
|
||||
fakeT := &ignoreFatalT{T: t}
|
||||
SetFeatureGateEmulationVersionDuringTest(fakeT, gate, newEmulationVersion)
|
||||
// Calling t.Parallel in subtest will resume the main test body
|
||||
t.Parallel()
|
||||
// Leaked from main test
|
||||
assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion))
|
||||
})
|
||||
// Leaked from subtest
|
||||
assert.True(t, gate.EmulationVersion().EqualTo(newEmulationVersion))
|
||||
fakeT := &ignoreFatalT{T: t}
|
||||
SetFeatureGateEmulationVersionDuringTest(fakeT, gate, originalEmulationVersion)
|
||||
assert.True(t, fakeT.fatalRecorded)
|
||||
}
|
||||
|
||||
func TestNoLeakFromSameEmulationVersionToMainTest(t *gotest.T) {
|
||||
t.Cleanup(cleanup)
|
||||
originalEmulationVersion := version.MustParse("1.31")
|
||||
newEmulationVersion := version.MustParse("1.31")
|
||||
gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion)
|
||||
assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion))
|
||||
|
||||
// Subtest setting feature gate and calling parallel will leak it out
|
||||
t.Run("LeakingSubtest", func(t *gotest.T) {
|
||||
SetFeatureGateEmulationVersionDuringTest(t, gate, newEmulationVersion)
|
||||
// Calling t.Parallel in subtest will resume the main test body
|
||||
t.Parallel()
|
||||
// Leaked from main test
|
||||
assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion))
|
||||
})
|
||||
// Leaked from subtest
|
||||
assert.True(t, gate.EmulationVersion().EqualTo(newEmulationVersion))
|
||||
SetFeatureGateEmulationVersionDuringTest(t, gate, originalEmulationVersion)
|
||||
}
|
||||
|
||||
func TestDetectEmulationVersionLeakToOtherSubtest(t *gotest.T) {
|
||||
t.Cleanup(cleanup)
|
||||
originalEmulationVersion := version.MustParse("1.31")
|
||||
newEmulationVersion := version.MustParse("1.30")
|
||||
gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion)
|
||||
assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion))
|
||||
|
||||
subtestName := "Subtest"
|
||||
// Subtest setting feature gate and calling parallel will leak it out
|
||||
t.Run(subtestName, func(t *gotest.T) {
|
||||
fakeT := &ignoreFatalT{T: t}
|
||||
SetFeatureGateEmulationVersionDuringTest(fakeT, gate, newEmulationVersion)
|
||||
t.Parallel()
|
||||
})
|
||||
// Add suffix to name to prevent tests with the same prefix.
|
||||
t.Run(subtestName+"Suffix", func(t *gotest.T) {
|
||||
// Leaked newEmulationVersion
|
||||
assert.True(t, gate.EmulationVersion().EqualTo(newEmulationVersion))
|
||||
|
||||
fakeT := &ignoreFatalT{T: t}
|
||||
SetFeatureGateEmulationVersionDuringTest(fakeT, gate, originalEmulationVersion)
|
||||
assert.True(t, fakeT.fatalRecorded)
|
||||
})
|
||||
}
|
||||
|
||||
type ignoreFatalT struct {
|
||||
*gotest.T
|
||||
fatalRecorded bool
|
||||
@@ -248,3 +444,9 @@ func (f *ignoreFatalT) Fatalf(format string, args ...any) {
|
||||
f.fatalRecorded = true
|
||||
f.T.Logf("[IGNORED] "+format, args...)
|
||||
}
|
||||
|
||||
func cleanup() {
|
||||
featureFlagOverride = map[featuregate.Feature]string{}
|
||||
emulationVersionOverride = ""
|
||||
emulationVersionOverrideValue = nil
|
||||
}
|
||||
|
||||
@@ -61,3 +61,10 @@ var (
|
||||
|
||||
buildDate = "1970-01-01T00:00:00Z" // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ')
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultKubeBinaryVersion is the hard coded k8 binary version based on the latest K8s release.
|
||||
// It is supposed to be consistent with gitMajor and gitMinor, except for local tests, where gitMajor and gitMinor are "".
|
||||
// Should update for each minor release!
|
||||
DefaultKubeBinaryVersion = "1.31"
|
||||
)
|
||||
|
||||
@@ -31,7 +31,6 @@ import (
|
||||
genericapiserver "k8s.io/apiserver/pkg/server"
|
||||
"k8s.io/apiserver/pkg/server/filters"
|
||||
genericoptions "k8s.io/apiserver/pkg/server/options"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
utilversion "k8s.io/apiserver/pkg/util/version"
|
||||
"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
|
||||
"k8s.io/kube-aggregator/pkg/apiserver"
|
||||
@@ -81,10 +80,7 @@ func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions)
|
||||
}
|
||||
cmd.SetContext(ctx)
|
||||
|
||||
fs := cmd.Flags()
|
||||
utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs)
|
||||
|
||||
o.AddFlags(fs)
|
||||
o.AddFlags(cmd.Flags())
|
||||
return cmd
|
||||
}
|
||||
|
||||
@@ -99,13 +95,8 @@ func (o *AggregatorOptions) AddFlags(fs *pflag.FlagSet) {
|
||||
|
||||
// NewDefaultOptions builds a "normal" set of options. You wouldn't normally expose this, but hyperkube isn't cobra compatible
|
||||
func NewDefaultOptions(out, err io.Writer) *AggregatorOptions {
|
||||
// effectiveVersion is used to set what apis and feature gates the generic api server is compatible with.
|
||||
// You can also have the flag setting the effectiveVersion of the aggregator apiserver, and
|
||||
// having a mapping from the aggregator apiserver version to generic apiserver version.
|
||||
effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
|
||||
utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
|
||||
o := &AggregatorOptions{
|
||||
ServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion),
|
||||
ServerRunOptions: genericoptions.NewServerRunOptions(),
|
||||
RecommendedOptions: genericoptions.NewRecommendedOptions(
|
||||
defaultEtcdPathPrefix,
|
||||
aggregatorscheme.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion),
|
||||
|
||||
@@ -7,6 +7,7 @@ go 1.22.0
|
||||
require (
|
||||
github.com/google/gofuzz v1.2.0
|
||||
github.com/spf13/cobra v1.8.1
|
||||
github.com/stretchr/testify v1.9.0
|
||||
k8s.io/apimachinery v0.0.0
|
||||
k8s.io/apiserver v0.0.0
|
||||
k8s.io/client-go v0.0.0
|
||||
@@ -55,6 +56,7 @@ require (
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/prometheus/client_golang v1.19.0 // indirect
|
||||
github.com/prometheus/client_model v0.6.0 // indirect
|
||||
github.com/prometheus/common v0.48.0 // indirect
|
||||
|
||||
1
staging/src/k8s.io/sample-apiserver/go.sum
generated
1
staging/src/k8s.io/sample-apiserver/go.sum
generated
@@ -170,6 +170,7 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
|
||||
@@ -113,7 +113,7 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti
|
||||
o.RecommendedOptions.AddFlags(flags)
|
||||
|
||||
wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2")
|
||||
wardleFeatureGate := featuregate.NewVersionedFeatureGate(version.MustParse("1.2"))
|
||||
wardleFeatureGate := utilfeature.DefaultFeatureGate.CopyKnownFeatures()
|
||||
utilruntime.Must(wardleFeatureGate.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
|
||||
"BanFlunder": {
|
||||
{Version: version.MustParse("1.2"), Default: true, PreRelease: featuregate.GA},
|
||||
|
||||
@@ -29,8 +29,6 @@ import (
|
||||
netutils "k8s.io/utils/net"
|
||||
|
||||
utilerrors "k8s.io/apimachinery/pkg/util/errors"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
utilversion "k8s.io/apiserver/pkg/util/version"
|
||||
apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app"
|
||||
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
@@ -65,9 +63,7 @@ func NewAPIServer(storageConfig storagebackend.Config) *APIServer {
|
||||
func (a *APIServer) Start(ctx context.Context) error {
|
||||
const tokenFilePath = "known_tokens.csv"
|
||||
|
||||
featureGate := utilfeature.DefaultFeatureGate
|
||||
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
|
||||
o := options.NewServerRunOptions(featureGate, effectiveVersion)
|
||||
o := options.NewServerRunOptions()
|
||||
o.Etcd.StorageConfig = a.storageConfig
|
||||
_, ipnet, err := netutils.ParseCIDRSloppy(clusterIPRange)
|
||||
if err != nil {
|
||||
|
||||
@@ -27,9 +27,11 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
"k8s.io/apiserver/pkg/storage"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
utilversion "k8s.io/apiserver/pkg/util/version"
|
||||
"k8s.io/client-go/dynamic"
|
||||
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
|
||||
@@ -37,9 +39,13 @@ import (
|
||||
|
||||
// TestCrossGroupStorage tests to make sure that all objects stored in an expected location in etcd can be converted/read.
|
||||
func TestCrossGroupStorage(t *testing.T) {
|
||||
testRegistry := utilversion.NewComponentGlobalsRegistry()
|
||||
utilruntime.Must(testRegistry.Register("test", utilversion.NewEffectiveVersion("0.0"), utilfeature.DefaultFeatureGate.DeepCopy()))
|
||||
|
||||
apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) {
|
||||
// force enable all resources so we can check storage.
|
||||
opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("0.0")
|
||||
opts.GenericServerRunOptions.ComponentName = "test"
|
||||
opts.GenericServerRunOptions.ComponentGlobalsRegistry = testRegistry
|
||||
})
|
||||
defer apiServer.Cleanup()
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apiserver/pkg/util/feature"
|
||||
utilversion "k8s.io/apiserver/pkg/util/version"
|
||||
@@ -75,8 +76,12 @@ var allowMissingTestdataFixtures = map[schema.GroupVersionKind]bool{
|
||||
func TestEtcdStoragePath(t *testing.T) {
|
||||
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true)
|
||||
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)
|
||||
testRegistry := utilversion.NewComponentGlobalsRegistry()
|
||||
utilruntime.Must(testRegistry.Register("test", utilversion.NewEffectiveVersion("0.0"), feature.DefaultFeatureGate.DeepCopy()))
|
||||
|
||||
apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) {
|
||||
opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("0.0")
|
||||
opts.GenericServerRunOptions.ComponentName = "test"
|
||||
opts.GenericServerRunOptions.ComponentGlobalsRegistry = testRegistry
|
||||
})
|
||||
defer apiServer.Cleanup()
|
||||
defer dumpEtcdKVOnFailure(t, apiServer.KV)
|
||||
|
||||
@@ -40,8 +40,6 @@ import (
|
||||
utilerrors "k8s.io/apimachinery/pkg/util/errors"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
genericapiserveroptions "k8s.io/apiserver/pkg/server/options"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
utilversion "k8s.io/apiserver/pkg/util/version"
|
||||
cacheddiscovery "k8s.io/client-go/discovery/cached/memory"
|
||||
"k8s.io/client-go/dynamic"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
@@ -93,9 +91,7 @@ func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRu
|
||||
t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err)
|
||||
}
|
||||
|
||||
featureGate := utilfeature.DefaultFeatureGate
|
||||
effectiveVersion := utilversion.DefaultBuildEffectiveVersion()
|
||||
opts := options.NewServerRunOptions(featureGate, effectiveVersion)
|
||||
opts := options.NewServerRunOptions()
|
||||
opts.Options.SecureServing.Listener = listener
|
||||
opts.Options.SecureServing.ServerCert.CertDirectory = certDir
|
||||
opts.Options.ServiceAccountSigningKeyFile = saSigningKeyFile.Name()
|
||||
|
||||
@@ -35,8 +35,6 @@ import (
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
genericapiserver "k8s.io/apiserver/pkg/server"
|
||||
genericapiserveroptions "k8s.io/apiserver/pkg/server/options"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
utilversion "k8s.io/apiserver/pkg/util/version"
|
||||
client "k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/util/cert"
|
||||
@@ -137,9 +135,7 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) (
|
||||
t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err)
|
||||
}
|
||||
|
||||
featureGate := utilfeature.DefaultFeatureGate
|
||||
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
|
||||
opts := options.NewServerRunOptions(featureGate, effectiveVersion)
|
||||
opts := options.NewServerRunOptions()
|
||||
opts.SecureServing.Listener = listener
|
||||
opts.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1")
|
||||
opts.SecureServing.ServerCert.CertDirectory = certDir
|
||||
|
||||
Reference in New Issue
Block a user