Merge pull request #122891 from siyuanfoundation/api-comp-ver1

apimachinery: API Emulation Versioning
This commit is contained in:
Kubernetes Prow Robot
2024-06-25 20:04:48 -07:00
committed by GitHub
85 changed files with 4718 additions and 456 deletions

View File

@@ -63,7 +63,7 @@ type Extra struct {
MasterCount int
}
// NewServerRunOptions creates a new ServerRunOptions object with default parameters
// NewServerRunOptions creates and returns ServerRunOptions according to the given featureGate and effectiveVersion of the server binary to run.
func NewServerRunOptions() *ServerRunOptions {
s := ServerRunOptions{
Options: controlplaneapiserver.NewOptions(),

View File

@@ -27,13 +27,16 @@ import (
"github.com/spf13/pflag"
noopoteltrace "go.opentelemetry.io/otel/trace/noop"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/admission"
apiserveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/etcd3"
"k8s.io/apiserver/pkg/storage/storagebackend"
utilversion "k8s.io/apiserver/pkg/util/version"
auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics"
kapi "k8s.io/kubernetes/pkg/apis/core"
@@ -45,7 +48,13 @@ import (
)
func TestAddFlags(t *testing.T) {
componentGlobalsRegistry := utilversion.DefaultComponentGlobalsRegistry
t.Cleanup(func() {
componentGlobalsRegistry.Reset()
})
fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
utilruntime.Must(componentGlobalsRegistry.Register("test", utilversion.NewEffectiveVersion("1.32"), featuregate.NewFeatureGate()))
s := NewServerRunOptions()
for _, f := range s.Flags().FlagSets {
fs.AddFlagSet(f)
@@ -121,8 +130,10 @@ func TestAddFlags(t *testing.T) {
"--storage-backend=etcd3",
"--service-cluster-ip-range=192.168.128.0/17",
"--lease-reuse-duration-seconds=100",
"--emulated-version=test=1.31",
}
fs.Parse(args)
utilruntime.Must(componentGlobalsRegistry.Set())
// This is a snapshot of expected options parsed by args.
expected := &ServerRunOptions{
@@ -136,6 +147,8 @@ func TestAddFlags(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
ComponentGlobalsRegistry: componentGlobalsRegistry,
ComponentName: utilversion.DefaultKubeComponent,
},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &apiserveroptions.AdmissionOptions{
@@ -337,4 +350,8 @@ func TestAddFlags(t *testing.T) {
if !reflect.DeepEqual(expected, s) {
t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{})))
}
testEffectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor("test")
if testEffectiveVersion.EmulationVersion().String() != "1.31" {
t.Errorf("Got emulation version %s, wanted %s", testEffectiveVersion.EmulationVersion().String(), "1.31")
}
}

View File

@@ -26,6 +26,7 @@ import (
utilfeature "k8s.io/apiserver/pkg/util/feature"
netutils "k8s.io/utils/net"
"k8s.io/apimachinery/pkg/util/version"
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
"k8s.io/kubernetes/pkg/controlplane/reconcilers"
"k8s.io/kubernetes/pkg/features"
@@ -139,5 +140,14 @@ func (s CompletedOptions) Validate() []error {
errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount))
}
// TODO: remove in 1.32
// emulationVersion is introduced in 1.31, so it is only allowed to be equal to the binary version at 1.31.
effectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor(s.GenericServerRunOptions.ComponentName)
binaryVersion := version.MajorMinor(effectiveVersion.BinaryVersion().Major(), effectiveVersion.BinaryVersion().Minor())
if binaryVersion.EqualTo(version.MajorMinor(1, 31)) && !effectiveVersion.EmulationVersion().EqualTo(binaryVersion) {
errs = append(errs, fmt.Errorf("emulation version needs to be equal to binary version(%s) in compatibility-version alpha, got %s",
binaryVersion.String(), effectiveVersion.EmulationVersion().String()))
}
return errs
}

View File

@@ -36,6 +36,7 @@ import (
serverstorage "k8s.io/apiserver/pkg/server/storage"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/apiserver/pkg/util/notfoundhandler"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/apiserver/pkg/util/webhook"
clientgoinformers "k8s.io/client-go/informers"
"k8s.io/client-go/rest"
@@ -63,7 +64,10 @@ func init() {
// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
_, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
utilversion.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
s := options.NewServerRunOptions()
cmd := &cobra.Command{
Use: "kube-apiserver",
Long: `The Kubernetes API server validates and configures data
@@ -74,6 +78,9 @@ cluster's shared state through which all other components interact.`,
// stop printing usage when the command errors
SilenceUsage: true,
PersistentPreRunE: func(*cobra.Command, []string) error {
if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil {
return err
}
// silence client-go warnings.
// kube-apiserver loopback clients should not log self-issued warnings.
rest.SetDefaultWarningHandler(rest.NoWarnings{})
@@ -82,10 +89,9 @@ cluster's shared state through which all other components interact.`,
RunE: func(cmd *cobra.Command, args []string) error {
verflag.PrintAndExitIfRequested()
fs := cmd.Flags()
// Activate logging as soon as possible, after that
// show flags with the final logging configuration.
if err := logsapi.ValidateAndApply(s.Logs, utilfeature.DefaultFeatureGate); err != nil {
if err := logsapi.ValidateAndApply(s.Logs, featureGate); err != nil {
return err
}
cliflag.PrintFlags(fs)
@@ -101,7 +107,7 @@ cluster's shared state through which all other components interact.`,
return utilerrors.NewAggregate(errs)
}
// add feature enablement metrics
utilfeature.DefaultMutableFeatureGate.AddMetrics()
featureGate.AddMetrics()
return Run(cmd.Context(), completedOptions)
},
Args: func(cmd *cobra.Command, args []string) error {

View File

@@ -43,16 +43,19 @@ import (
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
serveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/apiserver/pkg/storageversion"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
clientgotransport "k8s.io/client-go/transport"
"k8s.io/client-go/util/cert"
"k8s.io/client-go/util/keyutil"
featuregatetesting "k8s.io/component-base/featuregate/testing"
logsapi "k8s.io/component-base/logs/api/v1"
"k8s.io/klog/v2"
"k8s.io/kube-aggregator/pkg/apiserver"
@@ -98,6 +101,9 @@ type TestServerInstanceOptions struct {
// We specify this as on option to pass a common proxyCA to multiple apiservers to simulate
// an apiserver version skew scenario where all apiservers use the same proxyCA to verify client connections.
ProxyCA *ProxyCA
// Set the BinaryVersion of server effective version.
// Default to 1.31
BinaryVersion string
}
// TestServer return values supplied by kube-test-ApiServer
@@ -177,7 +183,18 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
fs := pflag.NewFlagSet("test", pflag.PanicOnError)
featureGate := utilfeature.DefaultMutableFeatureGate
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
if instanceOptions.BinaryVersion != "" {
effectiveVersion = utilversion.NewEffectiveVersion(instanceOptions.BinaryVersion)
}
// need to call SetFeatureGateEmulationVersionDuringTest to reset the feature gate emulation version at the end of the test.
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, featureGate, effectiveVersion.EmulationVersion())
utilversion.DefaultComponentGlobalsRegistry.Reset()
utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate))
s := options.NewServerRunOptions()
for _, f := range s.Flags().FlagSets {
fs.AddFlagSet(f)
}
@@ -321,6 +338,10 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
return result, err
}
if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil {
return result, err
}
saSigningKeyFile, err := os.CreateTemp("/tmp", "insecure_test_key")
if err != nil {
t.Fatalf("create temp file failed: %v", err)

View File

@@ -17,7 +17,6 @@ limitations under the License.
package apiserver
import (
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
apiextensionsoptions "k8s.io/apiextensions-apiserver/pkg/cmd/server/options"
@@ -27,6 +26,7 @@ import (
"k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/util/webhook"
"k8s.io/client-go/informers"
v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
)

View File

@@ -88,7 +88,7 @@ func (s *Server) InstallAPIs(restStorageProviders ...RESTStorageProvider) error
nonLegacy := []*genericapiserver.APIGroupInfo{}
// used later in the loop to filter the served resource by those that have expired.
resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(*s.GenericAPIServer.Version)
resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(s.GenericAPIServer.EffectiveVersion.EmulationVersion())
if err != nil {
return err
}

View File

@@ -46,7 +46,6 @@ import (
clientgoinformers "k8s.io/client-go/informers"
clientgoclientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/util/keyutil"
"k8s.io/component-base/version"
aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
openapicommon "k8s.io/kube-openapi/pkg/common"
@@ -172,9 +171,6 @@ func BuildGenericConfig(
sets.NewString("attach", "exec", "proxy", "log", "portforward"),
)
kubeVersion := version.Get()
genericConfig.Version = &kubeVersion
if genericConfig.EgressSelector != nil {
s.Etcd.StorageConfig.Transport.EgressLookup = genericConfig.EgressSelector.Lookup
}
@@ -185,7 +181,9 @@ func BuildGenericConfig(
}
storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
storageFactoryConfig.CurrentVersion = genericConfig.EffectiveVersion
storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig
storageFactoryConfig.DefaultResourceEncoding.SetEffectiveVersion(genericConfig.EffectiveVersion)
storageFactory, lastErr = storageFactoryConfig.Complete(s.Etcd).New()
if lastErr != nil {
return

View File

@@ -66,7 +66,7 @@ func TestBuildGenericConfig(t *testing.T) {
t.Errorf("There are different StorageObjectCountTracker in genericConfig and storageFactory")
}
restOptions, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""})
restOptions, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}, nil)
if err != nil {
t.Fatal(err)
}

View File

@@ -202,6 +202,10 @@ func (o *Options) Complete(alternateDNS []string, alternateIPs []net.IP) (Comple
Options: *o,
}
if err := completed.GenericServerRunOptions.Complete(); err != nil {
return CompletedOptions{}, err
}
// set defaults
if err := completed.GenericServerRunOptions.DefaultAdvertiseAddress(completed.SecureServing.SecureServingOptions); err != nil {
return CompletedOptions{}, err

View File

@@ -26,13 +26,16 @@ import (
"github.com/spf13/pflag"
noopoteltrace "go.opentelemetry.io/otel/trace/noop"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/admission"
apiserveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/etcd3"
"k8s.io/apiserver/pkg/storage/storagebackend"
utilversion "k8s.io/apiserver/pkg/util/version"
auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics"
netutils "k8s.io/utils/net"
@@ -41,7 +44,12 @@ import (
)
func TestAddFlags(t *testing.T) {
componentGlobalsRegistry := utilversion.DefaultComponentGlobalsRegistry
t.Cleanup(func() {
componentGlobalsRegistry.Reset()
})
fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
utilruntime.Must(componentGlobalsRegistry.Register("test", utilversion.NewEffectiveVersion("1.32"), featuregate.NewFeatureGate()))
s := NewOptions()
var fss cliflag.NamedFlagSets
s.AddFlags(&fss)
@@ -108,8 +116,10 @@ func TestAddFlags(t *testing.T) {
"--request-timeout=2m",
"--storage-backend=etcd3",
"--lease-reuse-duration-seconds=100",
"--emulated-version=test=1.31",
}
fs.Parse(args)
utilruntime.Must(componentGlobalsRegistry.Set())
// This is a snapshot of expected options parsed by args.
expected := &Options{
@@ -122,6 +132,8 @@ func TestAddFlags(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
ComponentGlobalsRegistry: componentGlobalsRegistry,
ComponentName: utilversion.DefaultKubeComponent,
},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &apiserveroptions.AdmissionOptions{
@@ -292,4 +304,9 @@ func TestAddFlags(t *testing.T) {
if !reflect.DeepEqual(expected, s) {
t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{})))
}
testEffectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor("test")
if testEffectiveVersion.EmulationVersion().String() != "1.31" {
t.Errorf("Got emulation version %s, wanted %s", testEffectiveVersion.EmulationVersion().String(), "1.31")
}
}

View File

@@ -100,6 +100,7 @@ func validateUnknownVersionInteroperabilityProxyFlags(options *Options) []error
func (s *Options) Validate() []error {
var errs []error
errs = append(errs, s.GenericServerRunOptions.Validate()...)
errs = append(errs, s.Etcd.Validate()...)
errs = append(errs, validateAPIPriorityAndFairness(s)...)
errs = append(errs, s.SecureServing.Validate()...)

View File

@@ -23,6 +23,7 @@ import (
kubeapiserveradmission "k8s.io/apiserver/pkg/admission"
genericoptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/component-base/featuregate"
basemetrics "k8s.io/component-base/metrics"
"k8s.io/kubernetes/pkg/features"
@@ -200,7 +201,7 @@ func TestValidateOptions(t *testing.T) {
name: "validate master count equal 0",
expectErrors: true,
options: &Options{
GenericServerRunOptions: &genericoptions.ServerRunOptions{},
GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry},
Etcd: &genericoptions.EtcdOptions{},
SecureServing: &genericoptions.SecureServingOptionsWithLoopback{},
Audit: &genericoptions.AuditOptions{},
@@ -227,7 +228,7 @@ func TestValidateOptions(t *testing.T) {
name: "validate token request enable not attempted",
expectErrors: true,
options: &Options{
GenericServerRunOptions: &genericoptions.ServerRunOptions{},
GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry},
Etcd: &genericoptions.EtcdOptions{},
SecureServing: &genericoptions.SecureServingOptionsWithLoopback{},
Audit: &genericoptions.AuditOptions{},

View File

@@ -82,7 +82,7 @@ func BuildPeerProxy(versionedInformer clientgoinformers.SharedInformerFactory, s
// The peer endpoint leases are used to find network locations of apiservers for peer proxy
func CreatePeerEndpointLeaseReconciler(c genericapiserver.Config, storageFactory serverstorage.StorageFactory) (reconcilers.PeerEndpointLeaseReconciler, error) {
ttl := DefaultPeerEndpointReconcilerTTL
config, err := storageFactory.NewConfig(api.Resource("apiServerPeerIPInfo"))
config, err := storageFactory.NewConfig(api.Resource("apiServerPeerIPInfo"), &api.Endpoints{})
if err != nil {
return nil, fmt.Errorf("error creating storage factory config: %w", err)
}

View File

@@ -214,7 +214,7 @@ func (c *Config) createLeaseReconciler() reconcilers.EndpointReconciler {
endpointsAdapter := reconcilers.NewEndpointsAdapter(endpointClient, endpointSliceClient)
ttl := c.Extra.MasterEndpointReconcileTTL
config, err := c.ControlPlane.StorageFactory.NewConfig(api.Resource("apiServerIPInfo"))
config, err := c.ControlPlane.StorageFactory.NewConfig(api.Resource("apiServerIPInfo"), &api.Endpoints{})
if err != nil {
klog.Fatalf("Error creating storage factory config: %v", err)
}

View File

@@ -20,6 +20,7 @@ import (
"context"
"crypto/tls"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
@@ -55,6 +56,7 @@ import (
serverstorage "k8s.io/apiserver/pkg/server/storage"
etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
"k8s.io/apiserver/pkg/util/openapi"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/discovery"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
@@ -103,7 +105,9 @@ func setUp(t *testing.T) (*etcd3testing.EtcdTestServer, Config, *assert.Assertio
},
}
config.ControlPlane.Generic.EffectiveVersion = utilversion.DefaultKubeEffectiveVersion()
storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
storageFactoryConfig.DefaultResourceEncoding.SetEffectiveVersion(config.ControlPlane.Generic.EffectiveVersion)
storageConfig.StorageObjectCountTracker = config.ControlPlane.Generic.StorageObjectCountTracker
resourceEncoding := resourceconfig.MergeResourceEncodingConfigs(storageFactoryConfig.DefaultResourceEncoding, storageFactoryConfig.ResourceEncodingOverrides)
storageFactory := serverstorage.NewDefaultStorageFactory(*storageConfig, "application/vnd.kubernetes.protobuf", storageFactoryConfig.Serializer, resourceEncoding, DefaultAPIResourceConfigSource(), nil)
@@ -115,9 +119,7 @@ func setUp(t *testing.T) (*etcd3testing.EtcdTestServer, Config, *assert.Assertio
t.Fatal(err)
}
kubeVersion := kubeversion.Get()
config.ControlPlane.Generic.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
config.ControlPlane.Generic.Version = &kubeVersion
config.ControlPlane.StorageFactory = storageFactory
config.ControlPlane.Generic.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}}
config.ControlPlane.Generic.PublicAddress = netutils.ParseIPSloppy("192.168.10.4")
@@ -240,9 +242,13 @@ func TestVersion(t *testing.T) {
if err != nil {
t.Errorf("unexpected error: %v", err)
}
expectedInfo := kubeversion.Get()
kubeVersion := utilversion.DefaultKubeEffectiveVersion().BinaryVersion()
expectedInfo.Major = fmt.Sprintf("%d", kubeVersion.Major())
expectedInfo.Minor = fmt.Sprintf("%d", kubeVersion.Minor())
if !reflect.DeepEqual(kubeversion.Get(), info) {
t.Errorf("Expected %#v, Got %#v", kubeversion.Get(), info)
if !reflect.DeepEqual(expectedInfo, info) {
t.Errorf("Expected %#v, Got %#v", expectedInfo, info)
}
}

View File

@@ -19,6 +19,8 @@ package features
import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
clientfeatures "k8s.io/client-go/features"
"k8s.io/component-base/featuregate"
)
@@ -78,8 +80,7 @@ func TestClientAdapterAdd(t *testing.T) {
t.Errorf("expected feature %q not found", name)
continue
}
if actual != expected {
if diff := cmp.Diff(actual, expected, cmpopts.IgnoreFields(featuregate.FeatureSpec{}, "Version")); diff != "" {
t.Errorf("expected feature %q spec %#v, got spec %#v", name, expected, actual)
}
}

View File

@@ -946,6 +946,7 @@ const (
func init() {
runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates))
runtime.Must(utilfeature.DefaultMutableFeatureGate.AddVersioned(defaultVersionedKubernetesFeatureGates))
// Register all client-go features with kube's feature gate instance and make all client-go
// feature checks use kube's instance. The effect is that for kube binaries, client-go

View File

@@ -62,6 +62,9 @@ func TestAllRegisteredFeaturesExpected(t *testing.T) {
if err := knownFeatureGates.Add(defaultKubernetesFeatureGates); err != nil {
t.Fatal(err)
}
if err := knownFeatureGates.AddVersioned(defaultVersionedKubernetesFeatureGates); err != nil {
t.Fatal(err)
}
knownFeatures := knownFeatureGates.GetAll()
for registeredFeature := range registeredFeatures {

View File

@@ -0,0 +1,34 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package features
import (
"k8s.io/component-base/featuregate"
)
// defaultVersionedKubernetesFeatureGates consists of all known Kubernetes-specific feature keys with VersionedSpecs.
// To add a new feature, define a key for it and add it here. The features will be
// available throughout Kubernetes binaries.
//
// Entries are separated from each other with blank lines to avoid sweeping gofmt changes
// when adding or removing one entry.
var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{
// Example:
// genericfeatures.EmulationVersion: {
// {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha},
// },
}

View File

@@ -25,6 +25,7 @@ import (
"k8s.io/apiserver/pkg/server/resourceconfig"
serverstorage "k8s.io/apiserver/pkg/server/storage"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/apiserver/pkg/util/version"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/apis/apps"
"k8s.io/kubernetes/pkg/apis/certificates"
@@ -93,6 +94,7 @@ type StorageFactoryConfig struct {
Serializer runtime.StorageSerializer
ResourceEncodingOverrides []schema.GroupVersionResource
EtcdServersOverrides []string
CurrentVersion version.EffectiveVersion
}
// Complete completes the StorageFactoryConfig with provided etcdOptions returning completedStorageFactoryConfig.

View File

@@ -321,7 +321,7 @@ func (p *legacyProvider) NewRESTStorage(apiResourceConfigSource serverstorage.AP
func (c *Config) newServiceIPAllocators() (registries rangeRegistries, primaryClusterIPAllocator ipallocator.Interface, clusterIPAllocators map[api.IPFamily]ipallocator.Interface, nodePortAllocator *portallocator.PortAllocator, err error) {
clusterIPAllocators = map[api.IPFamily]ipallocator.Interface{}
serviceStorageConfig, err := c.StorageFactory.NewConfig(api.Resource("services"))
serviceStorageConfig, err := c.StorageFactory.NewConfig(api.Resource("services"), &api.Service{})
if err != nil {
return rangeRegistries{}, nil, nil, nil, err
}

View File

@@ -19,6 +19,7 @@ package rest
import (
"testing"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/server/storage"
"k8s.io/apiserver/pkg/storage/storagebackend"
@@ -40,7 +41,7 @@ func TestGetServersToValidate(t *testing.T) {
type fakeStorageFactory struct{}
func (f fakeStorageFactory) NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
func (f fakeStorageFactory) NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
return nil, nil
}

View File

@@ -44,7 +44,7 @@ func NewEtcdStorageForResource(t *testing.T, resource schema.GroupResource) (*st
if err != nil {
t.Fatalf("Error while making storage factory: %v", err)
}
resourceConfig, err := factory.NewConfig(resource)
resourceConfig, err := factory.NewConfig(resource, nil)
if err != nil {
t.Fatalf("Error while finding storage destination: %v", err)
}

View File

@@ -40,7 +40,6 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/version"
"k8s.io/apiserver/pkg/endpoints/discovery"
"k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
genericregistry "k8s.io/apiserver/pkg/registry/generic"
@@ -118,12 +117,6 @@ func (cfg *Config) Complete() CompletedConfig {
}
c.GenericConfig.EnableDiscovery = false
if c.GenericConfig.Version == nil {
c.GenericConfig.Version = &version.Info{
Major: "0",
Minor: "1",
}
}
return CompletedConfig{&c}
}

View File

@@ -1178,8 +1178,9 @@ type crdConversionRESTOptionsGetter struct {
preserveUnknownFields bool
}
func (t crdConversionRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
ret, err := t.RESTOptionsGetter.GetRESTOptions(resource)
func (t crdConversionRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) {
// Explicitly ignore example, we override storageconfig below
ret, err := t.RESTOptionsGetter.GetRESTOptions(resource, nil)
if err == nil {
d := schemaCoercingDecoder{delegate: ret.StorageConfig.Codec, validator: unstructuredSchemaCoercer{
// drop invalid fields while decoding old CRs (before we haven't had any ObjectMeta validation)

View File

@@ -92,7 +92,7 @@ func (o CustomResourceDefinitionsServerOptions) Validate() error {
// Complete fills in missing options.
func (o *CustomResourceDefinitionsServerOptions) Complete() error {
return nil
return o.ServerRunOptions.Complete()
}
// Config returns an apiextensions-apiserver configuration.

View File

@@ -24,6 +24,7 @@ import (
"k8s.io/apiextensions-apiserver/pkg/cmd/server/options"
genericapiserver "k8s.io/apiserver/pkg/server"
utilversion "k8s.io/apiserver/pkg/util/version"
)
func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command {
@@ -32,6 +33,9 @@ func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command
cmd := &cobra.Command{
Short: "Launch an API extensions API server",
Long: "Launch an API extensions API server",
PersistentPreRunE: func(*cobra.Command, []string) error {
return utilversion.DefaultComponentGlobalsRegistry.Set()
},
RunE: func(c *cobra.Command, args []string) error {
if err := o.Complete(); err != nil {
return err

View File

@@ -31,11 +31,14 @@ import (
extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
"k8s.io/apiextensions-apiserver/pkg/cmd/server/options"
generatedopenapi "k8s.io/apiextensions-apiserver/pkg/generated/openapi"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/storage/storagebackend"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/apiserver/pkg/util/openapi"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
logsapi "k8s.io/component-base/logs/api/v1"
@@ -120,6 +123,10 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin
fs := pflag.NewFlagSet("test", pflag.PanicOnError)
featureGate := utilfeature.DefaultMutableFeatureGate
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
utilversion.DefaultComponentGlobalsRegistry.Reset()
utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate))
s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr)
s.AddFlags(fs)
@@ -143,6 +150,10 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin
fs.Parse(customFlags)
if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil {
return result, err
}
if err := s.Complete(); err != nil {
return result, fmt.Errorf("failed to set default options: %v", err)
}

View File

@@ -183,7 +183,7 @@ func testWebhookConverter(t *testing.T, watchCache bool) {
crd := multiVersionFixture.DeepCopy()
RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd, nil, nil)
restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural})
restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}, nil)
if err != nil {
t.Fatal(err)
}

View File

@@ -659,7 +659,7 @@ func TestCustomResourceDefaultingOfMetaFields(t *testing.T) {
// get persisted object
RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd, nil, nil)
restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural})
restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}, nil)
if err != nil {
t.Fatal(err)
}

View File

@@ -156,7 +156,7 @@ func StartDefaultServerWithClientsAndEtcd(t servertesting.Logger, extraFlags ...
}
RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd, resourceTransformers, nil)
restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "hopefully-ignored-group", Resource: "hopefully-ignored-resources"})
restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "hopefully-ignored-group", Resource: "hopefully-ignored-resources"}, nil)
if err != nil {
return nil, nil, nil, nil, "", err
}

View File

@@ -133,7 +133,7 @@ func TestInvalidObjectMetaInStorage(t *testing.T) {
}
RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd, nil, nil)
restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: noxuDefinition.Spec.Group, Resource: noxuDefinition.Spec.Names.Plural})
restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: noxuDefinition.Spec.Group, Resource: noxuDefinition.Spec.Names.Plural}, nil)
if err != nil {
t.Fatal(err)
}

View File

@@ -326,7 +326,7 @@ func TestPruningFromStorage(t *testing.T) {
t.Fatal(err)
}
restOptions, err := completedConfig.GenericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural})
restOptions, err := completedConfig.GenericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}, nil)
if err != nil {
t.Fatal(err)
}

View File

@@ -23,6 +23,8 @@ import (
"regexp"
"strconv"
"strings"
apimachineryversion "k8s.io/apimachinery/pkg/version"
)
// Version is an opaque representation of a version number
@@ -31,6 +33,7 @@ type Version struct {
semver bool
preRelease string
buildMetadata string
info apimachineryversion.Info
}
var (
@@ -145,6 +148,43 @@ func MustParseGeneric(str string) *Version {
return v
}
// Parse tries to do ParseSemantic first to keep more information.
// If ParseSemantic fails, it would just do ParseGeneric.
func Parse(str string) (*Version, error) {
v, err := parse(str, true)
if err != nil {
return parse(str, false)
}
return v, err
}
// MustParse is like Parse except that it panics on error
func MustParse(str string) *Version {
v, err := Parse(str)
if err != nil {
panic(err)
}
return v
}
// ParseMajorMinor parses a "generic" version string and returns a version with the major and minor version.
func ParseMajorMinor(str string) (*Version, error) {
v, err := ParseGeneric(str)
if err != nil {
return nil, err
}
return MajorMinor(v.Major(), v.Minor()), nil
}
// MustParseMajorMinor is like ParseMajorMinor except that it panics on error
func MustParseMajorMinor(str string) *Version {
v, err := ParseMajorMinor(str)
if err != nil {
panic(err)
}
return v
}
// ParseSemantic parses a version string that exactly obeys the syntax and semantics of
// the "Semantic Versioning" specification (http://semver.org/) (although it ignores
// leading and trailing whitespace, and allows the version to be preceded by "v"). For
@@ -215,6 +255,32 @@ func (v *Version) WithMinor(minor uint) *Version {
return &result
}
// SubtractMinor returns the version with offset from the original minor, with the same major and no patch.
// If -offset >= current minor, the minor would be 0.
func (v *Version) OffsetMinor(offset int) *Version {
var minor uint
if offset >= 0 {
minor = v.Minor() + uint(offset)
} else {
diff := uint(-offset)
if diff < v.Minor() {
minor = v.Minor() - diff
}
}
return MajorMinor(v.Major(), minor)
}
// SubtractMinor returns the version diff minor versions back, with the same major and no patch.
// If diff >= current minor, the minor would be 0.
func (v *Version) SubtractMinor(diff uint) *Version {
return v.OffsetMinor(-int(diff))
}
// AddMinor returns the version diff minor versions forward, with the same major and no patch.
func (v *Version) AddMinor(diff uint) *Version {
return v.OffsetMinor(int(diff))
}
// WithPatch returns copy of the version object with requested patch number
func (v *Version) WithPatch(patch uint) *Version {
result := *v
@@ -224,6 +290,9 @@ func (v *Version) WithPatch(patch uint) *Version {
// WithPreRelease returns copy of the version object with requested prerelease
func (v *Version) WithPreRelease(preRelease string) *Version {
if len(preRelease) == 0 {
return v
}
result := *v
result.components = []uint{v.Major(), v.Minor(), v.Patch()}
result.preRelease = preRelease
@@ -345,6 +414,17 @@ func onlyZeros(array []uint) bool {
return true
}
// EqualTo tests if a version is equal to a given version.
func (v *Version) EqualTo(other *Version) bool {
if v == nil {
return other == nil
}
if other == nil {
return false
}
return v.compareInternal(other) == 0
}
// AtLeast tests if a version is at least equal to a given minimum version. If both
// Versions are Semantic Versions, this will use the Semantic Version comparison
// algorithm. Otherwise, it will compare only the numeric components, with non-present
@@ -360,6 +440,11 @@ func (v *Version) LessThan(other *Version) bool {
return v.compareInternal(other) == -1
}
// GreaterThan tests if a version is greater than a given version.
func (v *Version) GreaterThan(other *Version) bool {
return v.compareInternal(other) == 1
}
// Compare compares v against a version string (which will be parsed as either Semantic
// or non-Semantic depending on v). On success it returns -1 if v is less than other, 1 if
// it is greater than other, or 0 if they are equal.
@@ -370,3 +455,30 @@ func (v *Version) Compare(other string) (int, error) {
}
return v.compareInternal(ov), nil
}
// WithInfo returns copy of the version object with requested info
func (v *Version) WithInfo(info apimachineryversion.Info) *Version {
result := *v
result.info = info
return &result
}
func (v *Version) Info() *apimachineryversion.Info {
if v == nil {
return nil
}
// in case info is empty, or the major and minor in info is different from the actual major and minor
v.info.Major = itoa(v.Major())
v.info.Minor = itoa(v.Minor())
if v.info.GitVersion == "" {
v.info.GitVersion = v.String()
}
return &v.info
}
func itoa(i uint) string {
if i == 0 {
return ""
}
return strconv.Itoa(int(i))
}

View File

@@ -452,3 +452,99 @@ func TestHighestSupportedVersion(t *testing.T) {
}
}
}
func TestOffsetMinor(t *testing.T) {
var tests = []struct {
version string
diff int
expectedComponents []uint
}{
{
version: "1.0.2",
diff: -3,
expectedComponents: []uint{1, 0},
},
{
version: "1.3.2-alpha+001",
diff: -2,
expectedComponents: []uint{1, 1},
},
{
version: "1.3.2-alpha+001",
diff: -3,
expectedComponents: []uint{1, 0},
},
{
version: "1.20",
diff: -5,
expectedComponents: []uint{1, 15},
},
{
version: "1.20",
diff: 5,
expectedComponents: []uint{1, 25},
},
}
for _, test := range tests {
version, _ := ParseGeneric(test.version)
if !reflect.DeepEqual(test.expectedComponents, version.OffsetMinor(test.diff).Components()) {
t.Error("parse returned un'expected components")
}
}
}
func TestParse(t *testing.T) {
var tests = []struct {
version string
expectErr bool
expectedComponents []uint
expectedPreRelease string
expectedBuildMetadata string
}{
{
version: "1.0.2",
expectedComponents: []uint{1, 0, 2},
},
{
version: "1.0.2-alpha+001",
expectedComponents: []uint{1, 0, 2},
expectedPreRelease: "alpha",
expectedBuildMetadata: "001",
},
{
version: "1.2",
expectedComponents: []uint{1, 2},
},
{
version: "1.0.2-beta+exp.sha.5114f85",
expectedComponents: []uint{1, 0, 2},
expectedPreRelease: "beta",
expectedBuildMetadata: "exp.sha.5114f85",
},
{
version: "a.b.c",
expectErr: true,
},
}
for _, test := range tests {
version, err := Parse(test.version)
if test.expectErr {
if err == nil {
t.Fatalf("got no err, expected err")
}
continue
}
if !reflect.DeepEqual(test.expectedComponents, version.Components()) {
t.Error("parse returned un'expected components")
}
if test.expectedPreRelease != version.PreRelease() {
t.Errorf("parse returned version.PreRelease %s, expected %s", test.expectedPreRelease, version.PreRelease())
}
if test.expectedBuildMetadata != version.BuildMetadata() {
t.Errorf("parse returned version.BuildMetadata %s, expected %s", test.expectedBuildMetadata, version.BuildMetadata())
}
}
}

View File

@@ -30,20 +30,27 @@ import (
"k8s.io/apimachinery/pkg/util/version"
celconfig "k8s.io/apiserver/pkg/apis/cel"
"k8s.io/apiserver/pkg/cel/library"
utilversion "k8s.io/apiserver/pkg/util/version"
)
// DefaultCompatibilityVersion returns a default compatibility version for use with EnvSet
// that guarantees compatibility with CEL features/libraries/parameters understood by
// an n-1 version
// the api server min compatibility version
//
// This default will be set to no more than n-1 the current Kubernetes major.minor version.
// This default will be set to no more than the current Kubernetes major.minor version.
//
// Note that a default version number less than n-1 indicates a wider range of version
// compatibility than strictly required for rollback. A wide range of compatibility is
// desirable because it means that CEL expressions are portable across a wider range
// of Kubernetes versions.
// Note that a default version number less than n-1 the current Kubernetes major.minor version
// indicates a wider range of version compatibility than strictly required for rollback.
// A wide range of compatibility is desirable because it means that CEL expressions are portable
// across a wider range of Kubernetes versions.
// A default version number equal to the current Kubernetes major.minor version
// indicates fast forward CEL features that can be used when rollback is no longer needed.
func DefaultCompatibilityVersion() *version.Version {
return version.MajorMinor(1, 30)
effectiveVer := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent)
if effectiveVer == nil {
effectiveVer = utilversion.DefaultKubeEffectiveVersion()
}
return effectiveVer.MinCompatibilityVersion()
}
var baseOpts = append(baseOptsWithoutStrictCost, StrictCostOpt)

View File

@@ -322,6 +322,17 @@ const (
func init() {
runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates))
runtime.Must(utilfeature.DefaultMutableFeatureGate.AddVersioned(defaultVersionedKubernetesFeatureGates))
}
// defaultVersionedKubernetesFeatureGates consists of all known Kubernetes-specific feature keys with VersionedSpecs.
// To add a new feature, define a key for it above and add it here. The features will be
// available throughout Kubernetes binaries.
var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{
// Example:
// EmulationVersion: {
// {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha},
// },
}
// defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys.

View File

@@ -19,6 +19,7 @@ package generic
import (
"time"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/storage"
"k8s.io/apiserver/pkg/storage/storagebackend"
@@ -39,12 +40,15 @@ type RESTOptions struct {
}
// Implement RESTOptionsGetter so that RESTOptions can directly be used when available (i.e. tests)
func (opts RESTOptions) GetRESTOptions(schema.GroupResource) (RESTOptions, error) {
func (opts RESTOptions) GetRESTOptions(schema.GroupResource, runtime.Object) (RESTOptions, error) {
return opts, nil
}
type RESTOptionsGetter interface {
GetRESTOptions(resource schema.GroupResource) (RESTOptions, error)
// GetRESTOptions returns the RESTOptions for the given resource and example object.
// The example object is used to determine the storage version for the resource.
// If the example object is nil, the storage version will be determined by the resource's default storage version.
GetRESTOptions(resource schema.GroupResource, example runtime.Object) (RESTOptions, error)
}
// StoreOptions is set of configuration options used to complete generic registries.

View File

@@ -1518,7 +1518,7 @@ func (e *Store) CompleteWithOptions(options *generic.StoreOptions) error {
return err
}
opts, err := options.RESTOptions.GetRESTOptions(e.DefaultQualifiedResource)
opts, err := options.RESTOptions.GetRESTOptions(e.DefaultQualifiedResource, e.NewFunc())
if err != nil {
return err
}

View File

@@ -42,8 +42,8 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/version"
utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup"
"k8s.io/apimachinery/pkg/version"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/authentication/authenticator"
@@ -70,8 +70,10 @@ import (
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/informers"
restclient "k8s.io/client-go/rest"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics/features"
"k8s.io/component-base/metrics/prometheus/slis"
@@ -147,8 +149,11 @@ type Config struct {
// done values in this values for this map are ignored.
PostStartHooks map[string]PostStartHookConfigEntry
// Version will enable the /version endpoint if non-nil
Version *version.Info
// EffectiveVersion determines which apis and features are available
// based on when the api/feature lifecyle.
EffectiveVersion utilversion.EffectiveVersion
// FeatureGate is a way to plumb feature gate through if you have them.
FeatureGate featuregate.FeatureGate
// AuditBackend is where audit events are sent to.
AuditBackend audit.Backend
// AuditPolicyRuleEvaluator makes the decision of whether and how to audit log a request.
@@ -585,7 +590,7 @@ func (c *Config) AddPostStartHookOrDie(name string, hook PostStartHookFunc) {
}
}
func completeOpenAPI(config *openapicommon.Config, version *version.Info) {
func completeOpenAPI(config *openapicommon.Config, version *version.Version) {
if config == nil {
return
}
@@ -624,7 +629,7 @@ func completeOpenAPI(config *openapicommon.Config, version *version.Info) {
}
}
func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Info) {
func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Version) {
if config == nil {
return
}
@@ -676,6 +681,9 @@ func (c *Config) ShutdownInitiatedNotify() <-chan struct{} {
// Complete fills in any fields not set that are required to have valid data and can be derived
// from other fields. If you're going to `ApplyOptions`, do that first. It's mutating the receiver.
func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedConfig {
if c.FeatureGate == nil {
c.FeatureGate = utilfeature.DefaultFeatureGate
}
if len(c.ExternalAddress) == 0 && c.PublicAddress != nil {
c.ExternalAddress = c.PublicAddress.String()
}
@@ -691,9 +699,8 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo
}
c.ExternalAddress = net.JoinHostPort(c.ExternalAddress, strconv.Itoa(port))
}
completeOpenAPI(c.OpenAPIConfig, c.Version)
completeOpenAPIV3(c.OpenAPIV3Config, c.Version)
completeOpenAPI(c.OpenAPIConfig, c.EffectiveVersion.EmulationVersion())
completeOpenAPIV3(c.OpenAPIV3Config, c.EffectiveVersion.EmulationVersion())
if c.DiscoveryAddresses == nil {
c.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: c.ExternalAddress}
@@ -711,7 +718,7 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo
} else {
c.EquivalentResourceRegistry = runtime.NewEquivalentResourceRegistryWithIdentity(func(groupResource schema.GroupResource) string {
// use the storage prefix as the key if possible
if opts, err := c.RESTOptionsGetter.GetRESTOptions(groupResource); err == nil {
if opts, err := c.RESTOptionsGetter.GetRESTOptions(groupResource, nil); err == nil {
return opts.ResourcePrefix
}
// otherwise return "" to use the default key (parent GV name)
@@ -819,12 +826,13 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
APIServerID: c.APIServerID,
StorageVersionManager: c.StorageVersionManager,
Version: c.Version,
EffectiveVersion: c.EffectiveVersion,
FeatureGate: c.FeatureGate,
muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{},
}
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
if c.FeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
manager := c.AggregatedDiscoveryGroupManager
if manager == nil {
manager = discoveryendpoint.NewResourceManager("apis")
@@ -1039,14 +1047,14 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
handler = genericfilters.WithRetryAfter(handler, c.lifecycleSignals.NotAcceptingNewRequest.Signaled())
}
handler = genericfilters.WithHTTPLogging(handler)
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) {
if c.FeatureGate.Enabled(genericfeatures.APIServerTracing) {
handler = genericapifilters.WithTracing(handler, c.TracerProvider)
}
handler = genericapifilters.WithLatencyTrackers(handler)
// WithRoutine will execute future handlers in a separate goroutine and serving
// handler in current goroutine to minimize the stack memory usage. It must be
// after WithPanicRecover() to be protected from panics.
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServingWithRoutine) {
if c.FeatureGate.Enabled(genericfeatures.APIServingWithRoutine) {
handler = genericfilters.WithRoutine(handler, c.LongRunningFunc)
}
handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver)
@@ -1087,10 +1095,10 @@ func installAPI(s *GenericAPIServer, c *Config) {
}
}
routes.Version{Version: c.Version}.Install(s.Handler.GoRestfulContainer)
routes.Version{Version: c.EffectiveVersion.BinaryVersion().Info()}.Install(s.Handler.GoRestfulContainer)
if c.EnableDiscovery {
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
if c.FeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(s.DiscoveryGroupManager, s.AggregatedDiscoveryGroupManager)
s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/apis", metav1.APIGroupList{}))
} else {

View File

@@ -40,6 +40,8 @@ import (
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/server/healthz"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/client-go/rest"
@@ -90,6 +92,7 @@ func TestNewWithDelegate(t *testing.T) {
delegateConfig.PublicAddress = netutils.ParseIPSloppy("192.168.10.4")
delegateConfig.LegacyAPIGroupPrefixes = sets.NewString("/api")
delegateConfig.LoopbackClientConfig = &rest.Config{}
delegateConfig.EffectiveVersion = utilversion.NewEffectiveVersion("")
clientset := fake.NewSimpleClientset()
if clientset == nil {
t.Fatal("unable to create fake client set")
@@ -122,6 +125,7 @@ func TestNewWithDelegate(t *testing.T) {
wrappingConfig.PublicAddress = netutils.ParseIPSloppy("192.168.10.4")
wrappingConfig.LegacyAPIGroupPrefixes = sets.NewString("/api")
wrappingConfig.LoopbackClientConfig = &rest.Config{}
wrappingConfig.EffectiveVersion = utilversion.NewEffectiveVersion("")
wrappingConfig.HealthzChecks = append(wrappingConfig.HealthzChecks, healthz.NamedCheck("wrapping-health", func(r *http.Request) error {
return fmt.Errorf("wrapping failed healthcheck")
@@ -305,6 +309,7 @@ func TestAuthenticationAuditAnnotationsDefaultChain(t *testing.T) {
LongRunningFunc: func(_ *http.Request, _ *request.RequestInfo) bool { return false },
lifecycleSignals: newLifecycleSignals(),
TracerProvider: tracing.NewNoopTracerProvider(),
FeatureGate: utilfeature.DefaultFeatureGate,
}
h := DefaultBuildHandlerChain(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

View File

@@ -17,6 +17,7 @@ limitations under the License.
package server
import (
"fmt"
"os"
"strconv"
"strings"
@@ -25,16 +26,15 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
apimachineryversion "k8s.io/apimachinery/pkg/version"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/klog/v2"
)
// resourceExpirationEvaluator holds info for deciding if a particular rest.Storage needs to excluded from the API
type resourceExpirationEvaluator struct {
currentMajor int
currentMinor int
isAlpha bool
currentVersion *apimachineryversion.Version
isAlpha bool
// This is usually set for testing for which tests need to be removed. This prevent insta-failing CI.
// Set KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA to see what will be removed when we tag beta
strictRemovedHandlingInAlpha bool
@@ -53,30 +53,17 @@ type ResourceExpirationEvaluator interface {
ShouldServeForVersion(majorRemoved, minorRemoved int) bool
}
func NewResourceExpirationEvaluator(currentVersion apimachineryversion.Info) (ResourceExpirationEvaluator, error) {
func NewResourceExpirationEvaluator(currentVersion *apimachineryversion.Version) (ResourceExpirationEvaluator, error) {
if currentVersion == nil {
return nil, fmt.Errorf("empty NewResourceExpirationEvaluator currentVersion")
}
klog.V(1).Infof("NewResourceExpirationEvaluator with currentVersion: %s.", currentVersion)
ret := &resourceExpirationEvaluator{
strictRemovedHandlingInAlpha: false,
}
if len(currentVersion.Major) > 0 {
currentMajor64, err := strconv.ParseInt(currentVersion.Major, 10, 32)
if err != nil {
return nil, err
}
ret.currentMajor = int(currentMajor64)
}
if len(currentVersion.Minor) > 0 {
// split the "normal" + and - for semver stuff
minorString := strings.Split(currentVersion.Minor, "+")[0]
minorString = strings.Split(minorString, "-")[0]
minorString = strings.Split(minorString, ".")[0]
currentMinor64, err := strconv.ParseInt(minorString, 10, 32)
if err != nil {
return nil, err
}
ret.currentMinor = int(currentMinor64)
}
ret.isAlpha = strings.Contains(currentVersion.GitVersion, "alpha")
// Only keeps the major and minor versions from input version.
ret.currentVersion = apimachineryversion.MajorMinor(currentVersion.Major(), currentVersion.Minor())
ret.isAlpha = strings.Contains(currentVersion.PreRelease(), "alpha")
if envString, ok := os.LookupEnv("KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA"); !ok {
// do nothing
@@ -112,6 +99,16 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio
return false
}
introduced, ok := versionedPtr.(introducedInterface)
// skip the introduced check for test when currentVersion is 0.0 to test all apis
if ok && (e.currentVersion.Major() > 0 || e.currentVersion.Minor() > 0) {
majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced()
verIntroduced := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced))
if e.currentVersion.LessThan(verIntroduced) {
return false
}
}
removed, ok := versionedPtr.(removedInterface)
if !ok {
return true
@@ -121,16 +118,11 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio
}
func (e *resourceExpirationEvaluator) ShouldServeForVersion(majorRemoved, minorRemoved int) bool {
if e.currentMajor < majorRemoved {
removedVer := apimachineryversion.MajorMinor(uint(majorRemoved), uint(minorRemoved))
if removedVer.GreaterThan(e.currentVersion) {
return true
}
if e.currentMajor > majorRemoved {
return false
}
if e.currentMinor < minorRemoved {
return true
}
if e.currentMinor > minorRemoved {
if removedVer.LessThan(e.currentVersion) {
return false
}
// at this point major and minor are equal, so this API should be removed when the current release GAs.
@@ -152,6 +144,11 @@ type removedInterface interface {
APILifecycleRemoved() (major, minor int)
}
// Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
type introducedInterface interface {
APILifecycleIntroduced() (major, minor int)
}
// removeDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted.
// versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage) {
@@ -171,6 +168,8 @@ func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versi
}
klog.V(1).Infof("Removing resource %v.%v.%v because it is time to stop serving it per APILifecycle.", resourceName, apiVersion, groupName)
storage := versionToResource[resourceName]
storage.Destroy()
delete(versionToResource, resourceName)
}
versionedResourcesStorageMap[apiVersion] = versionToResource

View File

@@ -25,57 +25,41 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/dump"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/version"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/registry/rest"
)
func Test_newResourceExpirationEvaluator(t *testing.T) {
tests := []struct {
name string
currentVersion version.Info
currentVersion string
expected resourceExpirationEvaluator
expectedErr string
}{
{
name: "beta",
currentVersion: version.Info{
Major: "1",
Minor: "20+",
GitVersion: "v1.20.0-beta.0.62+a5d22854a2ac21",
},
expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20},
name: "beta",
currentVersion: "v1.20.0-beta.0.62+a5d22854a2ac21",
expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)},
},
{
name: "alpha",
currentVersion: version.Info{
Major: "1",
Minor: "20+",
GitVersion: "v1.20.0-alpha.0.62+a5d22854a2ac21",
},
expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20, isAlpha: true},
name: "alpha",
currentVersion: "v1.20.0-alpha.0.62+a5d22854a2ac21",
expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20), isAlpha: true},
},
{
name: "maintenance",
currentVersion: version.Info{
Major: "1",
Minor: "20+",
GitVersion: "v1.20.1",
},
expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20},
name: "maintenance",
currentVersion: "v1.20.1",
expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)},
},
{
name: "bad",
currentVersion: version.Info{
Major: "1",
Minor: "20something+",
GitVersion: "v1.20.1",
},
expectedErr: `strconv.ParseInt: parsing "20something": invalid syntax`,
name: "no v prefix",
currentVersion: "1.20.1",
expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
actual, actualErr := NewResourceExpirationEvaluator(tt.currentVersion)
actual, actualErr := NewResourceExpirationEvaluator(apimachineryversion.MustParse(tt.currentVersion))
checkErr(t, actualErr, tt.expectedErr)
if actualErr != nil {
@@ -90,12 +74,12 @@ func Test_newResourceExpirationEvaluator(t *testing.T) {
}
}
func storageRemovedIn(major, minor int) removedInStorage {
return removedInStorage{major: major, minor: minor}
func storageRemovedIn(major, minor int) *removedInStorage {
return &removedInStorage{major: major, minor: minor}
}
func storageNeverRemoved() removedInStorage {
return removedInStorage{neverRemoved: true}
func storageNeverRemoved() *removedInStorage {
return &removedInStorage{neverRemoved: true}
}
type removedInStorage struct {
@@ -103,23 +87,23 @@ type removedInStorage struct {
neverRemoved bool
}
func (r removedInStorage) New() runtime.Object {
func (r *removedInStorage) New() runtime.Object {
if r.neverRemoved {
return neverRemovedObj{}
return &defaultObj{}
}
return removedInObj{major: r.major, minor: r.minor}
return &removedInObj{major: r.major, minor: r.minor}
}
func (r removedInStorage) Destroy() {
func (r *removedInStorage) Destroy() {
}
type neverRemovedObj struct {
type defaultObj struct {
}
func (r neverRemovedObj) GetObjectKind() schema.ObjectKind {
func (r *defaultObj) GetObjectKind() schema.ObjectKind {
panic("don't do this")
}
func (r neverRemovedObj) DeepCopyObject() runtime.Object {
func (r *defaultObj) DeepCopyObject() runtime.Object {
panic("don't do this either")
}
@@ -127,13 +111,45 @@ type removedInObj struct {
major, minor int
}
func (r removedInObj) GetObjectKind() schema.ObjectKind {
func (r *removedInObj) GetObjectKind() schema.ObjectKind {
panic("don't do this")
}
func (r removedInObj) DeepCopyObject() runtime.Object {
func (r *removedInObj) DeepCopyObject() runtime.Object {
panic("don't do this either")
}
func (r removedInObj) APILifecycleRemoved() (major, minor int) {
func (r *removedInObj) APILifecycleRemoved() (major, minor int) {
return r.major, r.minor
}
func storageIntroducedIn(major, minor int) *introducedInStorage {
return &introducedInStorage{major: major, minor: minor}
}
type introducedInStorage struct {
major, minor int
}
func (r *introducedInStorage) New() runtime.Object {
if r.major == 0 && r.minor == 0 {
return &defaultObj{}
}
return &IntroducedInObj{major: r.major, minor: r.minor}
}
func (r *introducedInStorage) Destroy() {
}
type IntroducedInObj struct {
major, minor int
}
func (r *IntroducedInObj) GetObjectKind() schema.ObjectKind {
panic("don't do this")
}
func (r *IntroducedInObj) DeepCopyObject() runtime.Object {
panic("don't do this either")
}
func (r *IntroducedInObj) APILifecycleIntroduced() (major, minor int) {
return r.major, r.minor
}
@@ -147,8 +163,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-curr",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageRemovedIn(1, 20),
expected: false,
@@ -156,8 +171,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-curr-but-deferred",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
serveRemovedAPIsOneMoreRelease: true,
},
restStorage: storageRemovedIn(1, 20),
@@ -166,9 +180,8 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-curr-but-alpha",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
isAlpha: true,
currentVersion: apimachineryversion.MajorMinor(1, 20),
isAlpha: true,
},
restStorage: storageRemovedIn(1, 20),
expected: true,
@@ -176,8 +189,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-curr-but-alpha-but-strict",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
isAlpha: true,
strictRemovedHandlingInAlpha: true,
},
@@ -187,8 +199,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-prev-deferral-does-not-help",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 21,
currentVersion: apimachineryversion.MajorMinor(1, 21),
serveRemovedAPIsOneMoreRelease: true,
},
restStorage: storageRemovedIn(1, 20),
@@ -197,8 +208,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-prev-major",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 2,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(2, 20),
serveRemovedAPIsOneMoreRelease: true,
},
restStorage: storageRemovedIn(1, 20),
@@ -207,8 +217,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-future",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageRemovedIn(1, 21),
expected: true,
@@ -216,12 +225,43 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "never-removed",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageNeverRemoved(),
expected: true,
},
{
name: "introduced-in-curr",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageIntroducedIn(1, 20),
expected: true,
},
{
name: "introduced-in-prev-major",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageIntroducedIn(1, 19),
expected: true,
},
{
name: "introduced-in-future",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageIntroducedIn(1, 21),
expected: false,
},
{
name: "missing-introduced",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageIntroducedIn(0, 0),
expected: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -269,8 +309,7 @@ func Test_removeDeletedKinds(t *testing.T) {
{
name: "remove-one-of-two",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
@@ -287,8 +326,7 @@ func Test_removeDeletedKinds(t *testing.T) {
{
name: "remove-nested-not-expired",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
@@ -306,8 +344,7 @@ func Test_removeDeletedKinds(t *testing.T) {
{
name: "remove-all-of-version",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {

View File

@@ -40,7 +40,6 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup"
"k8s.io/apimachinery/pkg/version"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/authorization/authorizer"
@@ -52,8 +51,9 @@ import (
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/server/routes"
"k8s.io/apiserver/pkg/storageversion"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
restclient "k8s.io/client-go/rest"
"k8s.io/component-base/featuregate"
"k8s.io/klog/v2"
openapibuilder3 "k8s.io/kube-openapi/pkg/builder3"
openapicommon "k8s.io/kube-openapi/pkg/common"
@@ -236,8 +236,11 @@ type GenericAPIServer struct {
// StorageVersionManager holds the storage versions of the API resources installed by this server.
StorageVersionManager storageversion.Manager
// Version will enable the /version endpoint if non-nil
Version *version.Info
// EffectiveVersion determines which apis and features are available
// based on when the api/feature lifecyle.
EffectiveVersion utilversion.EffectiveVersion
// FeatureGate is a way to plumb feature gate through if you have them.
FeatureGate featuregate.FeatureGate
// lifecycleSignals provides access to the various signals that happen during the life cycle of the apiserver.
lifecycleSignals lifecycleSignals
@@ -776,7 +779,7 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A
}
resourceInfos = append(resourceInfos, r...)
if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
if s.FeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
// Aggregated discovery only aggregates resources under /apis
if apiPrefix == APIGroupPrefix {
s.AggregatedDiscoveryGroupManager.AddGroupVersion(
@@ -804,8 +807,8 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A
s.RegisterDestroyFunc(apiGroupInfo.destroyStorage)
if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionAPI) &&
utilfeature.DefaultFeatureGate.Enabled(features.APIServerIdentity) {
if s.FeatureGate.Enabled(features.StorageVersionAPI) &&
s.FeatureGate.Enabled(features.APIServerIdentity) {
// API installation happens before we start listening on the handlers,
// therefore it is safe to register ResourceInfos here. The handler will block
// write requests until the storage versions of the targeting resources are updated.
@@ -835,7 +838,7 @@ func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo
// Install the version handler.
// Add a handler at /<apiPrefix> to enumerate the supported api versions.
legacyRootAPIHandler := discovery.NewLegacyRootAPIHandler(s.discoveryAddresses, s.Serializer, apiPrefix)
if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
if s.FeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(legacyRootAPIHandler, s.AggregatedLegacyDiscoveryGroupManager)
s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/api", metav1.APIVersions{}))
} else {

View File

@@ -48,6 +48,7 @@ import (
openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
"k8s.io/apiserver/pkg/registry/rest"
genericfilters "k8s.io/apiserver/pkg/server/filters"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/apiserver/pkg/warning"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake"
@@ -137,7 +138,7 @@ func setUp(t *testing.T) (Config, *assert.Assertions) {
if clientset == nil {
t.Fatal("unable to create fake client set")
}
config.EffectiveVersion = utilversion.NewEffectiveVersion("")
config.OpenAPIConfig = DefaultOpenAPIConfig(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme()))
config.OpenAPIConfig.Info.Version = "unversioned"
config.OpenAPIV3Config = DefaultOpenAPIV3Config(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme()))
@@ -459,7 +460,9 @@ func TestNotRestRoutesHaveAuth(t *testing.T) {
config.EnableProfiling = true
kubeVersion := fakeVersion()
config.Version = &kubeVersion
effectiveVersion := utilversion.NewEffectiveVersion(kubeVersion.String())
effectiveVersion.Set(effectiveVersion.BinaryVersion().WithInfo(kubeVersion), effectiveVersion.EmulationVersion(), effectiveVersion.MinCompatibilityVersion())
config.EffectiveVersion = effectiveVersion
s, err := config.Complete(nil).New("test", NewEmptyDelegate())
if err != nil {
@@ -586,7 +589,7 @@ func fakeVersion() version.Info {
return version.Info{
Major: "42",
Minor: "42",
GitVersion: "42",
GitVersion: "42.42",
GitCommit: "34973274ccef6ab4dfaaf86599792fa9c3fe4689",
GitTreeState: "Dirty",
BuildDate: time.Now().String(),

View File

@@ -32,46 +32,41 @@ func (f fakeGroupRegistry) IsGroupRegistered(group string) bool {
func TestAPIEnablementOptionsValidate(t *testing.T) {
testCases := []struct {
name string
testOptions *APIEnablementOptions
expectErr string
name string
runtimeConfig cliflag.ConfigurationMap
expectErr string
}{
{
name: "test when options is nil",
},
{
name: "test when invalid key with only api/all=false",
testOptions: &APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{"api/all": "false"},
},
expectErr: "invalid key with only api/all=false",
name: "test when invalid key with only api/all=false",
runtimeConfig: cliflag.ConfigurationMap{"api/all": "false"},
expectErr: "invalid key with only api/all=false",
},
{
name: "test when ConfigurationMap key is invalid",
testOptions: &APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{"apiall": "false"},
},
expectErr: "runtime-config invalid key",
name: "test when ConfigurationMap key is invalid",
runtimeConfig: cliflag.ConfigurationMap{"apiall": "false"},
expectErr: "runtime-config invalid key",
},
{
name: "test when unknown api groups",
testOptions: &APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{"api/v1": "true"},
},
expectErr: "unknown api groups",
name: "test when unknown api groups",
runtimeConfig: cliflag.ConfigurationMap{"api/v1": "true"},
expectErr: "unknown api groups",
},
{
name: "test when valid api groups",
testOptions: &APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{"apiregistration.k8s.io/v1beta1": "true"},
},
name: "test when valid api groups",
runtimeConfig: cliflag.ConfigurationMap{"apiregistration.k8s.io/v1beta1": "true"},
},
}
testGroupRegistry := fakeGroupRegistry{}
for _, testcase := range testCases {
t.Run(testcase.name, func(t *testing.T) {
errs := testcase.testOptions.Validate(testGroupRegistry)
testOptions := &APIEnablementOptions{
RuntimeConfig: testcase.runtimeConfig,
}
errs := testOptions.Validate(testGroupRegistry)
if len(testcase.expectErr) != 0 && !strings.Contains(utilerrors.NewAggregate(errs).Error(), testcase.expectErr) {
t.Errorf("got err: %v, expected err: %s", errs, testcase.expectErr)
}

View File

@@ -383,8 +383,8 @@ type StorageFactoryRestOptionsFactory struct {
StorageFactory serverstorage.StorageFactory
}
func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
storageConfig, err := f.StorageFactory.NewConfig(resource)
func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) {
storageConfig, err := f.StorageFactory.NewConfig(resource, example)
if err != nil {
return generic.RESTOptions{}, fmt.Errorf("unable to find storage destination for %v, due to %v", resource, err.Error())
}
@@ -469,7 +469,7 @@ type SimpleStorageFactory struct {
StorageConfig storagebackend.Config
}
func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
return s.StorageConfig.ForResource(resource), nil
}
@@ -493,8 +493,8 @@ type transformerStorageFactory struct {
resourceTransformers storagevalue.ResourceTransformers
}
func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
config, err := t.delegate.NewConfig(resource)
func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
config, err := t.delegate.NewConfig(resource, example)
if err != nil {
return nil, err
}

View File

@@ -437,7 +437,7 @@ func TestRestOptionsStorageObjectCountTracker(t *testing.T) {
if err := etcdOptions.ApplyTo(serverConfig); err != nil {
t.Fatalf("Failed to apply etcd options error: %v", err)
}
restOptions, err := serverConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""})
restOptions, err := serverConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}, nil)
if err != nil {
t.Fatal(err)
}

View File

@@ -25,8 +25,10 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/server"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"github.com/spf13/pflag"
)
@@ -89,9 +91,24 @@ type ServerRunOptions struct {
// This grace period is orthogonal to other grace periods, and
// it is not overridden by any other grace period.
ShutdownWatchTerminationGracePeriod time.Duration
// ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored.
ComponentGlobalsRegistry utilversion.ComponentGlobalsRegistry
// ComponentName is name under which the server's global variabled are registered in the ComponentGlobalsRegistry.
ComponentName string
}
func NewServerRunOptions() *ServerRunOptions {
if utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) == nil {
featureGate := utilfeature.DefaultMutableFeatureGate
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate))
}
return NewServerRunOptionsForComponent(utilversion.DefaultKubeComponent, utilversion.DefaultComponentGlobalsRegistry)
}
func NewServerRunOptionsForComponent(componentName string, componentGlobalsRegistry utilversion.ComponentGlobalsRegistry) *ServerRunOptions {
defaults := server.NewConfig(serializer.CodecFactory{})
return &ServerRunOptions{
MaxRequestsInFlight: defaults.MaxRequestsInFlight,
@@ -104,11 +121,16 @@ func NewServerRunOptions() *ServerRunOptions {
JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes,
MaxRequestBodyBytes: defaults.MaxRequestBodyBytes,
ShutdownSendRetryAfter: false,
ComponentName: componentName,
ComponentGlobalsRegistry: componentGlobalsRegistry,
}
}
// ApplyTo applies the run options to the method receiver and returns self
func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
if err := s.ComponentGlobalsRegistry.SetFallback(); err != nil {
return err
}
c.CorsAllowedOriginList = s.CorsAllowedOriginList
c.HSTSDirectives = s.HSTSDirectives
c.ExternalAddress = s.ExternalHost
@@ -124,6 +146,8 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
c.PublicAddress = s.AdvertiseAddress
c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter
c.ShutdownWatchTerminationGracePeriod = s.ShutdownWatchTerminationGracePeriod
c.EffectiveVersion = s.ComponentGlobalsRegistry.EffectiveVersionFor(s.ComponentName)
c.FeatureGate = s.ComponentGlobalsRegistry.FeatureGateFor(s.ComponentName)
return nil
}
@@ -196,6 +220,9 @@ func (s *ServerRunOptions) Validate() []error {
if err := validateCorsAllowedOriginList(s.CorsAllowedOriginList); err != nil {
errors = append(errors, err)
}
if errs := s.ComponentGlobalsRegistry.Validate(); len(errs) != 0 {
errors = append(errors, errs...)
}
return errors
}
@@ -337,5 +364,10 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
"This option, if set, represents the maximum amount of grace period the apiserver will wait "+
"for active watch request(s) to drain during the graceful server shutdown window.")
utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
s.ComponentGlobalsRegistry.AddFlags(fs)
}
// Complete fills missing fields with defaults.
func (s *ServerRunOptions) Complete() error {
return s.ComponentGlobalsRegistry.SetFallback()
}

View File

@@ -23,10 +23,21 @@ import (
"time"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/version"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
netutils "k8s.io/utils/net"
)
func TestServerRunOptionsValidate(t *testing.T) {
testRegistry := utilversion.NewComponentGlobalsRegistry()
featureGate := utilfeature.DefaultFeatureGate.DeepCopy()
effectiveVersion := utilversion.NewEffectiveVersion("1.30")
effectiveVersion.SetEmulationVersion(version.MajorMinor(1, 32))
testComponent := "test"
utilruntime.Must(testRegistry.Register(testComponent, effectiveVersion, featureGate))
testCases := []struct {
name string
testOptions *ServerRunOptions
@@ -43,6 +54,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--max-requests-inflight can not be negative value",
},
@@ -57,6 +69,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--max-mutating-requests-inflight can not be negative value",
},
@@ -71,6 +84,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--request-timeout can not be negative value",
},
@@ -85,6 +99,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: -1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--min-request-timeout can not be negative value",
},
@@ -99,6 +114,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: -10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value",
},
@@ -113,6 +129,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: -10 * 1024 * 1024,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "ServerRunOptions.MaxRequestBodyBytes can not be negative value",
},
@@ -128,6 +145,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
LivezGracePeriod: -time.Second,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--livez-grace-period can not be a negative value",
},
@@ -143,6 +161,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ShutdownDelayDuration: -time.Second,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--shutdown-delay-duration can not be negative value",
},
@@ -158,9 +177,27 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
expectErr: "--strict-transport-security-directives invalid, allowed values: max-age=expireTime, includeSubDomains, preload. see https://tools.ietf.org/html/rfc6797#section-6.1 for more information",
},
{
name: "Test when emulation version is invalid",
testOptions: &ServerRunOptions{
AdvertiseAddress: netutils.ParseIPSloppy("192.168.10.10"),
CorsAllowedOriginList: []string{"^10.10.10.100$", "^10.10.10.200$"},
HSTSDirectives: []string{"max-age=31536000", "includeSubDomains", "preload"},
MaxRequestsInFlight: 400,
MaxMutatingRequestsInFlight: 200,
RequestTimeout: time.Duration(2) * time.Minute,
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentName: testComponent,
ComponentGlobalsRegistry: testRegistry,
},
expectErr: "emulation version 1.32 is not between [1.29, 1.30.0]",
},
{
name: "Test when ServerRunOptions is valid",
testOptions: &ServerRunOptions{
@@ -173,6 +210,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
},
},
}

View File

@@ -43,6 +43,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/version"
"k8s.io/apiserver/pkg/server"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/discovery"
restclient "k8s.io/client-go/rest"
cliflag "k8s.io/component-base/cli/flag"
@@ -276,9 +277,8 @@ func TestServerRunWithSNI(t *testing.T) {
// launch server
config := setUp(t)
v := fakeVersion()
config.Version = &v
config.EffectiveVersion = utilversion.NewEffectiveVersion(v.String())
config.EnableIndex = true
secureOptions := (&SecureServingOptions{
@@ -463,11 +463,9 @@ func certSignature(cert tls.Certificate) (string, error) {
func fakeVersion() version.Info {
return version.Info{
Major: "42",
Minor: "42",
GitVersion: "42",
GitCommit: "34973274ccef6ab4dfaaf86599792fa9c3fe4689",
GitTreeState: "Dirty",
Major: "42",
Minor: "42",
GitVersion: "42.42",
}
}

View File

@@ -21,6 +21,8 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/util/version"
)
type ResourceEncodingConfig interface {
@@ -33,10 +35,15 @@ type ResourceEncodingConfig interface {
InMemoryEncodingFor(schema.GroupResource) (schema.GroupVersion, error)
}
type CompatibilityResourceEncodingConfig interface {
BackwardCompatibileStorageEncodingFor(schema.GroupResource, runtime.Object) (schema.GroupVersion, error)
}
type DefaultResourceEncodingConfig struct {
// resources records the overriding encoding configs for individual resources.
resources map[schema.GroupResource]*OverridingResourceEncoding
scheme *runtime.Scheme
resources map[schema.GroupResource]*OverridingResourceEncoding
scheme *runtime.Scheme
effectiveVersion version.EffectiveVersion
}
type OverridingResourceEncoding struct {
@@ -47,7 +54,7 @@ type OverridingResourceEncoding struct {
var _ ResourceEncodingConfig = &DefaultResourceEncodingConfig{}
func NewDefaultResourceEncodingConfig(scheme *runtime.Scheme) *DefaultResourceEncodingConfig {
return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme}
return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme, effectiveVersion: version.DefaultKubeEffectiveVersion()}
}
func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored schema.GroupResource, externalEncodingVersion, internalVersion schema.GroupVersion) {
@@ -57,6 +64,10 @@ func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored
}
}
func (o *DefaultResourceEncodingConfig) SetEffectiveVersion(effectiveVersion version.EffectiveVersion) {
o.effectiveVersion = effectiveVersion
}
func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) {
if !o.scheme.IsGroupRegistered(resource.Group) {
return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
@@ -71,6 +82,24 @@ func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.Group
return o.scheme.PrioritizedVersionsForGroup(resource.Group)[0], nil
}
func (o *DefaultResourceEncodingConfig) BackwardCompatibileStorageEncodingFor(resource schema.GroupResource, example runtime.Object) (schema.GroupVersion, error) {
if !o.scheme.IsGroupRegistered(resource.Group) {
return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
}
// Always respect overrides
resourceOverride, resourceExists := o.resources[resource]
if resourceExists {
return resourceOverride.ExternalResourceEncoding, nil
}
return emulatedStorageVersion(
o.scheme.PrioritizedVersionsForGroup(resource.Group)[0],
example,
o.effectiveVersion,
o.scheme)
}
func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) {
if !o.scheme.IsGroupRegistered(resource.Group) {
return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
@@ -82,3 +111,79 @@ func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.Grou
}
return schema.GroupVersion{Group: resource.Group, Version: runtime.APIVersionInternal}, nil
}
// Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
type introducedInterface interface {
APILifecycleIntroduced() (major, minor int)
}
func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example runtime.Object, effectiveVersion version.EffectiveVersion, scheme *runtime.Scheme) (schema.GroupVersion, error) {
if example == nil || effectiveVersion == nil {
return binaryVersionOfResource, nil
}
// Look up example in scheme to find all objects of the same Group-Kind
// Use the highest priority version for that group-kind whose lifecycle window
// includes the current emulation version.
// If no version is found, use the binary version
// (in this case the API should be disabled anyway)
gvks, _, err := scheme.ObjectKinds(example)
if err != nil {
return schema.GroupVersion{}, err
} else if len(gvks) == 0 {
// Probably shouldn't happen if err is non-nil
return schema.GroupVersion{}, fmt.Errorf("object %T has no GVKs registered in scheme", example)
}
// VersionsForGroupKind returns versions in priority order
versions := scheme.VersionsForGroupKind(schema.GroupKind{Group: gvks[0].Group, Kind: gvks[0].Kind})
compatibilityVersion := effectiveVersion.MinCompatibilityVersion()
for _, gv := range versions {
if gv.Version == runtime.APIVersionInternal {
continue
}
gvk := schema.GroupVersionKind{
Group: gv.Group,
Version: gv.Version,
Kind: gvks[0].Kind,
}
exampleOfGVK, err := scheme.New(gvk)
if err != nil {
return schema.GroupVersion{}, err
}
// If it was introduced after current compatibility version, don't use it
// skip the introduced check for test when currentVersion is 0.0 to test all apis
if introduced, hasIntroduced := exampleOfGVK.(introducedInterface); hasIntroduced && (compatibilityVersion.Major() > 0 || compatibilityVersion.Minor() > 0) {
// API resource lifecycles should be relative to k8s api version
majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced()
introducedVer := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced))
if introducedVer.GreaterThan(compatibilityVersion) {
continue
}
}
// versions is returned in priority order, so just use first result
return gvk.GroupVersion(), nil
}
// Getting here means we're serving a version that is unknown to the
// min-compatibility-version server.
//
// This is only expected to happen when serving an alpha API type due
// to missing pre-release lifecycle information
// (which doesn't happen by default), or when emulation-version and
// min-compatibility-version are several versions apart so a beta or GA API
// was being served which didn't exist at all in min-compatibility-version.
//
// In the alpha case - we do not support compatibility versioning of
// alpha types and recommend users do not mix the two.
// In the skip-level case - The version of apiserver we are retaining
// compatibility with has no knowledge of the type,
// so storing it in another type is no issue.
return binaryVersionOfResource, nil
}

View File

@@ -42,7 +42,7 @@ type Backend struct {
type StorageFactory interface {
// New finds the storage destination for the given group and resource. It will
// return an error if the group has no storage destination configured.
NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error)
NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error)
// ResourcePrefix returns the overridden resource prefix for the GroupResource
// This allows for cohabitation of resources with different native types and provides
@@ -226,7 +226,7 @@ func (s *DefaultStorageFactory) getStorageGroupResource(groupResource schema.Gro
// New finds the storage destination for the given group and resource. It will
// return an error if the group has no storage destination configured.
func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
chosenStorageResource := s.getStorageGroupResource(groupResource)
// operate on copy
@@ -244,14 +244,23 @@ func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*
}
var err error
codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource)
if err != nil {
return nil, err
if backwardCompatibleInterface, ok := s.ResourceEncodingConfig.(CompatibilityResourceEncodingConfig); ok {
codecConfig.StorageVersion, err = backwardCompatibleInterface.BackwardCompatibileStorageEncodingFor(groupResource, example)
if err != nil {
return nil, err
}
} else {
codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource)
if err != nil {
return nil, err
}
}
codecConfig.MemoryVersion, err = s.ResourceEncodingConfig.InMemoryEncodingFor(groupResource)
if err != nil {
return nil, err
}
codecConfig.Config = storageConfig
storageConfig.Codec, storageConfig.EncodeVersioner, err = s.newStorageCodecFn(codecConfig)

View File

@@ -26,10 +26,12 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/apis/example"
exampleinstall "k8s.io/apiserver/pkg/apis/example/install"
examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/apiserver/pkg/util/version"
)
var (
@@ -118,7 +120,7 @@ func TestConfigurableStorageFactory(t *testing.T) {
f.SetEtcdLocation(example.Resource("*"), []string{"/server2"})
f.SetEtcdPrefix(example.Resource("test"), "/prefix_for_test")
config, err := f.NewConfig(example.Resource("test"))
config, err := f.NewConfig(example.Resource("test"), nil)
if err != nil {
t.Fatal(err)
}
@@ -163,7 +165,7 @@ func TestUpdateEtcdOverrides(t *testing.T) {
storageFactory.SetEtcdLocation(test.resource, test.servers)
var err error
config, err := storageFactory.NewConfig(test.resource)
config, err := storageFactory.NewConfig(test.resource, nil)
if err != nil {
t.Errorf("%d: unexpected error %v", i, err)
continue
@@ -173,7 +175,7 @@ func TestUpdateEtcdOverrides(t *testing.T) {
continue
}
config, err = storageFactory.NewConfig(schema.GroupResource{Group: examplev1.GroupName, Resource: "unlikely"})
config, err = storageFactory.NewConfig(schema.GroupResource{Group: examplev1.GroupName, Resource: "unlikely"}, nil)
if err != nil {
t.Errorf("%d: unexpected error %v", i, err)
continue
@@ -244,3 +246,241 @@ func TestConfigs(t *testing.T) {
}
}
}
var introducedLifecycles = map[reflect.Type]*apimachineryversion.Version{}
var removedLifecycles = map[reflect.Type]*apimachineryversion.Version{}
type fakeLifecycler[T, V any] struct {
metav1.TypeMeta
metav1.ObjectMeta
}
type removedLifecycler[T, V any] struct {
fakeLifecycler[T, V]
}
func (f *fakeLifecycler[T, V]) GetObjectKind() schema.ObjectKind { return f }
func (f *fakeLifecycler[T, V]) DeepCopyObject() runtime.Object { return f }
func (f *fakeLifecycler[T, V]) APILifecycleIntroduced() (major, minor int) {
if introduced, ok := introducedLifecycles[reflect.TypeOf(f)]; ok {
return int(introduced.Major()), int(introduced.Minor())
}
panic("no lifecycle version set")
}
func (f *removedLifecycler[T, V]) APILifecycleRemoved() (major, minor int) {
if removed, ok := removedLifecycles[reflect.TypeOf(f)]; ok {
return int(removed.Major()), int(removed.Minor())
}
panic("no lifecycle version set")
}
func registerFakeLifecycle[T, V any](sch *runtime.Scheme, group, introduced, removed string) {
f := fakeLifecycler[T, V]{}
introducedLifecycles[reflect.TypeOf(&f)] = apimachineryversion.MustParseSemantic(introduced)
var res runtime.Object
if removed != "" {
removedLifecycles[reflect.TypeOf(&f)] = apimachineryversion.MustParseSemantic(removed)
res = &removedLifecycler[T, V]{fakeLifecycler: f}
} else {
res = &f
}
var v V
var t T
sch.AddKnownTypeWithName(
schema.GroupVersionKind{
Group: group,
Version: strings.ToLower(reflect.TypeOf(v).Name()),
Kind: reflect.TypeOf(t).Name(),
},
res,
)
// Also ensure internal version is registered
// If it is registertd multiple times, it will ignore subsequent registrations
internalInstance := &fakeLifecycler[T, struct{}]{}
sch.AddKnownTypeWithName(
schema.GroupVersionKind{
Group: group,
Version: runtime.APIVersionInternal,
Kind: reflect.TypeOf(t).Name(),
},
internalInstance,
)
}
func TestStorageFactoryCompatibilityVersion(t *testing.T) {
// Creates a scheme with stub types for unit test
sch := runtime.NewScheme()
codecs := serializer.NewCodecFactory(sch)
type Internal = struct{}
type V1beta1 struct{}
type V1beta2 struct{}
type V1beta3 struct{}
type V1 struct{}
type Pod struct{}
type FlowSchema struct{}
type ValidatingAdmisisonPolicy struct{}
type CronJob struct{}
// Order dictates priority order
registerFakeLifecycle[FlowSchema, V1](sch, "flowcontrol.apiserver.k8s.io", "1.29.0", "")
registerFakeLifecycle[FlowSchema, V1beta3](sch, "flowcontrol.apiserver.k8s.io", "1.26.0", "1.32.0")
registerFakeLifecycle[FlowSchema, V1beta2](sch, "flowcontrol.apiserver.k8s.io", "1.23.0", "1.29.0")
registerFakeLifecycle[FlowSchema, V1beta1](sch, "flowcontrol.apiserver.k8s.io", "1.20.0", "1.26.0")
registerFakeLifecycle[CronJob, V1](sch, "batch", "1.21.0", "")
registerFakeLifecycle[CronJob, V1beta1](sch, "batch", "1.8.0", "1.21.0")
registerFakeLifecycle[ValidatingAdmisisonPolicy, V1](sch, "admissionregistration.k8s.io", "1.30.0", "")
registerFakeLifecycle[ValidatingAdmisisonPolicy, V1beta1](sch, "admissionregistration.k8s.io", "1.28.0", "1.34.0")
registerFakeLifecycle[Pod, V1](sch, "", "1.31.0", "")
// FlowSchema
// - v1beta1: 1.20.0 - 1.23.0
// - v1beta2: 1.23.0 - 1.26.0
// - v1beta3: 1.26.0 - 1.30.0
// - v1: 1.29.0+
// CronJob
// - v1beta1: 1.8.0 - 1.21.0
// - v1: 1.21.0+
// ValidatingAdmissionPolicy
// - v1beta1: 1.28.0 - 1.31.0
// - v1: 1.30.0+
testcases := []struct {
effectiveVersion string
example runtime.Object
expectedVersion schema.GroupVersion
}{
{
// Basic case. Beta version for long time
effectiveVersion: "1.14.0",
example: &fakeLifecycler[CronJob, Internal]{},
expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1beta1"},
},
{
// Basic case. Beta version for long time
effectiveVersion: "1.20.0",
example: &fakeLifecycler[CronJob, Internal]{},
expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1beta1"},
},
{
// Basic case. GA version for long time
effectiveVersion: "1.28.0",
example: &fakeLifecycler[CronJob, Internal]{},
expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1"},
},
{
// Basic core/v1
effectiveVersion: "1.31.0",
example: &fakeLifecycler[Pod, Internal]{},
expectedVersion: schema.GroupVersion{Group: "", Version: "v1"},
},
{
// Corner case: 1.1.0 has no flowcontrol. Options are to error
// out or to use the latest version. This test assumes the latter.
effectiveVersion: "1.1.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1"},
},
{
effectiveVersion: "1.21.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1"},
},
{
// v2Beta1 introduced this version, but minCompatibility should
// force v1beta1
effectiveVersion: "1.23.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1"},
},
{
effectiveVersion: "1.24.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2"},
},
{
effectiveVersion: "1.26.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2"},
},
{
effectiveVersion: "1.27.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta3"},
},
{
// GA API introduced 1.29 but must keep storing in v1beta3 for downgrades
effectiveVersion: "1.29.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta3"},
},
{
// Version after GA api is introduced
effectiveVersion: "1.30.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1"},
},
{
effectiveVersion: "1.30.0",
example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{},
expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1beta1"},
},
{
effectiveVersion: "1.31.0",
example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{},
expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1"},
},
{
effectiveVersion: "1.29.0",
example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{},
expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1beta1"},
},
}
for _, tc := range testcases {
gvks, _, err := sch.ObjectKinds(tc.example)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
gvk := gvks[0]
t.Run(gvk.GroupKind().String()+"@"+tc.effectiveVersion, func(t *testing.T) {
config := NewDefaultResourceEncodingConfig(sch)
config.SetEffectiveVersion(version.NewEffectiveVersion(tc.effectiveVersion))
f := NewDefaultStorageFactory(
storagebackend.Config{},
"",
codecs,
config,
NewResourceConfig(),
nil)
cfg, err := f.NewConfig(schema.GroupResource{
Group: gvk.Group,
Resource: gvk.Kind, // doesnt really matter here
}, tc.example)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
gvks, _, err := sch.ObjectKinds(tc.example)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
expectEncodeVersioner := runtime.NewMultiGroupVersioner(tc.expectedVersion,
schema.GroupKind{
Group: gvks[0].Group,
}, schema.GroupKind{
Group: gvks[0].Group,
})
if cfg.EncodeVersioner.Identifier() != expectEncodeVersioner.Identifier() {
t.Errorf("expected %v, got %v", expectEncodeVersioner, cfg.EncodeVersioner)
}
})
}
}

View File

@@ -25,7 +25,7 @@ var (
// Only top-level commands/options setup and the k8s.io/component-base/featuregate/testing package should make use of this.
// Tests that need to modify feature gates for the duration of their test should use:
// featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.<FeatureName>, <value>)
DefaultMutableFeatureGate featuregate.MutableFeatureGate = featuregate.NewFeatureGate()
DefaultMutableFeatureGate featuregate.MutableVersionedFeatureGate = featuregate.NewFeatureGate()
// DefaultFeatureGate is a shared global FeatureGate.
// Top-level commands/options setup that needs to modify this feature gate should use DefaultMutableFeatureGate.

View File

@@ -0,0 +1,454 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package version
import (
"fmt"
"sort"
"strings"
"sync"
"github.com/spf13/pflag"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/version"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/featuregate"
"k8s.io/klog/v2"
)
// DefaultComponentGlobalsRegistry is the global var to store the effective versions and feature gates for all components for easy access.
// Example usage:
// // register the component effective version and feature gate first
// _, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
// wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2")
// wardleFeatureGate := featuregate.NewFeatureGate()
// utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate, false))
//
// cmd := &cobra.Command{
// ...
// // call DefaultComponentGlobalsRegistry.Set() in PersistentPreRunE
// PersistentPreRunE: func(*cobra.Command, []string) error {
// if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil {
// return err
// }
// ...
// },
// RunE: func(c *cobra.Command, args []string) error {
// // call utilversion.DefaultComponentGlobalsRegistry.Validate() somewhere
// },
// }
//
// flags := cmd.Flags()
// // add flags
// utilversion.DefaultComponentGlobalsRegistry.AddFlags(flags)
var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGlobalsRegistry()
const (
DefaultKubeComponent = "kube"
klogLevel = 2
)
type VersionMapping func(from *version.Version) *version.Version
// ComponentGlobals stores the global variables for a component for easy access.
type ComponentGlobals struct {
effectiveVersion MutableEffectiveVersion
featureGate featuregate.MutableVersionedFeatureGate
// emulationVersionMapping contains the mapping from the emulation version of this component
// to the emulation version of another component.
emulationVersionMapping map[string]VersionMapping
// dependentEmulationVersion stores whether or not this component's EmulationVersion is dependent through mapping on another component.
// If true, the emulation version cannot be set from the flag, or version mapping from another component.
dependentEmulationVersion bool
// minCompatibilityVersionMapping contains the mapping from the min compatibility version of this component
// to the min compatibility version of another component.
minCompatibilityVersionMapping map[string]VersionMapping
// dependentMinCompatibilityVersion stores whether or not this component's MinCompatibilityVersion is dependent through mapping on another component
// If true, the min compatibility version cannot be set from the flag, or version mapping from another component.
dependentMinCompatibilityVersion bool
}
type ComponentGlobalsRegistry interface {
// EffectiveVersionFor returns the EffectiveVersion registered under the component.
// Returns nil if the component is not registered.
EffectiveVersionFor(component string) EffectiveVersion
// FeatureGateFor returns the FeatureGate registered under the component.
// Returns nil if the component is not registered.
FeatureGateFor(component string) featuregate.FeatureGate
// Register registers the EffectiveVersion and FeatureGate for a component.
// returns error if the component is already registered.
Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error
// ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry.
// Otherwise, the provided variables would be registered under the component, and the same variables would be returned.
ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate)
// AddFlags adds flags of "--emulated-version" and "--feature-gates"
AddFlags(fs *pflag.FlagSet)
// Set sets the flags for all global variables for all components registered.
Set() error
// SetFallback calls Set() if it has never been called.
SetFallback() error
// Validate calls the Validate() function for all the global variables for all components registered.
Validate() []error
// Reset removes all stored ComponentGlobals, configurations, and version mappings.
Reset()
// SetEmulationVersionMapping sets the mapping from the emulation version of one component
// to the emulation version of another component.
// Once set, the emulation version of the toComponent will be determined by the emulation version of the fromComponent,
// and cannot be set from cmd flags anymore.
// For a given component, its emulation version can only depend on one other component, no multiple dependency is allowed.
SetEmulationVersionMapping(fromComponent, toComponent string, f VersionMapping) error
}
type componentGlobalsRegistry struct {
componentGlobals map[string]*ComponentGlobals
mutex sync.RWMutex
// list of component name to emulation version set from the flag.
emulationVersionConfig []string
// map of component name to the list of feature gates set from the flag.
featureGatesConfig map[string][]string
// set stores if the Set() function for the registry is already called.
set bool
}
func NewComponentGlobalsRegistry() *componentGlobalsRegistry {
return &componentGlobalsRegistry{
componentGlobals: make(map[string]*ComponentGlobals),
emulationVersionConfig: nil,
featureGatesConfig: nil,
}
}
func (r *componentGlobalsRegistry) Reset() {
r.mutex.RLock()
defer r.mutex.RUnlock()
r.componentGlobals = make(map[string]*ComponentGlobals)
r.emulationVersionConfig = nil
r.featureGatesConfig = nil
r.set = false
}
func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion {
r.mutex.RLock()
defer r.mutex.RUnlock()
globals, ok := r.componentGlobals[component]
if !ok {
return nil
}
return globals.effectiveVersion
}
func (r *componentGlobalsRegistry) FeatureGateFor(component string) featuregate.FeatureGate {
r.mutex.RLock()
defer r.mutex.RUnlock()
globals, ok := r.componentGlobals[component]
if !ok {
return nil
}
return globals.featureGate
}
func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error {
if _, ok := r.componentGlobals[component]; ok {
return fmt.Errorf("component globals of %s already registered", component)
}
if featureGate != nil {
if err := featureGate.SetEmulationVersion(effectiveVersion.EmulationVersion()); err != nil {
return err
}
}
c := ComponentGlobals{
effectiveVersion: effectiveVersion,
featureGate: featureGate,
emulationVersionMapping: make(map[string]VersionMapping),
minCompatibilityVersionMapping: make(map[string]VersionMapping),
}
r.componentGlobals[component] = &c
return nil
}
func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error {
if effectiveVersion == nil {
return fmt.Errorf("cannot register nil effectiveVersion")
}
r.mutex.Lock()
defer r.mutex.Unlock()
return r.unsafeRegister(component, effectiveVersion, featureGate)
}
func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) {
r.mutex.Lock()
defer r.mutex.Unlock()
globals, ok := r.componentGlobals[component]
if ok {
return globals.effectiveVersion, globals.featureGate
}
utilruntime.Must(r.unsafeRegister(component, effectiveVersion, featureGate))
return effectiveVersion, featureGate
}
func (r *componentGlobalsRegistry) unsafeKnownFeatures() []string {
var known []string
for component, globals := range r.componentGlobals {
if globals.featureGate == nil {
continue
}
for _, f := range globals.featureGate.KnownFeatures() {
known = append(known, component+":"+f)
}
}
sort.Strings(known)
return known
}
func (r *componentGlobalsRegistry) unsafeVersionFlagOptions(isEmulation bool) []string {
var vs []string
for component, globals := range r.componentGlobals {
binaryVer := globals.effectiveVersion.BinaryVersion()
if isEmulation {
if globals.dependentEmulationVersion {
continue
}
// emulated version could be between binaryMajor.{binaryMinor} and binaryMajor.{binaryMinor}
// TODO: change to binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} in 1.32
vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component,
binaryVer.SubtractMinor(0).String(), binaryVer.String(), globals.effectiveVersion.EmulationVersion().String()))
} else {
if globals.dependentMinCompatibilityVersion {
continue
}
// min compatibility version could be between binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor}
vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component,
binaryVer.SubtractMinor(1).String(), binaryVer.String(), globals.effectiveVersion.MinCompatibilityVersion().String()))
}
}
sort.Strings(vs)
return vs
}
func (r *componentGlobalsRegistry) AddFlags(fs *pflag.FlagSet) {
if r == nil {
return
}
r.mutex.Lock()
defer r.mutex.Unlock()
for _, globals := range r.componentGlobals {
if globals.featureGate != nil {
globals.featureGate.Close()
}
}
if r.emulationVersionConfig != nil || r.featureGatesConfig != nil {
klog.Warning("calling componentGlobalsRegistry.AddFlags more than once, the registry will be set by the latest flags")
}
r.emulationVersionConfig = []string{}
r.featureGatesConfig = make(map[string][]string)
fs.StringSliceVar(&r.emulationVersionConfig, "emulated-version", r.emulationVersionConfig, ""+
"The versions different components emulate their capabilities (APIs, features, ...) of.\n"+
"If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+
"Version format could only be major.minor, for example: '--emulated-version=wardle=1.2,kube=1.31'. Options are:\n"+strings.Join(r.unsafeVersionFlagOptions(true), "\n")+
"If the component is not specified, defaults to \"kube\"")
fs.Var(cliflag.NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(&r.featureGatesConfig), "feature-gates", "Comma-separated list of component:key=value pairs that describe feature gates for alpha/experimental features of different components.\n"+
"If the component is not specified, defaults to \"kube\". This flag can be repeatedly invoked. For example: --feature-gates 'wardle:featureA=true,wardle:featureB=false' --feature-gates 'kube:featureC=true'"+
"Options are:\n"+strings.Join(r.unsafeKnownFeatures(), "\n"))
}
type componentVersion struct {
component string
ver *version.Version
}
// getFullEmulationVersionConfig expands the given version config with version registered version mapping,
// and returns the map of component to Version.
func (r *componentGlobalsRegistry) getFullEmulationVersionConfig(
versionConfigMap map[string]*version.Version) (map[string]*version.Version, error) {
result := map[string]*version.Version{}
setQueue := []componentVersion{}
for comp, ver := range versionConfigMap {
if _, ok := r.componentGlobals[comp]; !ok {
return result, fmt.Errorf("component not registered: %s", comp)
}
klog.V(klogLevel).Infof("setting version %s=%s", comp, ver.String())
setQueue = append(setQueue, componentVersion{comp, ver})
}
for len(setQueue) > 0 {
cv := setQueue[0]
if _, visited := result[cv.component]; visited {
return result, fmt.Errorf("setting version of %s more than once, probably version mapping loop", cv.component)
}
setQueue = setQueue[1:]
result[cv.component] = cv.ver
for toComp, f := range r.componentGlobals[cv.component].emulationVersionMapping {
toVer := f(cv.ver)
if toVer == nil {
return result, fmt.Errorf("got nil version from mapping of %s=%s to component:%s", cv.component, cv.ver.String(), toComp)
}
klog.V(klogLevel).Infof("setting version %s=%s from version mapping of %s=%s", toComp, toVer.String(), cv.component, cv.ver.String())
setQueue = append(setQueue, componentVersion{toComp, toVer})
}
}
return result, nil
}
func toVersionMap(versionConfig []string) (map[string]*version.Version, error) {
m := map[string]*version.Version{}
for _, compVer := range versionConfig {
// default to "kube" of component is not specified
k := "kube"
v := compVer
if strings.Contains(compVer, "=") {
arr := strings.SplitN(compVer, "=", 2)
if len(arr) != 2 {
return m, fmt.Errorf("malformed pair, expect string=string")
}
k = strings.TrimSpace(arr[0])
v = strings.TrimSpace(arr[1])
}
ver, err := version.Parse(v)
if err != nil {
return m, err
}
if ver.Patch() != 0 {
return m, fmt.Errorf("patch version not allowed, got: %s=%s", k, ver.String())
}
if existingVer, ok := m[k]; ok {
return m, fmt.Errorf("duplicate version flag, %s=%s and %s=%s", k, existingVer.String(), k, ver.String())
}
m[k] = ver
}
return m, nil
}
func (r *componentGlobalsRegistry) SetFallback() error {
r.mutex.Lock()
set := r.set
r.mutex.Unlock()
if set {
return nil
}
klog.Warning("setting componentGlobalsRegistry in SetFallback. We recommend calling componentGlobalsRegistry.Set()" +
" right after parsing flags to avoid using feature gates before their final values are set by the flags.")
return r.Set()
}
func (r *componentGlobalsRegistry) Set() error {
r.mutex.Lock()
defer r.mutex.Unlock()
r.set = true
emulationVersionConfigMap, err := toVersionMap(r.emulationVersionConfig)
if err != nil {
return err
}
for comp := range emulationVersionConfigMap {
if _, ok := r.componentGlobals[comp]; !ok {
return fmt.Errorf("component not registered: %s", comp)
}
// only components without any dependencies can be set from the flag.
if r.componentGlobals[comp].dependentEmulationVersion {
return fmt.Errorf("EmulationVersion of %s is set by mapping, cannot set it by flag", comp)
}
}
if emulationVersions, err := r.getFullEmulationVersionConfig(emulationVersionConfigMap); err != nil {
return err
} else {
for comp, ver := range emulationVersions {
r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(ver)
}
}
// Set feature gate emulation version before setting feature gate flag values.
for comp, globals := range r.componentGlobals {
if globals.featureGate == nil {
continue
}
klog.V(klogLevel).Infof("setting %s:feature gate emulation version to %s", comp, globals.effectiveVersion.EmulationVersion().String())
if err := globals.featureGate.SetEmulationVersion(globals.effectiveVersion.EmulationVersion()); err != nil {
return err
}
}
for comp, fg := range r.featureGatesConfig {
if comp == "" {
if _, ok := r.featureGatesConfig[DefaultKubeComponent]; ok {
return fmt.Errorf("set kube feature gates with default empty prefix or kube: prefix consistently, do not mix use")
}
comp = DefaultKubeComponent
}
if _, ok := r.componentGlobals[comp]; !ok {
return fmt.Errorf("component not registered: %s", comp)
}
featureGate := r.componentGlobals[comp].featureGate
if featureGate == nil {
return fmt.Errorf("component featureGate not registered: %s", comp)
}
flagVal := strings.Join(fg, ",")
klog.V(klogLevel).Infof("setting %s:feature-gates=%s", comp, flagVal)
if err := featureGate.Set(flagVal); err != nil {
return err
}
}
return nil
}
func (r *componentGlobalsRegistry) Validate() []error {
var errs []error
r.mutex.Lock()
defer r.mutex.Unlock()
for _, globals := range r.componentGlobals {
errs = append(errs, globals.effectiveVersion.Validate()...)
if globals.featureGate != nil {
errs = append(errs, globals.featureGate.Validate()...)
}
}
return errs
}
func (r *componentGlobalsRegistry) SetEmulationVersionMapping(fromComponent, toComponent string, f VersionMapping) error {
if f == nil {
return nil
}
klog.V(klogLevel).Infof("setting EmulationVersion mapping from %s to %s", fromComponent, toComponent)
r.mutex.Lock()
defer r.mutex.Unlock()
if _, ok := r.componentGlobals[fromComponent]; !ok {
return fmt.Errorf("component not registered: %s", fromComponent)
}
if _, ok := r.componentGlobals[toComponent]; !ok {
return fmt.Errorf("component not registered: %s", toComponent)
}
// check multiple dependency
if r.componentGlobals[toComponent].dependentEmulationVersion {
return fmt.Errorf("mapping of %s already exists from another component", toComponent)
}
r.componentGlobals[toComponent].dependentEmulationVersion = true
versionMapping := r.componentGlobals[fromComponent].emulationVersionMapping
if _, ok := versionMapping[toComponent]; ok {
return fmt.Errorf("EmulationVersion from %s to %s already exists", fromComponent, toComponent)
}
versionMapping[toComponent] = f
klog.V(klogLevel).Infof("setting the default EmulationVersion of %s based on mapping from the default EmulationVersion of %s", fromComponent, toComponent)
defaultFromVersion := r.componentGlobals[fromComponent].effectiveVersion.EmulationVersion()
emulationVersions, err := r.getFullEmulationVersionConfig(map[string]*version.Version{fromComponent: defaultFromVersion})
if err != nil {
return err
}
for comp, ver := range emulationVersions {
r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(ver)
}
return nil
}

View File

@@ -0,0 +1,418 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package version
import (
"fmt"
"strings"
"testing"
"github.com/spf13/pflag"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/component-base/featuregate"
)
const (
testComponent = "test"
)
func TestEffectiveVersionRegistry(t *testing.T) {
r := NewComponentGlobalsRegistry()
ver1 := NewEffectiveVersion("1.31")
ver2 := NewEffectiveVersion("1.28")
if r.EffectiveVersionFor(testComponent) != nil {
t.Fatalf("expected nil EffectiveVersion initially")
}
if err := r.Register(testComponent, ver1, nil); err != nil {
t.Fatalf("expected no error to register new component, but got err: %v", err)
}
if !r.EffectiveVersionFor(testComponent).EqualTo(ver1) {
t.Fatalf("expected EffectiveVersionFor to return the version registered")
}
// overwrite
if err := r.Register(testComponent, ver2, nil); err == nil {
t.Fatalf("expected error to register existing component when override is false")
}
if !r.EffectiveVersionFor(testComponent).EqualTo(ver1) {
t.Fatalf("expected EffectiveVersionFor to return the version overridden")
}
}
func testRegistry(t *testing.T) *componentGlobalsRegistry {
r := NewComponentGlobalsRegistry()
verKube := NewEffectiveVersion("1.31")
fgKube := featuregate.NewVersionedFeatureGate(version.MustParse("0.0"))
err := fgKube.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
"kubeA": {
{Version: version.MustParse("1.31"), Default: true, LockToDefault: true, PreRelease: featuregate.GA},
{Version: version.MustParse("1.28"), Default: false, PreRelease: featuregate.Beta},
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
},
"kubeB": {
{Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha},
},
"commonC": {
{Version: version.MustParse("1.29"), Default: true, PreRelease: featuregate.Beta},
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
},
})
if err != nil {
t.Fatal(err)
}
verTest := NewEffectiveVersion("2.8")
fgTest := featuregate.NewVersionedFeatureGate(version.MustParse("0.0"))
err = fgTest.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
"testA": {
{Version: version.MustParse("2.10"), Default: true, PreRelease: featuregate.GA},
{Version: version.MustParse("2.8"), Default: false, PreRelease: featuregate.Beta},
{Version: version.MustParse("2.7"), Default: false, PreRelease: featuregate.Alpha},
},
"testB": {
{Version: version.MustParse("2.9"), Default: false, PreRelease: featuregate.Alpha},
},
"commonC": {
{Version: version.MustParse("2.9"), Default: true, PreRelease: featuregate.Beta},
{Version: version.MustParse("2.7"), Default: false, PreRelease: featuregate.Alpha},
},
})
if err != nil {
t.Fatal(err)
}
utilruntime.Must(r.Register(DefaultKubeComponent, verKube, fgKube))
utilruntime.Must(r.Register(testComponent, verTest, fgTest))
return r
}
func TestVersionFlagOptions(t *testing.T) {
r := testRegistry(t)
emuVers := strings.Join(r.unsafeVersionFlagOptions(true), "\n")
expectedEmuVers := "kube=1.31..1.31 (default=1.31)\ntest=2.8..2.8 (default=2.8)"
if emuVers != expectedEmuVers {
t.Errorf("wanted emulation version flag options to be: %s, got %s", expectedEmuVers, emuVers)
}
minCompVers := strings.Join(r.unsafeVersionFlagOptions(false), "\n")
expectedMinCompVers := "kube=1.30..1.31 (default=1.30)\ntest=2.7..2.8 (default=2.7)"
if minCompVers != expectedMinCompVers {
t.Errorf("wanted min compatibility version flag options to be: %s, got %s", expectedMinCompVers, minCompVers)
}
}
func TestVersionFlagOptionsWithMapping(t *testing.T) {
r := testRegistry(t)
utilruntime.Must(r.SetEmulationVersionMapping(testComponent, DefaultKubeComponent,
func(from *version.Version) *version.Version { return from.OffsetMinor(3) }))
emuVers := strings.Join(r.unsafeVersionFlagOptions(true), "\n")
expectedEmuVers := "test=2.8..2.8 (default=2.8)"
if emuVers != expectedEmuVers {
t.Errorf("wanted emulation version flag options to be: %s, got %s", expectedEmuVers, emuVers)
}
minCompVers := strings.Join(r.unsafeVersionFlagOptions(false), "\n")
expectedMinCompVers := "kube=1.30..1.31 (default=1.30)\ntest=2.7..2.8 (default=2.7)"
if minCompVers != expectedMinCompVers {
t.Errorf("wanted min compatibility version flag options to be: %s, got %s", expectedMinCompVers, minCompVers)
}
}
func TestVersionedFeatureGateFlag(t *testing.T) {
r := testRegistry(t)
known := strings.Join(r.unsafeKnownFeatures(), "\n")
expectedKnown := "kube:AllAlpha=true|false (ALPHA - default=false)\n" +
"kube:AllBeta=true|false (BETA - default=false)\n" +
"kube:commonC=true|false (BETA - default=true)\n" +
"kube:kubeB=true|false (ALPHA - default=false)\n" +
"test:AllAlpha=true|false (ALPHA - default=false)\n" +
"test:AllBeta=true|false (BETA - default=false)\n" +
"test:commonC=true|false (ALPHA - default=false)\n" +
"test:testA=true|false (BETA - default=false)"
if known != expectedKnown {
t.Errorf("wanted min compatibility version flag options to be:\n%s, got:\n%s", expectedKnown, known)
}
}
func TestFlags(t *testing.T) {
tests := []struct {
name string
flags []string
parseError string
expectedKubeEmulationVersion string
expectedTestEmulationVersion string
expectedKubeFeatureValues map[featuregate.Feature]bool
expectedTestFeatureValues map[featuregate.Feature]bool
}{
{
name: "setting kube emulation version",
flags: []string{"--emulated-version=kube=1.30"},
expectedKubeEmulationVersion: "1.30",
},
{
name: "setting kube emulation version twice",
flags: []string{
"--emulated-version=kube=1.30",
"--emulated-version=kube=1.32",
},
parseError: "duplicate version flag, kube=1.30 and kube=1.32",
},
{
name: "prefix v ok",
flags: []string{"--emulated-version=kube=v1.30"},
expectedKubeEmulationVersion: "1.30",
},
{
name: "patch version not ok",
flags: []string{"--emulated-version=kube=1.30.2"},
parseError: "patch version not allowed, got: kube=1.30.2",
},
{
name: "setting test emulation version",
flags: []string{"--emulated-version=test=2.7"},
expectedKubeEmulationVersion: "1.31",
expectedTestEmulationVersion: "2.7",
},
{
name: "version missing component default to kube",
flags: []string{"--emulated-version=1.30"},
expectedKubeEmulationVersion: "1.30",
},
{
name: "version missing component default to kube with duplicate",
flags: []string{"--emulated-version=1.30", "--emulated-version=kube=1.30"},
parseError: "duplicate version flag, kube=1.30 and kube=1.30",
},
{
name: "version unregistered component",
flags: []string{"--emulated-version=test3=1.31"},
parseError: "component not registered: test3",
},
{
name: "invalid version",
flags: []string{"--emulated-version=test=1.foo"},
parseError: "illegal version string \"1.foo\"",
},
{
name: "setting test feature flag",
flags: []string{
"--emulated-version=test=2.7",
"--feature-gates=test:testA=true",
},
expectedKubeEmulationVersion: "1.31",
expectedTestEmulationVersion: "2.7",
expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": true, "kubeB": false, "commonC": true},
expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": true, "testB": false, "commonC": false},
},
{
name: "setting future test feature flag",
flags: []string{
"--emulated-version=test=2.7",
"--feature-gates=test:testA=true,test:testB=true",
},
parseError: "cannot set feature gate testB to true, feature is PreAlpha at emulated version 2.7",
},
{
name: "setting kube feature flag",
flags: []string{
"--emulated-version=test=2.7",
"--emulated-version=kube=1.30",
"--feature-gates=kubeB=false,test:commonC=true",
"--feature-gates=commonC=false,kubeB=true",
},
expectedKubeEmulationVersion: "1.30",
expectedTestEmulationVersion: "2.7",
expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": false, "kubeB": true, "commonC": false},
expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": false, "testB": false, "commonC": true},
},
{
name: "setting kube feature flag with different prefix",
flags: []string{
"--emulated-version=test=2.7",
"--emulated-version=kube=1.30",
"--feature-gates=kube:kubeB=false,test:commonC=true",
"--feature-gates=commonC=false,kubeB=true",
},
parseError: "set kube feature gates with default empty prefix or kube: prefix consistently, do not mix use",
},
{
name: "setting locked kube feature flag",
flags: []string{
"--emulated-version=test=2.7",
"--feature-gates=kubeA=false",
},
parseError: "cannot set feature gate kubeA to false, feature is locked to true",
},
{
name: "setting unknown test feature flag",
flags: []string{
"--emulated-version=test=2.7",
"--feature-gates=test:testD=true",
},
parseError: "unrecognized feature gate: testD",
},
{
name: "setting unknown component feature flag",
flags: []string{
"--emulated-version=test=2.7",
"--feature-gates=test3:commonC=true",
},
parseError: "component not registered: test3",
},
}
for i, test := range tests {
t.Run(test.name, func(t *testing.T) {
fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError)
r := testRegistry(t)
r.AddFlags(fs)
err := fs.Parse(test.flags)
if err == nil {
err = r.Set()
}
if test.parseError != "" {
if err == nil || !strings.Contains(err.Error(), test.parseError) {
t.Fatalf("%d: Parse() expected: %v, got: %v", i, test.parseError, err)
}
return
}
if err != nil {
t.Fatalf("%d: Parse() expected: nil, got: %v", i, err)
}
if len(test.expectedKubeEmulationVersion) > 0 {
assertVersionEqualTo(t, r.EffectiveVersionFor(DefaultKubeComponent).EmulationVersion(), test.expectedKubeEmulationVersion)
}
if len(test.expectedTestEmulationVersion) > 0 {
assertVersionEqualTo(t, r.EffectiveVersionFor(testComponent).EmulationVersion(), test.expectedTestEmulationVersion)
}
for f, v := range test.expectedKubeFeatureValues {
if r.FeatureGateFor(DefaultKubeComponent).Enabled(f) != v {
t.Errorf("%d: expected kube feature Enabled(%s)=%v", i, f, v)
}
}
for f, v := range test.expectedTestFeatureValues {
if r.FeatureGateFor(testComponent).Enabled(f) != v {
t.Errorf("%d: expected test feature Enabled(%s)=%v", i, f, v)
}
}
})
}
}
func TestVersionMapping(t *testing.T) {
r := NewComponentGlobalsRegistry()
ver1 := NewEffectiveVersion("0.58")
ver2 := NewEffectiveVersion("1.28")
ver3 := NewEffectiveVersion("2.10")
utilruntime.Must(r.Register("test1", ver1, nil))
utilruntime.Must(r.Register("test2", ver2, nil))
utilruntime.Must(r.Register("test3", ver3, nil))
assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58")
assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28")
assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.10")
utilruntime.Must(r.SetEmulationVersionMapping("test2", "test3",
func(from *version.Version) *version.Version {
return version.MajorMinor(from.Major()+1, from.Minor()-19)
}))
utilruntime.Must(r.SetEmulationVersionMapping("test1", "test2",
func(from *version.Version) *version.Version {
return version.MajorMinor(from.Major()+1, from.Minor()-28)
}))
assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58")
assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.30")
assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.11")
fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError)
r.AddFlags(fs)
if err := fs.Parse([]string{fmt.Sprintf("--emulated-version=%s", "test1=0.56")}); err != nil {
t.Fatal(err)
return
}
if err := r.Set(); err != nil {
t.Fatal(err)
return
}
assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.56")
assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28")
assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.09")
}
func TestVersionMappingWithMultipleDependency(t *testing.T) {
r := NewComponentGlobalsRegistry()
ver1 := NewEffectiveVersion("0.58")
ver2 := NewEffectiveVersion("1.28")
ver3 := NewEffectiveVersion("2.10")
utilruntime.Must(r.Register("test1", ver1, nil))
utilruntime.Must(r.Register("test2", ver2, nil))
utilruntime.Must(r.Register("test3", ver3, nil))
assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58")
assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28")
assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.10")
utilruntime.Must(r.SetEmulationVersionMapping("test1", "test2",
func(from *version.Version) *version.Version {
return version.MajorMinor(from.Major()+1, from.Minor()-28)
}))
err := r.SetEmulationVersionMapping("test3", "test2",
func(from *version.Version) *version.Version {
return version.MajorMinor(from.Major()-1, from.Minor()+19)
})
if err == nil {
t.Errorf("expect error when setting 2nd mapping to test2")
}
}
func TestVersionMappingWithCyclicDependency(t *testing.T) {
r := NewComponentGlobalsRegistry()
ver1 := NewEffectiveVersion("0.58")
ver2 := NewEffectiveVersion("1.28")
ver3 := NewEffectiveVersion("2.10")
utilruntime.Must(r.Register("test1", ver1, nil))
utilruntime.Must(r.Register("test2", ver2, nil))
utilruntime.Must(r.Register("test3", ver3, nil))
assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58")
assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28")
assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.10")
utilruntime.Must(r.SetEmulationVersionMapping("test1", "test2",
func(from *version.Version) *version.Version {
return version.MajorMinor(from.Major()+1, from.Minor()-28)
}))
utilruntime.Must(r.SetEmulationVersionMapping("test2", "test3",
func(from *version.Version) *version.Version {
return version.MajorMinor(from.Major()+1, from.Minor()-19)
}))
err := r.SetEmulationVersionMapping("test3", "test1",
func(from *version.Version) *version.Version {
return version.MajorMinor(from.Major()-2, from.Minor()+48)
})
if err == nil {
t.Errorf("expect cyclic version mapping error")
}
}
func assertVersionEqualTo(t *testing.T, ver *version.Version, expectedVer string) {
if ver.EqualTo(version.MustParse(expectedVer)) {
return
}
t.Errorf("expected: %s, got %s", expectedVer, ver.String())
}

View File

@@ -0,0 +1,157 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package version
import (
"fmt"
"sync/atomic"
"k8s.io/apimachinery/pkg/util/version"
baseversion "k8s.io/component-base/version"
)
type EffectiveVersion interface {
BinaryVersion() *version.Version
EmulationVersion() *version.Version
MinCompatibilityVersion() *version.Version
EqualTo(other EffectiveVersion) bool
String() string
Validate() []error
}
type MutableEffectiveVersion interface {
EffectiveVersion
Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version)
SetEmulationVersion(emulationVersion *version.Version)
SetMinCompatibilityVersion(minCompatibilityVersion *version.Version)
}
type effectiveVersion struct {
binaryVersion atomic.Pointer[version.Version]
// If the emulationVersion is set by the users, it could only contain major and minor versions.
// In tests, emulationVersion could be the same as the binary version, or set directly,
// which can have "alpha" as pre-release to continue serving expired apis while we clean up the test.
emulationVersion atomic.Pointer[version.Version]
// minCompatibilityVersion could only contain major and minor versions.
minCompatibilityVersion atomic.Pointer[version.Version]
}
func (m *effectiveVersion) BinaryVersion() *version.Version {
return m.binaryVersion.Load()
}
func (m *effectiveVersion) EmulationVersion() *version.Version {
ver := m.emulationVersion.Load()
if ver != nil {
// Emulation version can have "alpha" as pre-release to continue serving expired apis while we clean up the test.
// The pre-release should not be accessible to the users.
return ver.WithPreRelease(m.BinaryVersion().PreRelease())
}
return ver
}
func (m *effectiveVersion) MinCompatibilityVersion() *version.Version {
return m.minCompatibilityVersion.Load()
}
func (m *effectiveVersion) EqualTo(other EffectiveVersion) bool {
return m.BinaryVersion().EqualTo(other.BinaryVersion()) && m.EmulationVersion().EqualTo(other.EmulationVersion()) && m.MinCompatibilityVersion().EqualTo(other.MinCompatibilityVersion())
}
func (m *effectiveVersion) String() string {
if m == nil {
return "<nil>"
}
return fmt.Sprintf("{BinaryVersion: %s, EmulationVersion: %s, MinCompatibilityVersion: %s}",
m.BinaryVersion().String(), m.EmulationVersion().String(), m.MinCompatibilityVersion().String())
}
func majorMinor(ver *version.Version) *version.Version {
if ver == nil {
return ver
}
return version.MajorMinor(ver.Major(), ver.Minor())
}
func (m *effectiveVersion) Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) {
m.binaryVersion.Store(binaryVersion)
m.emulationVersion.Store(majorMinor(emulationVersion))
m.minCompatibilityVersion.Store(majorMinor(minCompatibilityVersion))
}
func (m *effectiveVersion) SetEmulationVersion(emulationVersion *version.Version) {
m.emulationVersion.Store(majorMinor(emulationVersion))
}
func (m *effectiveVersion) SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) {
m.minCompatibilityVersion.Store(majorMinor(minCompatibilityVersion))
}
func (m *effectiveVersion) Validate() []error {
var errs []error
// Validate only checks the major and minor versions.
binaryVersion := m.binaryVersion.Load().WithPatch(0)
emulationVersion := m.emulationVersion.Load()
minCompatibilityVersion := m.minCompatibilityVersion.Load()
// emulationVersion can only be 1.{binaryMinor-1}...1.{binaryMinor}.
maxEmuVer := binaryVersion
minEmuVer := binaryVersion.SubtractMinor(1)
if emulationVersion.GreaterThan(maxEmuVer) || emulationVersion.LessThan(minEmuVer) {
errs = append(errs, fmt.Errorf("emulation version %s is not between [%s, %s]", emulationVersion.String(), minEmuVer.String(), maxEmuVer.String()))
}
// minCompatibilityVersion can only be 1.{binaryMinor-1} for alpha.
maxCompVer := binaryVersion.SubtractMinor(1)
minCompVer := binaryVersion.SubtractMinor(1)
if minCompatibilityVersion.GreaterThan(maxCompVer) || minCompatibilityVersion.LessThan(minCompVer) {
errs = append(errs, fmt.Errorf("minCompatibilityVersion version %s is not between [%s, %s]", minCompatibilityVersion.String(), minCompVer.String(), maxCompVer.String()))
}
return errs
}
func newEffectiveVersion(binaryVersion *version.Version) MutableEffectiveVersion {
effective := &effectiveVersion{}
compatVersion := binaryVersion.SubtractMinor(1)
effective.Set(binaryVersion, binaryVersion, compatVersion)
return effective
}
func NewEffectiveVersion(binaryVer string) MutableEffectiveVersion {
if binaryVer == "" {
return &effectiveVersion{}
}
binaryVersion := version.MustParse(binaryVer)
return newEffectiveVersion(binaryVersion)
}
// DefaultBuildEffectiveVersion returns the MutableEffectiveVersion based on the
// current build information.
func DefaultBuildEffectiveVersion() MutableEffectiveVersion {
verInfo := baseversion.Get()
binaryVersion := version.MustParse(verInfo.String()).WithInfo(verInfo)
if binaryVersion.Major() == 0 && binaryVersion.Minor() == 0 {
return DefaultKubeEffectiveVersion()
}
return newEffectiveVersion(binaryVersion)
}
// DefaultKubeEffectiveVersion returns the MutableEffectiveVersion based on the
// latest K8s release.
func DefaultKubeEffectiveVersion() MutableEffectiveVersion {
binaryVersion := version.MustParse(baseversion.DefaultKubeBinaryVersion).WithInfo(baseversion.Get())
return newEffectiveVersion(binaryVersion)
}

View File

@@ -0,0 +1,107 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package version
import (
"testing"
"k8s.io/apimachinery/pkg/util/version"
)
func TestValidate(t *testing.T) {
tests := []struct {
name string
binaryVersion string
emulationVersion string
minCompatibilityVersion string
expectErrors bool
}{
{
name: "patch version diff ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.1",
minCompatibilityVersion: "v1.31.5",
},
{
name: "emulation version one minor lower than binary ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.31.0",
},
{
name: "emulation version two minor lower than binary not ok",
binaryVersion: "v1.33.2",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.32.0",
expectErrors: true,
},
{
name: "emulation version one minor higher than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.33.0",
minCompatibilityVersion: "v1.31.0",
expectErrors: true,
},
{
name: "emulation version two minor higher than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.34.0",
minCompatibilityVersion: "v1.31.0",
expectErrors: true,
},
{
name: "compatibility version same as binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.32.0",
expectErrors: true,
},
{
name: "compatibility version two minor lower than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.30.0",
expectErrors: true,
},
{
name: "compatibility version one minor higher than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.33.0",
expectErrors: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
binaryVersion := version.MustParseGeneric(test.binaryVersion)
effective := &effectiveVersion{}
emulationVersion := version.MustParseGeneric(test.emulationVersion)
minCompatibilityVersion := version.MustParseGeneric(test.minCompatibilityVersion)
effective.Set(binaryVersion, emulationVersion, minCompatibilityVersion)
errs := effective.Validate()
if len(errs) > 0 && !test.expectErrors {
t.Errorf("expected no errors, errors found %+v", errs)
}
if len(errs) == 0 && test.expectErrors {
t.Errorf("expected errors, no errors found")
}
})
}
}

View File

@@ -33,8 +33,9 @@ import (
// while still allowing the distribution of key-value pairs across multiple flag invocations.
// For example: `--flag "a:hello" --flag "b:again" --flag "b:beautiful" --flag "c:world"` results in `{"a": ["hello"], "b": ["again", "beautiful"], "c": ["world"]}`
type ColonSeparatedMultimapStringString struct {
Multimap *map[string][]string
initialized bool // set to true after the first Set call
Multimap *map[string][]string
initialized bool // set to true after the first Set call
allowDefaultEmptyKey bool
}
// NewColonSeparatedMultimapStringString takes a pointer to a map[string][]string and returns the
@@ -43,6 +44,12 @@ func NewColonSeparatedMultimapStringString(m *map[string][]string) *ColonSeparat
return &ColonSeparatedMultimapStringString{Multimap: m}
}
// NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey takes a pointer to a map[string][]string and returns the
// ColonSeparatedMultimapStringString flag parsing shim for that map. It allows default empty key with no colon in the flag.
func NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(m *map[string][]string) *ColonSeparatedMultimapStringString {
return &ColonSeparatedMultimapStringString{Multimap: m, allowDefaultEmptyKey: true}
}
// Set implements github.com/spf13/pflag.Value
func (m *ColonSeparatedMultimapStringString) Set(value string) error {
if m.Multimap == nil {
@@ -58,11 +65,16 @@ func (m *ColonSeparatedMultimapStringString) Set(value string) error {
continue
}
kv := strings.SplitN(pair, ":", 2)
if len(kv) != 2 {
return fmt.Errorf("malformed pair, expect string:string")
var k, v string
if m.allowDefaultEmptyKey && len(kv) == 1 {
v = strings.TrimSpace(kv[0])
} else {
if len(kv) != 2 {
return fmt.Errorf("malformed pair, expect string:string")
}
k = strings.TrimSpace(kv[0])
v = strings.TrimSpace(kv[1])
}
k := strings.TrimSpace(kv[0])
v := strings.TrimSpace(kv[1])
(*m.Multimap)[k] = append((*m.Multimap)[k], v)
}
return nil

View File

@@ -98,6 +98,21 @@ func TestSetColonSeparatedMultimapStringString(t *testing.T) {
&ColonSeparatedMultimapStringString{
initialized: true,
Multimap: &map[string][]string{}}, ""},
{"empty key no colon", []string{"foo"},
NewColonSeparatedMultimapStringString(&nilMap),
&ColonSeparatedMultimapStringString{
initialized: true,
Multimap: &map[string][]string{
"": {"foo"},
}}, "malformed pair, expect string:string"},
{"empty key no colon allowed", []string{"foo"},
NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(&nilMap),
&ColonSeparatedMultimapStringString{
initialized: true,
allowDefaultEmptyKey: true,
Multimap: &map[string][]string{
"": {"foo"},
}}, ""},
{"empty key", []string{":foo"},
NewColonSeparatedMultimapStringString(&nilMap),
&ColonSeparatedMultimapStringString{

View File

@@ -19,6 +19,7 @@ package featuregate
import (
"context"
"fmt"
"reflect"
"sort"
"strconv"
"strings"
@@ -27,8 +28,11 @@ import (
"github.com/spf13/pflag"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/naming"
"k8s.io/apimachinery/pkg/util/version"
featuremetrics "k8s.io/component-base/metrics/prometheus/feature"
baseversion "k8s.io/component-base/version"
"k8s.io/klog/v2"
)
@@ -52,13 +56,13 @@ const (
var (
// The generic features.
defaultFeatures = map[Feature]FeatureSpec{
allAlphaGate: {Default: false, PreRelease: Alpha},
allBetaGate: {Default: false, PreRelease: Beta},
defaultFeatures = map[Feature]VersionedSpecs{
allAlphaGate: {{Default: false, PreRelease: Alpha, Version: version.MajorMinor(0, 0)}},
allBetaGate: {{Default: false, PreRelease: Beta, Version: version.MajorMinor(0, 0)}},
}
// Special handling for a few gates.
specialFeatures = map[Feature]func(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool){
specialFeatures = map[Feature]func(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, cVer *version.Version){
allAlphaGate: setUnsetAlphaGates,
allBetaGate: setUnsetBetaGates,
}
@@ -69,13 +73,28 @@ type FeatureSpec struct {
Default bool
// LockToDefault indicates that the feature is locked to its default and cannot be changed
LockToDefault bool
// PreRelease indicates the maturity level of the feature
// PreRelease indicates the current maturity level of the feature
PreRelease prerelease
// Version indicates the earliest version from which this FeatureSpec is valid.
// If multiple FeatureSpecs exist for a Feature, the one with the highest version that is less
// than or equal to the effective version of the component is used.
Version *version.Version
}
type VersionedSpecs []FeatureSpec
func (g VersionedSpecs) Len() int { return len(g) }
func (g VersionedSpecs) Less(i, j int) bool {
return g[i].Version.LessThan(g[j].Version)
}
func (g VersionedSpecs) Swap(i, j int) { g[i], g[j] = g[j], g[i] }
type PromotionVersionMapping map[prerelease]string
type prerelease string
const (
PreAlpha = prerelease("PRE-ALPHA")
// Values for PreRelease.
Alpha = prerelease("ALPHA")
Beta = prerelease("BETA")
@@ -94,7 +113,12 @@ type FeatureGate interface {
// DeepCopy returns a deep copy of the FeatureGate object, such that gates can be
// set on the copy without mutating the original. This is useful for validating
// config against potential feature gate changes before committing those changes.
DeepCopy() MutableFeatureGate
DeepCopy() MutableVersionedFeatureGate
// CopyKnownFeatures returns a partial copy of the FeatureGate object, with all the known features and overrides.
// This is useful for creating a new instance of feature gate without inheriting all the enabled configurations of the base feature gate.
CopyKnownFeatures() MutableVersionedFeatureGate
// Validate checks if the flag gates are valid at the emulated version.
Validate() []error
}
// MutableFeatureGate parses and stores flag gates for known features from
@@ -104,6 +128,8 @@ type MutableFeatureGate interface {
// AddFlag adds a flag for setting global feature gates to the specified FlagSet.
AddFlag(fs *pflag.FlagSet)
// Close sets closed to true, and prevents subsequent calls to Add
Close()
// Set parses and stores flag gates for known features
// from a string like feature1=true,feature2=false,...
Set(value string) error
@@ -128,25 +154,73 @@ type MutableFeatureGate interface {
OverrideDefault(name Feature, override bool) error
}
// MutableVersionedFeatureGate parses and stores flag gates for known features from
// a string like feature1=true,feature2=false,...
// MutableVersionedFeatureGate sets options based on the emulated version of the featured gate.
type MutableVersionedFeatureGate interface {
MutableFeatureGate
// EmulationVersion returns the version the feature gate is set to emulate.
// If set, the feature gate would enable/disable features based on
// feature availability and pre-release at the emulated version instead of the binary version.
EmulationVersion() *version.Version
// SetEmulationVersion overrides the emulationVersion of the feature gate.
// Otherwise, the emulationVersion will be the same as the binary version.
// If set, the feature defaults and availability will be as if the binary is at the emulated version.
SetEmulationVersion(emulationVersion *version.Version) error
// GetAll returns a copy of the map of known feature names to versioned feature specs.
GetAllVersioned() map[Feature]VersionedSpecs
// AddVersioned adds versioned feature specs to the featureGate.
AddVersioned(features map[Feature]VersionedSpecs) error
// OverrideDefaultAtVersion sets a local override for the registered default value of a named
// feature for the prerelease lifecycle the given version is at.
// If the feature has not been previously registered (e.g. by a call to Add),
// has a locked default, or if the gate has already registered itself with a FlagSet, a non-nil
// error is returned.
//
// When two or more components consume a common feature, one component can override its
// default at runtime in order to adopt new defaults before or after the other
// components. For example, a new feature can be evaluated with a limited blast radius by
// overriding its default to true for a limited number of components without simultaneously
// changing its default for all consuming components.
OverrideDefaultAtVersion(name Feature, override bool, ver *version.Version) error
// ExplicitlySet returns true if the feature value is explicitly set instead of
// being derived from the default values or special features.
ExplicitlySet(name Feature) bool
// ResetFeatureValueToDefault resets the value of the feature back to the default value.
ResetFeatureValueToDefault(name Feature) error
}
// featureGate implements FeatureGate as well as pflag.Value for flag parsing.
type featureGate struct {
featureGateName string
special map[Feature]func(map[Feature]FeatureSpec, map[Feature]bool, bool)
special map[Feature]func(map[Feature]VersionedSpecs, map[Feature]bool, bool, *version.Version)
// lock guards writes to known, enabled, and reads/writes of closed
// lock guards writes to all below fields.
lock sync.Mutex
// known holds a map[Feature]FeatureSpec
known atomic.Value
// enabled holds a map[Feature]bool
enabled atomic.Value
// enabledRaw holds a raw map[string]bool of the parsed flag.
// It keeps the original values of "special" features like "all alpha gates",
// while enabled keeps the values of all resolved features.
enabledRaw atomic.Value
// closed is set to true when AddFlag is called, and prevents subsequent calls to Add
closed bool
// queriedFeatures stores all the features that have been queried through the Enabled interface.
// It is reset when SetEmulationVersion is called.
queriedFeatures atomic.Value
emulationVersion atomic.Pointer[version.Version]
}
func setUnsetAlphaGates(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool) {
func setUnsetAlphaGates(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, cVer *version.Version) {
for k, v := range known {
if v.PreRelease == Alpha {
if k == "AllAlpha" || k == "AllBeta" {
continue
}
featureSpec := featureSpecAtEmulationVersion(v, cVer)
if featureSpec.PreRelease == Alpha {
if _, found := enabled[k]; !found {
enabled[k] = val
}
@@ -154,9 +228,13 @@ func setUnsetAlphaGates(known map[Feature]FeatureSpec, enabled map[Feature]bool,
}
}
func setUnsetBetaGates(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool) {
func setUnsetBetaGates(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, cVer *version.Version) {
for k, v := range known {
if v.PreRelease == Beta {
if k == "AllAlpha" || k == "AllBeta" {
continue
}
featureSpec := featureSpecAtEmulationVersion(v, cVer)
if featureSpec.PreRelease == Beta {
if _, found := enabled[k]; !found {
enabled[k] = val
}
@@ -171,8 +249,10 @@ var _ pflag.Value = &featureGate{}
// call chains, so they'd be unhelpful as names.
var internalPackages = []string{"k8s.io/component-base/featuregate/feature_gate.go"}
func NewFeatureGate() *featureGate {
known := map[Feature]FeatureSpec{}
// NewVersionedFeatureGate creates a feature gate with the emulation version set to the provided version.
// SetEmulationVersion can be called after to change emulation version to a desired value.
func NewVersionedFeatureGate(emulationVersion *version.Version) *featureGate {
known := map[Feature]VersionedSpecs{}
for k, v := range defaultFeatures {
known[k] = v
}
@@ -183,10 +263,19 @@ func NewFeatureGate() *featureGate {
}
f.known.Store(known)
f.enabled.Store(map[Feature]bool{})
f.enabledRaw.Store(map[string]bool{})
f.emulationVersion.Store(emulationVersion)
f.queriedFeatures.Store(map[Feature]struct{}{})
klog.V(1).Infof("new feature gate with emulationVersion=%s", f.emulationVersion.Load().String())
return f
}
// NewFeatureGate creates a feature gate with the current binary version.
func NewFeatureGate() *featureGate {
binaryVersison := version.MustParse(baseversion.DefaultKubeBinaryVersion)
return NewVersionedFeatureGate(binaryVersison)
}
// Set parses a string of the form "key1=value1,key2=value2,..." into a
// map[string]bool of known keys or returns an error.
func (f *featureGate) Set(value string) error {
@@ -210,35 +299,52 @@ func (f *featureGate) Set(value string) error {
return f.SetFromMap(m)
}
// SetFromMap stores flag gates for known features from a map[string]bool or returns an error
func (f *featureGate) SetFromMap(m map[string]bool) error {
// Validate checks if the flag gates are valid at the emulated version.
func (f *featureGate) Validate() []error {
f.lock.Lock()
defer f.lock.Unlock()
// Copy existing state
known := map[Feature]FeatureSpec{}
for k, v := range f.known.Load().(map[Feature]FeatureSpec) {
known[k] = v
m, ok := f.enabledRaw.Load().(map[string]bool)
if !ok {
return []error{fmt.Errorf("cannot cast enabledRaw to map[string]bool")}
}
enabled := map[Feature]bool{}
for k, v := range f.enabled.Load().(map[Feature]bool) {
enabled[k] = v
return f.unsafeSetFromMap(enabled, m, f.EmulationVersion())
}
// unsafeSetFromMap stores flag gates for known features from a map[string]bool into an enabled map.
func (f *featureGate) unsafeSetFromMap(enabled map[Feature]bool, m map[string]bool, emulationVersion *version.Version) []error {
var errs []error
// Copy existing state
known := map[Feature]VersionedSpecs{}
for k, v := range f.known.Load().(map[Feature]VersionedSpecs) {
sort.Sort(v)
known[k] = v
}
for k, v := range m {
k := Feature(k)
featureSpec, ok := known[k]
key := Feature(k)
versionedSpecs, ok := known[key]
if !ok {
return fmt.Errorf("unrecognized feature gate: %s", k)
// early return if encounters an unknown feature.
errs = append(errs, fmt.Errorf("unrecognized feature gate: %s", k))
return errs
}
featureSpec := featureSpecAtEmulationVersion(versionedSpecs, emulationVersion)
if featureSpec.LockToDefault && featureSpec.Default != v {
return fmt.Errorf("cannot set feature gate %v to %v, feature is locked to %v", k, v, featureSpec.Default)
errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is locked to %v", k, v, featureSpec.Default))
continue
}
enabled[k] = v
// Handle "special" features like "all alpha gates"
if fn, found := f.special[k]; found {
fn(known, enabled, v)
if fn, found := f.special[key]; found {
fn(known, enabled, v, emulationVersion)
enabled[key] = v
continue
}
if featureSpec.PreRelease == PreAlpha {
errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is PreAlpha at emulated version %s", k, v, emulationVersion.String()))
continue
}
enabled[key] = v
if featureSpec.PreRelease == Deprecated {
klog.Warningf("Setting deprecated feature gate %s=%t. It will be removed in a future release.", k, v)
@@ -246,13 +352,39 @@ func (f *featureGate) SetFromMap(m map[string]bool) error {
klog.Warningf("Setting GA feature gate %s=%t. It will be removed in a future release.", k, v)
}
}
return errs
}
// Persist changes
f.known.Store(known)
f.enabled.Store(enabled)
// SetFromMap stores flag gates for known features from a map[string]bool or returns an error
func (f *featureGate) SetFromMap(m map[string]bool) error {
f.lock.Lock()
defer f.lock.Unlock()
klog.V(1).Infof("feature gates: %v", f.enabled)
return nil
// Copy existing state
enabled := map[Feature]bool{}
for k, v := range f.enabled.Load().(map[Feature]bool) {
enabled[k] = v
}
enabledRaw := map[string]bool{}
for k, v := range f.enabledRaw.Load().(map[string]bool) {
enabledRaw[k] = v
}
// Update enabledRaw first.
// SetFromMap might be called when emulationVersion is not finalized yet, and we do not know the final state of enabled.
// But the flags still need to be saved.
for k, v := range m {
enabledRaw[k] = v
}
f.enabledRaw.Store(enabledRaw)
errs := f.unsafeSetFromMap(enabled, enabledRaw, f.EmulationVersion())
if len(errs) == 0 {
// Persist changes
f.enabled.Store(enabled)
klog.V(1).Infof("feature gates: %v", f.enabled)
}
return utilerrors.NewAggregate(errs)
}
// String returns a string containing all enabled feature gates, formatted as "key1=value1,key2=value2,...".
@@ -271,6 +403,17 @@ func (f *featureGate) Type() string {
// Add adds features to the featureGate.
func (f *featureGate) Add(features map[Feature]FeatureSpec) error {
vs := map[Feature]VersionedSpecs{}
for name, spec := range features {
// if no version is provided for the FeatureSpec, it is defaulted to version 0.0 so that it can be enabled/disabled regardless of emulation version.
spec.Version = version.MajorMinor(0, 0)
vs[name] = VersionedSpecs{spec}
}
return f.AddVersioned(vs)
}
// AddVersioned adds versioned feature specs to the featureGate.
func (f *featureGate) AddVersioned(features map[Feature]VersionedSpecs) error {
f.lock.Lock()
defer f.lock.Unlock()
@@ -279,20 +422,21 @@ func (f *featureGate) Add(features map[Feature]FeatureSpec) error {
}
// Copy existing state
known := map[Feature]FeatureSpec{}
for k, v := range f.known.Load().(map[Feature]FeatureSpec) {
known := map[Feature]VersionedSpecs{}
for k, v := range f.known.Load().(map[Feature]VersionedSpecs) {
known[k] = v
}
for name, spec := range features {
for name, specs := range features {
sort.Sort(specs)
if existingSpec, found := known[name]; found {
if existingSpec == spec {
sort.Sort(existingSpec)
if reflect.DeepEqual(existingSpec, specs) {
continue
}
return fmt.Errorf("feature gate %q with different spec already exists: %v", name, existingSpec)
}
known[name] = spec
known[name] = specs
}
// Persist updated state
@@ -302,6 +446,10 @@ func (f *featureGate) Add(features map[Feature]FeatureSpec) error {
}
func (f *featureGate) OverrideDefault(name Feature, override bool) error {
return f.OverrideDefaultAtVersion(name, override, f.EmulationVersion())
}
func (f *featureGate) OverrideDefaultAtVersion(name Feature, override bool, ver *version.Version) error {
f.lock.Lock()
defer f.lock.Unlock()
@@ -309,17 +457,22 @@ func (f *featureGate) OverrideDefault(name Feature, override bool) error {
return fmt.Errorf("cannot override default for feature %q: gates already added to a flag set", name)
}
known := map[Feature]FeatureSpec{}
for name, spec := range f.known.Load().(map[Feature]FeatureSpec) {
known[name] = spec
known := map[Feature]VersionedSpecs{}
for k, v := range f.known.Load().(map[Feature]VersionedSpecs) {
sort.Sort(v)
known[k] = v
}
spec, ok := known[name]
switch {
case !ok:
specs, ok := known[name]
if !ok {
return fmt.Errorf("cannot override default: feature %q is not registered", name)
}
spec := featureSpecAtEmulationVersion(specs, ver)
switch {
case spec.LockToDefault:
return fmt.Errorf("cannot override default: feature %q default is locked to %t", name, spec.Default)
case spec.PreRelease == PreAlpha:
return fmt.Errorf("cannot override default: feature %q is not available before version %s", name, ver.String())
case spec.PreRelease == Deprecated:
klog.Warningf("Overriding default of deprecated feature gate %s=%t. It will be removed in a future release.", name, override)
case spec.PreRelease == GA:
@@ -327,42 +480,154 @@ func (f *featureGate) OverrideDefault(name Feature, override bool) error {
}
spec.Default = override
known[name] = spec
known[name] = specs
f.known.Store(known)
return nil
}
// GetAll returns a copy of the map of known feature names to feature specs.
// GetAll returns a copy of the map of known feature names to feature specs for the current emulationVersion.
func (f *featureGate) GetAll() map[Feature]FeatureSpec {
retval := map[Feature]FeatureSpec{}
for k, v := range f.known.Load().(map[Feature]FeatureSpec) {
f.lock.Lock()
versionedSpecs := f.GetAllVersioned()
emuVer := f.EmulationVersion()
f.lock.Unlock()
for k, v := range versionedSpecs {
spec := featureSpecAtEmulationVersion(v, emuVer)
if spec.PreRelease == PreAlpha {
// The feature is not available at the emulation version.
continue
}
retval[k] = *spec
}
return retval
}
// GetAllVersioned returns a copy of the map of known feature names to versioned feature specs.
func (f *featureGate) GetAllVersioned() map[Feature]VersionedSpecs {
retval := map[Feature]VersionedSpecs{}
for k, v := range f.known.Load().(map[Feature]VersionedSpecs) {
retval[k] = v
}
return retval
}
// Enabled returns true if the key is enabled. If the key is not known, this call will panic.
func (f *featureGate) Enabled(key Feature) bool {
if v, ok := f.enabled.Load().(map[Feature]bool)[key]; ok {
return v
func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) error {
if emulationVersion.EqualTo(f.EmulationVersion()) {
return nil
}
if v, ok := f.known.Load().(map[Feature]FeatureSpec)[key]; ok {
return v.Default
f.lock.Lock()
defer f.lock.Unlock()
klog.V(1).Infof("set feature gate emulationVersion to %s", emulationVersion.String())
// Copy existing state
enabledRaw := map[string]bool{}
for k, v := range f.enabledRaw.Load().(map[string]bool) {
enabledRaw[k] = v
}
// enabled map should be reset whenever emulationVersion is changed.
enabled := map[Feature]bool{}
errs := f.unsafeSetFromMap(enabled, enabledRaw, emulationVersion)
queriedFeatures := f.queriedFeatures.Load().(map[Feature]struct{})
known := f.known.Load().(map[Feature]VersionedSpecs)
for feature := range queriedFeatures {
newVal := featureEnabled(feature, enabled, known, emulationVersion)
oldVal := featureEnabled(feature, f.enabled.Load().(map[Feature]bool), known, f.EmulationVersion())
if newVal != oldVal {
klog.Warningf("SetEmulationVersion will change already queried feature:%s from %v to %v", feature, oldVal, newVal)
}
}
panic(fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName))
if len(errs) == 0 {
// Persist changes
f.enabled.Store(enabled)
f.emulationVersion.Store(emulationVersion)
f.queriedFeatures.Store(map[Feature]struct{}{})
}
return utilerrors.NewAggregate(errs)
}
func (f *featureGate) EmulationVersion() *version.Version {
return f.emulationVersion.Load()
}
// featureSpec returns the featureSpec at the EmulationVersion if the key exists, an error otherwise.
// This is useful to keep multiple implementations of a feature based on the PreRelease or Version info.
func (f *featureGate) featureSpec(key Feature) (FeatureSpec, error) {
if v, ok := f.known.Load().(map[Feature]VersionedSpecs)[key]; ok {
featureSpec := f.featureSpecAtEmulationVersion(v)
return *featureSpec, nil
}
return FeatureSpec{}, fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName)
}
func (f *featureGate) unsafeRecordQueried(key Feature) {
queriedFeatures := map[Feature]struct{}{}
for k := range f.queriedFeatures.Load().(map[Feature]struct{}) {
queriedFeatures[k] = struct{}{}
}
if _, ok := queriedFeatures[key]; ok {
return
}
queriedFeatures[key] = struct{}{}
f.queriedFeatures.Store(queriedFeatures)
}
func featureEnabled(key Feature, enabled map[Feature]bool, known map[Feature]VersionedSpecs, emulationVersion *version.Version) bool {
// check explicitly set enabled list
if v, ok := enabled[key]; ok {
return v
}
if v, ok := known[key]; ok {
return featureSpecAtEmulationVersion(v, emulationVersion).Default
}
panic(fmt.Errorf("feature %q is not registered in FeatureGate", key))
}
// Enabled returns true if the key is enabled. If the key is not known, this call will panic.
func (f *featureGate) Enabled(key Feature) bool {
// TODO: ideally we should lock the feature gate in this call to be safe, need to evaluate how much performance impact locking would have.
v := featureEnabled(key, f.enabled.Load().(map[Feature]bool), f.known.Load().(map[Feature]VersionedSpecs), f.EmulationVersion())
f.unsafeRecordQueried(key)
return v
}
func (f *featureGate) featureSpecAtEmulationVersion(v VersionedSpecs) *FeatureSpec {
return featureSpecAtEmulationVersion(v, f.EmulationVersion())
}
func featureSpecAtEmulationVersion(v VersionedSpecs, emulationVersion *version.Version) *FeatureSpec {
i := len(v) - 1
for ; i >= 0; i-- {
if v[i].Version.GreaterThan(emulationVersion) {
continue
}
return &v[i]
}
return &FeatureSpec{
Default: false,
PreRelease: PreAlpha,
Version: version.MajorMinor(0, 0),
}
}
// Close sets closed to true, and prevents subsequent calls to Add
func (f *featureGate) Close() {
f.lock.Lock()
f.closed = true
f.lock.Unlock()
}
// AddFlag adds a flag for setting global feature gates to the specified FlagSet.
func (f *featureGate) AddFlag(fs *pflag.FlagSet) {
f.lock.Lock()
// TODO(mtaufen): Shouldn't we just close it on the first Set/SetFromMap instead?
// Not all components expose a feature gates flag using this AddFlag method, and
// in the future, all components will completely stop exposing a feature gates flag,
// in favor of componentconfig.
f.closed = true
f.lock.Unlock()
f.Close()
known := f.KnownFeatures()
fs.Var(f, flagName, ""+
@@ -377,32 +642,52 @@ func (f *featureGate) AddMetrics() {
}
// KnownFeatures returns a slice of strings describing the FeatureGate's known features.
// Deprecated and GA features are hidden from the list.
// preAlpha, Deprecated and GA features are hidden from the list.
func (f *featureGate) KnownFeatures() []string {
var known []string
for k, v := range f.known.Load().(map[Feature]FeatureSpec) {
if v.PreRelease == GA || v.PreRelease == Deprecated {
for k, v := range f.known.Load().(map[Feature]VersionedSpecs) {
if k == "AllAlpha" || k == "AllBeta" {
known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, v[0].PreRelease, v[0].Default))
continue
}
known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, v.PreRelease, v.Default))
featureSpec := f.featureSpecAtEmulationVersion(v)
if featureSpec.PreRelease == GA || featureSpec.PreRelease == Deprecated || featureSpec.PreRelease == PreAlpha {
continue
}
known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, featureSpec.PreRelease, featureSpec.Default))
}
sort.Strings(known)
return known
}
// CopyKnownFeatures returns a partial copy of the FeatureGate object, with all the known features and overrides.
// This is useful for creating a new instance of feature gate without inheriting all the enabled configurations of the base feature gate.
func (f *featureGate) CopyKnownFeatures() MutableVersionedFeatureGate {
fg := NewVersionedFeatureGate(f.EmulationVersion())
known := f.GetAllVersioned()
fg.known.Store(known)
return fg
}
// DeepCopy returns a deep copy of the FeatureGate object, such that gates can be
// set on the copy without mutating the original. This is useful for validating
// config against potential feature gate changes before committing those changes.
func (f *featureGate) DeepCopy() MutableFeatureGate {
func (f *featureGate) DeepCopy() MutableVersionedFeatureGate {
f.lock.Lock()
defer f.lock.Unlock()
// Copy existing state.
known := map[Feature]FeatureSpec{}
for k, v := range f.known.Load().(map[Feature]FeatureSpec) {
known := map[Feature]VersionedSpecs{}
for k, v := range f.known.Load().(map[Feature]VersionedSpecs) {
known[k] = v
}
enabled := map[Feature]bool{}
for k, v := range f.enabled.Load().(map[Feature]bool) {
enabled[k] = v
}
enabledRaw := map[string]bool{}
for k, v := range f.enabledRaw.Load().(map[string]bool) {
enabledRaw[k] = v
}
// Construct a new featureGate around the copied state.
// Note that specialFeatures is treated as immutable by convention,
@@ -411,9 +696,48 @@ func (f *featureGate) DeepCopy() MutableFeatureGate {
special: specialFeatures,
closed: f.closed,
}
fg.emulationVersion.Store(f.EmulationVersion())
fg.known.Store(known)
fg.enabled.Store(enabled)
fg.enabledRaw.Store(enabledRaw)
fg.queriedFeatures.Store(map[Feature]struct{}{})
return fg
}
// ExplicitlySet returns true if the feature value is explicitly set instead of
// being derived from the default values or special features.
func (f *featureGate) ExplicitlySet(name Feature) bool {
enabledRaw := f.enabledRaw.Load().(map[string]bool)
_, ok := enabledRaw[string(name)]
return ok
}
// ResetFeatureValueToDefault resets the value of the feature back to the default value.
func (f *featureGate) ResetFeatureValueToDefault(name Feature) error {
f.lock.Lock()
defer f.lock.Unlock()
enabled := map[Feature]bool{}
for k, v := range f.enabled.Load().(map[Feature]bool) {
enabled[k] = v
}
enabledRaw := map[string]bool{}
for k, v := range f.enabledRaw.Load().(map[string]bool) {
enabledRaw[k] = v
}
_, inEnabled := enabled[name]
if inEnabled {
delete(enabled, name)
}
_, inEnabledRaw := enabledRaw[string(name)]
if inEnabledRaw {
delete(enabledRaw, string(name))
}
// some features could be in enabled map but not enabledRaw map,
// for example some Alpha feature when AllAlpha is set.
if inEnabledRaw && !inEnabled {
return fmt.Errorf("feature:%s was explicitly set, but not in enabled map", name)
}
f.enabled.Store(enabled)
f.enabledRaw.Store(enabledRaw)
return nil
}

View File

@@ -20,14 +20,16 @@ import (
"fmt"
"strings"
"sync"
"testing"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/component-base/featuregate"
)
var (
overrideLock sync.Mutex
featureFlagOverride map[featuregate.Feature]string
overrideLock sync.Mutex
featureFlagOverride map[featuregate.Feature]string
emulationVersionOverride string
emulationVersionOverrideValue *version.Version
)
func init() {
@@ -42,10 +44,12 @@ func init() {
// Example use:
//
// featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.<FeatureName>, true)
func SetFeatureGateDuringTest(tb testing.TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) {
func SetFeatureGateDuringTest(tb TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) {
tb.Helper()
detectParallelOverrideCleanup := detectParallelOverride(tb, f)
originalValue := gate.Enabled(f)
originalEmuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion()
originalExplicitlySet := gate.(featuregate.MutableVersionedFeatureGate).ExplicitlySet(f)
// Specially handle AllAlpha and AllBeta
if f == "AllAlpha" || f == "AllBeta" {
@@ -67,13 +71,46 @@ func SetFeatureGateDuringTest(tb testing.TB, gate featuregate.FeatureGate, f fea
tb.Cleanup(func() {
tb.Helper()
detectParallelOverrideCleanup()
if err := gate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, originalValue)); err != nil {
tb.Errorf("error restoring %s=%v: %v", f, originalValue, err)
emuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion()
if !emuVer.EqualTo(originalEmuVer) {
tb.Fatalf("change of feature gate emulation version from %s to %s in the chain of SetFeatureGateDuringTest is not allowed\nuse SetFeatureGateEmulationVersionDuringTest to change emulation version in tests",
originalEmuVer.String(), emuVer.String())
}
if originalExplicitlySet {
if err := gate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, originalValue)); err != nil {
tb.Errorf("error restoring %s=%v: %v", f, originalValue, err)
}
} else {
if err := gate.(featuregate.MutableVersionedFeatureGate).ResetFeatureValueToDefault(f); err != nil {
tb.Errorf("error restoring %s=%v: %v", f, originalValue, err)
}
}
})
}
func detectParallelOverride(tb testing.TB, f featuregate.Feature) func() {
// SetFeatureGateEmulationVersionDuringTest sets the specified gate to the specified emulation version for duration of the test.
// Fails when it detects second call to set a different emulation version or is unable to set or restore emulation version.
// WARNING: Can leak set variable when called in test calling t.Parallel(), however second attempt to set a different emulation version will cause fatal.
// Example use:
// featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31"))
func SetFeatureGateEmulationVersionDuringTest(tb TB, gate featuregate.FeatureGate, ver *version.Version) {
tb.Helper()
detectParallelOverrideCleanup := detectParallelOverrideEmulationVersion(tb, ver)
originalEmuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion()
if err := gate.(featuregate.MutableVersionedFeatureGate).SetEmulationVersion(ver); err != nil {
tb.Fatalf("failed to set emulation version to %s during test", ver.String())
}
tb.Cleanup(func() {
tb.Helper()
detectParallelOverrideCleanup()
if err := gate.(featuregate.MutableVersionedFeatureGate).SetEmulationVersion(originalEmuVer); err != nil {
tb.Fatalf("failed to restore emulation version to %s during test", originalEmuVer.String())
}
})
}
func detectParallelOverride(tb TB, f featuregate.Feature) func() {
tb.Helper()
overrideLock.Lock()
defer overrideLock.Unlock()
@@ -94,7 +131,44 @@ func detectParallelOverride(tb testing.TB, f featuregate.Feature) func() {
}
}
func sameTestOrSubtest(tb testing.TB, testName string) bool {
func detectParallelOverrideEmulationVersion(tb TB, ver *version.Version) func() {
tb.Helper()
overrideLock.Lock()
defer overrideLock.Unlock()
beforeOverrideTestName := emulationVersionOverride
beforeOverrideValue := emulationVersionOverrideValue
if ver.EqualTo(beforeOverrideValue) {
return func() {}
}
if beforeOverrideTestName != "" && !sameTestOrSubtest(tb, beforeOverrideTestName) {
tb.Fatalf("Detected parallel setting of a feature gate emulation version by both %q and %q", beforeOverrideTestName, tb.Name())
}
emulationVersionOverride = tb.Name()
emulationVersionOverrideValue = ver
return func() {
tb.Helper()
overrideLock.Lock()
defer overrideLock.Unlock()
if afterOverrideTestName := emulationVersionOverride; afterOverrideTestName != tb.Name() {
tb.Fatalf("Detected parallel setting of a feature gate emulation version between both %q and %q", afterOverrideTestName, tb.Name())
}
emulationVersionOverride = beforeOverrideTestName
emulationVersionOverrideValue = beforeOverrideValue
}
}
func sameTestOrSubtest(tb TB, testName string) bool {
// Assumes that "/" is not used in test names.
return tb.Name() == testName || strings.HasPrefix(tb.Name(), testName+"/")
}
type TB interface {
Cleanup(func())
Error(args ...any)
Errorf(format string, args ...any)
Fatal(args ...any)
Fatalf(format string, args ...any)
Helper()
Name() string
}

View File

@@ -21,6 +21,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/component-base/featuregate"
)
@@ -71,6 +72,7 @@ func TestSpecialGates(t *gotest.T) {
expect(t, gate, before)
t.Cleanup(func() {
expect(t, gate, before)
cleanup()
})
SetFeatureGateDuringTest(t, gate, "AllAlpha", true)
@@ -156,10 +158,107 @@ func TestSetFeatureGateInTest(t *gotest.T) {
assert.True(t, gate.Enabled("feature"))
}
func TestDetectLeakToMainTest(t *gotest.T) {
t.Cleanup(func() {
featureFlagOverride = map[featuregate.Feature]string{}
func TestSpecialGatesVersioned(t *gotest.T) {
originalEmulationVersion := version.MustParse("1.31")
gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion)
err := gate.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
"alpha_default_on": {
{Version: version.MustParse("1.27"), Default: true, PreRelease: featuregate.Alpha},
},
"alpha_default_off": {
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
},
"beta_default_on": {
{Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta},
},
"beta_default_on_set_off": {
{Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta},
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
},
"beta_default_off": {
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Beta},
},
"beta_default_off_set_on": {
{Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Beta},
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
},
})
require.NoError(t, err)
require.NoError(t, gate.Set("beta_default_on_set_off=false"))
require.NoError(t, gate.Set("beta_default_off_set_on=true"))
before := map[featuregate.Feature]bool{
"AllAlpha": false,
"AllBeta": false,
"alpha_default_on": true,
"alpha_default_off": false,
"beta_default_on": true,
"beta_default_on_set_off": false,
"beta_default_off": false,
"beta_default_off_set_on": true,
}
expect(t, gate, before)
t.Cleanup(func() {
expect(t, gate, before)
cleanup()
})
t.Run("OverwriteInSubtest", func(t *gotest.T) {
SetFeatureGateDuringTest(t, gate, "AllAlpha", true)
expect(t, gate, map[featuregate.Feature]bool{
"AllAlpha": true,
"AllBeta": false,
"alpha_default_on": true,
"alpha_default_off": true,
"beta_default_on": true,
"beta_default_on_set_off": false,
"beta_default_off": false,
"beta_default_off_set_on": true,
})
SetFeatureGateDuringTest(t, gate, "AllBeta", true)
expect(t, gate, map[featuregate.Feature]bool{
"AllAlpha": true,
"AllBeta": true,
"alpha_default_on": true,
"alpha_default_off": true,
"beta_default_on": true,
"beta_default_on_set_off": true,
"beta_default_off": true,
"beta_default_off_set_on": true,
})
})
expect(t, gate, before)
t.Run("OverwriteInSubtestAtDifferentVersion", func(t *gotest.T) {
SetFeatureGateEmulationVersionDuringTest(t, gate, version.MustParse("1.28"))
SetFeatureGateDuringTest(t, gate, "AllAlpha", true)
expect(t, gate, map[featuregate.Feature]bool{
"AllAlpha": true,
"AllBeta": false,
"alpha_default_on": true,
"alpha_default_off": true,
"beta_default_on": false,
"beta_default_on_set_off": true,
"beta_default_off": false,
"beta_default_off_set_on": true,
})
})
}
func TestDetectLeakToMainTest(t *gotest.T) {
t.Cleanup(cleanup)
gate := featuregate.NewFeatureGate()
err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{
"feature": {PreRelease: featuregate.Alpha, Default: false},
@@ -183,9 +282,7 @@ func TestDetectLeakToMainTest(t *gotest.T) {
}
func TestDetectLeakToOtherSubtest(t *gotest.T) {
t.Cleanup(func() {
featureFlagOverride = map[featuregate.Feature]string{}
})
t.Cleanup(cleanup)
gate := featuregate.NewFeatureGate()
err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{
"feature": {PreRelease: featuregate.Alpha, Default: false},
@@ -211,9 +308,7 @@ func TestDetectLeakToOtherSubtest(t *gotest.T) {
}
func TestCannotDetectLeakFromSubtest(t *gotest.T) {
t.Cleanup(func() {
featureFlagOverride = map[featuregate.Feature]string{}
})
t.Cleanup(cleanup)
gate := featuregate.NewFeatureGate()
err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{
"feature": {PreRelease: featuregate.Alpha, Default: false},
@@ -230,6 +325,107 @@ func TestCannotDetectLeakFromSubtest(t *gotest.T) {
assert.True(t, gate.Enabled("feature"))
}
func TestCannotDetectLeakFromTwoSubtestsWithDifferentFeatures(t *gotest.T) {
t.Cleanup(cleanup)
gate := featuregate.NewFeatureGate()
err := gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{
"feature1": {PreRelease: featuregate.Alpha, Default: false},
"feature2": {PreRelease: featuregate.Alpha, Default: false},
})
require.NoError(t, err)
assert.False(t, gate.Enabled("feature1"))
assert.False(t, gate.Enabled("feature2"))
subtestName := "Subtest"
// Subtest setting feature gate and calling parallel will leak it out
t.Run(subtestName, func(t *gotest.T) {
SetFeatureGateDuringTest(t, gate, "feature1", true)
t.Parallel()
assert.True(t, gate.Enabled("feature1"))
assert.False(t, gate.Enabled("feature2"))
})
// Leaked true
assert.True(t, gate.Enabled("feature1"))
assert.False(t, gate.Enabled("feature2"))
// Add suffix to name to prevent tests with the same prefix.
t.Run(subtestName+"Suffix", func(t *gotest.T) {
// Leaked true
assert.True(t, gate.Enabled("feature1"))
assert.False(t, gate.Enabled("feature2"))
SetFeatureGateDuringTest(t, gate, "feature2", true)
assert.True(t, gate.Enabled("feature1"))
assert.True(t, gate.Enabled("feature2"))
})
}
func TestDetectEmulationVersionLeakToMainTest(t *gotest.T) {
t.Cleanup(cleanup)
originalEmulationVersion := version.MustParse("1.31")
newEmulationVersion := version.MustParse("1.30")
gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion)
assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion))
// Subtest setting feature gate and calling parallel will leak it out
t.Run("LeakingSubtest", func(t *gotest.T) {
fakeT := &ignoreFatalT{T: t}
SetFeatureGateEmulationVersionDuringTest(fakeT, gate, newEmulationVersion)
// Calling t.Parallel in subtest will resume the main test body
t.Parallel()
// Leaked from main test
assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion))
})
// Leaked from subtest
assert.True(t, gate.EmulationVersion().EqualTo(newEmulationVersion))
fakeT := &ignoreFatalT{T: t}
SetFeatureGateEmulationVersionDuringTest(fakeT, gate, originalEmulationVersion)
assert.True(t, fakeT.fatalRecorded)
}
func TestNoLeakFromSameEmulationVersionToMainTest(t *gotest.T) {
t.Cleanup(cleanup)
originalEmulationVersion := version.MustParse("1.31")
newEmulationVersion := version.MustParse("1.31")
gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion)
assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion))
// Subtest setting feature gate and calling parallel will leak it out
t.Run("LeakingSubtest", func(t *gotest.T) {
SetFeatureGateEmulationVersionDuringTest(t, gate, newEmulationVersion)
// Calling t.Parallel in subtest will resume the main test body
t.Parallel()
// Leaked from main test
assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion))
})
// Leaked from subtest
assert.True(t, gate.EmulationVersion().EqualTo(newEmulationVersion))
SetFeatureGateEmulationVersionDuringTest(t, gate, originalEmulationVersion)
}
func TestDetectEmulationVersionLeakToOtherSubtest(t *gotest.T) {
t.Cleanup(cleanup)
originalEmulationVersion := version.MustParse("1.31")
newEmulationVersion := version.MustParse("1.30")
gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion)
assert.True(t, gate.EmulationVersion().EqualTo(originalEmulationVersion))
subtestName := "Subtest"
// Subtest setting feature gate and calling parallel will leak it out
t.Run(subtestName, func(t *gotest.T) {
fakeT := &ignoreFatalT{T: t}
SetFeatureGateEmulationVersionDuringTest(fakeT, gate, newEmulationVersion)
t.Parallel()
})
// Add suffix to name to prevent tests with the same prefix.
t.Run(subtestName+"Suffix", func(t *gotest.T) {
// Leaked newEmulationVersion
assert.True(t, gate.EmulationVersion().EqualTo(newEmulationVersion))
fakeT := &ignoreFatalT{T: t}
SetFeatureGateEmulationVersionDuringTest(fakeT, gate, originalEmulationVersion)
assert.True(t, fakeT.fatalRecorded)
})
}
type ignoreFatalT struct {
*gotest.T
fatalRecorded bool
@@ -248,3 +444,9 @@ func (f *ignoreFatalT) Fatalf(format string, args ...any) {
f.fatalRecorded = true
f.T.Logf("[IGNORED] "+format, args...)
}
func cleanup() {
featureFlagOverride = map[featuregate.Feature]string{}
emulationVersionOverride = ""
emulationVersionOverrideValue = nil
}

View File

@@ -61,3 +61,10 @@ var (
buildDate = "1970-01-01T00:00:00Z" // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ')
)
const (
// DefaultKubeBinaryVersion is the hard coded k8 binary version based on the latest K8s release.
// It is supposed to be consistent with gitMajor and gitMinor, except for local tests, where gitMajor and gitMinor are "".
// Should update for each minor release!
DefaultKubeBinaryVersion = "1.31"
)

View File

@@ -38,7 +38,6 @@ import (
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/transport"
"k8s.io/component-base/tracing"
"k8s.io/component-base/version"
v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
v1helper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper"
"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
@@ -185,8 +184,6 @@ func (cfg *Config) Complete() CompletedConfig {
// the kube aggregator wires its own discovery mechanism
// TODO eventually collapse this by extracting all of the discovery out
c.GenericConfig.EnableDiscovery = false
version := version.Get()
c.GenericConfig.Version = &version
return CompletedConfig{&c}
}
@@ -247,7 +244,7 @@ func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.Deleg
}
// used later to filter the served resource by those that have expired.
resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(*c.GenericConfig.Version)
resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(s.GenericAPIServer.EffectiveVersion.EmulationVersion())
if err != nil {
return nil, err
}

View File

@@ -31,6 +31,7 @@ import (
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/filters"
genericoptions "k8s.io/apiserver/pkg/server/options"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
"k8s.io/kube-aggregator/pkg/apiserver"
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
@@ -61,6 +62,9 @@ func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions)
cmd := &cobra.Command{
Short: "Launch a API aggregator and proxy server",
Long: "Launch a API aggregator and proxy server",
PersistentPreRunE: func(*cobra.Command, []string) error {
return utilversion.DefaultComponentGlobalsRegistry.Set()
},
RunE: func(c *cobra.Command, args []string) error {
if err := o.Complete(); err != nil {
return err
@@ -117,7 +121,7 @@ func (o AggregatorOptions) Validate(args []string) error {
// Complete fills in missing Options.
func (o *AggregatorOptions) Complete() error {
return nil
return o.ServerRunOptions.Complete()
}
// RunAggregator runs the API Aggregator.

View File

@@ -7,6 +7,7 @@ go 1.22.0
require (
github.com/google/gofuzz v1.2.0
github.com/spf13/cobra v1.8.1
github.com/stretchr/testify v1.9.0
k8s.io/apimachinery v0.0.0
k8s.io/apiserver v0.0.0
k8s.io/client-go v0.0.0
@@ -55,6 +56,7 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.19.0 // indirect
github.com/prometheus/client_model v0.6.0 // indirect
github.com/prometheus/common v0.48.0 // indirect

View File

@@ -170,6 +170,7 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

View File

@@ -18,6 +18,7 @@ limitations under the License.
// +k8s:deepcopy-gen=package
// +k8s:conversion-gen=k8s.io/sample-apiserver/pkg/apis/wardle
// +k8s:defaulter-gen=TypeMeta
// +k8s:prerelease-lifecycle-gen=true
// +groupName=wardle.example.com
// Package v1alpha1 is the v1alpha1 version of the API.

View File

@@ -19,6 +19,8 @@ package v1alpha1
import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +k8s:prerelease-lifecycle-gen:introduced=1.0
// +k8s:prerelease-lifecycle-gen:removed=1.10
// FlunderList is a list of Flunder objects.
type FlunderList struct {
@@ -47,6 +49,8 @@ type FlunderStatus struct {
// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +k8s:prerelease-lifecycle-gen:introduced=1.0
// +k8s:prerelease-lifecycle-gen:removed=1.10
type Flunder struct {
metav1.TypeMeta `json:",inline"`
@@ -59,6 +63,8 @@ type Flunder struct {
// +genclient
// +genclient:nonNamespaced
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +k8s:prerelease-lifecycle-gen:introduced=1.0
// +k8s:prerelease-lifecycle-gen:removed=1.10
type Fischer struct {
metav1.TypeMeta `json:",inline"`
@@ -71,6 +77,8 @@ type Fischer struct {
// +genclient:nonNamespaced
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +k8s:prerelease-lifecycle-gen:introduced=1.0
// +k8s:prerelease-lifecycle-gen:removed=1.10
// FischerList is a list of Fischer objects.
type FischerList struct {

View File

@@ -0,0 +1,94 @@
//go:build !ignore_autogenerated
// +build !ignore_autogenerated
/*
Copyright The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Code generated by prerelease-lifecycle-gen. DO NOT EDIT.
package v1alpha1
// APILifecycleIntroduced is an autogenerated function, returning the release in which the API struct was introduced as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
func (in *Fischer) APILifecycleIntroduced() (major, minor int) {
return 1, 0
}
// APILifecycleDeprecated is an autogenerated function, returning the release in which the API struct was or will be deprecated as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:deprecated" tags in types.go or "k8s:prerelease-lifecycle-gen:introduced" plus three minor.
func (in *Fischer) APILifecycleDeprecated() (major, minor int) {
return 1, 3
}
// APILifecycleRemoved is an autogenerated function, returning the release in which the API is no longer served as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:removed" tags in types.go or "k8s:prerelease-lifecycle-gen:deprecated" plus three minor.
func (in *Fischer) APILifecycleRemoved() (major, minor int) {
return 1, 10
}
// APILifecycleIntroduced is an autogenerated function, returning the release in which the API struct was introduced as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
func (in *FischerList) APILifecycleIntroduced() (major, minor int) {
return 1, 0
}
// APILifecycleDeprecated is an autogenerated function, returning the release in which the API struct was or will be deprecated as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:deprecated" tags in types.go or "k8s:prerelease-lifecycle-gen:introduced" plus three minor.
func (in *FischerList) APILifecycleDeprecated() (major, minor int) {
return 1, 3
}
// APILifecycleRemoved is an autogenerated function, returning the release in which the API is no longer served as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:removed" tags in types.go or "k8s:prerelease-lifecycle-gen:deprecated" plus three minor.
func (in *FischerList) APILifecycleRemoved() (major, minor int) {
return 1, 10
}
// APILifecycleIntroduced is an autogenerated function, returning the release in which the API struct was introduced as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
func (in *Flunder) APILifecycleIntroduced() (major, minor int) {
return 1, 0
}
// APILifecycleDeprecated is an autogenerated function, returning the release in which the API struct was or will be deprecated as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:deprecated" tags in types.go or "k8s:prerelease-lifecycle-gen:introduced" plus three minor.
func (in *Flunder) APILifecycleDeprecated() (major, minor int) {
return 1, 3
}
// APILifecycleRemoved is an autogenerated function, returning the release in which the API is no longer served as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:removed" tags in types.go or "k8s:prerelease-lifecycle-gen:deprecated" plus three minor.
func (in *Flunder) APILifecycleRemoved() (major, minor int) {
return 1, 10
}
// APILifecycleIntroduced is an autogenerated function, returning the release in which the API struct was introduced as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
func (in *FlunderList) APILifecycleIntroduced() (major, minor int) {
return 1, 0
}
// APILifecycleDeprecated is an autogenerated function, returning the release in which the API struct was or will be deprecated as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:deprecated" tags in types.go or "k8s:prerelease-lifecycle-gen:introduced" plus three minor.
func (in *FlunderList) APILifecycleDeprecated() (major, minor int) {
return 1, 3
}
// APILifecycleRemoved is an autogenerated function, returning the release in which the API is no longer served as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:removed" tags in types.go or "k8s:prerelease-lifecycle-gen:deprecated" plus three minor.
func (in *FlunderList) APILifecycleRemoved() (major, minor int) {
return 1, 10
}

View File

@@ -21,7 +21,6 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/version"
"k8s.io/apiserver/pkg/registry/rest"
genericapiserver "k8s.io/apiserver/pkg/server"
@@ -37,7 +36,8 @@ var (
Scheme = runtime.NewScheme()
// Codecs provides methods for retrieving codecs and serializers for specific
// versions and content types.
Codecs = serializer.NewCodecFactory(Scheme)
Codecs = serializer.NewCodecFactory(Scheme)
WardleComponentName = "wardle"
)
func init() {
@@ -91,11 +91,6 @@ func (cfg *Config) Complete() CompletedConfig {
&cfg.ExtraConfig,
}
c.GenericConfig.Version = &version.Info{
Major: "1",
Minor: "0",
}
return CompletedConfig{&c}
}

View File

@@ -27,11 +27,15 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/endpoints/openapi"
genericapiserver "k8s.io/apiserver/pkg/server"
genericoptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/component-base/featuregate"
"k8s.io/sample-apiserver/pkg/admission/plugin/banflunder"
"k8s.io/sample-apiserver/pkg/admission/wardleinitializer"
"k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1"
@@ -55,6 +59,16 @@ type WardleServerOptions struct {
AlternateDNS []string
}
func wardleEmulationVersionToKubeEmulationVersion(ver *version.Version) *version.Version {
if ver.Major() != 1 {
return nil
}
kubeVer := utilversion.DefaultKubeEffectiveVersion().BinaryVersion()
// "1.1" maps to kubeVer
offset := int(ver.Minor()) - 1
return kubeVer.OffsetMinor(offset)
}
// NewWardleServerOptions returns a new WardleServerOptions
func NewWardleServerOptions(out, errOut io.Writer) *WardleServerOptions {
o := &WardleServerOptions{
@@ -77,6 +91,9 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti
cmd := &cobra.Command{
Short: "Launch a wardle API server",
Long: "Launch a wardle API server",
PersistentPreRunE: func(*cobra.Command, []string) error {
return utilversion.DefaultComponentGlobalsRegistry.Set()
},
RunE: func(c *cobra.Command, args []string) error {
if err := o.Complete(); err != nil {
return err
@@ -94,7 +111,21 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti
flags := cmd.Flags()
o.RecommendedOptions.AddFlags(flags)
utilfeature.DefaultMutableFeatureGate.AddFlag(flags)
wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2")
wardleFeatureGate := utilfeature.DefaultFeatureGate.CopyKnownFeatures()
utilruntime.Must(wardleFeatureGate.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
"BanFlunder": {
{Version: version.MustParse("1.2"), Default: true, PreRelease: featuregate.GA},
{Version: version.MustParse("1.1"), Default: false, PreRelease: featuregate.Beta},
{Version: version.MustParse("1.0"), Default: false, PreRelease: featuregate.Alpha},
},
}))
utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate))
_, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.SetEmulationVersionMapping(apiserver.WardleComponentName, utilversion.DefaultKubeComponent, wardleEmulationVersionToKubeEmulationVersion))
utilversion.DefaultComponentGlobalsRegistry.AddFlags(flags)
return cmd
}
@@ -103,17 +134,19 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti
func (o WardleServerOptions) Validate(args []string) error {
errors := []error{}
errors = append(errors, o.RecommendedOptions.Validate()...)
errors = append(errors, utilversion.DefaultComponentGlobalsRegistry.Validate()...)
return utilerrors.NewAggregate(errors)
}
// Complete fills in fields required to have valid data
func (o *WardleServerOptions) Complete() error {
// register admission plugins
banflunder.Register(o.RecommendedOptions.Admission.Plugins)
// add admission plugins to the RecommendedPluginOrder
o.RecommendedOptions.Admission.RecommendedPluginOrder = append(o.RecommendedOptions.Admission.RecommendedPluginOrder, "BanFlunder")
if utilversion.DefaultComponentGlobalsRegistry.FeatureGateFor(apiserver.WardleComponentName).Enabled("BanFlunder") {
// register admission plugins
banflunder.Register(o.RecommendedOptions.Admission.Plugins)
// add admission plugins to the RecommendedPluginOrder
o.RecommendedOptions.Admission.RecommendedPluginOrder = append(o.RecommendedOptions.Admission.RecommendedPluginOrder, "BanFlunder")
}
return nil
}
@@ -144,6 +177,9 @@ func (o *WardleServerOptions) Config() (*apiserver.Config, error) {
serverConfig.OpenAPIV3Config.Info.Title = "Wardle"
serverConfig.OpenAPIV3Config.Info.Version = "0.1"
serverConfig.FeatureGate = utilversion.DefaultComponentGlobalsRegistry.FeatureGateFor(apiserver.WardleComponentName)
serverConfig.EffectiveVersion = utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(apiserver.WardleComponentName)
if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil {
return nil, err
}

View File

@@ -0,0 +1,58 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package server
import (
"testing"
"k8s.io/apimachinery/pkg/util/version"
utilversion "k8s.io/apiserver/pkg/util/version"
"github.com/stretchr/testify/assert"
)
func TestWardleEmulationVersionToKubeEmulationVersion(t *testing.T) {
defaultKubeEffectiveVersion := utilversion.DefaultKubeEffectiveVersion()
testCases := []struct {
desc string
wardleEmulationVer *version.Version
expectedKubeEmulationVer *version.Version
}{
{
desc: "same version as than kube binary",
wardleEmulationVer: version.MajorMinor(1, 1),
expectedKubeEmulationVer: defaultKubeEffectiveVersion.BinaryVersion(),
},
{
desc: "1 version higher than kube binary",
wardleEmulationVer: version.MajorMinor(1, 2),
expectedKubeEmulationVer: defaultKubeEffectiveVersion.BinaryVersion().OffsetMinor(1),
},
{
desc: "no mapping",
wardleEmulationVer: version.MajorMinor(2, 10),
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
mappedKubeEmulationVer := wardleEmulationVersionToKubeEmulationVersion(tc.wardleEmulationVer)
assert.True(t, mappedKubeEmulationVer.EqualTo(tc.expectedKubeEmulationVer))
})
}
}

View File

@@ -32,6 +32,7 @@ import (
"testing"
"time"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -55,16 +56,20 @@ import (
"k8s.io/apimachinery/pkg/watch"
"k8s.io/apiserver/pkg/endpoints/handlers"
"k8s.io/apiserver/pkg/storage/storagebackend"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/discovery/cached/memory"
"k8s.io/client-go/dynamic"
clientset "k8s.io/client-go/kubernetes"
appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
"k8s.io/client-go/metadata"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/restmapper"
"k8s.io/client-go/tools/pager"
"k8s.io/klog/v2"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
"k8s.io/kubernetes/pkg/controlplane"
"k8s.io/kubernetes/pkg/kubeapiserver"
"k8s.io/kubernetes/test/integration"
"k8s.io/kubernetes/test/integration/etcd"
"k8s.io/kubernetes/test/integration/framework"
@@ -2929,6 +2934,340 @@ func TestDedupOwnerReferences(t *testing.T) {
}
}
func TestEmulatedStorageVersion(t *testing.T) {
validVap := &admissionregistrationv1.ValidatingAdmissionPolicy{
Spec: admissionregistrationv1.ValidatingAdmissionPolicySpec{
MatchConstraints: &admissionregistrationv1.MatchResources{
ResourceRules: []admissionregistrationv1.NamedRuleWithOperations{
{
ResourceNames: []string{"foo"},
RuleWithOperations: admissionregistrationv1.RuleWithOperations{
Operations: []admissionregistrationv1.OperationType{"CREATE"},
Rule: admissionregistrationv1.Rule{
APIGroups: []string{"*"},
APIVersions: []string{"*"},
Resources: []string{"*"},
},
},
},
},
},
Validations: []admissionregistrationv1.Validation{
{
Expression: "true",
Message: "always valid",
},
},
},
}
type testCase struct {
name string
emulatedVersion string
gvr schema.GroupVersionResource
object runtime.Object
expectedStorageVersion schema.GroupVersion
}
cases := []testCase{
{
name: "vap first ga release",
emulatedVersion: "1.30",
gvr: schema.GroupVersionResource{
Group: "admissionregistration.k8s.io",
Version: "v1",
Resource: "validatingadmissionpolicies",
},
object: validVap,
expectedStorageVersion: schema.GroupVersion{
Group: "admissionregistration.k8s.io",
Version: "v1beta1",
},
},
{
name: "vap after ga release",
emulatedVersion: "1.31",
gvr: schema.GroupVersionResource{
Group: "admissionregistration.k8s.io",
Version: "v1beta1",
Resource: "validatingadmissionpolicies",
},
object: validVap,
expectedStorageVersion: schema.GroupVersion{
Group: "admissionregistration.k8s.io",
Version: "v1",
},
},
}
// Group cases by their emulated version
groupedCases := map[string][]testCase{}
for _, c := range cases {
groupedCases[c.emulatedVersion] = append(groupedCases[c.emulatedVersion], c)
}
for emulatedVersion, cases := range groupedCases {
t.Run(emulatedVersion, func(t *testing.T) {
server := kubeapiservertesting.StartTestServerOrDie(
t, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: emulatedVersion},
[]string{"--emulated-version=kube=" + emulatedVersion, `--storage-media-type=application/json`}, framework.SharedEtcd())
defer server.TearDownFn()
client := clientset.NewForConfigOrDie(server.ClientConfig)
dynamicClient := dynamic.NewForConfigOrDie(server.ClientConfig)
// create test namespace
testNamespace := "test-emulated-storage-version"
_, err := client.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: testNamespace,
},
}, metav1.CreateOptions{})
if err != nil {
t.Fatalf("failed to create test ns: %v", err)
}
restMapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(client.Discovery()))
for i, c := range cases {
t.Run(c.name, func(t *testing.T) {
gvk, err := restMapper.KindFor(c.gvr)
if err != nil {
t.Fatalf("failed to get GVK: %v", err)
}
c.object.GetObjectKind().SetGroupVersionKind(gvk)
mapping, err := restMapper.RESTMapping(gvk.GroupKind(), gvk.Version)
if err != nil {
t.Fatalf("failed to get RESTMapping: %v", err)
} else if mapping.Resource != c.gvr {
t.Fatalf("expected resource %v, got %v", c.gvr, mapping.Resource)
}
asUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(c.object)
if err != nil {
t.Fatalf("failed to convert object to unstructured: %v", err)
}
uns := &unstructured.Unstructured{
Object: asUnstructured,
}
uns.SetName(fmt.Sprintf("test-object%d", i))
ns := testNamespace
if mapping.Scope.Name() == meta.RESTScopeNameRoot {
ns = ""
}
// create object
created, err := dynamicClient.Resource(c.gvr).Namespace(ns).Create(context.TODO(), uns, metav1.CreateOptions{})
if err != nil {
t.Fatalf("failed to create object: %v", err)
}
// Fetch object from ETCD
// Use this poor man's way to get the etcd path. This wont
// work for all resources, but should work for most we want
// to test against
resourcePrefix := mapping.Resource.Resource
if special, ok := kubeapiserver.SpecialDefaultResourcePrefixes[c.gvr.GroupResource()]; ok {
resourcePrefix = special
}
etcdPathComponents := []string{
"/",
server.EtcdStoragePrefix,
resourcePrefix,
}
if len(ns) > 0 {
etcdPathComponents = append(etcdPathComponents, ns)
}
etcdPathComponents = append(etcdPathComponents, created.GetName())
etcdPath := path.Join(etcdPathComponents...)
fetched, err := server.EtcdClient.Get(context.TODO(), etcdPath)
if err != nil {
t.Fatalf("failed to fetch object from etcd: %v", err)
} else if fetched.More || fetched.Count != 1 || len(fetched.Kvs) != 1 {
t.Fatalf("unexpected fetched response: %v", fetched)
}
storedObject := &metav1.PartialObjectMetadata{}
err = json.Unmarshal(fetched.Kvs[0].Value, storedObject)
if err != nil {
t.Fatalf("failed to decode object: %v", err)
} else if storedObject.GroupVersionKind().GroupVersion() != c.expectedStorageVersion {
t.Fatalf("expected storage version %s, got %s", c.expectedStorageVersion.String(), storedObject.GroupVersionKind().GroupVersion().String())
}
})
}
})
}
}
// TestAllowedEmulationVersions tests the TestServer can start without problem for all allowed emulation versions.
func TestAllowedEmulationVersions(t *testing.T) {
tcs := []struct {
name string
emulationVersion string
}{
{
name: "default",
emulationVersion: utilversion.DefaultKubeEffectiveVersion().EmulationVersion().String(),
},
}
for _, tc := range tcs {
t.Run(tc.emulationVersion, func(t *testing.T) {
server := kubeapiservertesting.StartTestServerOrDie(t, nil,
[]string{fmt.Sprintf("--emulated-version=kube=%s", tc.emulationVersion)}, framework.SharedEtcd())
defer server.TearDownFn()
rt, err := restclient.TransportFor(server.ClientConfig)
if err != nil {
t.Fatal(err)
}
req, err := http.NewRequest("GET", server.ClientConfig.Host+"/", nil)
if err != nil {
t.Fatal(err)
}
resp, err := rt.RoundTrip(req)
if err != nil {
t.Fatal(err)
}
expectedStatusCode := 200
if resp.StatusCode != expectedStatusCode {
t.Errorf("expect status code: %d, got : %d\n", expectedStatusCode, resp.StatusCode)
}
defer func() {
_ = resp.Body.Close()
}()
})
}
}
func TestEnableEmulationVersion(t *testing.T) {
server := kubeapiservertesting.StartTestServerOrDie(t,
&kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"},
[]string{"--emulated-version=kube=1.31"}, framework.SharedEtcd())
defer server.TearDownFn()
rt, err := restclient.TransportFor(server.ClientConfig)
if err != nil {
t.Fatal(err)
}
tcs := []struct {
path string
expectedStatusCode int
}{
{
path: "/",
expectedStatusCode: 200,
},
{
path: "/apis/apps/v1/deployments",
expectedStatusCode: 200,
},
{
path: "/apis/flowcontrol.apiserver.k8s.io/v1/flowschemas",
expectedStatusCode: 200,
},
{
path: "/apis/flowcontrol.apiserver.k8s.io/v1beta1/flowschemas", // introduced at 1.20, removed at 1.26
expectedStatusCode: 404,
},
{
path: "/apis/flowcontrol.apiserver.k8s.io/v1beta2/flowschemas", // introduced at 1.23, removed at 1.29
expectedStatusCode: 404,
},
{
path: "/apis/flowcontrol.apiserver.k8s.io/v1beta3/flowschemas", // introduced at 1.26, removed at 1.32
expectedStatusCode: 200,
},
}
for _, tc := range tcs {
t.Run(tc.path, func(t *testing.T) {
req, err := http.NewRequest("GET", server.ClientConfig.Host+tc.path, nil)
if err != nil {
t.Fatal(err)
}
resp, err := rt.RoundTrip(req)
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != tc.expectedStatusCode {
t.Errorf("expect status code: %d, got : %d\n", tc.expectedStatusCode, resp.StatusCode)
}
defer func() {
_ = resp.Body.Close()
}()
})
}
}
func TestDisableEmulationVersion(t *testing.T) {
server := kubeapiservertesting.StartTestServerOrDie(t,
&kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"},
[]string{}, framework.SharedEtcd())
defer server.TearDownFn()
rt, err := restclient.TransportFor(server.ClientConfig)
if err != nil {
t.Fatal(err)
}
tcs := []struct {
path string
expectedStatusCode int
}{
{
path: "/",
expectedStatusCode: 200,
},
{
path: "/apis/apps/v1/deployments",
expectedStatusCode: 200,
},
{
path: "/apis/flowcontrol.apiserver.k8s.io/v1/flowschemas",
expectedStatusCode: 200,
},
{
path: "/apis/flowcontrol.apiserver.k8s.io/v1beta1/flowschemas", // introduced at 1.20, removed at 1.26
expectedStatusCode: 404,
},
{
path: "/apis/flowcontrol.apiserver.k8s.io/v1beta2/flowschemas", // introduced at 1.23, removed at 1.29
expectedStatusCode: 404,
},
{
path: "/apis/flowcontrol.apiserver.k8s.io/v1beta3/flowschemas", // introduced at 1.26, removed at 1.32
expectedStatusCode: 404,
},
}
for _, tc := range tcs {
t.Run(tc.path, func(t *testing.T) {
req, err := http.NewRequest("GET", server.ClientConfig.Host+tc.path, nil)
if err != nil {
t.Fatal(err)
}
resp, err := rt.RoundTrip(req)
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != tc.expectedStatusCode {
t.Errorf("expect status code: %d, got : %d\n", tc.expectedStatusCode, resp.StatusCode)
}
defer func() {
_ = resp.Body.Close()
}()
})
}
}
type dependentClient struct {
t *testing.T
client dynamic.ResourceInterface

View File

@@ -29,6 +29,7 @@ import (
rbacapi "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/watch"
@@ -80,8 +81,8 @@ type testRESTOptionsGetter struct {
config *controlplane.Config
}
func (getter *testRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
storageConfig, err := getter.config.ControlPlane.Extra.StorageFactory.NewConfig(resource)
func (getter *testRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) {
storageConfig, err := getter.config.ControlPlane.Extra.StorageFactory.NewConfig(resource, example)
if err != nil {
return generic.RESTOptions{}, fmt.Errorf("failed to get storage: %v", err)
}

View File

@@ -48,6 +48,7 @@ import (
clientset "k8s.io/client-go/kubernetes"
"k8s.io/utils/pointer"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/component-base/version"
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
"k8s.io/kubernetes/pkg/api/legacyscheme"
@@ -65,7 +66,12 @@ func TestClient(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if e, a := version.Get(), *info; !reflect.DeepEqual(e, a) {
expectedInfo := version.Get()
kubeVersion := utilversion.DefaultKubeEffectiveVersion().BinaryVersion()
expectedInfo.Major = fmt.Sprintf("%d", kubeVersion.Major())
expectedInfo.Minor = fmt.Sprintf("%d", kubeVersion.Minor())
if e, a := expectedInfo, *info; !reflect.DeepEqual(e, a) {
t.Errorf("expected %#v, got %#v", e, a)
}

View File

@@ -1219,7 +1219,7 @@ func getRESTOptionsGetterForSecrets(t testing.TB, test *transformTest) generic.R
t.Fatal("not REST options found")
}
opts, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: "secrets"})
opts, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: "secrets"}, nil)
if err != nil {
t.Fatal(err)
}

View File

@@ -115,7 +115,9 @@ func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool,
return nil, fmt.Errorf("failed to read config file: %w", err)
}
if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(tb, nil, e.getEncryptionOptions(reload), e.storageConfig); err != nil {
if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(
tb, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "0.0"},
e.getEncryptionOptions(reload), e.storageConfig); err != nil {
e.cleanUp()
return nil, fmt.Errorf("failed to start KubeAPI server: %w", err)
}

View File

@@ -27,17 +27,25 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/apiserver/pkg/storage"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/dynamic"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
)
// TestCrossGroupStorage tests to make sure that all objects stored in an expected location in etcd can be converted/read.
func TestCrossGroupStorage(t *testing.T) {
testRegistry := utilversion.NewComponentGlobalsRegistry()
utilruntime.Must(testRegistry.Register("test", utilversion.NewEffectiveVersion("0.0"), utilfeature.DefaultFeatureGate.DeepCopy()))
apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) {
// force enable all resources so we can check storage.
opts.GenericServerRunOptions.ComponentName = "test"
opts.GenericServerRunOptions.ComponentGlobalsRegistry = testRegistry
})
defer apiServer.Cleanup()

View File

@@ -34,8 +34,10 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/dynamic"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
@@ -74,7 +76,12 @@ var allowMissingTestdataFixtures = map[schema.GroupVersionKind]bool{
func TestEtcdStoragePath(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true)
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)
testRegistry := utilversion.NewComponentGlobalsRegistry()
utilruntime.Must(testRegistry.Register("test", utilversion.NewEffectiveVersion("0.0"), feature.DefaultFeatureGate.DeepCopy()))
apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) {
opts.GenericServerRunOptions.ComponentName = "test"
opts.GenericServerRunOptions.ComponentGlobalsRegistry = testRegistry
})
defer apiServer.Cleanup()
defer dumpEtcdKVOnFailure(t, apiServer.KV)

View File

@@ -226,6 +226,18 @@ func TestAPIServiceWaitOnStart(t *testing.T) {
}
func TestAggregatedAPIServer(t *testing.T) {
t.Run("WithoutWardleFeatureGateAtV1.2", func(t *testing.T) {
testAggregatedAPIServer(t, false, "1.2")
})
t.Run("WithoutWardleFeatureGateAtV1.1", func(t *testing.T) {
testAggregatedAPIServer(t, false, "1.1")
})
t.Run("WithWardleFeatureGateAtV1.1", func(t *testing.T) {
testAggregatedAPIServer(t, true, "1.1")
})
}
func testAggregatedAPIServer(t *testing.T, enableWardleFeatureGate bool, emulationVersion string) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
t.Cleanup(cancel)
@@ -240,7 +252,7 @@ func TestAggregatedAPIServer(t *testing.T) {
// endpoints cannot have loopback IPs so we need to override the resolver itself
t.Cleanup(app.SetServiceResolverForTests(staticURLServiceResolver(fmt.Sprintf("https://127.0.0.1:%d", wardlePort))))
testServer := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true}, nil, framework.SharedEtcd())
testServer := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, BinaryVersion: "1.32"}, nil, framework.SharedEtcd())
defer testServer.TearDownFn()
kubeClientConfig := rest.CopyConfig(testServer.ClientConfig)
// force json because everything speaks it
@@ -286,13 +298,18 @@ func TestAggregatedAPIServer(t *testing.T) {
o.RecommendedOptions.SecureServing.Listener = listener
o.RecommendedOptions.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1")
wardleCmd := sampleserver.NewCommandStartWardleServer(ctx, o)
wardleCmd.SetArgs([]string{
args := []string{
"--authentication-kubeconfig", wardleToKASKubeConfigFile,
"--authorization-kubeconfig", wardleToKASKubeConfigFile,
"--etcd-servers", framework.GetEtcdURL(),
"--cert-dir", wardleCertDir,
"--kubeconfig", wardleToKASKubeConfigFile,
})
"--emulated-version", fmt.Sprintf("wardle=%s", emulationVersion),
}
if enableWardleFeatureGate {
args = append(args, "--feature-gates", "wardle:BanFlunder=true")
}
wardleCmd.SetArgs(args)
if err := wardleCmd.Execute(); err != nil {
t.Error(err)
}
@@ -386,10 +403,13 @@ func TestAggregatedAPIServer(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "panda",
},
DisallowedFlunders: []string{"badname"},
}, metav1.CreateOptions{})
if err != nil {
t.Fatal(err)
}
// clean up data after test is done
defer wardleClient.Fischers().Delete(ctx, "panda", metav1.DeleteOptions{})
fischersList, err := wardleClient.Fischers().List(ctx, metav1.ListOptions{})
if err != nil {
t.Fatal(err)
@@ -401,17 +421,41 @@ func TestAggregatedAPIServer(t *testing.T) {
t.Error("expected non-empty resource version for fischer list")
}
_, err = wardleClient.Flunders(metav1.NamespaceSystem).Create(ctx, &wardlev1alpha1.Flunder{
ObjectMeta: metav1.ObjectMeta{
Name: "badname",
},
}, metav1.CreateOptions{})
banFlunder := enableWardleFeatureGate || emulationVersion == "1.2"
if banFlunder && err == nil {
t.Fatal("expect flunder:badname not admitted when wardle feature gates are specified")
}
if !banFlunder {
if err != nil {
t.Fatal("expect flunder:badname admitted when wardle feature gates are not specified")
} else {
defer wardleClient.Flunders(metav1.NamespaceSystem).Delete(ctx, "badname", metav1.DeleteOptions{})
}
}
_, err = wardleClient.Flunders(metav1.NamespaceSystem).Create(ctx, &wardlev1alpha1.Flunder{
ObjectMeta: metav1.ObjectMeta{
Name: "panda",
},
}, metav1.CreateOptions{})
if err != nil {
t.Fatal(err)
}
defer wardleClient.Flunders(metav1.NamespaceSystem).Delete(ctx, "panda", metav1.DeleteOptions{})
flunderList, err := wardleClient.Flunders(metav1.NamespaceSystem).List(ctx, metav1.ListOptions{})
if err != nil {
t.Fatal(err)
}
if len(flunderList.Items) != 1 {
t.Errorf("expected one flunder: %#v", flunderList.Items)
expectedFlunderCount := 2
if banFlunder {
expectedFlunderCount = 1
}
if len(flunderList.Items) != expectedFlunderCount {
t.Errorf("expected %d flunder: %#v", expectedFlunderCount, flunderList.Items)
}
if len(flunderList.ResourceVersion) == 0 {
t.Error("expected non-empty resource version for flunder list")