apiserver: Add API emulation versioning.

Co-authored-by: Siyuan Zhang <sizhang@google.com>
Co-authored-by: Joe Betz <jpbetz@google.com>
Co-authored-by: Alex Zielenski <zielenski@google.com>

Signed-off-by: Siyuan Zhang <sizhang@google.com>
This commit is contained in:
Siyuan Zhang
2024-01-19 16:07:00 -08:00
parent d0579b6f9c
commit 403301bfdf
86 changed files with 3420 additions and 427 deletions

View File

@@ -26,6 +26,8 @@ import (
utilnet "k8s.io/apimachinery/pkg/util/net"
cliflag "k8s.io/component-base/cli/flag"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/component-base/featuregate"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/cluster/ports"
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
@@ -63,10 +65,10 @@ type Extra struct {
MasterCount int
}
// NewServerRunOptions creates a new ServerRunOptions object with default parameters
func NewServerRunOptions() *ServerRunOptions {
// NewServerRunOptions creates and returns ServerRunOptions according to the given featureGate and effectiveVersion of the server binary to run.
func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions {
s := ServerRunOptions{
Options: controlplaneapiserver.NewOptions(),
Options: controlplaneapiserver.NewOptions(featureGate, effectiveVersion),
CloudProvider: kubeoptions.NewCloudProviderOptions(),
Extra: Extra{

View File

@@ -31,9 +31,11 @@ import (
apiserveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/etcd3"
"k8s.io/apiserver/pkg/storage/storagebackend"
utilversion "k8s.io/apiserver/pkg/util/version"
auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics"
kapi "k8s.io/kubernetes/pkg/apis/core"
@@ -46,10 +48,15 @@ import (
func TestAddFlags(t *testing.T) {
fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
s := NewServerRunOptions()
featureGate := featuregate.NewFeatureGate()
effectiveVersion := utilversion.NewEffectiveVersion("1.32")
s := NewServerRunOptions(featureGate, effectiveVersion)
for _, f := range s.Flags().FlagSets {
fs.AddFlagSet(f)
}
featureGate.AddFlag(fs, "")
effectiveVersion.AddFlags(fs, "")
args := []string{
"--enable-admission-plugins=AlwaysDeny",
@@ -121,6 +128,7 @@ func TestAddFlags(t *testing.T) {
"--storage-backend=etcd3",
"--service-cluster-ip-range=192.168.128.0/17",
"--lease-reuse-duration-seconds=100",
"--emulated-version=1.31",
}
fs.Parse(args)
@@ -136,6 +144,8 @@ func TestAddFlags(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &apiserveroptions.AdmissionOptions{
@@ -337,4 +347,8 @@ func TestAddFlags(t *testing.T) {
if !reflect.DeepEqual(expected, s) {
t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{})))
}
if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" {
t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31")
}
}

View File

@@ -36,6 +36,7 @@ import (
serverstorage "k8s.io/apiserver/pkg/server/storage"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/apiserver/pkg/util/notfoundhandler"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/apiserver/pkg/util/webhook"
clientgoinformers "k8s.io/client-go/informers"
"k8s.io/client-go/rest"
@@ -63,7 +64,10 @@ func init() {
// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
s := options.NewServerRunOptions()
effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
utilversion.ComponentGenericAPIServer, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
s := options.NewServerRunOptions(featureGate, effectiveVersion)
cmd := &cobra.Command{
Use: "kube-apiserver",
Long: `The Kubernetes API server validates and configures data
@@ -83,9 +87,13 @@ cluster's shared state through which all other components interact.`,
verflag.PrintAndExitIfRequested()
fs := cmd.Flags()
if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil {
return err
}
// Activate logging as soon as possible, after that
// show flags with the final logging configuration.
if err := logsapi.ValidateAndApply(s.Logs, utilfeature.DefaultFeatureGate); err != nil {
if err := logsapi.ValidateAndApply(s.Logs, featureGate); err != nil {
return err
}
cliflag.PrintFlags(fs)
@@ -101,7 +109,7 @@ cluster's shared state through which all other components interact.`,
return utilerrors.NewAggregate(errs)
}
// add feature enablement metrics
utilfeature.DefaultMutableFeatureGate.AddMetrics()
featureGate.AddMetrics()
return Run(cmd.Context(), completedOptions)
},
Args: func(cmd *cobra.Command, args []string) error {
@@ -118,6 +126,9 @@ cluster's shared state through which all other components interact.`,
fs := cmd.Flags()
namedFlagSets := s.Flags()
verflag.AddFlags(namedFlagSets.FlagSet("global"))
featureGate.AddFlag(namedFlagSets.FlagSet("global"), "")
effectiveVersion.AddFlags(namedFlagSets.FlagSet("global"), "")
globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags())
options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic"))
for _, f := range namedFlagSets.FlagSets {

View File

@@ -48,6 +48,7 @@ import (
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/apiserver/pkg/storageversion"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
clientgotransport "k8s.io/client-go/transport"
@@ -98,6 +99,9 @@ type TestServerInstanceOptions struct {
// We specify this as on option to pass a common proxyCA to multiple apiservers to simulate
// an apiserver version skew scenario where all apiservers use the same proxyCA to verify client connections.
ProxyCA *ProxyCA
// Set the BinaryVersion of server effective version.
// Default to 1.31
BinaryVersion string
}
// TestServer return values supplied by kube-test-ApiServer
@@ -177,10 +181,21 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
fs := pflag.NewFlagSet("test", pflag.PanicOnError)
s := options.NewServerRunOptions()
featureGate := utilfeature.DefaultMutableFeatureGate
binaryVersion := utilversion.DefaultKubeEffectiveVersion().BinaryVersion().String()
if instanceOptions.BinaryVersion != "" {
binaryVersion = instanceOptions.BinaryVersion
}
effectiveVersion := utilversion.NewEffectiveVersion(binaryVersion)
_ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.ComponentGenericAPIServer, effectiveVersion, featureGate, true)
s := options.NewServerRunOptions(featureGate, effectiveVersion)
for _, f := range s.Flags().FlagSets {
fs.AddFlagSet(f)
}
featureGate.AddFlag(fs, "")
effectiveVersion.AddFlags(fs, "")
s.SecureServing.Listener, s.SecureServing.BindPort, err = createLocalhostListenerOnFreePort()
if err != nil {
@@ -321,6 +336,10 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
return result, err
}
if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil {
return result, err
}
saSigningKeyFile, err := os.CreateTemp("/tmp", "insecure_test_key")
if err != nil {
t.Fatalf("create temp file failed: %v", err)

View File

@@ -273,7 +273,7 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy
fs := fss.FlagSet("misc")
fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
fs.StringVar(&s.Generic.ClientConnection.Kubeconfig, "kubeconfig", s.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).")
utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"))
utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"), "")
return fss
}

View File

@@ -189,7 +189,7 @@ func (o *Options) initFlags() {
o.Authorization.AddFlags(nfs.FlagSet("authorization"))
o.Deprecated.AddFlags(nfs.FlagSet("deprecated"))
options.BindLeaderElectionFlags(o.LeaderElection, nfs.FlagSet("leader election"))
utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate"))
utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate"), "")
o.Metrics.AddFlags(nfs.FlagSet("metrics"))
logsapi.AddFlags(o.Logs, nfs.FlagSet("logs"))

View File

@@ -17,7 +17,6 @@ limitations under the License.
package apiserver
import (
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
apiextensionsoptions "k8s.io/apiextensions-apiserver/pkg/cmd/server/options"
@@ -27,6 +26,7 @@ import (
"k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/util/webhook"
"k8s.io/client-go/informers"
v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
)

View File

@@ -88,7 +88,7 @@ func (s *Server) InstallAPIs(restStorageProviders ...RESTStorageProvider) error
nonLegacy := []*genericapiserver.APIGroupInfo{}
// used later in the loop to filter the served resource by those that have expired.
resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(*s.GenericAPIServer.Version)
resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(s.GenericAPIServer.EffectiveVersion.EmulationVersion())
if err != nil {
return err
}

View File

@@ -185,7 +185,9 @@ func BuildGenericConfig(
}
storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
storageFactoryConfig.CurrentVersion = genericConfig.EffectiveVersion
storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig
storageFactoryConfig.DefaultResourceEncoding.SetEffectiveVersion(genericConfig.EffectiveVersion)
storageFactory, lastErr = storageFactoryConfig.Complete(s.Etcd).New()
if lastErr != nil {
return

View File

@@ -24,6 +24,8 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
apiserveroptions "k8s.io/apiserver/pkg/server/options"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/component-base/featuregate"
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
@@ -32,7 +34,9 @@ import (
)
func TestBuildGenericConfig(t *testing.T) {
opts := options.NewOptions()
featureGate := featuregate.NewFeatureGate()
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
opts := options.NewOptions(featureGate, effectiveVersion)
s := (&apiserveroptions.SecureServingOptions{
BindAddress: netutils.ParseIPSloppy("127.0.0.1"),
}).WithLoopback()
@@ -66,7 +70,7 @@ func TestBuildGenericConfig(t *testing.T) {
t.Errorf("There are different StorageObjectCountTracker in genericConfig and storageFactory")
}
restOptions, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""})
restOptions, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}, nil)
if err != nil {
t.Fatal(err)
}

View File

@@ -28,8 +28,10 @@ import (
peerreconcilers "k8s.io/apiserver/pkg/reconcilers"
genericoptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/storagebackend"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/util/keyutil"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
logsapi "k8s.io/component-base/logs/api/v1"
"k8s.io/component-base/metrics"
@@ -98,9 +100,9 @@ type CompletedOptions struct {
}
// NewOptions creates a new ServerRunOptions object with default parameters
func NewOptions() *Options {
func NewOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *Options {
s := Options{
GenericServerRunOptions: genericoptions.NewServerRunOptions(),
GenericServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion),
Etcd: genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)),
SecureServing: kubeoptions.NewSecureServingOptions(),
Audit: genericoptions.NewAuditOptions(),
@@ -202,6 +204,10 @@ func (o *Options) Complete(alternateDNS []string, alternateIPs []net.IP) (Comple
Options: *o,
}
if err := completed.GenericServerRunOptions.Complete(); err != nil {
return CompletedOptions{}, err
}
// set defaults
if err := completed.GenericServerRunOptions.DefaultAdvertiseAddress(completed.SecureServing.SecureServingOptions); err != nil {
return CompletedOptions{}, err

View File

@@ -30,9 +30,11 @@ import (
apiserveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/etcd3"
"k8s.io/apiserver/pkg/storage/storagebackend"
utilversion "k8s.io/apiserver/pkg/util/version"
auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics"
netutils "k8s.io/utils/net"
@@ -42,12 +44,16 @@ import (
func TestAddFlags(t *testing.T) {
fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
s := NewOptions()
featureGate := featuregate.NewFeatureGate()
effectiveVersion := utilversion.NewEffectiveVersion("1.32")
s := NewOptions(featureGate, effectiveVersion)
var fss cliflag.NamedFlagSets
s.AddFlags(&fss)
for _, f := range fss.FlagSets {
fs.AddFlagSet(f)
}
featureGate.AddFlag(fs, "")
effectiveVersion.AddFlags(fs, "")
args := []string{
"--enable-admission-plugins=AlwaysDeny",
@@ -108,6 +114,7 @@ func TestAddFlags(t *testing.T) {
"--request-timeout=2m",
"--storage-backend=etcd3",
"--lease-reuse-duration-seconds=100",
"--emulated-version=1.31",
}
fs.Parse(args)
@@ -122,6 +129,8 @@ func TestAddFlags(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &apiserveroptions.AdmissionOptions{
@@ -292,4 +301,8 @@ func TestAddFlags(t *testing.T) {
if !reflect.DeepEqual(expected, s) {
t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{})))
}
if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" {
t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31")
}
}

View File

@@ -100,6 +100,7 @@ func validateUnknownVersionInteroperabilityProxyFlags(options *Options) []error
func (s *Options) Validate() []error {
var errs []error
errs = append(errs, s.GenericServerRunOptions.Validate()...)
errs = append(errs, s.Etcd.Validate()...)
errs = append(errs, validateAPIPriorityAndFairness(s)...)
errs = append(errs, s.SecureServing.Validate()...)

View File

@@ -23,6 +23,7 @@ import (
kubeapiserveradmission "k8s.io/apiserver/pkg/admission"
genericoptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/component-base/featuregate"
basemetrics "k8s.io/component-base/metrics"
"k8s.io/kubernetes/pkg/features"
@@ -200,7 +201,7 @@ func TestValidateOptions(t *testing.T) {
name: "validate master count equal 0",
expectErrors: true,
options: &Options{
GenericServerRunOptions: &genericoptions.ServerRunOptions{},
GenericServerRunOptions: &genericoptions.ServerRunOptions{FeatureGate: utilfeature.DefaultFeatureGate.DeepCopy(), EffectiveVersion: utilversion.NewEffectiveVersion("1.32")},
Etcd: &genericoptions.EtcdOptions{},
SecureServing: &genericoptions.SecureServingOptionsWithLoopback{},
Audit: &genericoptions.AuditOptions{},
@@ -227,7 +228,7 @@ func TestValidateOptions(t *testing.T) {
name: "validate token request enable not attempted",
expectErrors: true,
options: &Options{
GenericServerRunOptions: &genericoptions.ServerRunOptions{},
GenericServerRunOptions: &genericoptions.ServerRunOptions{FeatureGate: utilfeature.DefaultFeatureGate.DeepCopy(), EffectiveVersion: utilversion.NewEffectiveVersion("1.32")},
Etcd: &genericoptions.EtcdOptions{},
SecureServing: &genericoptions.SecureServingOptionsWithLoopback{},
Audit: &genericoptions.AuditOptions{},

View File

@@ -82,7 +82,7 @@ func BuildPeerProxy(versionedInformer clientgoinformers.SharedInformerFactory, s
// The peer endpoint leases are used to find network locations of apiservers for peer proxy
func CreatePeerEndpointLeaseReconciler(c genericapiserver.Config, storageFactory serverstorage.StorageFactory) (reconcilers.PeerEndpointLeaseReconciler, error) {
ttl := DefaultPeerEndpointReconcilerTTL
config, err := storageFactory.NewConfig(api.Resource("apiServerPeerIPInfo"))
config, err := storageFactory.NewConfig(api.Resource("apiServerPeerIPInfo"), &api.Endpoints{})
if err != nil {
return nil, fmt.Errorf("error creating storage factory config: %w", err)
}

View File

@@ -214,7 +214,7 @@ func (c *Config) createLeaseReconciler() reconcilers.EndpointReconciler {
endpointsAdapter := reconcilers.NewEndpointsAdapter(endpointClient, endpointSliceClient)
ttl := c.Extra.MasterEndpointReconcileTTL
config, err := c.ControlPlane.StorageFactory.NewConfig(api.Resource("apiServerIPInfo"))
config, err := c.ControlPlane.StorageFactory.NewConfig(api.Resource("apiServerIPInfo"), &api.Endpoints{})
if err != nil {
klog.Fatalf("Error creating storage factory config: %v", err)
}

View File

@@ -55,6 +55,7 @@ import (
serverstorage "k8s.io/apiserver/pkg/server/storage"
etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
"k8s.io/apiserver/pkg/util/openapi"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/discovery"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
@@ -103,7 +104,9 @@ func setUp(t *testing.T) (*etcd3testing.EtcdTestServer, Config, *assert.Assertio
},
}
config.ControlPlane.Generic.EffectiveVersion = utilversion.DefaultKubeEffectiveVersion()
storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
storageFactoryConfig.DefaultResourceEncoding.SetEffectiveVersion(config.ControlPlane.Generic.EffectiveVersion)
storageConfig.StorageObjectCountTracker = config.ControlPlane.Generic.StorageObjectCountTracker
resourceEncoding := resourceconfig.MergeResourceEncodingConfigs(storageFactoryConfig.DefaultResourceEncoding, storageFactoryConfig.ResourceEncodingOverrides)
storageFactory := serverstorage.NewDefaultStorageFactory(*storageConfig, "application/vnd.kubernetes.protobuf", storageFactoryConfig.Serializer, resourceEncoding, DefaultAPIResourceConfigSource(), nil)

View File

@@ -19,6 +19,8 @@ package features
import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
clientfeatures "k8s.io/client-go/features"
"k8s.io/component-base/featuregate"
)
@@ -78,8 +80,7 @@ func TestClientAdapterAdd(t *testing.T) {
t.Errorf("expected feature %q not found", name)
continue
}
if actual != expected {
if diff := cmp.Diff(actual, expected, cmpopts.IgnoreFields(featuregate.FeatureSpec{}, "Version")); diff != "" {
t.Errorf("expected feature %q spec %#v, got spec %#v", name, expected, actual)
}
}

View File

@@ -946,6 +946,7 @@ const (
func init() {
runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates))
runtime.Must(utilfeature.DefaultMutableFeatureGate.AddVersioned(defaultVersionedKubernetesFeatureGates))
// Register all client-go features with kube's feature gate instance and make all client-go
// feature checks use kube's instance. The effect is that for kube binaries, client-go

View File

@@ -62,6 +62,9 @@ func TestAllRegisteredFeaturesExpected(t *testing.T) {
if err := knownFeatureGates.Add(defaultKubernetesFeatureGates); err != nil {
t.Fatal(err)
}
if err := knownFeatureGates.AddVersioned(defaultVersionedKubernetesFeatureGates); err != nil {
t.Fatal(err)
}
knownFeatures := knownFeatureGates.GetAll()
for registeredFeature := range registeredFeatures {

View File

@@ -0,0 +1,34 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package features
import (
"k8s.io/component-base/featuregate"
)
// defaultVersionedKubernetesFeatureGates consists of all known Kubernetes-specific feature keys with VersionedSpecs.
// To add a new feature, define a key for it and add it here. The features will be
// available throughout Kubernetes binaries.
//
// Entries are separated from each other with blank lines to avoid sweeping gofmt changes
// when adding or removing one entry.
var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{
// Example:
// genericfeatures.EmulationVersion: {
// {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha},
// },
}

View File

@@ -25,6 +25,7 @@ import (
"k8s.io/apiserver/pkg/server/resourceconfig"
serverstorage "k8s.io/apiserver/pkg/server/storage"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/apiserver/pkg/util/version"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/apis/apps"
"k8s.io/kubernetes/pkg/apis/certificates"
@@ -93,6 +94,7 @@ type StorageFactoryConfig struct {
Serializer runtime.StorageSerializer
ResourceEncodingOverrides []schema.GroupVersionResource
EtcdServersOverrides []string
CurrentVersion version.EffectiveVersion
}
// Complete completes the StorageFactoryConfig with provided etcdOptions returning completedStorageFactoryConfig.

View File

@@ -321,7 +321,7 @@ func (p *legacyProvider) NewRESTStorage(apiResourceConfigSource serverstorage.AP
func (c *Config) newServiceIPAllocators() (registries rangeRegistries, primaryClusterIPAllocator ipallocator.Interface, clusterIPAllocators map[api.IPFamily]ipallocator.Interface, nodePortAllocator *portallocator.PortAllocator, err error) {
clusterIPAllocators = map[api.IPFamily]ipallocator.Interface{}
serviceStorageConfig, err := c.StorageFactory.NewConfig(api.Resource("services"))
serviceStorageConfig, err := c.StorageFactory.NewConfig(api.Resource("services"), &api.Service{})
if err != nil {
return rangeRegistries{}, nil, nil, nil, err
}

View File

@@ -19,6 +19,7 @@ package rest
import (
"testing"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/server/storage"
"k8s.io/apiserver/pkg/storage/storagebackend"
@@ -40,7 +41,7 @@ func TestGetServersToValidate(t *testing.T) {
type fakeStorageFactory struct{}
func (f fakeStorageFactory) NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
func (f fakeStorageFactory) NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
return nil, nil
}

View File

@@ -44,7 +44,7 @@ func NewEtcdStorageForResource(t *testing.T, resource schema.GroupResource) (*st
if err != nil {
t.Fatalf("Error while making storage factory: %v", err)
}
resourceConfig, err := factory.NewConfig(resource)
resourceConfig, err := factory.NewConfig(resource, nil)
if err != nil {
t.Fatalf("Error while finding storage destination: %v", err)
}

View File

@@ -1178,8 +1178,9 @@ type crdConversionRESTOptionsGetter struct {
preserveUnknownFields bool
}
func (t crdConversionRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
ret, err := t.RESTOptionsGetter.GetRESTOptions(resource)
func (t crdConversionRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) {
// Explicitly ignore example, we override storageconfig below
ret, err := t.RESTOptionsGetter.GetRESTOptions(resource, nil)
if err == nil {
d := schemaCoercingDecoder{delegate: ret.StorageConfig.Codec, validator: unstructuredSchemaCoercer{
// drop invalid fields while decoding old CRs (before we haven't had any ObjectMeta validation)

View File

@@ -39,9 +39,11 @@ import (
flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
"k8s.io/apiserver/pkg/util/openapi"
"k8s.io/apiserver/pkg/util/proxy"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/apiserver/pkg/util/webhook"
scheme "k8s.io/client-go/kubernetes/scheme"
corev1 "k8s.io/client-go/listers/core/v1"
"k8s.io/component-base/featuregate"
netutils "k8s.io/utils/net"
)
@@ -58,9 +60,9 @@ type CustomResourceDefinitionsServerOptions struct {
}
// NewCustomResourceDefinitionsServerOptions creates default options of an apiextensions-apiserver.
func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer) *CustomResourceDefinitionsServerOptions {
func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer, featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *CustomResourceDefinitionsServerOptions {
o := &CustomResourceDefinitionsServerOptions{
ServerRunOptions: genericoptions.NewServerRunOptions(),
ServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion),
RecommendedOptions: genericoptions.NewRecommendedOptions(
defaultEtcdPathPrefix,
apiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion),
@@ -92,7 +94,7 @@ func (o CustomResourceDefinitionsServerOptions) Validate() error {
// Complete fills in missing options.
func (o *CustomResourceDefinitionsServerOptions) Complete() error {
return nil
return o.ServerRunOptions.Complete()
}
// Config returns an apiextensions-apiserver configuration.

View File

@@ -24,15 +24,26 @@ import (
"k8s.io/apiextensions-apiserver/pkg/cmd/server/options"
genericapiserver "k8s.io/apiserver/pkg/server"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
)
func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command {
o := options.NewCustomResourceDefinitionsServerOptions(out, errOut)
// effectiveVersion is used to set what apis and feature gates the generic api server is compatible with.
// You can also have the flag setting the effectiveVersion of the apiextensions apiserver, and
// having a mapping from the apiextensions apiserver version to generic apiserver version.
effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
utilversion.ComponentGenericAPIServer, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
o := options.NewCustomResourceDefinitionsServerOptions(out, errOut, featureGate, effectiveVersion)
cmd := &cobra.Command{
Short: "Launch an API extensions API server",
Long: "Launch an API extensions API server",
RunE: func(c *cobra.Command, args []string) error {
if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil {
return err
}
if err := o.Complete(); err != nil {
return err
}
@@ -48,6 +59,8 @@ func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command
cmd.SetContext(ctx)
fs := cmd.Flags()
featureGate.AddFlag(fs, "")
effectiveVersion.AddFlags(fs, "")
o.AddFlags(fs)
return cmd
}

View File

@@ -35,7 +35,9 @@ import (
openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/storage/storagebackend"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/apiserver/pkg/util/openapi"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
logsapi "k8s.io/component-base/logs/api/v1"
@@ -120,7 +122,13 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin
fs := pflag.NewFlagSet("test", pflag.PanicOnError)
s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr)
featureGate := utilfeature.DefaultMutableFeatureGate
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
_ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.ComponentGenericAPIServer, effectiveVersion, featureGate, true)
s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr, featureGate, effectiveVersion)
featureGate.AddFlag(fs, "")
effectiveVersion.AddFlags(fs, "")
s.AddFlags(fs)
s.RecommendedOptions.SecureServing.Listener, s.RecommendedOptions.SecureServing.BindPort, err = createLocalhostListenerOnFreePort()
@@ -143,6 +151,10 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin
fs.Parse(customFlags)
if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil {
return result, err
}
if err := s.Complete(); err != nil {
return result, fmt.Errorf("failed to set default options: %v", err)
}

View File

@@ -183,7 +183,7 @@ func testWebhookConverter(t *testing.T, watchCache bool) {
crd := multiVersionFixture.DeepCopy()
RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd, nil, nil)
restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural})
restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}, nil)
if err != nil {
t.Fatal(err)
}

View File

@@ -659,7 +659,7 @@ func TestCustomResourceDefaultingOfMetaFields(t *testing.T) {
// get persisted object
RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd, nil, nil)
restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural})
restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}, nil)
if err != nil {
t.Fatal(err)
}

View File

@@ -156,7 +156,7 @@ func StartDefaultServerWithClientsAndEtcd(t servertesting.Logger, extraFlags ...
}
RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd, resourceTransformers, nil)
restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "hopefully-ignored-group", Resource: "hopefully-ignored-resources"})
restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "hopefully-ignored-group", Resource: "hopefully-ignored-resources"}, nil)
if err != nil {
return nil, nil, nil, nil, "", err
}

View File

@@ -133,7 +133,7 @@ func TestInvalidObjectMetaInStorage(t *testing.T) {
}
RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd, nil, nil)
restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: noxuDefinition.Spec.Group, Resource: noxuDefinition.Spec.Names.Plural})
restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: noxuDefinition.Spec.Group, Resource: noxuDefinition.Spec.Names.Plural}, nil)
if err != nil {
t.Fatal(err)
}

View File

@@ -326,7 +326,7 @@ func TestPruningFromStorage(t *testing.T) {
t.Fatal(err)
}
restOptions, err := completedConfig.GenericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural})
restOptions, err := completedConfig.GenericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}, nil)
if err != nil {
t.Fatal(err)
}

View File

@@ -145,6 +145,43 @@ func MustParseGeneric(str string) *Version {
return v
}
// Parse tries to do ParseSemantic first to keep more information.
// If ParseSemantic fails, it would just do ParseGeneric.
func Parse(str string) (*Version, error) {
v, err := parse(str, true)
if err != nil {
return parse(str, false)
}
return v, err
}
// MustParse is like Parse except that it panics on error
func MustParse(str string) *Version {
v, err := Parse(str)
if err != nil {
panic(err)
}
return v
}
// ParseMajorMinor parses a "generic" version string and returns a version with the major and minor version.
func ParseMajorMinor(str string) (*Version, error) {
v, err := ParseGeneric(str)
if err != nil {
return nil, err
}
return MajorMinor(v.Major(), v.Minor()), nil
}
// MustParseMajorMinor is like ParseMajorMinor except that it panics on error
func MustParseMajorMinor(str string) *Version {
v, err := ParseMajorMinor(str)
if err != nil {
panic(err)
}
return v
}
// ParseSemantic parses a version string that exactly obeys the syntax and semantics of
// the "Semantic Versioning" specification (http://semver.org/) (although it ignores
// leading and trailing whitespace, and allows the version to be preceded by "v"). For
@@ -215,6 +252,21 @@ func (v *Version) WithMinor(minor uint) *Version {
return &result
}
// SubtractMinor returns the version diff minor versions back, with the same major and no patch.
// If diff >= current minor, the minor would be 0.
func (v *Version) SubtractMinor(diff uint) *Version {
var minor uint
if diff < v.Minor() {
minor = v.Minor() - diff
}
return MajorMinor(v.Major(), minor)
}
// AddMinor returns the version diff minor versions forward, with the same major and no patch.
func (v *Version) AddMinor(diff uint) *Version {
return MajorMinor(v.Major(), v.Minor()+diff)
}
// WithPatch returns copy of the version object with requested patch number
func (v *Version) WithPatch(patch uint) *Version {
result := *v
@@ -224,6 +276,9 @@ func (v *Version) WithPatch(patch uint) *Version {
// WithPreRelease returns copy of the version object with requested prerelease
func (v *Version) WithPreRelease(preRelease string) *Version {
if len(preRelease) == 0 {
return v
}
result := *v
result.components = []uint{v.Major(), v.Minor(), v.Patch()}
result.preRelease = preRelease
@@ -345,6 +400,17 @@ func onlyZeros(array []uint) bool {
return true
}
// EqualTo tests if a version is equal to a given version.
func (v *Version) EqualTo(other *Version) bool {
if v == nil {
return other == nil
}
if other == nil {
return false
}
return v.compareInternal(other) == 0
}
// AtLeast tests if a version is at least equal to a given minimum version. If both
// Versions are Semantic Versions, this will use the Semantic Version comparison
// algorithm. Otherwise, it will compare only the numeric components, with non-present
@@ -360,6 +426,11 @@ func (v *Version) LessThan(other *Version) bool {
return v.compareInternal(other) == -1
}
// GreaterThan tests if a version is greater than a given version.
func (v *Version) GreaterThan(other *Version) bool {
return v.compareInternal(other) == 1
}
// Compare compares v against a version string (which will be parsed as either Semantic
// or non-Semantic depending on v). On success it returns -1 if v is less than other, 1 if
// it is greater than other, or 0 if they are equal.

View File

@@ -452,3 +452,94 @@ func TestHighestSupportedVersion(t *testing.T) {
}
}
}
func TestSubtractMinor(t *testing.T) {
var tests = []struct {
version string
diff uint
expectedComponents []uint
}{
{
version: "1.0.2",
diff: 3,
expectedComponents: []uint{1, 0},
},
{
version: "1.3.2-alpha+001",
diff: 2,
expectedComponents: []uint{1, 1},
},
{
version: "1.3.2-alpha+001",
diff: 3,
expectedComponents: []uint{1, 0},
},
{
version: "1.20",
diff: 5,
expectedComponents: []uint{1, 15},
},
}
for _, test := range tests {
version, _ := ParseGeneric(test.version)
if !reflect.DeepEqual(test.expectedComponents, version.SubtractMinor(test.diff).Components()) {
t.Error("parse returned un'expected components")
}
}
}
func TestParse(t *testing.T) {
var tests = []struct {
version string
expectErr bool
expectedComponents []uint
expectedPreRelease string
expectedBuildMetadata string
}{
{
version: "1.0.2",
expectedComponents: []uint{1, 0, 2},
},
{
version: "1.0.2-alpha+001",
expectedComponents: []uint{1, 0, 2},
expectedPreRelease: "alpha",
expectedBuildMetadata: "001",
},
{
version: "1.2",
expectedComponents: []uint{1, 2},
},
{
version: "1.0.2-beta+exp.sha.5114f85",
expectedComponents: []uint{1, 0, 2},
expectedPreRelease: "beta",
expectedBuildMetadata: "exp.sha.5114f85",
},
{
version: "a.b.c",
expectErr: true,
},
}
for _, test := range tests {
version, err := Parse(test.version)
if test.expectErr {
if err == nil {
t.Fatalf("got no err, expected err")
}
continue
}
if !reflect.DeepEqual(test.expectedComponents, version.Components()) {
t.Error("parse returned un'expected components")
}
if test.expectedPreRelease != version.PreRelease() {
t.Errorf("parse returned version.PreRelease %s, expected %s", test.expectedPreRelease, version.PreRelease())
}
if test.expectedBuildMetadata != version.BuildMetadata() {
t.Errorf("parse returned version.BuildMetadata %s, expected %s", test.expectedBuildMetadata, version.BuildMetadata())
}
}
}

View File

@@ -30,20 +30,27 @@ import (
"k8s.io/apimachinery/pkg/util/version"
celconfig "k8s.io/apiserver/pkg/apis/cel"
"k8s.io/apiserver/pkg/cel/library"
utilversion "k8s.io/apiserver/pkg/util/version"
)
// DefaultCompatibilityVersion returns a default compatibility version for use with EnvSet
// that guarantees compatibility with CEL features/libraries/parameters understood by
// an n-1 version
// the api server min compatibility version
//
// This default will be set to no more than n-1 the current Kubernetes major.minor version.
// This default will be set to no more than the current Kubernetes major.minor version.
//
// Note that a default version number less than n-1 indicates a wider range of version
// compatibility than strictly required for rollback. A wide range of compatibility is
// desirable because it means that CEL expressions are portable across a wider range
// of Kubernetes versions.
// Note that a default version number less than n-1 the current Kubernetes major.minor version
// indicates a wider range of version compatibility than strictly required for rollback.
// A wide range of compatibility is desirable because it means that CEL expressions are portable
// across a wider range of Kubernetes versions.
// A default version number equal to the current Kubernetes major.minor version
// indicates fast forward CEL features that can be used when rollback is no longer needed.
func DefaultCompatibilityVersion() *version.Version {
return version.MajorMinor(1, 30)
effectiveVer := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer)
if effectiveVer == nil {
effectiveVer = utilversion.DefaultKubeEffectiveVersion()
}
return effectiveVer.MinCompatibilityVersion()
}
var baseOpts = append(baseOptsWithoutStrictCost, StrictCostOpt)

View File

@@ -322,6 +322,17 @@ const (
func init() {
runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates))
runtime.Must(utilfeature.DefaultMutableFeatureGate.AddVersioned(defaultVersionedKubernetesFeatureGates))
}
// defaultVersionedKubernetesFeatureGates consists of all known Kubernetes-specific feature keys with VersionedSpecs.
// To add a new feature, define a key for it above and add it here. The features will be
// available throughout Kubernetes binaries.
var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{
// Example:
// EmulationVersion: {
// {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha},
// },
}
// defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys.

View File

@@ -19,6 +19,7 @@ package generic
import (
"time"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/storage"
"k8s.io/apiserver/pkg/storage/storagebackend"
@@ -39,12 +40,15 @@ type RESTOptions struct {
}
// Implement RESTOptionsGetter so that RESTOptions can directly be used when available (i.e. tests)
func (opts RESTOptions) GetRESTOptions(schema.GroupResource) (RESTOptions, error) {
func (opts RESTOptions) GetRESTOptions(schema.GroupResource, runtime.Object) (RESTOptions, error) {
return opts, nil
}
type RESTOptionsGetter interface {
GetRESTOptions(resource schema.GroupResource) (RESTOptions, error)
// GetRESTOptions returns the RESTOptions for the given resource and example object.
// The example object is used to determine the storage version for the resource.
// If the example object is nil, the storage version will be determined by the resource's default storage version.
GetRESTOptions(resource schema.GroupResource, example runtime.Object) (RESTOptions, error)
}
// StoreOptions is set of configuration options used to complete generic registries.

View File

@@ -1518,7 +1518,7 @@ func (e *Store) CompleteWithOptions(options *generic.StoreOptions) error {
return err
}
opts, err := options.RESTOptions.GetRESTOptions(e.DefaultQualifiedResource)
opts, err := options.RESTOptions.GetRESTOptions(e.DefaultQualifiedResource, e.NewFunc())
if err != nil {
return err
}

View File

@@ -42,8 +42,9 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/version"
utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup"
"k8s.io/apimachinery/pkg/version"
apimachineryversion "k8s.io/apimachinery/pkg/version"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/authentication/authenticator"
@@ -70,8 +71,10 @@ import (
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/informers"
restclient "k8s.io/client-go/rest"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics/features"
"k8s.io/component-base/metrics/prometheus/slis"
@@ -148,7 +151,12 @@ type Config struct {
PostStartHooks map[string]PostStartHookConfigEntry
// Version will enable the /version endpoint if non-nil
Version *version.Info
Version *apimachineryversion.Info
// EffectiveVersion determines which apis and features are available
// based on when the api/feature lifecyle.
EffectiveVersion utilversion.EffectiveVersion
// FeatureGate is a way to plumb feature gate through if you have them.
FeatureGate featuregate.FeatureGate
// AuditBackend is where audit events are sent to.
AuditBackend audit.Backend
// AuditPolicyRuleEvaluator makes the decision of whether and how to audit log a request.
@@ -585,7 +593,7 @@ func (c *Config) AddPostStartHookOrDie(name string, hook PostStartHookFunc) {
}
}
func completeOpenAPI(config *openapicommon.Config, version *version.Info) {
func completeOpenAPI(config *openapicommon.Config, version *version.Version) {
if config == nil {
return
}
@@ -624,7 +632,7 @@ func completeOpenAPI(config *openapicommon.Config, version *version.Info) {
}
}
func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Info) {
func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Version) {
if config == nil {
return
}
@@ -676,6 +684,9 @@ func (c *Config) ShutdownInitiatedNotify() <-chan struct{} {
// Complete fills in any fields not set that are required to have valid data and can be derived
// from other fields. If you're going to `ApplyOptions`, do that first. It's mutating the receiver.
func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedConfig {
if c.FeatureGate == nil {
c.FeatureGate = utilfeature.DefaultFeatureGate
}
if len(c.ExternalAddress) == 0 && c.PublicAddress != nil {
c.ExternalAddress = c.PublicAddress.String()
}
@@ -691,9 +702,12 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo
}
c.ExternalAddress = net.JoinHostPort(c.ExternalAddress, strconv.Itoa(port))
}
completeOpenAPI(c.OpenAPIConfig, c.Version)
completeOpenAPIV3(c.OpenAPIV3Config, c.Version)
var ver *version.Version
if c.EffectiveVersion != nil {
ver = c.EffectiveVersion.EmulationVersion()
}
completeOpenAPI(c.OpenAPIConfig, ver)
completeOpenAPIV3(c.OpenAPIV3Config, ver)
if c.DiscoveryAddresses == nil {
c.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: c.ExternalAddress}
@@ -711,7 +725,7 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo
} else {
c.EquivalentResourceRegistry = runtime.NewEquivalentResourceRegistryWithIdentity(func(groupResource schema.GroupResource) string {
// use the storage prefix as the key if possible
if opts, err := c.RESTOptionsGetter.GetRESTOptions(groupResource); err == nil {
if opts, err := c.RESTOptionsGetter.GetRESTOptions(groupResource, nil); err == nil {
return opts.ResourcePrefix
}
// otherwise return "" to use the default key (parent GV name)
@@ -819,7 +833,9 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
APIServerID: c.APIServerID,
StorageVersionManager: c.StorageVersionManager,
Version: c.Version,
EffectiveVersion: c.EffectiveVersion,
Version: c.Version,
FeatureGate: c.FeatureGate,
muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{},
}

View File

@@ -17,6 +17,7 @@ limitations under the License.
package server
import (
"fmt"
"os"
"strconv"
"strings"
@@ -25,16 +26,15 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
apimachineryversion "k8s.io/apimachinery/pkg/version"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/klog/v2"
)
// resourceExpirationEvaluator holds info for deciding if a particular rest.Storage needs to excluded from the API
type resourceExpirationEvaluator struct {
currentMajor int
currentMinor int
isAlpha bool
currentVersion *apimachineryversion.Version
isAlpha bool
// This is usually set for testing for which tests need to be removed. This prevent insta-failing CI.
// Set KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA to see what will be removed when we tag beta
strictRemovedHandlingInAlpha bool
@@ -53,30 +53,17 @@ type ResourceExpirationEvaluator interface {
ShouldServeForVersion(majorRemoved, minorRemoved int) bool
}
func NewResourceExpirationEvaluator(currentVersion apimachineryversion.Info) (ResourceExpirationEvaluator, error) {
func NewResourceExpirationEvaluator(currentVersion *apimachineryversion.Version) (ResourceExpirationEvaluator, error) {
if currentVersion == nil {
return nil, fmt.Errorf("empty NewResourceExpirationEvaluator currentVersion")
}
klog.V(1).Infof("NewResourceExpirationEvaluator with currentVersion: %s.", currentVersion)
ret := &resourceExpirationEvaluator{
strictRemovedHandlingInAlpha: false,
}
if len(currentVersion.Major) > 0 {
currentMajor64, err := strconv.ParseInt(currentVersion.Major, 10, 32)
if err != nil {
return nil, err
}
ret.currentMajor = int(currentMajor64)
}
if len(currentVersion.Minor) > 0 {
// split the "normal" + and - for semver stuff
minorString := strings.Split(currentVersion.Minor, "+")[0]
minorString = strings.Split(minorString, "-")[0]
minorString = strings.Split(minorString, ".")[0]
currentMinor64, err := strconv.ParseInt(minorString, 10, 32)
if err != nil {
return nil, err
}
ret.currentMinor = int(currentMinor64)
}
ret.isAlpha = strings.Contains(currentVersion.GitVersion, "alpha")
// Only keeps the major and minor versions from input version.
ret.currentVersion = apimachineryversion.MajorMinor(currentVersion.Major(), currentVersion.Minor())
ret.isAlpha = strings.Contains(currentVersion.PreRelease(), "alpha")
if envString, ok := os.LookupEnv("KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA"); !ok {
// do nothing
@@ -112,6 +99,16 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio
return false
}
introduced, ok := versionedPtr.(introducedInterface)
// skip the introduced check for test where currentVersion is 0.0
if ok && (e.currentVersion.Major() > 0 || e.currentVersion.Minor() > 0) {
majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced()
verIntroduced := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced))
if e.currentVersion.LessThan(verIntroduced) {
return false
}
}
removed, ok := versionedPtr.(removedInterface)
if !ok {
return true
@@ -121,16 +118,11 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio
}
func (e *resourceExpirationEvaluator) ShouldServeForVersion(majorRemoved, minorRemoved int) bool {
if e.currentMajor < majorRemoved {
removedVer := apimachineryversion.MajorMinor(uint(majorRemoved), uint(minorRemoved))
if removedVer.GreaterThan(e.currentVersion) {
return true
}
if e.currentMajor > majorRemoved {
return false
}
if e.currentMinor < minorRemoved {
return true
}
if e.currentMinor > minorRemoved {
if removedVer.LessThan(e.currentVersion) {
return false
}
// at this point major and minor are equal, so this API should be removed when the current release GAs.
@@ -152,6 +144,11 @@ type removedInterface interface {
APILifecycleRemoved() (major, minor int)
}
// Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
type introducedInterface interface {
APILifecycleIntroduced() (major, minor int)
}
// removeDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted.
// versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage) {
@@ -171,6 +168,8 @@ func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versi
}
klog.V(1).Infof("Removing resource %v.%v.%v because it is time to stop serving it per APILifecycle.", resourceName, apiVersion, groupName)
storage := versionToResource[resourceName]
storage.Destroy()
delete(versionToResource, resourceName)
}
versionedResourcesStorageMap[apiVersion] = versionToResource

View File

@@ -25,57 +25,41 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/dump"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/version"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/registry/rest"
)
func Test_newResourceExpirationEvaluator(t *testing.T) {
tests := []struct {
name string
currentVersion version.Info
currentVersion string
expected resourceExpirationEvaluator
expectedErr string
}{
{
name: "beta",
currentVersion: version.Info{
Major: "1",
Minor: "20+",
GitVersion: "v1.20.0-beta.0.62+a5d22854a2ac21",
},
expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20},
name: "beta",
currentVersion: "v1.20.0-beta.0.62+a5d22854a2ac21",
expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)},
},
{
name: "alpha",
currentVersion: version.Info{
Major: "1",
Minor: "20+",
GitVersion: "v1.20.0-alpha.0.62+a5d22854a2ac21",
},
expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20, isAlpha: true},
name: "alpha",
currentVersion: "v1.20.0-alpha.0.62+a5d22854a2ac21",
expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20), isAlpha: true},
},
{
name: "maintenance",
currentVersion: version.Info{
Major: "1",
Minor: "20+",
GitVersion: "v1.20.1",
},
expected: resourceExpirationEvaluator{currentMajor: 1, currentMinor: 20},
name: "maintenance",
currentVersion: "v1.20.1",
expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)},
},
{
name: "bad",
currentVersion: version.Info{
Major: "1",
Minor: "20something+",
GitVersion: "v1.20.1",
},
expectedErr: `strconv.ParseInt: parsing "20something": invalid syntax`,
name: "no v prefix",
currentVersion: "1.20.1",
expected: resourceExpirationEvaluator{currentVersion: apimachineryversion.MajorMinor(1, 20)},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
actual, actualErr := NewResourceExpirationEvaluator(tt.currentVersion)
actual, actualErr := NewResourceExpirationEvaluator(apimachineryversion.MustParse(tt.currentVersion))
checkErr(t, actualErr, tt.expectedErr)
if actualErr != nil {
@@ -90,12 +74,12 @@ func Test_newResourceExpirationEvaluator(t *testing.T) {
}
}
func storageRemovedIn(major, minor int) removedInStorage {
return removedInStorage{major: major, minor: minor}
func storageRemovedIn(major, minor int) *removedInStorage {
return &removedInStorage{major: major, minor: minor}
}
func storageNeverRemoved() removedInStorage {
return removedInStorage{neverRemoved: true}
func storageNeverRemoved() *removedInStorage {
return &removedInStorage{neverRemoved: true}
}
type removedInStorage struct {
@@ -103,23 +87,23 @@ type removedInStorage struct {
neverRemoved bool
}
func (r removedInStorage) New() runtime.Object {
func (r *removedInStorage) New() runtime.Object {
if r.neverRemoved {
return neverRemovedObj{}
return &defaultObj{}
}
return removedInObj{major: r.major, minor: r.minor}
return &removedInObj{major: r.major, minor: r.minor}
}
func (r removedInStorage) Destroy() {
func (r *removedInStorage) Destroy() {
}
type neverRemovedObj struct {
type defaultObj struct {
}
func (r neverRemovedObj) GetObjectKind() schema.ObjectKind {
func (r *defaultObj) GetObjectKind() schema.ObjectKind {
panic("don't do this")
}
func (r neverRemovedObj) DeepCopyObject() runtime.Object {
func (r *defaultObj) DeepCopyObject() runtime.Object {
panic("don't do this either")
}
@@ -127,13 +111,45 @@ type removedInObj struct {
major, minor int
}
func (r removedInObj) GetObjectKind() schema.ObjectKind {
func (r *removedInObj) GetObjectKind() schema.ObjectKind {
panic("don't do this")
}
func (r removedInObj) DeepCopyObject() runtime.Object {
func (r *removedInObj) DeepCopyObject() runtime.Object {
panic("don't do this either")
}
func (r removedInObj) APILifecycleRemoved() (major, minor int) {
func (r *removedInObj) APILifecycleRemoved() (major, minor int) {
return r.major, r.minor
}
func storageIntroducedIn(major, minor int) *introducedInStorage {
return &introducedInStorage{major: major, minor: minor}
}
type introducedInStorage struct {
major, minor int
}
func (r *introducedInStorage) New() runtime.Object {
if r.major == 0 && r.minor == 0 {
return &defaultObj{}
}
return &IntroducedInObj{major: r.major, minor: r.minor}
}
func (r *introducedInStorage) Destroy() {
}
type IntroducedInObj struct {
major, minor int
}
func (r *IntroducedInObj) GetObjectKind() schema.ObjectKind {
panic("don't do this")
}
func (r *IntroducedInObj) DeepCopyObject() runtime.Object {
panic("don't do this either")
}
func (r *IntroducedInObj) APILifecycleIntroduced() (major, minor int) {
return r.major, r.minor
}
@@ -147,8 +163,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-curr",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageRemovedIn(1, 20),
expected: false,
@@ -156,8 +171,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-curr-but-deferred",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
serveRemovedAPIsOneMoreRelease: true,
},
restStorage: storageRemovedIn(1, 20),
@@ -166,9 +180,8 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-curr-but-alpha",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
isAlpha: true,
currentVersion: apimachineryversion.MajorMinor(1, 20),
isAlpha: true,
},
restStorage: storageRemovedIn(1, 20),
expected: true,
@@ -176,8 +189,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-curr-but-alpha-but-strict",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
isAlpha: true,
strictRemovedHandlingInAlpha: true,
},
@@ -187,8 +199,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-prev-deferral-does-not-help",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 21,
currentVersion: apimachineryversion.MajorMinor(1, 21),
serveRemovedAPIsOneMoreRelease: true,
},
restStorage: storageRemovedIn(1, 20),
@@ -197,8 +208,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-prev-major",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 2,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(2, 20),
serveRemovedAPIsOneMoreRelease: true,
},
restStorage: storageRemovedIn(1, 20),
@@ -207,8 +217,7 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "removed-in-future",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageRemovedIn(1, 21),
expected: true,
@@ -216,12 +225,43 @@ func Test_resourceExpirationEvaluator_shouldServe(t *testing.T) {
{
name: "never-removed",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageNeverRemoved(),
expected: true,
},
{
name: "introduced-in-curr",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageIntroducedIn(1, 20),
expected: true,
},
{
name: "introduced-in-prev-major",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageIntroducedIn(1, 19),
expected: true,
},
{
name: "introduced-in-future",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageIntroducedIn(1, 21),
expected: false,
},
{
name: "missing-introduced",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
restStorage: storageIntroducedIn(0, 0),
expected: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -269,8 +309,7 @@ func Test_removeDeletedKinds(t *testing.T) {
{
name: "remove-one-of-two",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
@@ -287,8 +326,7 @@ func Test_removeDeletedKinds(t *testing.T) {
{
name: "remove-nested-not-expired",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
@@ -306,8 +344,7 @@ func Test_removeDeletedKinds(t *testing.T) {
{
name: "remove-all-of-version",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentMajor: 1,
currentMinor: 20,
currentVersion: apimachineryversion.MajorMinor(1, 20),
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {

View File

@@ -53,7 +53,9 @@ import (
"k8s.io/apiserver/pkg/server/routes"
"k8s.io/apiserver/pkg/storageversion"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
restclient "k8s.io/client-go/rest"
"k8s.io/component-base/featuregate"
"k8s.io/klog/v2"
openapibuilder3 "k8s.io/kube-openapi/pkg/builder3"
openapicommon "k8s.io/kube-openapi/pkg/common"
@@ -238,6 +240,11 @@ type GenericAPIServer struct {
// Version will enable the /version endpoint if non-nil
Version *version.Info
// EffectiveVersion determines which apis and features are available
// based on when the api/feature lifecyle.
EffectiveVersion utilversion.EffectiveVersion
// FeatureGate is a way to plumb feature gate through if you have them.
FeatureGate featuregate.FeatureGate
// lifecycleSignals provides access to the various signals that happen during the life cycle of the apiserver.
lifecycleSignals lifecycleSignals

View File

@@ -48,6 +48,7 @@ import (
openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
"k8s.io/apiserver/pkg/registry/rest"
genericfilters "k8s.io/apiserver/pkg/server/filters"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/apiserver/pkg/warning"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake"
@@ -460,6 +461,7 @@ func TestNotRestRoutesHaveAuth(t *testing.T) {
kubeVersion := fakeVersion()
config.Version = &kubeVersion
config.EffectiveVersion = utilversion.NewEffectiveVersion(kubeVersion.String())
s, err := config.Complete(nil).New("test", NewEmptyDelegate())
if err != nil {
@@ -586,7 +588,7 @@ func fakeVersion() version.Info {
return version.Info{
Major: "42",
Minor: "42",
GitVersion: "42",
GitVersion: "42.42",
GitCommit: "34973274ccef6ab4dfaaf86599792fa9c3fe4689",
GitTreeState: "Dirty",
BuildDate: time.Now().String(),

View File

@@ -32,46 +32,42 @@ func (f fakeGroupRegistry) IsGroupRegistered(group string) bool {
func TestAPIEnablementOptionsValidate(t *testing.T) {
testCases := []struct {
name string
testOptions *APIEnablementOptions
expectErr string
name string
testOptions *APIEnablementOptions
runtimeConfig cliflag.ConfigurationMap
expectErr string
}{
{
name: "test when options is nil",
},
{
name: "test when invalid key with only api/all=false",
testOptions: &APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{"api/all": "false"},
},
expectErr: "invalid key with only api/all=false",
name: "test when invalid key with only api/all=false",
runtimeConfig: cliflag.ConfigurationMap{"api/all": "false"},
expectErr: "invalid key with only api/all=false",
},
{
name: "test when ConfigurationMap key is invalid",
testOptions: &APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{"apiall": "false"},
},
expectErr: "runtime-config invalid key",
name: "test when ConfigurationMap key is invalid",
runtimeConfig: cliflag.ConfigurationMap{"apiall": "false"},
expectErr: "runtime-config invalid key",
},
{
name: "test when unknown api groups",
testOptions: &APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{"api/v1": "true"},
},
expectErr: "unknown api groups",
name: "test when unknown api groups",
runtimeConfig: cliflag.ConfigurationMap{"api/v1": "true"},
expectErr: "unknown api groups",
},
{
name: "test when valid api groups",
testOptions: &APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{"apiregistration.k8s.io/v1beta1": "true"},
},
name: "test when valid api groups",
runtimeConfig: cliflag.ConfigurationMap{"apiregistration.k8s.io/v1beta1": "true"},
},
}
testGroupRegistry := fakeGroupRegistry{}
for _, testcase := range testCases {
t.Run(testcase.name, func(t *testing.T) {
errs := testcase.testOptions.Validate(testGroupRegistry)
testOptions := &APIEnablementOptions{
RuntimeConfig: testcase.runtimeConfig,
}
errs := testOptions.Validate(testGroupRegistry)
if len(testcase.expectErr) != 0 && !strings.Contains(utilerrors.NewAggregate(errs).Error(), testcase.expectErr) {
t.Errorf("got err: %v, expected err: %s", errs, testcase.expectErr)
}

View File

@@ -383,8 +383,8 @@ type StorageFactoryRestOptionsFactory struct {
StorageFactory serverstorage.StorageFactory
}
func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
storageConfig, err := f.StorageFactory.NewConfig(resource)
func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) {
storageConfig, err := f.StorageFactory.NewConfig(resource, example)
if err != nil {
return generic.RESTOptions{}, fmt.Errorf("unable to find storage destination for %v, due to %v", resource, err.Error())
}
@@ -469,7 +469,7 @@ type SimpleStorageFactory struct {
StorageConfig storagebackend.Config
}
func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
return s.StorageConfig.ForResource(resource), nil
}
@@ -493,8 +493,8 @@ type transformerStorageFactory struct {
resourceTransformers storagevalue.ResourceTransformers
}
func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
config, err := t.delegate.NewConfig(resource)
func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
config, err := t.delegate.NewConfig(resource, example)
if err != nil {
return nil, err
}

View File

@@ -437,7 +437,7 @@ func TestRestOptionsStorageObjectCountTracker(t *testing.T) {
if err := etcdOptions.ApplyTo(serverConfig); err != nil {
t.Fatalf("Failed to apply etcd options error: %v", err)
}
restOptions, err := serverConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""})
restOptions, err := serverConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}, nil)
if err != nil {
t.Fatal(err)
}

View File

@@ -26,7 +26,8 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apiserver/pkg/server"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/component-base/featuregate"
"github.com/spf13/pflag"
)
@@ -89,9 +90,13 @@ type ServerRunOptions struct {
// This grace period is orthogonal to other grace periods, and
// it is not overridden by any other grace period.
ShutdownWatchTerminationGracePeriod time.Duration
// FeatureGate are the featuregate to install on the CLI
FeatureGate featuregate.FeatureGate
EffectiveVersion utilversion.EffectiveVersion
}
func NewServerRunOptions() *ServerRunOptions {
func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions {
defaults := server.NewConfig(serializer.CodecFactory{})
return &ServerRunOptions{
MaxRequestsInFlight: defaults.MaxRequestsInFlight,
@@ -104,6 +109,8 @@ func NewServerRunOptions() *ServerRunOptions {
JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes,
MaxRequestBodyBytes: defaults.MaxRequestBodyBytes,
ShutdownSendRetryAfter: false,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
}
}
@@ -124,6 +131,8 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
c.PublicAddress = s.AdvertiseAddress
c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter
c.ShutdownWatchTerminationGracePeriod = s.ShutdownWatchTerminationGracePeriod
c.EffectiveVersion = s.EffectiveVersion
c.FeatureGate = s.FeatureGate
return nil
}
@@ -196,6 +205,14 @@ func (s *ServerRunOptions) Validate() []error {
if err := validateCorsAllowedOriginList(s.CorsAllowedOriginList); err != nil {
errors = append(errors, err)
}
if s.FeatureGate != nil {
if errs := s.FeatureGate.Validate(); len(errs) != 0 {
errors = append(errors, errs...)
}
}
if errs := s.EffectiveVersion.Validate(); len(errs) != 0 {
errors = append(errors, errs...)
}
return errors
}
@@ -336,6 +353,15 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
fs.DurationVar(&s.ShutdownWatchTerminationGracePeriod, "shutdown-watch-termination-grace-period", s.ShutdownWatchTerminationGracePeriod, ""+
"This option, if set, represents the maximum amount of grace period the apiserver will wait "+
"for active watch request(s) to drain during the graceful server shutdown window.")
utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
}
// Complete fills missing fields with defaults.
func (s *ServerRunOptions) Complete() error {
if s.FeatureGate == nil {
return fmt.Errorf("nil FeatureGate in ServerRunOptions")
}
if s.EffectiveVersion == nil {
return fmt.Errorf("nil EffectiveVersion in ServerRunOptions")
}
return nil
}

View File

@@ -23,10 +23,14 @@ import (
"time"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
netutils "k8s.io/utils/net"
)
func TestServerRunOptionsValidate(t *testing.T) {
featureGate := utilfeature.DefaultFeatureGate.DeepCopy()
effectiveVersion := utilversion.NewEffectiveVersion("1.30")
testCases := []struct {
name string
testOptions *ServerRunOptions
@@ -43,6 +47,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
expectErr: "--max-requests-inflight can not be negative value",
},
@@ -57,6 +63,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
expectErr: "--max-mutating-requests-inflight can not be negative value",
},
@@ -71,6 +79,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
expectErr: "--request-timeout can not be negative value",
},
@@ -85,6 +95,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: -1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
expectErr: "--min-request-timeout can not be negative value",
},
@@ -99,6 +111,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: -10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
expectErr: "ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value",
},
@@ -113,6 +127,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: -10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
expectErr: "ServerRunOptions.MaxRequestBodyBytes can not be negative value",
},
@@ -128,6 +144,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
LivezGracePeriod: -time.Second,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
expectErr: "--livez-grace-period can not be a negative value",
},
@@ -143,6 +161,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ShutdownDelayDuration: -time.Second,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
expectErr: "--shutdown-delay-duration can not be negative value",
},
@@ -158,6 +178,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
expectErr: "--strict-transport-security-directives invalid, allowed values: max-age=expireTime, includeSubDomains, preload. see https://tools.ietf.org/html/rfc6797#section-6.1 for more information",
},
@@ -173,6 +195,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
},
}
@@ -192,6 +216,8 @@ func TestServerRunOptionsValidate(t *testing.T) {
}
func TestValidateCorsAllowedOriginList(t *testing.T) {
featureGate := utilfeature.DefaultFeatureGate.DeepCopy()
effectiveVersion := utilversion.NewEffectiveVersion("1.30")
tests := []struct {
regexp [][]string
errShouldContain string
@@ -239,7 +265,7 @@ func TestValidateCorsAllowedOriginList(t *testing.T) {
for _, test := range tests {
for _, regexp := range test.regexp {
t.Run(fmt.Sprintf("regexp/%s", regexp), func(t *testing.T) {
options := NewServerRunOptions()
options := NewServerRunOptions(featureGate, effectiveVersion)
if errs := options.Validate(); len(errs) != 0 {
t.Fatalf("wrong test setup: %#v", errs)
}
@@ -263,6 +289,8 @@ func TestValidateCorsAllowedOriginList(t *testing.T) {
}
func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) {
featureGate := utilfeature.DefaultFeatureGate.DeepCopy()
effectiveVersion := utilversion.NewEffectiveVersion("1.30")
tests := []struct {
name string
optionsFn func() *ServerRunOptions
@@ -271,13 +299,13 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) {
{
name: "default should be valid",
optionsFn: func() *ServerRunOptions {
return NewServerRunOptions()
return NewServerRunOptions(featureGate, effectiveVersion)
},
},
{
name: "negative not allowed",
optionsFn: func() *ServerRunOptions {
o := NewServerRunOptions()
o := NewServerRunOptions(featureGate, effectiveVersion)
o.ShutdownWatchTerminationGracePeriod = -time.Second
return o
},
@@ -304,7 +332,7 @@ func TestServerRunOptionsWithShutdownWatchTerminationGracePeriod(t *testing.T) {
}
t.Run("default should be zero", func(t *testing.T) {
options := NewServerRunOptions()
options := NewServerRunOptions(featureGate, effectiveVersion)
if options.ShutdownWatchTerminationGracePeriod != time.Duration(0) {
t.Errorf("expected default of ShutdownWatchTerminationGracePeriod to be zero, but got: %s", options.ShutdownWatchTerminationGracePeriod)
}

View File

@@ -43,6 +43,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/version"
"k8s.io/apiserver/pkg/server"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/discovery"
restclient "k8s.io/client-go/rest"
cliflag "k8s.io/component-base/cli/flag"
@@ -276,9 +277,9 @@ func TestServerRunWithSNI(t *testing.T) {
// launch server
config := setUp(t)
v := fakeVersion()
config.Version = &v
config.EffectiveVersion = utilversion.NewEffectiveVersion(v.String())
config.EnableIndex = true
secureOptions := (&SecureServingOptions{
@@ -463,11 +464,9 @@ func certSignature(cert tls.Certificate) (string, error) {
func fakeVersion() version.Info {
return version.Info{
Major: "42",
Minor: "42",
GitVersion: "42",
GitCommit: "34973274ccef6ab4dfaaf86599792fa9c3fe4689",
GitTreeState: "Dirty",
Major: "42",
Minor: "42",
GitVersion: "42.42",
}
}

View File

@@ -21,6 +21,8 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/util/version"
)
type ResourceEncodingConfig interface {
@@ -33,10 +35,15 @@ type ResourceEncodingConfig interface {
InMemoryEncodingFor(schema.GroupResource) (schema.GroupVersion, error)
}
type CompatibilityResourceEncodingConfig interface {
BackwardCompatibileStorageEncodingFor(schema.GroupResource, runtime.Object) (schema.GroupVersion, error)
}
type DefaultResourceEncodingConfig struct {
// resources records the overriding encoding configs for individual resources.
resources map[schema.GroupResource]*OverridingResourceEncoding
scheme *runtime.Scheme
resources map[schema.GroupResource]*OverridingResourceEncoding
scheme *runtime.Scheme
effectiveVersion version.EffectiveVersion
}
type OverridingResourceEncoding struct {
@@ -47,7 +54,7 @@ type OverridingResourceEncoding struct {
var _ ResourceEncodingConfig = &DefaultResourceEncodingConfig{}
func NewDefaultResourceEncodingConfig(scheme *runtime.Scheme) *DefaultResourceEncodingConfig {
return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme}
return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme, effectiveVersion: version.DefaultKubeEffectiveVersion()}
}
func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored schema.GroupResource, externalEncodingVersion, internalVersion schema.GroupVersion) {
@@ -57,6 +64,10 @@ func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored
}
}
func (o *DefaultResourceEncodingConfig) SetEffectiveVersion(effectiveVersion version.EffectiveVersion) {
o.effectiveVersion = effectiveVersion
}
func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) {
if !o.scheme.IsGroupRegistered(resource.Group) {
return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
@@ -71,6 +82,24 @@ func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.Group
return o.scheme.PrioritizedVersionsForGroup(resource.Group)[0], nil
}
func (o *DefaultResourceEncodingConfig) BackwardCompatibileStorageEncodingFor(resource schema.GroupResource, example runtime.Object) (schema.GroupVersion, error) {
if !o.scheme.IsGroupRegistered(resource.Group) {
return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
}
// Always respect overrides
resourceOverride, resourceExists := o.resources[resource]
if resourceExists {
return resourceOverride.ExternalResourceEncoding, nil
}
return emulatedStorageVersion(
o.scheme.PrioritizedVersionsForGroup(resource.Group)[0],
example,
o.effectiveVersion,
o.scheme)
}
func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) {
if !o.scheme.IsGroupRegistered(resource.Group) {
return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
@@ -82,3 +111,78 @@ func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.Grou
}
return schema.GroupVersion{Group: resource.Group, Version: runtime.APIVersionInternal}, nil
}
// Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
type introducedInterface interface {
APILifecycleIntroduced() (major, minor int)
}
func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example runtime.Object, effectiveVersion version.EffectiveVersion, scheme *runtime.Scheme) (schema.GroupVersion, error) {
if example == nil || effectiveVersion == nil {
return binaryVersionOfResource, nil
}
// Look up example in scheme to find all objects of the same Group-Kind
// Use the highest priority version for that group-kind whose lifecycle window
// includes the current emulation version.
// If no version is found, use the binary version
// (in this case the API should be disabled anyway)
gvks, _, err := scheme.ObjectKinds(example)
if err != nil {
return schema.GroupVersion{}, err
} else if len(gvks) == 0 {
// Probably shouldn't happen if err is non-nil
return schema.GroupVersion{}, fmt.Errorf("object %T has no GVKs registered in scheme", example)
}
// VersionsForGroupKind returns versions in priority order
versions := scheme.VersionsForGroupKind(schema.GroupKind{Group: gvks[0].Group, Kind: gvks[0].Kind})
compatibilityVersion := effectiveVersion.MinCompatibilityVersion()
for _, gv := range versions {
if gv.Version == runtime.APIVersionInternal {
continue
}
gvk := schema.GroupVersionKind{
Group: gv.Group,
Version: gv.Version,
Kind: gvks[0].Kind,
}
exampleOfGVK, err := scheme.New(gvk)
if err != nil {
return schema.GroupVersion{}, err
}
// If it was introduced after current compatibility version, don't use it
if introduced, hasIntroduced := exampleOfGVK.(introducedInterface); hasIntroduced && (compatibilityVersion.Major() > 0 || compatibilityVersion.Minor() > 0) {
// API resource lifecycles should be relative to k8s api version
majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced()
introducedVer := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced))
if introducedVer.GreaterThan(compatibilityVersion) {
continue
}
}
// versions is returned in priority order, so just use first result
return gvk.GroupVersion(), nil
}
// Getting here means we're serving a version that is unknown to the
// min-compatibility-version server.
//
// This is only expected to happen when serving an alpha API type due
// to missing pre-release lifecycle information
// (which doesn't happen by default), or when emulation-version and
// min-compatibility-version are several versions apart so a beta or GA API
// was being served which didn't exist at all in min-compatibility-version.
//
// In the alpha case - we do not support compatibility versioning of
// alpha types and recommend users do not mix the two.
// In the skip-level case - The version of apiserver we are retaining
// compatibility with has no knowledge of the type,
// so storing it in another type is no issue.
return binaryVersionOfResource, nil
}

View File

@@ -42,7 +42,7 @@ type Backend struct {
type StorageFactory interface {
// New finds the storage destination for the given group and resource. It will
// return an error if the group has no storage destination configured.
NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error)
NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error)
// ResourcePrefix returns the overridden resource prefix for the GroupResource
// This allows for cohabitation of resources with different native types and provides
@@ -226,7 +226,7 @@ func (s *DefaultStorageFactory) getStorageGroupResource(groupResource schema.Gro
// New finds the storage destination for the given group and resource. It will
// return an error if the group has no storage destination configured.
func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
chosenStorageResource := s.getStorageGroupResource(groupResource)
// operate on copy
@@ -244,14 +244,23 @@ func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*
}
var err error
codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource)
if err != nil {
return nil, err
if backwardCompatibleInterface, ok := s.ResourceEncodingConfig.(CompatibilityResourceEncodingConfig); ok {
codecConfig.StorageVersion, err = backwardCompatibleInterface.BackwardCompatibileStorageEncodingFor(groupResource, example)
if err != nil {
return nil, err
}
} else {
codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource)
if err != nil {
return nil, err
}
}
codecConfig.MemoryVersion, err = s.ResourceEncodingConfig.InMemoryEncodingFor(groupResource)
if err != nil {
return nil, err
}
codecConfig.Config = storageConfig
storageConfig.Codec, storageConfig.EncodeVersioner, err = s.newStorageCodecFn(codecConfig)

View File

@@ -26,10 +26,12 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/apis/example"
exampleinstall "k8s.io/apiserver/pkg/apis/example/install"
examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/apiserver/pkg/util/version"
)
var (
@@ -118,7 +120,7 @@ func TestConfigurableStorageFactory(t *testing.T) {
f.SetEtcdLocation(example.Resource("*"), []string{"/server2"})
f.SetEtcdPrefix(example.Resource("test"), "/prefix_for_test")
config, err := f.NewConfig(example.Resource("test"))
config, err := f.NewConfig(example.Resource("test"), nil)
if err != nil {
t.Fatal(err)
}
@@ -163,7 +165,7 @@ func TestUpdateEtcdOverrides(t *testing.T) {
storageFactory.SetEtcdLocation(test.resource, test.servers)
var err error
config, err := storageFactory.NewConfig(test.resource)
config, err := storageFactory.NewConfig(test.resource, nil)
if err != nil {
t.Errorf("%d: unexpected error %v", i, err)
continue
@@ -173,7 +175,7 @@ func TestUpdateEtcdOverrides(t *testing.T) {
continue
}
config, err = storageFactory.NewConfig(schema.GroupResource{Group: examplev1.GroupName, Resource: "unlikely"})
config, err = storageFactory.NewConfig(schema.GroupResource{Group: examplev1.GroupName, Resource: "unlikely"}, nil)
if err != nil {
t.Errorf("%d: unexpected error %v", i, err)
continue
@@ -244,3 +246,241 @@ func TestConfigs(t *testing.T) {
}
}
}
var introducedLifecycles = map[reflect.Type]*apimachineryversion.Version{}
var removedLifecycles = map[reflect.Type]*apimachineryversion.Version{}
type fakeLifecycler[T, V any] struct {
metav1.TypeMeta
metav1.ObjectMeta
}
type removedLifecycler[T, V any] struct {
fakeLifecycler[T, V]
}
func (f *fakeLifecycler[T, V]) GetObjectKind() schema.ObjectKind { return f }
func (f *fakeLifecycler[T, V]) DeepCopyObject() runtime.Object { return f }
func (f *fakeLifecycler[T, V]) APILifecycleIntroduced() (major, minor int) {
if introduced, ok := introducedLifecycles[reflect.TypeOf(f)]; ok {
return int(introduced.Major()), int(introduced.Minor())
}
panic("no lifecycle version set")
}
func (f *removedLifecycler[T, V]) APILifecycleRemoved() (major, minor int) {
if removed, ok := removedLifecycles[reflect.TypeOf(f)]; ok {
return int(removed.Major()), int(removed.Minor())
}
panic("no lifecycle version set")
}
func registerFakeLifecycle[T, V any](sch *runtime.Scheme, group, introduced, removed string) {
f := fakeLifecycler[T, V]{}
introducedLifecycles[reflect.TypeOf(&f)] = apimachineryversion.MustParseSemantic(introduced)
var res runtime.Object
if removed != "" {
removedLifecycles[reflect.TypeOf(&f)] = apimachineryversion.MustParseSemantic(removed)
res = &removedLifecycler[T, V]{fakeLifecycler: f}
} else {
res = &f
}
var v V
var t T
sch.AddKnownTypeWithName(
schema.GroupVersionKind{
Group: group,
Version: strings.ToLower(reflect.TypeOf(v).Name()),
Kind: reflect.TypeOf(t).Name(),
},
res,
)
// Also ensure internal version is registered
// If it is registertd multiple times, it will ignore subsequent registrations
internalInstance := &fakeLifecycler[T, struct{}]{}
sch.AddKnownTypeWithName(
schema.GroupVersionKind{
Group: group,
Version: runtime.APIVersionInternal,
Kind: reflect.TypeOf(t).Name(),
},
internalInstance,
)
}
func TestStorageFactoryCompatibilityVersion(t *testing.T) {
// Creates a scheme with stub types for unit test
sch := runtime.NewScheme()
codecs := serializer.NewCodecFactory(sch)
type Internal = struct{}
type V1beta1 struct{}
type V1beta2 struct{}
type V1beta3 struct{}
type V1 struct{}
type Pod struct{}
type FlowSchema struct{}
type ValidatingAdmisisonPolicy struct{}
type CronJob struct{}
// Order dictates priority order
registerFakeLifecycle[FlowSchema, V1](sch, "flowcontrol.apiserver.k8s.io", "1.29.0", "")
registerFakeLifecycle[FlowSchema, V1beta3](sch, "flowcontrol.apiserver.k8s.io", "1.26.0", "1.32.0")
registerFakeLifecycle[FlowSchema, V1beta2](sch, "flowcontrol.apiserver.k8s.io", "1.23.0", "1.29.0")
registerFakeLifecycle[FlowSchema, V1beta1](sch, "flowcontrol.apiserver.k8s.io", "1.20.0", "1.26.0")
registerFakeLifecycle[CronJob, V1](sch, "batch", "1.21.0", "")
registerFakeLifecycle[CronJob, V1beta1](sch, "batch", "1.8.0", "1.21.0")
registerFakeLifecycle[ValidatingAdmisisonPolicy, V1](sch, "admissionregistration.k8s.io", "1.30.0", "")
registerFakeLifecycle[ValidatingAdmisisonPolicy, V1beta1](sch, "admissionregistration.k8s.io", "1.28.0", "1.34.0")
registerFakeLifecycle[Pod, V1](sch, "", "1.31.0", "")
// FlowSchema
// - v1beta1: 1.20.0 - 1.23.0
// - v1beta2: 1.23.0 - 1.26.0
// - v1beta3: 1.26.0 - 1.30.0
// - v1: 1.29.0+
// CronJob
// - v1beta1: 1.8.0 - 1.21.0
// - v1: 1.21.0+
// ValidatingAdmissionPolicy
// - v1beta1: 1.28.0 - 1.31.0
// - v1: 1.30.0+
testcases := []struct {
effectiveVersion string
example runtime.Object
expectedVersion schema.GroupVersion
}{
{
// Basic case. Beta version for long time
effectiveVersion: "1.14.0",
example: &fakeLifecycler[CronJob, Internal]{},
expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1beta1"},
},
{
// Basic case. Beta version for long time
effectiveVersion: "1.20.0",
example: &fakeLifecycler[CronJob, Internal]{},
expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1beta1"},
},
{
// Basic case. GA version for long time
effectiveVersion: "1.28.0",
example: &fakeLifecycler[CronJob, Internal]{},
expectedVersion: schema.GroupVersion{Group: "batch", Version: "v1"},
},
{
// Basic core/v1
effectiveVersion: "1.31.0",
example: &fakeLifecycler[Pod, Internal]{},
expectedVersion: schema.GroupVersion{Group: "", Version: "v1"},
},
{
// Corner case: 1.1.0 has no flowcontrol. Options are to error
// out or to use the latest version. This test assumes the latter.
effectiveVersion: "1.1.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1"},
},
{
effectiveVersion: "1.21.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1"},
},
{
// v2Beta1 introduced this version, but minCompatibility should
// force v1beta1
effectiveVersion: "1.23.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1"},
},
{
effectiveVersion: "1.24.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2"},
},
{
effectiveVersion: "1.26.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2"},
},
{
effectiveVersion: "1.27.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta3"},
},
{
// GA API introduced 1.29 but must keep storing in v1beta3 for downgrades
effectiveVersion: "1.29.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta3"},
},
{
// Version after GA api is introduced
effectiveVersion: "1.30.0",
example: &fakeLifecycler[FlowSchema, Internal]{},
expectedVersion: schema.GroupVersion{Group: "flowcontrol.apiserver.k8s.io", Version: "v1"},
},
{
effectiveVersion: "1.30.0",
example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{},
expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1beta1"},
},
{
effectiveVersion: "1.31.0",
example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{},
expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1"},
},
{
effectiveVersion: "1.29.0",
example: &fakeLifecycler[ValidatingAdmisisonPolicy, Internal]{},
expectedVersion: schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1beta1"},
},
}
for _, tc := range testcases {
gvks, _, err := sch.ObjectKinds(tc.example)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
gvk := gvks[0]
t.Run(gvk.GroupKind().String()+"@"+tc.effectiveVersion, func(t *testing.T) {
config := NewDefaultResourceEncodingConfig(sch)
config.SetEffectiveVersion(version.NewEffectiveVersion(tc.effectiveVersion))
f := NewDefaultStorageFactory(
storagebackend.Config{},
"",
codecs,
config,
NewResourceConfig(),
nil)
cfg, err := f.NewConfig(schema.GroupResource{
Group: gvk.Group,
Resource: gvk.Kind, // doesnt really matter here
}, tc.example)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
gvks, _, err := sch.ObjectKinds(tc.example)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
expectEncodeVersioner := runtime.NewMultiGroupVersioner(tc.expectedVersion,
schema.GroupKind{
Group: gvks[0].Group,
}, schema.GroupKind{
Group: gvks[0].Group,
})
if cfg.EncodeVersioner.Identifier() != expectEncodeVersioner.Identifier() {
t.Errorf("expected %v, got %v", expectEncodeVersioner, cfg.EncodeVersioner)
}
})
}
}

View File

@@ -25,7 +25,7 @@ var (
// Only top-level commands/options setup and the k8s.io/component-base/featuregate/testing package should make use of this.
// Tests that need to modify feature gates for the duration of their test should use:
// featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.<FeatureName>, <value>)
DefaultMutableFeatureGate featuregate.MutableFeatureGate = featuregate.NewFeatureGate()
DefaultMutableFeatureGate featuregate.MutableVersionedFeatureGate = featuregate.NewFeatureGate()
// DefaultFeatureGate is a shared global FeatureGate.
// Top-level commands/options setup that needs to modify this feature gate should use DefaultMutableFeatureGate.

View File

@@ -0,0 +1,142 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package version
import (
"fmt"
"sync"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/component-base/featuregate"
)
var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGlobalsRegistry()
const (
ComponentGenericAPIServer = "k8s.io/apiserver"
)
// ComponentGlobals stores the global variables for a component for easy access.
type ComponentGlobals struct {
effectiveVersion MutableEffectiveVersion
featureGate featuregate.MutableVersionedFeatureGate
}
type ComponentGlobalsRegistry interface {
// EffectiveVersionFor returns the EffectiveVersion registered under the component.
// Returns nil if the component is not registered.
EffectiveVersionFor(component string) EffectiveVersion
// FeatureGateFor returns the FeatureGate registered under the component.
// Returns nil if the component is not registered.
FeatureGateFor(component string) featuregate.FeatureGate
// Register registers the EffectiveVersion and FeatureGate for a component.
// Overrides existing ComponentGlobals if it is already in the registry if override is true,
// otherwise returns error if the component is already registered.
Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error
// ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry.
// Otherwise, the provided variables would be registered under the component, and the same variables would be returned.
ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate)
// SetAllComponents sets the emulation version for other global variables for all components registered.
SetAllComponents() error
// SetAllComponents calls the Validate() function for all the global variables for all components registered.
ValidateAllComponents() []error
}
type componentGlobalsRegistry struct {
componentGlobals map[string]ComponentGlobals
mutex sync.RWMutex
}
func NewComponentGlobalsRegistry() ComponentGlobalsRegistry {
return &componentGlobalsRegistry{componentGlobals: map[string]ComponentGlobals{}}
}
func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion {
r.mutex.RLock()
defer r.mutex.RUnlock()
globals, ok := r.componentGlobals[component]
if !ok {
return nil
}
return globals.effectiveVersion
}
func (r *componentGlobalsRegistry) FeatureGateFor(component string) featuregate.FeatureGate {
r.mutex.RLock()
defer r.mutex.RUnlock()
globals, ok := r.componentGlobals[component]
if !ok {
return nil
}
return globals.featureGate
}
func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error {
if _, ok := r.componentGlobals[component]; ok && !override {
return fmt.Errorf("component globals of %s already registered", component)
}
if featureGate != nil {
featureGate.DeferErrorsToValidation(true)
}
c := ComponentGlobals{effectiveVersion: effectiveVersion, featureGate: featureGate}
r.componentGlobals[component] = c
return nil
}
func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error {
r.mutex.Lock()
defer r.mutex.Unlock()
return r.unsafeRegister(component, effectiveVersion, featureGate, override)
}
func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) {
r.mutex.Lock()
defer r.mutex.Unlock()
globals, ok := r.componentGlobals[component]
if ok {
return globals.effectiveVersion, globals.featureGate
}
utilruntime.Must(r.unsafeRegister(component, effectiveVersion, featureGate, false))
return effectiveVersion, featureGate
}
func (r *componentGlobalsRegistry) SetAllComponents() error {
r.mutex.Lock()
defer r.mutex.Unlock()
for _, globals := range r.componentGlobals {
if globals.featureGate == nil {
continue
}
if err := globals.featureGate.SetEmulationVersion(globals.effectiveVersion.EmulationVersion()); err != nil {
return err
}
}
return nil
}
func (r *componentGlobalsRegistry) ValidateAllComponents() []error {
var errs []error
r.mutex.Lock()
defer r.mutex.Unlock()
for _, globals := range r.componentGlobals {
errs = append(errs, globals.effectiveVersion.Validate()...)
if globals.featureGate != nil {
errs = append(errs, globals.featureGate.Validate()...)
}
}
return errs
}

View File

@@ -0,0 +1,48 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package version
import (
"testing"
)
func TestEffectiveVersionRegistry(t *testing.T) {
r := NewComponentGlobalsRegistry()
testComponent := "test"
ver1 := NewEffectiveVersion("1.31")
ver2 := NewEffectiveVersion("1.28")
if r.EffectiveVersionFor(testComponent) != nil {
t.Fatalf("expected nil EffectiveVersion initially")
}
if err := r.Register(testComponent, ver1, nil, false); err != nil {
t.Fatalf("expected no error to register new component, but got err: %v", err)
}
if !r.EffectiveVersionFor(testComponent).EqualTo(ver1) {
t.Fatalf("expected EffectiveVersionFor to return the version registered")
}
// overwrite
if err := r.Register(testComponent, ver2, nil, false); err == nil {
t.Fatalf("expected error to register existing component when override is false")
}
if err := r.Register(testComponent, ver2, nil, true); err != nil {
t.Fatalf("expected no error to overriding existing component, but got err: %v", err)
}
if !r.EffectiveVersionFor(testComponent).EqualTo(ver2) {
t.Fatalf("expected EffectiveVersionFor to return the version overridden")
}
}

View File

@@ -0,0 +1,195 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package version
import (
"fmt"
"strings"
"sync/atomic"
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/util/version"
baseversion "k8s.io/component-base/version"
)
type EffectiveVersion interface {
BinaryVersion() *version.Version
EmulationVersion() *version.Version
MinCompatibilityVersion() *version.Version
EqualTo(other EffectiveVersion) bool
String() string
Validate() []error
}
type MutableEffectiveVersion interface {
EffectiveVersion
Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version)
SetEmulationVersion(emulationVersion *version.Version)
SetMinCompatibilityVersion(minCompatibilityVersion *version.Version)
// AddFlags adds the "{prefix}-emulated-version" to the flagset.
AddFlags(fs *pflag.FlagSet, prefix string)
}
type VersionVar struct {
val atomic.Pointer[version.Version]
}
// Set sets the flag value
func (v *VersionVar) Set(s string) error {
components := strings.Split(s, ".")
if len(components) != 2 {
return fmt.Errorf("version %s is not in the format of major.minor", s)
}
ver, err := version.ParseGeneric(s)
if err != nil {
return err
}
v.val.Store(ver)
return nil
}
// String returns the flag value
func (v *VersionVar) String() string {
ver := v.val.Load()
return ver.String()
}
// Type gets the flag type
func (v *VersionVar) Type() string {
return "version"
}
type effectiveVersion struct {
binaryVersion atomic.Pointer[version.Version]
// If the emulationVersion is set by the users, it could only contain major and minor versions.
// In tests, emulationVersion could be the same as the binary version, or set directly,
// which can have "alpha" as pre-release to continue serving expired apis while we clean up the test.
emulationVersion VersionVar
// minCompatibilityVersion could only contain major and minor versions.
minCompatibilityVersion VersionVar
}
func (m *effectiveVersion) BinaryVersion() *version.Version {
return m.binaryVersion.Load()
}
func (m *effectiveVersion) EmulationVersion() *version.Version {
// Emulation version can have "alpha" as pre-release to continue serving expired apis while we clean up the test.
// The pre-release should not be accessible to the users.
return m.emulationVersion.val.Load().WithPreRelease(m.BinaryVersion().PreRelease())
}
func (m *effectiveVersion) MinCompatibilityVersion() *version.Version {
return m.minCompatibilityVersion.val.Load()
}
func (m *effectiveVersion) EqualTo(other EffectiveVersion) bool {
return m.BinaryVersion().EqualTo(other.BinaryVersion()) && m.EmulationVersion().EqualTo(other.EmulationVersion()) && m.MinCompatibilityVersion().EqualTo(other.MinCompatibilityVersion())
}
func (m *effectiveVersion) String() string {
if m == nil {
return "<nil>"
}
return fmt.Sprintf("{BinaryVersion: %s, EmulationVersion: %s, MinCompatibilityVersion: %s}",
m.BinaryVersion().String(), m.EmulationVersion().String(), m.MinCompatibilityVersion().String())
}
func (m *effectiveVersion) Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) {
m.binaryVersion.Store(binaryVersion)
m.emulationVersion.val.Store(version.MajorMinor(emulationVersion.Major(), emulationVersion.Minor()))
m.minCompatibilityVersion.val.Store(version.MajorMinor(minCompatibilityVersion.Major(), minCompatibilityVersion.Minor()))
}
func (m *effectiveVersion) SetEmulationVersion(emulationVersion *version.Version) {
m.emulationVersion.val.Store(version.MajorMinor(emulationVersion.Major(), emulationVersion.Minor()))
}
func (m *effectiveVersion) SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) {
m.minCompatibilityVersion.val.Store(version.MajorMinor(minCompatibilityVersion.Major(), minCompatibilityVersion.Minor()))
}
func (m *effectiveVersion) Validate() []error {
var errs []error
// Validate only checks the major and minor versions.
binaryVersion := m.binaryVersion.Load().WithPatch(0)
emulationVersion := m.emulationVersion.val.Load()
minCompatibilityVersion := m.minCompatibilityVersion.val.Load()
// emulationVersion can only be 1.{binaryMinor-1}...1.{binaryMinor}.
maxEmuVer := binaryVersion
minEmuVer := binaryVersion.SubtractMinor(1)
// TODO: remove in 1.32
// emulationVersion is introduced in 1.31, so it cannot be lower than that.
// binaryVersion could be lower than 1.31 in tests. So we are only checking 1.31.
if binaryVersion.EqualTo(version.MajorMinor(1, 31)) {
minEmuVer = version.MajorMinor(1, 31)
}
if emulationVersion.GreaterThan(maxEmuVer) || emulationVersion.LessThan(minEmuVer) {
errs = append(errs, fmt.Errorf("emulation version %s is not between [%s, %s]", emulationVersion.String(), minEmuVer.String(), maxEmuVer.String()))
}
// minCompatibilityVersion can only be 1.{binaryMinor-1} for alpha.
maxCompVer := binaryVersion.SubtractMinor(1)
minCompVer := binaryVersion.SubtractMinor(1)
if minCompatibilityVersion.GreaterThan(maxCompVer) || minCompatibilityVersion.LessThan(minCompVer) {
errs = append(errs, fmt.Errorf("minCompatibilityVersion version %s is not between [%s, %s]", minCompatibilityVersion.String(), minCompVer.String(), maxCompVer.String()))
}
return errs
}
// AddFlags adds the "{prefix}-emulated-version" to the flagset.
func (m *effectiveVersion) AddFlags(fs *pflag.FlagSet, prefix string) {
if m == nil {
return
}
if len(prefix) > 0 && !strings.HasSuffix(prefix, "-") {
prefix += "-"
}
fs.Var(&m.emulationVersion, prefix+"emulated-version", ""+
"The version the K8s component emulates its capabilities (APIs, features, ...) of.\n"+
"If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+
"Any capabilities present in the binary version that were introduced after the emulated version will be unavailable and any capabilities removed after the emulated version will be available.\n"+
"This flag applies only to component capabilities, and does not disable bug fixes and performance improvements present in the binary version.\n"+
"Defaults to the binary version. The value should be between 1.{binaryMinorVersion-1} and 1.{binaryMinorVersion}.\n"+
"Format could only be major.minor")
}
func NewEffectiveVersion(binaryVer string) MutableEffectiveVersion {
effective := &effectiveVersion{}
binaryVersion := version.MustParse(binaryVer)
compatVersion := binaryVersion.SubtractMinor(1)
effective.Set(binaryVersion, binaryVersion, compatVersion)
return effective
}
// DefaultBuildEffectiveVersion returns the MutableEffectiveVersion based on the
// current build information.
func DefaultBuildEffectiveVersion() MutableEffectiveVersion {
verInfo := baseversion.Get()
ver := NewEffectiveVersion(verInfo.String())
if ver.BinaryVersion().Major() == 0 && ver.BinaryVersion().Minor() == 0 {
ver = DefaultKubeEffectiveVersion()
}
return ver
}
// DefaultKubeEffectiveVersion returns the MutableEffectiveVersion based on the
// latest K8s release.
// Should update for each minor release!
func DefaultKubeEffectiveVersion() MutableEffectiveVersion {
return NewEffectiveVersion("1.31")
}

View File

@@ -0,0 +1,180 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package version
import (
"fmt"
"strings"
"testing"
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/util/version"
)
func TestValidate(t *testing.T) {
tests := []struct {
name string
binaryVersion string
emulationVersion string
minCompatibilityVersion string
expectErrors bool
}{
{
name: "patch version diff ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.1",
minCompatibilityVersion: "v1.31.5",
},
{
name: "emulation version one minor lower than binary ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.31.0",
},
{
name: "binary version 1.31, emulation version lower than 1.31",
binaryVersion: "v1.31.2",
emulationVersion: "v1.30.0",
minCompatibilityVersion: "v1.30.0",
expectErrors: true,
},
{
name: "binary version 1.31, emulation version 1.31",
binaryVersion: "v1.31.2",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.30.0",
},
{
name: "binary version lower than 1.31",
binaryVersion: "v1.30.2",
emulationVersion: "v1.29.0",
minCompatibilityVersion: "v1.29.0",
},
{
name: "emulation version two minor lower than binary not ok",
binaryVersion: "v1.33.2",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.32.0",
expectErrors: true,
},
{
name: "emulation version one minor higher than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.33.0",
minCompatibilityVersion: "v1.31.0",
expectErrors: true,
},
{
name: "emulation version two minor higher than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.34.0",
minCompatibilityVersion: "v1.31.0",
expectErrors: true,
},
{
name: "compatibility version same as binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.32.0",
expectErrors: true,
},
{
name: "compatibility version two minor lower than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.30.0",
expectErrors: true,
},
{
name: "compatibility version one minor higher than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.33.0",
expectErrors: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
binaryVersion := version.MustParseGeneric(test.binaryVersion)
effective := &effectiveVersion{}
emulationVersion := version.MustParseGeneric(test.emulationVersion)
minCompatibilityVersion := version.MustParseGeneric(test.minCompatibilityVersion)
effective.Set(binaryVersion, emulationVersion, minCompatibilityVersion)
errs := effective.Validate()
if len(errs) > 0 && !test.expectErrors {
t.Errorf("expected no errors, errors found %+v", errs)
}
if len(errs) == 0 && test.expectErrors {
t.Errorf("expected errors, no errors found")
}
})
}
}
func TestEffectiveVersionsFlag(t *testing.T) {
tests := []struct {
name string
emulationVerson string
expectedEmulationVersion *version.Version
parseError string
}{
{
name: "major.minor ok",
emulationVerson: "1.30",
expectedEmulationVersion: version.MajorMinor(1, 30),
},
{
name: "v prefix ok",
emulationVerson: "v1.30",
expectedEmulationVersion: version.MajorMinor(1, 30),
},
{
name: "semantic version not ok",
emulationVerson: "1.30.1",
parseError: "version 1.30.1 is not in the format of major.minor",
},
{
name: "invalid version",
emulationVerson: "1.foo",
parseError: "illegal version string",
},
}
for i, test := range tests {
t.Run(test.name, func(t *testing.T) {
fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError)
effective := NewEffectiveVersion("1.30")
effective.AddFlags(fs, "test")
err := fs.Parse([]string{fmt.Sprintf("--test-emulated-version=%s", test.emulationVerson)})
if test.parseError != "" {
if !strings.Contains(err.Error(), test.parseError) {
t.Fatalf("%d: Parse() Expected %v, Got %v", i, test.parseError, err)
}
return
}
if err != nil {
t.Fatalf("%d: Parse() Expected nil, Got %v", i, err)
}
if !effective.EmulationVersion().EqualTo(test.expectedEmulationVersion) {
t.Errorf("%d: EmulationVersion Expected %s, Got %s", i, test.expectedEmulationVersion.String(), effective.EmulationVersion().String())
}
})
}
}

View File

@@ -164,7 +164,7 @@ func (o *CloudControllerManagerOptions) Flags(allControllers []string, disabledB
fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
fs.StringVar(&o.Generic.ClientConnection.Kubeconfig, "kubeconfig", o.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).")
fs.DurationVar(&o.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", o.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.")
utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"))
utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"), "")
return fss
}

View File

@@ -19,6 +19,7 @@ package featuregate
import (
"context"
"fmt"
"reflect"
"sort"
"strconv"
"strings"
@@ -27,8 +28,11 @@ import (
"github.com/spf13/pflag"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/naming"
"k8s.io/apimachinery/pkg/util/version"
featuremetrics "k8s.io/component-base/metrics/prometheus/feature"
baseversion "k8s.io/component-base/version"
"k8s.io/klog/v2"
)
@@ -52,13 +56,13 @@ const (
var (
// The generic features.
defaultFeatures = map[Feature]FeatureSpec{
allAlphaGate: {Default: false, PreRelease: Alpha},
allBetaGate: {Default: false, PreRelease: Beta},
defaultFeatures = map[Feature]VersionedSpecs{
allAlphaGate: {{Default: false, PreRelease: Alpha, Version: version.MajorMinor(0, 0)}},
allBetaGate: {{Default: false, PreRelease: Beta, Version: version.MajorMinor(0, 0)}},
}
// Special handling for a few gates.
specialFeatures = map[Feature]func(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool){
specialFeatures = map[Feature]func(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, cVer *version.Version){
allAlphaGate: setUnsetAlphaGates,
allBetaGate: setUnsetBetaGates,
}
@@ -69,13 +73,28 @@ type FeatureSpec struct {
Default bool
// LockToDefault indicates that the feature is locked to its default and cannot be changed
LockToDefault bool
// PreRelease indicates the maturity level of the feature
// PreRelease indicates the current maturity level of the feature
PreRelease prerelease
// Version indicates the earliest version from which this FeatureSpec is valid.
// If multiple FeatureSpecs exist for a Feature, the one with the highest version that is less
// than or equal to the effective version of the component is used.
Version *version.Version
}
type VersionedSpecs []FeatureSpec
func (g VersionedSpecs) Len() int { return len(g) }
func (g VersionedSpecs) Less(i, j int) bool {
return g[i].Version.LessThan(g[j].Version)
}
func (g VersionedSpecs) Swap(i, j int) { g[i], g[j] = g[j], g[i] }
type PromotionVersionMapping map[prerelease]string
type prerelease string
const (
PreAlpha = prerelease("PRE-ALPHA")
// Values for PreRelease.
Alpha = prerelease("ALPHA")
Beta = prerelease("BETA")
@@ -94,7 +113,10 @@ type FeatureGate interface {
// DeepCopy returns a deep copy of the FeatureGate object, such that gates can be
// set on the copy without mutating the original. This is useful for validating
// config against potential feature gate changes before committing those changes.
DeepCopy() MutableFeatureGate
DeepCopy() MutableVersionedFeatureGate
// Validate checks if the flag gates are valid at the emulated version.
// Should always be called after Set when DeferErrorsToValidation is set to true.
Validate() []error
}
// MutableFeatureGate parses and stores flag gates for known features from
@@ -103,7 +125,7 @@ type MutableFeatureGate interface {
FeatureGate
// AddFlag adds a flag for setting global feature gates to the specified FlagSet.
AddFlag(fs *pflag.FlagSet)
AddFlag(fs *pflag.FlagSet, prefix string)
// Set parses and stores flag gates for known features
// from a string like feature1=true,feature2=false,...
Set(value string) error
@@ -128,25 +150,70 @@ type MutableFeatureGate interface {
OverrideDefault(name Feature, override bool) error
}
// MutableVersionedFeatureGate parses and stores flag gates for known features from
// a string like feature1=true,feature2=false,...
// MutableVersionedFeatureGate sets options based on the emulated version of the featured gate.
type MutableVersionedFeatureGate interface {
MutableFeatureGate
// EmulationVersion returns the version the feature gate is set to emulate.
// If set, the feature gate would enable/disable features based on
// feature availability and pre-release at the emulated version instead of the binary version.
EmulationVersion() *version.Version
// SetEmulationVersion overrides the emulationVersion of the feature gate.
// Otherwise, the emulationVersion will be the same as the binary version.
// If set, the feature defaults and availability will be as if the binary is at the emulated version.
SetEmulationVersion(emulationVersion *version.Version) error
// DeferErrorsToValidation defers the errors of Set function to the Validate() function if true.
// This is used when the user wants to set the feature gate flag before the emulationVersion is finalized.
// Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true.
DeferErrorsToValidation(val bool)
// GetAll returns a copy of the map of known feature names to versioned feature specs.
GetAllVersioned() map[Feature]VersionedSpecs
// AddVersioned adds versioned feature specs to the featureGate.
AddVersioned(features map[Feature]VersionedSpecs) error
}
// MutableVersionedFeatureGateForTests is a feature gate interface that should only be used in tests.
type MutableVersionedFeatureGateForTests interface {
MutableVersionedFeatureGate
// Reset sets the enabled and enabledRaw to the input map.
Reset(m map[string]bool)
// EnabledRawMap returns the raw enable map from the feature gate.
EnabledRawMap() map[string]bool
}
// featureGate implements FeatureGate as well as pflag.Value for flag parsing.
type featureGate struct {
featureGateName string
special map[Feature]func(map[Feature]FeatureSpec, map[Feature]bool, bool)
special map[Feature]func(map[Feature]VersionedSpecs, map[Feature]bool, bool, *version.Version)
// lock guards writes to known, enabled, and reads/writes of closed
// lock guards writes to all below fields.
lock sync.Mutex
// known holds a map[Feature]FeatureSpec
known atomic.Value
// enabled holds a map[Feature]bool
enabled atomic.Value
// enabledRaw holds a raw map[string]bool of the parsed flag.
// It keeps the original values of "special" features like "all alpha gates",
// while enabled keeps the values of all resolved features.
enabledRaw atomic.Value
// closed is set to true when AddFlag is called, and prevents subsequent calls to Add
closed bool
// deferErrorsToValidation could be set to true to defer checking flag setting error,
// because the emulationVersion may not be the final emulationVersion when the flag is set.
// Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true.
deferErrorsToValidation bool
emulationVersion atomic.Pointer[version.Version]
}
func setUnsetAlphaGates(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool) {
func setUnsetAlphaGates(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, cVer *version.Version) {
for k, v := range known {
if v.PreRelease == Alpha {
if k == "AllAlpha" || k == "AllBeta" {
continue
}
currentVersion := getCurrentVersion(v, cVer)
if currentVersion.PreRelease == Alpha {
if _, found := enabled[k]; !found {
enabled[k] = val
}
@@ -154,9 +221,13 @@ func setUnsetAlphaGates(known map[Feature]FeatureSpec, enabled map[Feature]bool,
}
}
func setUnsetBetaGates(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool) {
func setUnsetBetaGates(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, cVer *version.Version) {
for k, v := range known {
if v.PreRelease == Beta {
if k == "AllAlpha" || k == "AllBeta" {
continue
}
currentVersion := getCurrentVersion(v, cVer)
if currentVersion.PreRelease == Beta {
if _, found := enabled[k]; !found {
enabled[k] = val
}
@@ -171,8 +242,10 @@ var _ pflag.Value = &featureGate{}
// call chains, so they'd be unhelpful as names.
var internalPackages = []string{"k8s.io/component-base/featuregate/feature_gate.go"}
func NewFeatureGate() *featureGate {
known := map[Feature]FeatureSpec{}
// NewVersionedFeatureGate creates a feature gate with the emulation version set to the provided version.
// SetEmulationVersion can be called after to change emulation version to a desired value.
func NewVersionedFeatureGate(emulationVersion *version.Version) *featureGate {
known := map[Feature]VersionedSpecs{}
for k, v := range defaultFeatures {
known[k] = v
}
@@ -183,10 +256,18 @@ func NewFeatureGate() *featureGate {
}
f.known.Store(known)
f.enabled.Store(map[Feature]bool{})
f.enabledRaw.Store(map[string]bool{})
f.emulationVersion.Store(emulationVersion)
klog.V(1).Infof("new feature gate with emulationVersion=%s", f.emulationVersion.Load().String())
return f
}
// NewFeatureGate creates a feature gate with the current binary version.
func NewFeatureGate() *featureGate {
binaryVersison := version.MustParse(baseversion.Get().String())
return NewVersionedFeatureGate(binaryVersison)
}
// Set parses a string of the form "key1=value1,key2=value2,..." into a
// map[string]bool of known keys or returns an error.
func (f *featureGate) Set(value string) error {
@@ -207,7 +288,68 @@ func (f *featureGate) Set(value string) error {
}
m[k] = boolValue
}
return f.SetFromMap(m)
err := f.SetFromMap(m)
// ignores SetFromMap error, because the emulationVersion may not be the final emulationVersion when the flag is set.
// Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true.
if f.deferErrorsToValidation {
return nil
}
return err
}
// Validate checks if the flag gates are valid at the emulated version.
// Should always be called after Set when DeferErrorsToValidation is set to true.
func (f *featureGate) Validate() []error {
m, ok := f.enabledRaw.Load().(map[string]bool)
if !ok {
return []error{fmt.Errorf("cannot cast enabledRaw to map[string]bool")}
}
enabled := map[Feature]bool{}
return f.unsafeSetFromMap(enabled, m)
}
// unsafeSetFromMap stores flag gates for known features from a map[string]bool into an enabled map.
func (f *featureGate) unsafeSetFromMap(enabled map[Feature]bool, m map[string]bool) []error {
var errs []error
// Copy existing state
known := map[Feature]VersionedSpecs{}
for k, v := range f.known.Load().(map[Feature]VersionedSpecs) {
sort.Sort(v)
known[k] = v
}
for k, v := range m {
key := Feature(k)
versionedSpecs, ok := known[key]
if !ok {
// early return if encounters an unknown feature.
errs = append(errs, fmt.Errorf("unrecognized feature gate: %s", k))
return errs
}
currentVersion := f.getCurrentVersion(versionedSpecs)
if currentVersion.LockToDefault && currentVersion.Default != v {
errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is locked to %v", k, v, currentVersion.Default))
continue
}
// Handle "special" features like "all alpha gates"
if fn, found := f.special[key]; found {
fn(known, enabled, v, f.emulationVersion.Load())
enabled[key] = v
continue
}
if currentVersion.PreRelease == PreAlpha {
errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is PreAlpha at emulated version %s", k, v, f.EmulationVersion().String()))
continue
}
enabled[key] = v
if currentVersion.PreRelease == Deprecated {
klog.Warningf("Setting deprecated feature gate %s=%t. It will be removed in a future release.", k, v)
} else if currentVersion.PreRelease == GA {
klog.Warningf("Setting GA feature gate %s=%t. It will be removed in a future release.", k, v)
}
}
return errs
}
// SetFromMap stores flag gates for known features from a map[string]bool or returns an error
@@ -216,43 +358,30 @@ func (f *featureGate) SetFromMap(m map[string]bool) error {
defer f.lock.Unlock()
// Copy existing state
known := map[Feature]FeatureSpec{}
for k, v := range f.known.Load().(map[Feature]FeatureSpec) {
known[k] = v
}
enabled := map[Feature]bool{}
for k, v := range f.enabled.Load().(map[Feature]bool) {
enabled[k] = v
}
for k, v := range m {
k := Feature(k)
featureSpec, ok := known[k]
if !ok {
return fmt.Errorf("unrecognized feature gate: %s", k)
}
if featureSpec.LockToDefault && featureSpec.Default != v {
return fmt.Errorf("cannot set feature gate %v to %v, feature is locked to %v", k, v, featureSpec.Default)
}
enabled[k] = v
// Handle "special" features like "all alpha gates"
if fn, found := f.special[k]; found {
fn(known, enabled, v)
}
if featureSpec.PreRelease == Deprecated {
klog.Warningf("Setting deprecated feature gate %s=%t. It will be removed in a future release.", k, v)
} else if featureSpec.PreRelease == GA {
klog.Warningf("Setting GA feature gate %s=%t. It will be removed in a future release.", k, v)
}
enabledRaw := map[string]bool{}
for k, v := range f.enabledRaw.Load().(map[string]bool) {
enabledRaw[k] = v
}
// Persist changes
f.known.Store(known)
f.enabled.Store(enabled)
// Update enabledRaw first.
// SetFromMap might be called when emulationVersion is not finalized yet, and we do not know the final state of enabled.
// But the flags still need to be saved.
for k, v := range m {
enabledRaw[k] = v
}
f.enabledRaw.Store(enabledRaw)
klog.V(1).Infof("feature gates: %v", f.enabled)
return nil
errs := f.unsafeSetFromMap(enabled, enabledRaw)
if len(errs) == 0 {
// Persist changes
f.enabled.Store(enabled)
klog.V(1).Infof("feature gates: %v", f.enabled)
}
return utilerrors.NewAggregate(errs)
}
// String returns a string containing all enabled feature gates, formatted as "key1=value1,key2=value2,...".
@@ -271,6 +400,17 @@ func (f *featureGate) Type() string {
// Add adds features to the featureGate.
func (f *featureGate) Add(features map[Feature]FeatureSpec) error {
vs := map[Feature]VersionedSpecs{}
for name, spec := range features {
// if no version is provided for the FeatureSpec, it is defaulted to version 0.0 so that it can be enabled/disabled regardless of emulation version.
spec.Version = version.MajorMinor(0, 0)
vs[name] = VersionedSpecs{spec}
}
return f.AddVersioned(vs)
}
// AddVersioned adds versioned feature specs to the featureGate.
func (f *featureGate) AddVersioned(features map[Feature]VersionedSpecs) error {
f.lock.Lock()
defer f.lock.Unlock()
@@ -279,20 +419,21 @@ func (f *featureGate) Add(features map[Feature]FeatureSpec) error {
}
// Copy existing state
known := map[Feature]FeatureSpec{}
for k, v := range f.known.Load().(map[Feature]FeatureSpec) {
known := map[Feature]VersionedSpecs{}
for k, v := range f.known.Load().(map[Feature]VersionedSpecs) {
known[k] = v
}
for name, spec := range features {
for name, specs := range features {
sort.Sort(specs)
if existingSpec, found := known[name]; found {
if existingSpec == spec {
sort.Sort(existingSpec)
if reflect.DeepEqual(existingSpec, specs) {
continue
}
return fmt.Errorf("feature gate %q with different spec already exists: %v", name, existingSpec)
}
known[name] = spec
known[name] = specs
}
// Persist updated state
@@ -309,17 +450,22 @@ func (f *featureGate) OverrideDefault(name Feature, override bool) error {
return fmt.Errorf("cannot override default for feature %q: gates already added to a flag set", name)
}
known := map[Feature]FeatureSpec{}
for name, spec := range f.known.Load().(map[Feature]FeatureSpec) {
known[name] = spec
known := map[Feature]VersionedSpecs{}
for k, v := range f.known.Load().(map[Feature]VersionedSpecs) {
sort.Sort(v)
known[k] = v
}
spec, ok := known[name]
switch {
case !ok:
specs, ok := known[name]
if !ok {
return fmt.Errorf("cannot override default: feature %q is not registered", name)
}
spec := f.getCurrentVersion(specs)
switch {
case spec.LockToDefault:
return fmt.Errorf("cannot override default: feature %q default is locked to %t", name, spec.Default)
case spec.PreRelease == PreAlpha:
return fmt.Errorf("cannot override default: feature %q is not available before emulation version %s", name, f.EmulationVersion().String())
case spec.PreRelease == Deprecated:
klog.Warningf("Overriding default of deprecated feature gate %s=%t. It will be removed in a future release.", name, override)
case spec.PreRelease == GA:
@@ -327,35 +473,112 @@ func (f *featureGate) OverrideDefault(name Feature, override bool) error {
}
spec.Default = override
known[name] = spec
known[name] = specs
f.known.Store(known)
return nil
}
// GetAll returns a copy of the map of known feature names to feature specs.
// GetAll returns a copy of the map of known feature names to feature specs for the current emulationVersion.
func (f *featureGate) GetAll() map[Feature]FeatureSpec {
retval := map[Feature]FeatureSpec{}
for k, v := range f.known.Load().(map[Feature]FeatureSpec) {
for k, v := range f.GetAllVersioned() {
spec := f.getCurrentVersion(v)
if spec.PreRelease == PreAlpha {
// The feature is not available at the emulation version.
continue
}
retval[k] = *f.getCurrentVersion(v)
}
return retval
}
// GetAllVersioned returns a copy of the map of known feature names to versioned feature specs.
func (f *featureGate) GetAllVersioned() map[Feature]VersionedSpecs {
retval := map[Feature]VersionedSpecs{}
for k, v := range f.known.Load().(map[Feature]VersionedSpecs) {
retval[k] = v
}
return retval
}
// DeferErrorsToValidation could be used to defer checking flag setting error,
// because the emulationVersion may not be the final emulationVersion when the flag is set.
// Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true.
func (f *featureGate) DeferErrorsToValidation(val bool) {
f.lock.Lock()
defer f.lock.Unlock()
f.deferErrorsToValidation = val
}
func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) error {
f.lock.Lock()
defer f.lock.Unlock()
klog.V(1).Infof("set feature gate emulationVersion to %s", emulationVersion.String())
f.emulationVersion.Store(emulationVersion)
// Copy existing state
enabledRaw := map[string]bool{}
for k, v := range f.enabledRaw.Load().(map[string]bool) {
enabledRaw[k] = v
}
// enabled map should be reset whenever emulationVersion is changed.
enabled := map[Feature]bool{}
errs := f.unsafeSetFromMap(enabled, enabledRaw)
if len(errs) == 0 {
// Persist changes
f.enabled.Store(enabled)
}
return utilerrors.NewAggregate(errs)
}
func (f *featureGate) EmulationVersion() *version.Version {
return f.emulationVersion.Load()
}
// FeatureSpec returns the FeatureSpec at the EmulationVersion if the key exists, an error otherwise.
func (f *featureGate) FeatureSpec(key Feature) (FeatureSpec, error) {
if v, ok := f.known.Load().(map[Feature]VersionedSpecs)[key]; ok {
currentVersion := f.getCurrentVersion(v)
return *currentVersion, nil
}
return FeatureSpec{}, fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName)
}
// Enabled returns true if the key is enabled. If the key is not known, this call will panic.
func (f *featureGate) Enabled(key Feature) bool {
// fallback to default behavior, since we don't have emulation version set
if v, ok := f.enabled.Load().(map[Feature]bool)[key]; ok {
return v
}
if v, ok := f.known.Load().(map[Feature]FeatureSpec)[key]; ok {
return v.Default
if v, ok := f.known.Load().(map[Feature]VersionedSpecs)[key]; ok {
return f.getCurrentVersion(v).Default
}
panic(fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName))
}
func (f *featureGate) getCurrentVersion(v VersionedSpecs) *FeatureSpec {
return getCurrentVersion(v, f.EmulationVersion())
}
func getCurrentVersion(v VersionedSpecs, emulationVersion *version.Version) *FeatureSpec {
i := len(v) - 1
for ; i >= 0; i-- {
if v[i].Version.GreaterThan(emulationVersion) {
continue
}
return &v[i]
}
return &FeatureSpec{
Default: false,
PreRelease: PreAlpha,
Version: version.MajorMinor(0, 0),
}
}
// AddFlag adds a flag for setting global feature gates to the specified FlagSet.
func (f *featureGate) AddFlag(fs *pflag.FlagSet) {
func (f *featureGate) AddFlag(fs *pflag.FlagSet, prefix string) {
f.lock.Lock()
// TODO(mtaufen): Shouldn't we just close it on the first Set/SetFromMap instead?
// Not all components expose a feature gates flag using this AddFlag method, and
@@ -365,7 +588,10 @@ func (f *featureGate) AddFlag(fs *pflag.FlagSet) {
f.lock.Unlock()
known := f.KnownFeatures()
fs.Var(f, flagName, ""+
if len(prefix) > 0 && !strings.HasSuffix(prefix, "-") {
prefix += "-"
}
fs.Var(f, prefix+flagName, ""+
"A set of key=value pairs that describe feature gates for alpha/experimental features. "+
"Options are:\n"+strings.Join(known, "\n"))
}
@@ -377,14 +603,19 @@ func (f *featureGate) AddMetrics() {
}
// KnownFeatures returns a slice of strings describing the FeatureGate's known features.
// Deprecated and GA features are hidden from the list.
// preAlpha, Deprecated and GA features are hidden from the list.
func (f *featureGate) KnownFeatures() []string {
var known []string
for k, v := range f.known.Load().(map[Feature]FeatureSpec) {
if v.PreRelease == GA || v.PreRelease == Deprecated {
for k, v := range f.known.Load().(map[Feature]VersionedSpecs) {
if k == "AllAlpha" || k == "AllBeta" {
known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, v[0].PreRelease, v[0].Default))
continue
}
known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, v.PreRelease, v.Default))
currentV := f.getCurrentVersion(v)
if currentV.PreRelease == GA || currentV.PreRelease == Deprecated || currentV.PreRelease == PreAlpha {
continue
}
known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, currentV.PreRelease, currentV.Default))
}
sort.Strings(known)
return known
@@ -393,16 +624,20 @@ func (f *featureGate) KnownFeatures() []string {
// DeepCopy returns a deep copy of the FeatureGate object, such that gates can be
// set on the copy without mutating the original. This is useful for validating
// config against potential feature gate changes before committing those changes.
func (f *featureGate) DeepCopy() MutableFeatureGate {
func (f *featureGate) DeepCopy() MutableVersionedFeatureGate {
// Copy existing state.
known := map[Feature]FeatureSpec{}
for k, v := range f.known.Load().(map[Feature]FeatureSpec) {
known := map[Feature]VersionedSpecs{}
for k, v := range f.known.Load().(map[Feature]VersionedSpecs) {
known[k] = v
}
enabled := map[Feature]bool{}
for k, v := range f.enabled.Load().(map[Feature]bool) {
enabled[k] = v
}
enabledRaw := map[string]bool{}
for k, v := range f.enabledRaw.Load().(map[string]bool) {
enabledRaw[k] = v
}
// Construct a new featureGate around the copied state.
// Note that specialFeatures is treated as immutable by convention,
@@ -411,9 +646,22 @@ func (f *featureGate) DeepCopy() MutableFeatureGate {
special: specialFeatures,
closed: f.closed,
}
fg.emulationVersion.Store(f.EmulationVersion())
fg.known.Store(known)
fg.enabled.Store(enabled)
fg.enabledRaw.Store(enabledRaw)
return fg
}
// Reset sets the enabled and enabledRaw to the input map.
func (f *featureGate) Reset(m map[string]bool) {
enabled := map[Feature]bool{}
enabledRaw := map[string]bool{}
f.enabled.Store(enabled)
f.enabledRaw.Store(enabledRaw)
_ = f.SetFromMap(m)
}
func (f *featureGate) EnabledRawMap() map[string]bool {
return f.enabledRaw.Load().(map[string]bool)
}

View File

@@ -45,7 +45,7 @@ func init() {
func SetFeatureGateDuringTest(tb testing.TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) {
tb.Helper()
detectParallelOverrideCleanup := detectParallelOverride(tb, f)
originalValue := gate.Enabled(f)
originalEnabled := gate.(featuregate.MutableVersionedFeatureGateForTests).EnabledRawMap()
// Specially handle AllAlpha and AllBeta
if f == "AllAlpha" || f == "AllBeta" {
@@ -67,9 +67,7 @@ func SetFeatureGateDuringTest(tb testing.TB, gate featuregate.FeatureGate, f fea
tb.Cleanup(func() {
tb.Helper()
detectParallelOverrideCleanup()
if err := gate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, originalValue)); err != nil {
tb.Errorf("error restoring %s=%v: %v", f, originalValue, err)
}
gate.(featuregate.MutableVersionedFeatureGateForTests).Reset(originalEnabled)
})
}

View File

@@ -67,7 +67,7 @@ func NewLoggerCommand() *cobra.Command {
},
}
logsapi.AddFeatureGates(featureGate)
featureGate.AddFlag(cmd.Flags())
featureGate.AddFlag(cmd.Flags(), "")
logsapi.AddFlags(c, cmd.Flags())
return cmd
}

View File

@@ -83,7 +83,7 @@ func NewLoggerCommand() *cobra.Command {
// Shouldn't happen.
panic(err)
}
featureGate.AddFlag(cmd.Flags())
featureGate.AddFlag(cmd.Flags(), "")
logsapi.AddFlags(c, cmd.Flags())
return cmd
}

View File

@@ -247,7 +247,7 @@ func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.Deleg
}
// used later to filter the served resource by those that have expired.
resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(*c.GenericConfig.Version)
resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(s.GenericAPIServer.EffectiveVersion.EmulationVersion())
if err != nil {
return nil, err
}

View File

@@ -31,6 +31,9 @@ import (
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/filters"
genericoptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/component-base/featuregate"
"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
"k8s.io/kube-aggregator/pkg/apiserver"
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
@@ -58,10 +61,15 @@ type AggregatorOptions struct {
// with a default AggregatorOptions.
func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions) *cobra.Command {
o := *defaults
featureGate := o.ServerRunOptions.FeatureGate.(featuregate.MutableVersionedFeatureGate)
effectiveVersion := o.ServerRunOptions.EffectiveVersion.(utilversion.MutableEffectiveVersion)
cmd := &cobra.Command{
Short: "Launch a API aggregator and proxy server",
Long: "Launch a API aggregator and proxy server",
RunE: func(c *cobra.Command, args []string) error {
if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil {
return err
}
if err := o.Complete(); err != nil {
return err
}
@@ -76,7 +84,11 @@ func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions)
}
cmd.SetContext(ctx)
o.AddFlags(cmd.Flags())
fs := cmd.Flags()
featureGate.AddFlag(fs, "")
effectiveVersion.AddFlags(fs, "")
o.AddFlags(fs)
return cmd
}
@@ -91,8 +103,13 @@ func (o *AggregatorOptions) AddFlags(fs *pflag.FlagSet) {
// NewDefaultOptions builds a "normal" set of options. You wouldn't normally expose this, but hyperkube isn't cobra compatible
func NewDefaultOptions(out, err io.Writer) *AggregatorOptions {
// effectiveVersion is used to set what apis and feature gates the generic api server is compatible with.
// You can also have the flag setting the effectiveVersion of the aggregator apiserver, and
// having a mapping from the aggregator apiserver version to generic apiserver version.
effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
utilversion.ComponentGenericAPIServer, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
o := &AggregatorOptions{
ServerRunOptions: genericoptions.NewServerRunOptions(),
ServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion),
RecommendedOptions: genericoptions.NewRecommendedOptions(
defaultEtcdPathPrefix,
aggregatorscheme.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion),
@@ -117,7 +134,7 @@ func (o AggregatorOptions) Validate(args []string) error {
// Complete fills in missing Options.
func (o *AggregatorOptions) Complete() error {
return nil
return o.ServerRunOptions.Complete()
}
// RunAggregator runs the API Aggregator.

View File

@@ -18,6 +18,7 @@ limitations under the License.
// +k8s:deepcopy-gen=package
// +k8s:conversion-gen=k8s.io/sample-apiserver/pkg/apis/wardle
// +k8s:defaulter-gen=TypeMeta
// +k8s:prerelease-lifecycle-gen=true
// +groupName=wardle.example.com
// Package v1alpha1 is the v1alpha1 version of the API.

View File

@@ -19,6 +19,8 @@ package v1alpha1
import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +k8s:prerelease-lifecycle-gen:introduced=1.0
// +k8s:prerelease-lifecycle-gen:removed=1.10
// FlunderList is a list of Flunder objects.
type FlunderList struct {
@@ -47,6 +49,8 @@ type FlunderStatus struct {
// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +k8s:prerelease-lifecycle-gen:introduced=1.0
// +k8s:prerelease-lifecycle-gen:removed=1.10
type Flunder struct {
metav1.TypeMeta `json:",inline"`
@@ -59,6 +63,8 @@ type Flunder struct {
// +genclient
// +genclient:nonNamespaced
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +k8s:prerelease-lifecycle-gen:introduced=1.0
// +k8s:prerelease-lifecycle-gen:removed=1.10
type Fischer struct {
metav1.TypeMeta `json:",inline"`
@@ -71,6 +77,8 @@ type Fischer struct {
// +genclient:nonNamespaced
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +k8s:prerelease-lifecycle-gen:introduced=1.0
// +k8s:prerelease-lifecycle-gen:removed=1.10
// FischerList is a list of Fischer objects.
type FischerList struct {

View File

@@ -0,0 +1,94 @@
//go:build !ignore_autogenerated
// +build !ignore_autogenerated
/*
Copyright The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Code generated by prerelease-lifecycle-gen. DO NOT EDIT.
package v1alpha1
// APILifecycleIntroduced is an autogenerated function, returning the release in which the API struct was introduced as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
func (in *Fischer) APILifecycleIntroduced() (major, minor int) {
return 1, 0
}
// APILifecycleDeprecated is an autogenerated function, returning the release in which the API struct was or will be deprecated as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:deprecated" tags in types.go or "k8s:prerelease-lifecycle-gen:introduced" plus three minor.
func (in *Fischer) APILifecycleDeprecated() (major, minor int) {
return 1, 3
}
// APILifecycleRemoved is an autogenerated function, returning the release in which the API is no longer served as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:removed" tags in types.go or "k8s:prerelease-lifecycle-gen:deprecated" plus three minor.
func (in *Fischer) APILifecycleRemoved() (major, minor int) {
return 1, 10
}
// APILifecycleIntroduced is an autogenerated function, returning the release in which the API struct was introduced as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
func (in *FischerList) APILifecycleIntroduced() (major, minor int) {
return 1, 0
}
// APILifecycleDeprecated is an autogenerated function, returning the release in which the API struct was or will be deprecated as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:deprecated" tags in types.go or "k8s:prerelease-lifecycle-gen:introduced" plus three minor.
func (in *FischerList) APILifecycleDeprecated() (major, minor int) {
return 1, 3
}
// APILifecycleRemoved is an autogenerated function, returning the release in which the API is no longer served as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:removed" tags in types.go or "k8s:prerelease-lifecycle-gen:deprecated" plus three minor.
func (in *FischerList) APILifecycleRemoved() (major, minor int) {
return 1, 10
}
// APILifecycleIntroduced is an autogenerated function, returning the release in which the API struct was introduced as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
func (in *Flunder) APILifecycleIntroduced() (major, minor int) {
return 1, 0
}
// APILifecycleDeprecated is an autogenerated function, returning the release in which the API struct was or will be deprecated as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:deprecated" tags in types.go or "k8s:prerelease-lifecycle-gen:introduced" plus three minor.
func (in *Flunder) APILifecycleDeprecated() (major, minor int) {
return 1, 3
}
// APILifecycleRemoved is an autogenerated function, returning the release in which the API is no longer served as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:removed" tags in types.go or "k8s:prerelease-lifecycle-gen:deprecated" plus three minor.
func (in *Flunder) APILifecycleRemoved() (major, minor int) {
return 1, 10
}
// APILifecycleIntroduced is an autogenerated function, returning the release in which the API struct was introduced as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
func (in *FlunderList) APILifecycleIntroduced() (major, minor int) {
return 1, 0
}
// APILifecycleDeprecated is an autogenerated function, returning the release in which the API struct was or will be deprecated as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:deprecated" tags in types.go or "k8s:prerelease-lifecycle-gen:introduced" plus three minor.
func (in *FlunderList) APILifecycleDeprecated() (major, minor int) {
return 1, 3
}
// APILifecycleRemoved is an autogenerated function, returning the release in which the API is no longer served as int versions of major and minor for comparison.
// It is controlled by "k8s:prerelease-lifecycle-gen:removed" tags in types.go or "k8s:prerelease-lifecycle-gen:deprecated" plus three minor.
func (in *FlunderList) APILifecycleRemoved() (major, minor int) {
return 1, 10
}

View File

@@ -17,6 +17,8 @@ limitations under the License.
package apiserver
import (
"strconv"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
@@ -25,6 +27,7 @@ import (
"k8s.io/apiserver/pkg/registry/rest"
genericapiserver "k8s.io/apiserver/pkg/server"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/sample-apiserver/pkg/apis/wardle"
"k8s.io/sample-apiserver/pkg/apis/wardle/install"
wardleregistry "k8s.io/sample-apiserver/pkg/registry"
@@ -37,7 +40,8 @@ var (
Scheme = runtime.NewScheme()
// Codecs provides methods for retrieving codecs and serializers for specific
// versions and content types.
Codecs = serializer.NewCodecFactory(Scheme)
Codecs = serializer.NewCodecFactory(Scheme)
WardleComponentName = "wardle-server"
)
func init() {
@@ -90,10 +94,10 @@ func (cfg *Config) Complete() CompletedConfig {
cfg.GenericConfig.Complete(),
&cfg.ExtraConfig,
}
wardleEffectiveVersion := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(WardleComponentName)
c.GenericConfig.Version = &version.Info{
Major: "1",
Minor: "0",
Major: strconv.Itoa(int(wardleEffectiveVersion.BinaryVersion().Major())),
Minor: strconv.Itoa(int(wardleEffectiveVersion.BinaryVersion().Minor())),
}
return CompletedConfig{&c}

View File

@@ -18,6 +18,7 @@ package server
import (
"context"
"errors"
"fmt"
"io"
"net"
@@ -27,11 +28,14 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/endpoints/openapi"
genericapiserver "k8s.io/apiserver/pkg/server"
genericoptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/sample-apiserver/pkg/admission/plugin/banflunder"
"k8s.io/sample-apiserver/pkg/admission/wardleinitializer"
"k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1"
@@ -55,6 +59,24 @@ type WardleServerOptions struct {
AlternateDNS []string
}
func mapWardleEffectiveVersionToKubeEffectiveVersion(registry utilversion.ComponentGlobalsRegistry) error {
wardleVer := registry.EffectiveVersionFor(apiserver.WardleComponentName)
kubeVer := registry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer).(utilversion.MutableEffectiveVersion)
// map from wardle emulation version to kube emulation version.
emulationVersionMap := map[string]string{
"1.2": kubeVer.BinaryVersion().AddMinor(1).String(),
"1.1": kubeVer.BinaryVersion().String(),
"1.0": kubeVer.BinaryVersion().SubtractMinor(1).String(),
}
wardleEmulationVer := wardleVer.EmulationVersion()
if kubeEmulationVer, ok := emulationVersionMap[wardleEmulationVer.String()]; ok {
kubeVer.SetEmulationVersion(version.MustParse(kubeEmulationVer))
} else {
return fmt.Errorf("cannot find mapping from wardle emulation version: %s to kube version", wardleVer.EmulationVersion().String())
}
return nil
}
// NewWardleServerOptions returns a new WardleServerOptions
func NewWardleServerOptions(out, errOut io.Writer) *WardleServerOptions {
o := &WardleServerOptions{
@@ -94,7 +116,14 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti
flags := cmd.Flags()
o.RecommendedOptions.AddFlags(flags)
utilfeature.DefaultMutableFeatureGate.AddFlag(flags)
wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2")
utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, nil, false))
_, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
utilversion.ComponentGenericAPIServer, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
wardleEffectiveVersion.AddFlags(flags, "wardle-")
featureGate.AddFlag(flags, "")
return cmd
}
@@ -103,6 +132,7 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti
func (o WardleServerOptions) Validate(args []string) error {
errors := []error{}
errors = append(errors, o.RecommendedOptions.Validate()...)
errors = append(errors, utilversion.DefaultComponentGlobalsRegistry.ValidateAllComponents()...)
return utilerrors.NewAggregate(errors)
}
@@ -114,6 +144,17 @@ func (o *WardleServerOptions) Complete() error {
// add admission plugins to the RecommendedPluginOrder
o.RecommendedOptions.Admission.RecommendedPluginOrder = append(o.RecommendedOptions.Admission.RecommendedPluginOrder, "BanFlunder")
// convert wardle effective version to kube effective version to be used in generic api server, and set the generic api server feature gate.
if err := mapWardleEffectiveVersionToKubeEffectiveVersion(utilversion.DefaultComponentGlobalsRegistry); err != nil {
return err
}
if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil {
return err
}
if errs := utilversion.DefaultComponentGlobalsRegistry.ValidateAllComponents(); len(errs) > 0 {
return errors.Join(errs...)
}
return nil
}
@@ -144,6 +185,9 @@ func (o *WardleServerOptions) Config() (*apiserver.Config, error) {
serverConfig.OpenAPIV3Config.Info.Title = "Wardle"
serverConfig.OpenAPIV3Config.Info.Version = "0.1"
serverConfig.FeatureGate = utilversion.DefaultComponentGlobalsRegistry.FeatureGateFor(utilversion.ComponentGenericAPIServer)
serverConfig.EffectiveVersion = utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer)
if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil {
return nil, err
}

View File

@@ -0,0 +1,66 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package server
import (
"testing"
"k8s.io/apimachinery/pkg/util/version"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/sample-apiserver/pkg/apiserver"
"github.com/stretchr/testify/assert"
)
func TestMapBinaryEffectiveVersionToKubeEffectiveVersion(t *testing.T) {
wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2")
defaultKubeEffectiveVersion := utilversion.DefaultKubeEffectiveVersion()
testCases := []struct {
desc string
wardleEmulationVer *version.Version
expectedKubeEmulationVer *version.Version
}{
{
desc: "1 version higher than kube binary",
wardleEmulationVer: version.MajorMinor(1, 2),
expectedKubeEmulationVer: defaultKubeEffectiveVersion.BinaryVersion().AddMinor(1),
},
{
desc: "no mapping",
wardleEmulationVer: version.MajorMinor(1, 10),
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
registry := utilversion.NewComponentGlobalsRegistry()
_ = registry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, nil, true)
_ = registry.Register(utilversion.ComponentGenericAPIServer, defaultKubeEffectiveVersion, nil, true)
wardleEffectiveVersion.SetEmulationVersion(tc.wardleEmulationVer)
err := mapWardleEffectiveVersionToKubeEffectiveVersion(registry)
if tc.expectedKubeEmulationVer == nil {
if err == nil {
t.Fatal("expected error, no error found")
}
} else {
assert.True(t, registry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer).EmulationVersion().EqualTo(tc.expectedKubeEmulationVer))
}
})
}
}

View File

@@ -88,7 +88,7 @@ func NewCommand() *cobra.Command {
fs = sharedFlagSets.FlagSet("other")
featureGate := featuregate.NewFeatureGate()
utilruntime.Must(logsapi.AddFeatureGates(featureGate))
featureGate.AddFlag(fs)
featureGate.AddFlag(fs, "")
fs = cmd.PersistentFlags()
for _, f := range sharedFlagSets.FlagSets {

View File

@@ -29,6 +29,8 @@ import (
netutils "k8s.io/utils/net"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
"k8s.io/kubernetes/test/e2e/framework"
@@ -63,7 +65,9 @@ func NewAPIServer(storageConfig storagebackend.Config) *APIServer {
func (a *APIServer) Start(ctx context.Context) error {
const tokenFilePath = "known_tokens.csv"
o := options.NewServerRunOptions()
featureGate := utilfeature.DefaultFeatureGate
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
o := options.NewServerRunOptions(featureGate, effectiveVersion)
o.Etcd.StorageConfig = a.storageConfig
_, ipnet, err := netutils.ParseCIDRSloppy(clusterIPRange)
if err != nil {

View File

@@ -32,6 +32,7 @@ import (
"testing"
"time"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -55,16 +56,19 @@ import (
"k8s.io/apimachinery/pkg/watch"
"k8s.io/apiserver/pkg/endpoints/handlers"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/client-go/discovery/cached/memory"
"k8s.io/client-go/dynamic"
clientset "k8s.io/client-go/kubernetes"
appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
"k8s.io/client-go/metadata"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/restmapper"
"k8s.io/client-go/tools/pager"
"k8s.io/klog/v2"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
"k8s.io/kubernetes/pkg/controlplane"
"k8s.io/kubernetes/pkg/kubeapiserver"
"k8s.io/kubernetes/test/integration"
"k8s.io/kubernetes/test/integration/etcd"
"k8s.io/kubernetes/test/integration/framework"
@@ -2929,6 +2933,298 @@ func TestDedupOwnerReferences(t *testing.T) {
}
}
func TestEmulatedStorageVersion(t *testing.T) {
validVap := &admissionregistrationv1.ValidatingAdmissionPolicy{
Spec: admissionregistrationv1.ValidatingAdmissionPolicySpec{
MatchConstraints: &admissionregistrationv1.MatchResources{
ResourceRules: []admissionregistrationv1.NamedRuleWithOperations{
{
ResourceNames: []string{"foo"},
RuleWithOperations: admissionregistrationv1.RuleWithOperations{
Operations: []admissionregistrationv1.OperationType{"CREATE"},
Rule: admissionregistrationv1.Rule{
APIGroups: []string{"*"},
APIVersions: []string{"*"},
Resources: []string{"*"},
},
},
},
},
},
Validations: []admissionregistrationv1.Validation{
{
Expression: "true",
Message: "always valid",
},
},
},
}
type testCase struct {
name string
emulatedVersion string
gvr schema.GroupVersionResource
object runtime.Object
expectedStorageVersion schema.GroupVersion
}
cases := []testCase{
{
name: "vap first ga release",
emulatedVersion: "1.30",
gvr: schema.GroupVersionResource{
Group: "admissionregistration.k8s.io",
Version: "v1",
Resource: "validatingadmissionpolicies",
},
object: validVap,
expectedStorageVersion: schema.GroupVersion{
Group: "admissionregistration.k8s.io",
Version: "v1beta1",
},
},
{
name: "vap after ga release",
emulatedVersion: "1.31",
gvr: schema.GroupVersionResource{
Group: "admissionregistration.k8s.io",
Version: "v1beta1",
Resource: "validatingadmissionpolicies",
},
object: validVap,
expectedStorageVersion: schema.GroupVersion{
Group: "admissionregistration.k8s.io",
Version: "v1",
},
},
}
// Group cases by their emulated version
groupedCases := map[string][]testCase{}
for _, c := range cases {
groupedCases[c.emulatedVersion] = append(groupedCases[c.emulatedVersion], c)
}
for emulatedVersion, cases := range groupedCases {
t.Run(emulatedVersion, func(t *testing.T) {
server := kubeapiservertesting.StartTestServerOrDie(
t, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: emulatedVersion},
[]string{"--emulated-version=" + emulatedVersion, `--storage-media-type=application/json`}, framework.SharedEtcd())
defer server.TearDownFn()
client := clientset.NewForConfigOrDie(server.ClientConfig)
dynamicClient := dynamic.NewForConfigOrDie(server.ClientConfig)
// create test namespace
testNamespace := "test-emulated-storage-version"
_, err := client.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: testNamespace,
},
}, metav1.CreateOptions{})
if err != nil {
t.Fatalf("failed to create test ns: %v", err)
}
restMapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(client.Discovery()))
for i, c := range cases {
t.Run(c.name, func(t *testing.T) {
gvk, err := restMapper.KindFor(c.gvr)
if err != nil {
t.Fatalf("failed to get GVK: %v", err)
}
c.object.GetObjectKind().SetGroupVersionKind(gvk)
mapping, err := restMapper.RESTMapping(gvk.GroupKind(), gvk.Version)
if err != nil {
t.Fatalf("failed to get RESTMapping: %v", err)
} else if mapping.Resource != c.gvr {
t.Fatalf("expected resource %v, got %v", c.gvr, mapping.Resource)
}
asUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(c.object)
if err != nil {
t.Fatalf("failed to convert object to unstructured: %v", err)
}
uns := &unstructured.Unstructured{
Object: asUnstructured,
}
uns.SetName(fmt.Sprintf("test-object%d", i))
ns := testNamespace
if mapping.Scope.Name() == meta.RESTScopeNameRoot {
ns = ""
}
// create object
created, err := dynamicClient.Resource(c.gvr).Namespace(ns).Create(context.TODO(), uns, metav1.CreateOptions{})
if err != nil {
t.Fatalf("failed to create object: %v", err)
}
// Fetch object from ETCD
// Use this poor man's way to get the etcd path. This wont
// work for all resources, but should work for most we want
// to test against
resourcePrefix := mapping.Resource.Resource
if special, ok := kubeapiserver.SpecialDefaultResourcePrefixes[c.gvr.GroupResource()]; ok {
resourcePrefix = special
}
etcdPathComponents := []string{
"/",
server.EtcdStoragePrefix,
resourcePrefix,
}
if len(ns) > 0 {
etcdPathComponents = append(etcdPathComponents, ns)
}
etcdPathComponents = append(etcdPathComponents, created.GetName())
etcdPath := path.Join(etcdPathComponents...)
fetched, err := server.EtcdClient.Get(context.TODO(), etcdPath)
if err != nil {
t.Fatalf("failed to fetch object from etcd: %v", err)
} else if fetched.More || fetched.Count != 1 || len(fetched.Kvs) != 1 {
t.Fatalf("unexpected fetched response: %v", fetched)
}
storedObject := &metav1.PartialObjectMetadata{}
err = json.Unmarshal(fetched.Kvs[0].Value, storedObject)
if err != nil {
t.Fatalf("failed to decode object: %v", err)
} else if storedObject.GroupVersionKind().GroupVersion() != c.expectedStorageVersion {
t.Fatalf("expected storage version %s, got %s", c.expectedStorageVersion.String(), storedObject.GroupVersionKind().GroupVersion().String())
}
})
}
})
}
}
func TestEnableEmulationVersion(t *testing.T) {
server := kubeapiservertesting.StartTestServerOrDie(t,
&kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"},
[]string{"--emulated-version=1.31"}, framework.SharedEtcd())
defer server.TearDownFn()
rt, err := restclient.TransportFor(server.ClientConfig)
if err != nil {
t.Fatal(err)
}
tcs := []struct {
path string
expectedStatusCode int
}{
{
path: "/",
expectedStatusCode: 200,
},
{
path: "/apis/apps/v1/deployments",
expectedStatusCode: 200,
},
{
path: "/apis/flowcontrol.apiserver.k8s.io/v1/flowschemas",
expectedStatusCode: 200,
},
{
path: "/apis/flowcontrol.apiserver.k8s.io/v1beta1/flowschemas", // introduced at 1.20, removed at 1.26
expectedStatusCode: 404,
},
{
path: "/apis/flowcontrol.apiserver.k8s.io/v1beta2/flowschemas", // introduced at 1.23, removed at 1.29
expectedStatusCode: 404,
},
{
path: "/apis/flowcontrol.apiserver.k8s.io/v1beta3/flowschemas", // introduced at 1.26, removed at 1.32
expectedStatusCode: 200,
},
}
for _, tc := range tcs {
t.Run(tc.path, func(t *testing.T) {
req, err := http.NewRequest("GET", server.ClientConfig.Host+tc.path, nil)
if err != nil {
t.Fatal(err)
}
resp, err := rt.RoundTrip(req)
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != tc.expectedStatusCode {
t.Errorf("expect status code: %d, got : %d\n", tc.expectedStatusCode, resp.StatusCode)
}
defer func() {
_ = resp.Body.Close()
}()
})
}
}
func TestDisableEmulationVersion(t *testing.T) {
server := kubeapiservertesting.StartTestServerOrDie(t,
&kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"},
[]string{}, framework.SharedEtcd())
defer server.TearDownFn()
rt, err := restclient.TransportFor(server.ClientConfig)
if err != nil {
t.Fatal(err)
}
tcs := []struct {
path string
expectedStatusCode int
}{
{
path: "/",
expectedStatusCode: 200,
},
{
path: "/apis/apps/v1/deployments",
expectedStatusCode: 200,
},
{
path: "/apis/flowcontrol.apiserver.k8s.io/v1/flowschemas",
expectedStatusCode: 200,
},
{
path: "/apis/flowcontrol.apiserver.k8s.io/v1beta1/flowschemas", // introduced at 1.20, removed at 1.26
expectedStatusCode: 404,
},
{
path: "/apis/flowcontrol.apiserver.k8s.io/v1beta2/flowschemas", // introduced at 1.23, removed at 1.29
expectedStatusCode: 404,
},
{
path: "/apis/flowcontrol.apiserver.k8s.io/v1beta3/flowschemas", // introduced at 1.26, removed at 1.32
expectedStatusCode: 404,
},
}
for _, tc := range tcs {
t.Run(tc.path, func(t *testing.T) {
req, err := http.NewRequest("GET", server.ClientConfig.Host+tc.path, nil)
if err != nil {
t.Fatal(err)
}
resp, err := rt.RoundTrip(req)
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != tc.expectedStatusCode {
t.Errorf("expect status code: %d, got : %d\n", tc.expectedStatusCode, resp.StatusCode)
}
defer func() {
_ = resp.Body.Close()
}()
})
}
}
type dependentClient struct {
t *testing.T
client dynamic.ResourceInterface

View File

@@ -29,6 +29,7 @@ import (
rbacapi "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/watch"
@@ -80,8 +81,8 @@ type testRESTOptionsGetter struct {
config *controlplane.Config
}
func (getter *testRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
storageConfig, err := getter.config.ControlPlane.Extra.StorageFactory.NewConfig(resource)
func (getter *testRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) {
storageConfig, err := getter.config.ControlPlane.Extra.StorageFactory.NewConfig(resource, example)
if err != nil {
return generic.RESTOptions{}, fmt.Errorf("failed to get storage: %v", err)
}

View File

@@ -1219,7 +1219,7 @@ func getRESTOptionsGetterForSecrets(t testing.TB, test *transformTest) generic.R
t.Fatal("not REST options found")
}
opts, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: "secrets"})
opts, err := genericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: "secrets"}, nil)
if err != nil {
t.Fatal(err)
}

View File

@@ -115,7 +115,9 @@ func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool,
return nil, fmt.Errorf("failed to read config file: %w", err)
}
if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(tb, nil, e.getEncryptionOptions(reload), e.storageConfig); err != nil {
if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(
tb, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "0.0"},
e.getEncryptionOptions(reload), e.storageConfig); err != nil {
e.cleanUp()
return nil, fmt.Errorf("failed to start KubeAPI server: %w", err)
}

View File

@@ -32,9 +32,11 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/dynamic"
apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
)
// TestOverlappingBuiltInResources ensures the list of group-resources the custom resource finalizer should skip is up to date
@@ -69,7 +71,9 @@ func TestOverlappingBuiltInResources(t *testing.T) {
// TestOverlappingCustomResourceAPIService ensures creating and deleting a custom resource overlapping with APIServices does not destroy APIService data
func TestOverlappingCustomResourceAPIService(t *testing.T) {
apiServer := StartRealAPIServerOrDie(t)
apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) {
opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.30")
})
defer apiServer.Cleanup()
apiServiceClient, err := apiregistrationclient.NewForConfig(apiServer.Config)
@@ -231,7 +235,9 @@ func TestOverlappingCustomResourceAPIService(t *testing.T) {
// TestOverlappingCustomResourceCustomResourceDefinition ensures creating and deleting a custom resource overlapping with CustomResourceDefinition does not destroy CustomResourceDefinition data
func TestOverlappingCustomResourceCustomResourceDefinition(t *testing.T) {
apiServer := StartRealAPIServerOrDie(t)
apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) {
opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.30")
})
defer apiServer.Cleanup()
crdClient, err := crdclient.NewForConfig(apiServer.Config)

View File

@@ -30,6 +30,7 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/apiserver/pkg/storage"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/dynamic"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
)
@@ -38,6 +39,7 @@ import (
func TestCrossGroupStorage(t *testing.T) {
apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) {
// force enable all resources so we can check storage.
opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.30")
})
defer apiServer.Cleanup()

View File

@@ -36,6 +36,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/dynamic"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
@@ -75,6 +76,7 @@ func TestEtcdStoragePath(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true)
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)
apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) {
opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("0.0")
})
defer apiServer.Cleanup()
defer dumpEtcdKVOnFailure(t, apiServer.KV)

View File

@@ -40,6 +40,8 @@ import (
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/wait"
genericapiserveroptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
cacheddiscovery "k8s.io/client-go/discovery/cached/memory"
"k8s.io/client-go/dynamic"
clientset "k8s.io/client-go/kubernetes"
@@ -91,7 +93,9 @@ func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRu
t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err)
}
opts := options.NewServerRunOptions()
featureGate := utilfeature.DefaultFeatureGate
effectiveVersion := utilversion.DefaultBuildEffectiveVersion()
opts := options.NewServerRunOptions(featureGate, effectiveVersion)
opts.Options.SecureServing.Listener = listener
opts.Options.SecureServing.ServerCert.CertDirectory = certDir
opts.Options.ServiceAccountSigningKeyFile = saSigningKeyFile.Name()

View File

@@ -292,6 +292,7 @@ func TestAggregatedAPIServer(t *testing.T) {
"--etcd-servers", framework.GetEtcdURL(),
"--cert-dir", wardleCertDir,
"--kubeconfig", wardleToKASKubeConfigFile,
"--wardle-emulated-version", "1.1",
})
if err := wardleCmd.Execute(); err != nil {
t.Error(err)

View File

@@ -35,6 +35,8 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
genericapiserver "k8s.io/apiserver/pkg/server"
genericapiserveroptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
client "k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/util/cert"
@@ -135,7 +137,9 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) (
t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err)
}
opts := options.NewServerRunOptions()
featureGate := utilfeature.DefaultFeatureGate
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
opts := options.NewServerRunOptions(featureGate, effectiveVersion)
opts.SecureServing.Listener = listener
opts.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1")
opts.SecureServing.ServerCert.CertDirectory = certDir

View File

@@ -29,6 +29,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/wait"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/kubernetes"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
@@ -122,6 +123,7 @@ func TestServiceAllocIPAddress(t *testing.T) {
ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
opts.ServiceClusterIPRanges = serviceCIDR
opts.GenericServerRunOptions.AdvertiseAddress = netutils.ParseIPSloppy("2001:db8::10")
opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.31")
opts.APIEnablement.RuntimeConfig.Set("networking.k8s.io/v1alpha1=true")
},
})