github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #122555 from p0lyn0mial/upstream-client-go-fg-provider-with-types

Browse Source 
client-go/features: introduce feature gates
This commit is contained in:
Kubernetes Prow Robot
2024-01-10 20:49:12 +01:00
committed by GitHub
parent 35bed806dc 04bbd3481f
commit 0341e8294a
5 changed files with 562 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
staging/src/k8s.io/client-go/features/envvar.go Normal file
View File

@@ -0,0 +1,138 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package features
import (
"fmt"
"os"
"strconv"
"sync"
"sync/atomic"
"k8s.io/apimachinery/pkg/util/naming"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/klog/v2"
)
// internalPackages are packages that ignored when creating a name for featureGates. These packages are in the common
// call chains, so they'd be unhelpful as names.
var internalPackages = []string{"k8s.io/client-go/features/envvar.go"}
var _ Gates = &envVarFeatureGates{}
// newEnvVarFeatureGates creates a feature gate that allows for registration
// of features and checking if the features are enabled.
//
// On the first call to Enabled, the effective state of all known features is loaded from
// environment variables. The environment variable read for a given feature is formed by
// concatenating the prefix "KUBE_FEATURE_" with the feature's name.
//
// For example, if you have a feature named "MyFeature"
// setting an environmental variable "KUBE_FEATURE_MyFeature"
// will allow you to configure the state of that feature.
//
// Please note that environmental variables can only be set to the boolean value.
// Incorrect values will be ignored and logged.
func newEnvVarFeatureGates(features map[Feature]FeatureSpec) *envVarFeatureGates {
known := map[Feature]FeatureSpec{}
for name, spec := range features {
known[name] = spec
}
fg := &envVarFeatureGates{
callSiteName: naming.GetNameFromCallsite(internalPackages...),
known: known,
}
fg.enabled.Store(map[Feature]bool{})
return fg
}
// envVarFeatureGates implements Gates and allows for feature registration.
type envVarFeatureGates struct {
// callSiteName holds the name of the file
// that created this instance
callSiteName string
// readEnvVarsOnce guards reading environmental variables
readEnvVarsOnce sync.Once
// known holds known feature gates
known map[Feature]FeatureSpec
// enabled holds a map[Feature]bool
// with values explicitly set via env var
enabled atomic.Value
// readEnvVars holds the boolean value which
// indicates whether readEnvVarsOnce has been called.
readEnvVars atomic.Bool
}
// Enabled returns true if the key is enabled. If the key is not known, this call will panic.
func (f *envVarFeatureGates) Enabled(key Feature) bool {
if v, ok := f.getEnabledMapFromEnvVar()[key]; ok {
return v
}
if v, ok := f.known[key]; ok {
return v.Default
}
panic(fmt.Errorf("feature %q is not registered in FeatureGates %q", key, f.callSiteName))
}
// getEnabledMapFromEnvVar will fill the enabled map on the first call.
// This is the only time a known feature can be set to a value
// read from the corresponding environmental variable.
func (f *envVarFeatureGates) getEnabledMapFromEnvVar() map[Feature]bool {
f.readEnvVarsOnce.Do(func() {
featureGatesState := map[Feature]bool{}
for feature, featureSpec := range f.known {
featureState, featureStateSet := os.LookupEnv(fmt.Sprintf("KUBE_FEATURE_%s", feature))
if !featureStateSet {
continue
}
boolVal, boolErr := strconv.ParseBool(featureState)
switch {
case boolErr != nil:
utilruntime.HandleError(fmt.Errorf("cannot set feature gate %q to %q, due to %v", feature, featureState, boolErr))
case featureSpec.LockToDefault:
if boolVal != featureSpec.Default {
utilruntime.HandleError(fmt.Errorf("cannot set feature gate %q to %q, feature is locked to %v", feature, featureState, featureSpec.Default))
break
}
featureGatesState[feature] = featureSpec.Default
default:
featureGatesState[feature] = boolVal
}
}
f.enabled.Store(featureGatesState)
f.readEnvVars.Store(true)
for feature, featureSpec := range f.known {
if featureState, ok := featureGatesState[feature]; ok {
klog.V(1).InfoS("Feature gate updated state", "feature", feature, "enabled", featureState)
continue
}
klog.V(1).InfoS("Feature gate default state", "feature", feature, "enabled", featureSpec.Default)
}
})
return f.enabled.Load().(map[Feature]bool)
}
func (f *envVarFeatureGates) hasAlreadyReadEnvVar() bool {
return f.readEnvVars.Load()
}

156
staging/src/k8s.io/client-go/features/envvar_test.go Normal file
View File

@@ -0,0 +1,156 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package features
import (
"fmt"
"testing"
"github.com/stretchr/testify/require"
)
func TestEnvVarFeatureGates(t *testing.T) {
defaultTestFeatures := map[Feature]FeatureSpec{
"TestAlpha": {
Default: false,
LockToDefault: false,
PreRelease: "Alpha",
},
"TestBeta": {
Default: true,
LockToDefault: false,
PreRelease: "Beta",
},
}
expectedDefaultFeaturesState := map[Feature]bool{"TestAlpha": false, "TestBeta": true}
copyExpectedStateMap := func(toCopy map[Feature]bool) map[Feature]bool {
m := map[Feature]bool{}
for k, v := range toCopy {
m[k] = v
}
return m
}
scenarios := []struct {
name string
features map[Feature]FeatureSpec
envVariables map[string]string
expectedFeaturesState map[Feature]bool
expectedInternalEnabledFeatureState map[Feature]bool
}{
{
name: "can add empty features",
},
{
name: "no env var, features get Defaults assigned",
features: defaultTestFeatures,
expectedFeaturesState: expectedDefaultFeaturesState,
},
{
name: "incorrect env var, feature gets Default assigned",
features: defaultTestFeatures,
envVariables: map[string]string{"TestAlpha": "true"},
expectedFeaturesState: expectedDefaultFeaturesState,
},
{
name: "correct env var changes the feature gets state",
features: defaultTestFeatures,
envVariables: map[string]string{"KUBE_FEATURE_TestAlpha": "true"},
expectedFeaturesState: func() map[Feature]bool {
expectedDefaultFeaturesStateCopy := copyExpectedStateMap(expectedDefaultFeaturesState)
expectedDefaultFeaturesStateCopy["TestAlpha"] = true
return expectedDefaultFeaturesStateCopy
}(),
expectedInternalEnabledFeatureState: map[Feature]bool{"TestAlpha": true},
},
{
name: "incorrect env var value gets ignored",
features: defaultTestFeatures,
envVariables: map[string]string{"KUBE_FEATURE_TestAlpha": "TrueFalse"},
expectedFeaturesState: expectedDefaultFeaturesState,
},
{
name: "empty env var value gets ignored",
features: defaultTestFeatures,
envVariables: map[string]string{"KUBE_FEATURE_TestAlpha": ""},
expectedFeaturesState: expectedDefaultFeaturesState,
},
{
name: "a feature LockToDefault wins",
features: map[Feature]FeatureSpec{
"TestAlpha": {
Default: true,
LockToDefault: true,
PreRelease: "Alpha",
},
},
envVariables: map[string]string{"KUBE_FEATURE_TestAlpha": "False"},
expectedFeaturesState: map[Feature]bool{"TestAlpha": true},
},
{
name: "setting a feature to LockToDefault changes the internal state",
features: map[Feature]FeatureSpec{
"TestAlpha": {
Default: true,
LockToDefault: true,
PreRelease: "Alpha",
},
},
envVariables: map[string]string{"KUBE_FEATURE_TestAlpha": "True"},
expectedFeaturesState: map[Feature]bool{"TestAlpha": true},
expectedInternalEnabledFeatureState: map[Feature]bool{"TestAlpha": true},
},
}
for _, scenario := range scenarios {
t.Run(scenario.name, func(t *testing.T) {
for k, v := range scenario.envVariables {
t.Setenv(k, v)
}
target := newEnvVarFeatureGates(scenario.features)
for expectedFeature, expectedValue := range scenario.expectedFeaturesState {
actualValue := target.Enabled(expectedFeature)
require.Equal(t, actualValue, expectedValue, "expected feature=%v, to be=%v, not=%v", expectedFeature, expectedValue, actualValue)
}
enabledInternalMap := target.enabled.Load().(map[Feature]bool)
require.Len(t, enabledInternalMap, len(scenario.expectedInternalEnabledFeatureState))
for expectedFeature, expectedInternalPresence := range scenario.expectedInternalEnabledFeatureState {
featureInternalValue, featureSet := enabledInternalMap[expectedFeature]
require.Equal(t, expectedInternalPresence, featureSet, "feature %v present = %v, expected = %v", expectedFeature, featureSet, expectedInternalPresence)
expectedFeatureInternalValue := scenario.expectedFeaturesState[expectedFeature]
require.Equal(t, expectedFeatureInternalValue, featureInternalValue)
}
})
}
}
func TestEnvVarFeatureGatesEnabledPanic(t *testing.T) {
target := newEnvVarFeatureGates(nil)
require.PanicsWithError(t, fmt.Errorf("feature %q is not registered in FeatureGates %q", "UnknownFeature", target.callSiteName).Error(), func() { target.Enabled("UnknownFeature") })
}
func TestHasAlreadyReadEnvVar(t *testing.T) {
target := newEnvVarFeatureGates(nil)
require.False(t, target.hasAlreadyReadEnvVar())
_ = target.getEnabledMapFromEnvVar()
require.True(t, target.hasAlreadyReadEnvVar())
}

149
staging/src/k8s.io/client-go/features/features.go Normal file
View File

@@ -0,0 +1,149 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package features
import (
"errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"sync/atomic"
)
// NOTE: types Feature, FeatureSpec, prerelease (and its values)
// were duplicated from the component-base repository
//
// for more information please refer to https://docs.google.com/document/d/1g9BGCRw-7ucUxO6OtCWbb3lfzUGA_uU9178wLdXAIfs
const (
// Values for PreRelease.
Alpha = prerelease("ALPHA")
Beta = prerelease("BETA")
GA = prerelease("")
// Deprecated
Deprecated = prerelease("DEPRECATED")
)
type prerelease string
type Feature string
type FeatureSpec struct {
// Default is the default enablement state for the feature
Default bool
// LockToDefault indicates that the feature is locked to its default and cannot be changed
LockToDefault bool
// PreRelease indicates the maturity level of the feature
PreRelease prerelease
}
// Gates indicates whether a given feature is enabled or not.
type Gates interface {
// Enabled returns true if the key is enabled.
Enabled(key Feature) bool
}
// Registry represents an external feature gates registry.
type Registry interface {
// Add adds existing feature gates to the provided registry.
//
// As of today, this method is used by AddFeaturesToExistingFeatureGates and
// ReplaceFeatureGates to take control of the features exposed by this library.
Add(map[Feature]FeatureSpec) error
}
// FeatureGates returns the feature gates exposed by this library.
//
// By default, only the default features gates will be returned.
// The default implementation allows controlling the features
// via environmental variables.
// For example, if you have a feature named "MyFeature"
// setting an environmental variable "KUBE_FEATURE_MyFeature"
// will allow you to configure the state of that feature.
//
// Please note that the actual set of the feature gates
// might be overwritten by calling ReplaceFeatureGates method.
func FeatureGates() Gates {
return featureGates.Load().(*featureGatesWrapper).Gates
}
// AddFeaturesToExistingFeatureGates adds the default feature gates to the provided registry.
// Usually this function is combined with ReplaceFeatureGates to take control of the
// features exposed by this library.
func AddFeaturesToExistingFeatureGates(registry Registry) error {
return registry.Add(defaultKubernetesFeatureGates)
}
// ReplaceFeatureGates overwrites the default implementation of the feature gates
// used by this library.
//
// Useful for binaries that would like to have full control of the features
// exposed by this library, such as allowing consumers of a binary
// to interact with the features via a command line flag.
//
// For example:
//
// // first, register client-go's features to your registry.
// clientgofeaturegate.AddFeaturesToExistingFeatureGates(utilfeature.DefaultMutableFeatureGate)
// // then replace client-go's feature gates implementation with your implementation
// clientgofeaturegate.ReplaceFeatureGates(utilfeature.DefaultMutableFeatureGate)
func ReplaceFeatureGates(newFeatureGates Gates) {
if replaceFeatureGatesWithWarningIndicator(newFeatureGates) {
utilruntime.HandleError(errors.New("the default feature gates implementation has already been used and now it's being overwritten. This might lead to unexpected behaviour. Check your initialization order"))
}
}
func replaceFeatureGatesWithWarningIndicator(newFeatureGates Gates) bool {
shouldProduceWarning := false
if defaultFeatureGates, ok := FeatureGates().(*envVarFeatureGates); ok {
if defaultFeatureGates.hasAlreadyReadEnvVar() {
shouldProduceWarning = true
}
}
wrappedFeatureGates := &featureGatesWrapper{newFeatureGates}
featureGates.Store(wrappedFeatureGates)
return shouldProduceWarning
}
func init() {
envVarGates := newEnvVarFeatureGates(defaultKubernetesFeatureGates)
wrappedFeatureGates := &featureGatesWrapper{envVarGates}
featureGates.Store(wrappedFeatureGates)
}
// featureGatesWrapper a thin wrapper to satisfy featureGates variable (atomic.Value).
// That is, all calls to Store for a given Value must use values of the same concrete type.
type featureGatesWrapper struct {
Gates
}
var (
// featureGates is a shared global FeatureGates.
//
// Top-level commands/options setup that needs to modify this feature gates
// should use AddFeaturesToExistingFeatureGates followed by ReplaceFeatureGates.
featureGates = &atomic.Value{}
)
// defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys.
//
// To add a new feature, define a key for it above and add it here. The features will be
// available throughout Kubernetes binaries.
var defaultKubernetesFeatureGates = map[Feature]FeatureSpec{}

40
staging/src/k8s.io/client-go/features/features_test.go Normal file
View File

@@ -0,0 +1,40 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package features
import (
"testing"
"github.com/stretchr/testify/require"
)
// TestAddFeaturesToExistingFeatureGates ensures that
// the defaultKubernetesFeatureGates are added to a test feature gates registry.
func TestAddFeaturesToExistingFeatureGates(t *testing.T) {
fakeFeatureGates := &fakeRegistry{}
require.NoError(t, AddFeaturesToExistingFeatureGates(fakeFeatureGates))
require.Equal(t, defaultKubernetesFeatureGates, fakeFeatureGates.specs)
}
type fakeRegistry struct {
specs map[Feature]FeatureSpec
}
func (f *fakeRegistry) Add(specs map[Feature]FeatureSpec) error {
f.specs = specs
return nil
}

79
staging/src/k8s.io/client-go/features/testing/features_init_test.go Normal file
View File

@@ -0,0 +1,79 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package testing
import (
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/require"
"k8s.io/client-go/features"
)
func TestDriveInitDefaultFeatureGates(t *testing.T) {
featureGates := features.FeatureGates()
assertFunctionPanicsWithMessage(t, func() { featureGates.Enabled("FakeFeatureGate") }, "features.FeatureGates().Enabled", fmt.Sprintf("feature %q is not registered in FeatureGate", "FakeFeatureGate"))
fakeFeatureGates := &alwaysEnabledFakeGates{}
require.True(t, fakeFeatureGates.Enabled("FakeFeatureGate"))
features.ReplaceFeatureGates(fakeFeatureGates)
featureGates = features.FeatureGates()
assertFeatureGatesType(t, featureGates)
require.True(t, featureGates.Enabled("FakeFeatureGate"))
}
type alwaysEnabledFakeGates struct{}
func (f *alwaysEnabledFakeGates) Enabled(features.Feature) bool {
return true
}
func assertFeatureGatesType(t *testing.T, fg features.Gates) {
_, ok := fg.(*alwaysEnabledFakeGates)
if !ok {
t.Fatalf("passed features.FeatureGates() is NOT of type *alwaysEnabledFakeGates, it is of type = %T", fg)
}
}
func assertFunctionPanicsWithMessage(t *testing.T, f func(), fName, errMessage string) {
didPanic, panicMessage := didFunctionPanic(f)
if !didPanic {
t.Fatalf("function %q did not panicked", fName)
}
panicError, ok := panicMessage.(error)
if !ok || !strings.Contains(panicError.Error(), errMessage) {
t.Fatalf("func %q should panic with error message:\t%#v\n\tPanic value:\t%#v\n", fName, errMessage, panicMessage)
}
}
func didFunctionPanic(f func()) (didPanic bool, panicMessage interface{}) {
didPanic = true
defer func() {
panicMessage = recover()
}()
f()
didPanic = false
return
}