Merge pull request #122555 from p0lyn0mial/upstream-client-go-fg-provider-with-types

client-go/features: introduce feature gates

staging/src/k8s.io/client-go/features/envvar.go

@@ -0,0 +1,138 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package features
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"sync"
+	"sync/atomic"
+
+	"k8s.io/apimachinery/pkg/util/naming"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/klog/v2"
+)
+
+// internalPackages are packages that ignored when creating a name for featureGates. These packages are in the common
+// call chains, so they'd be unhelpful as names.
+var internalPackages = []string{"k8s.io/client-go/features/envvar.go"}
+
+var _ Gates = &envVarFeatureGates{}
+
+// newEnvVarFeatureGates creates a feature gate that allows for registration
+// of features and checking if the features are enabled.
+//
+// On the first call to Enabled, the effective state of all known features is loaded from
+// environment variables. The environment variable read for a given feature is formed by
+// concatenating the prefix "KUBE_FEATURE_" with the feature's name.
+//
+// For example, if you have a feature named "MyFeature"
+// setting an environmental variable "KUBE_FEATURE_MyFeature"
+// will allow you to configure the state of that feature.
+//
+// Please note that environmental variables can only be set to the boolean value.
+// Incorrect values will be ignored and logged.
+func newEnvVarFeatureGates(features map[Feature]FeatureSpec) *envVarFeatureGates {
+	known := map[Feature]FeatureSpec{}
+	for name, spec := range features {
+		known[name] = spec
+	}
+
+	fg := &envVarFeatureGates{
+		callSiteName: naming.GetNameFromCallsite(internalPackages...),
+		known:        known,
+	}
+	fg.enabled.Store(map[Feature]bool{})
+
+	return fg
+}
+
+// envVarFeatureGates implements Gates and allows for feature registration.
+type envVarFeatureGates struct {
+	// callSiteName holds the name of the file
+	// that created this instance
+	callSiteName string
+
+	// readEnvVarsOnce guards reading environmental variables
+	readEnvVarsOnce sync.Once
+
+	// known holds known feature gates
+	known map[Feature]FeatureSpec
+
+	// enabled holds a map[Feature]bool
+	// with values explicitly set via env var
+	enabled atomic.Value
+
+	// readEnvVars holds the boolean value which
+	// indicates whether readEnvVarsOnce has been called.
+	readEnvVars atomic.Bool
+}
+
+// Enabled returns true if the key is enabled.  If the key is not known, this call will panic.
+func (f *envVarFeatureGates) Enabled(key Feature) bool {
+	if v, ok := f.getEnabledMapFromEnvVar()[key]; ok {
+		return v
+	}
+	if v, ok := f.known[key]; ok {
+		return v.Default
+	}
+	panic(fmt.Errorf("feature %q is not registered in FeatureGates %q", key, f.callSiteName))
+}
+
+// getEnabledMapFromEnvVar will fill the enabled map on the first call.
+// This is the only time a known feature can be set to a value
+// read from the corresponding environmental variable.
+func (f *envVarFeatureGates) getEnabledMapFromEnvVar() map[Feature]bool {
+	f.readEnvVarsOnce.Do(func() {
+		featureGatesState := map[Feature]bool{}
+		for feature, featureSpec := range f.known {
+			featureState, featureStateSet := os.LookupEnv(fmt.Sprintf("KUBE_FEATURE_%s", feature))
+			if !featureStateSet {
+				continue
+			}
+			boolVal, boolErr := strconv.ParseBool(featureState)
+			switch {
+			case boolErr != nil:
+				utilruntime.HandleError(fmt.Errorf("cannot set feature gate %q to %q, due to %v", feature, featureState, boolErr))
+			case featureSpec.LockToDefault:
+				if boolVal != featureSpec.Default {
+					utilruntime.HandleError(fmt.Errorf("cannot set feature gate %q to %q, feature is locked to %v", feature, featureState, featureSpec.Default))
+					break
+				}
+				featureGatesState[feature] = featureSpec.Default
+			default:
+				featureGatesState[feature] = boolVal
+			}
+		}
+		f.enabled.Store(featureGatesState)
+		f.readEnvVars.Store(true)
+
+		for feature, featureSpec := range f.known {
+			if featureState, ok := featureGatesState[feature]; ok {
+				klog.V(1).InfoS("Feature gate updated state", "feature", feature, "enabled", featureState)
+				continue
+			}
+			klog.V(1).InfoS("Feature gate default state", "feature", feature, "enabled", featureSpec.Default)
+		}
+	})
+	return f.enabled.Load().(map[Feature]bool)
+}
+
+func (f *envVarFeatureGates) hasAlreadyReadEnvVar() bool {
+	return f.readEnvVars.Load()
+}

staging/src/k8s.io/client-go/features/envvar_test.go

@@ -0,0 +1,156 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package features
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestEnvVarFeatureGates(t *testing.T) {
+	defaultTestFeatures := map[Feature]FeatureSpec{
+		"TestAlpha": {
+			Default:       false,
+			LockToDefault: false,
+			PreRelease:    "Alpha",
+		},
+		"TestBeta": {
+			Default:       true,
+			LockToDefault: false,
+			PreRelease:    "Beta",
+		},
+	}
+	expectedDefaultFeaturesState := map[Feature]bool{"TestAlpha": false, "TestBeta": true}
+
+	copyExpectedStateMap := func(toCopy map[Feature]bool) map[Feature]bool {
+		m := map[Feature]bool{}
+		for k, v := range toCopy {
+			m[k] = v
+		}
+		return m
+	}
+
+	scenarios := []struct {
+		name                                string
+		features                            map[Feature]FeatureSpec
+		envVariables                        map[string]string
+		expectedFeaturesState               map[Feature]bool
+		expectedInternalEnabledFeatureState map[Feature]bool
+	}{
+		{
+			name: "can add empty features",
+		},
+		{
+			name:                  "no env var, features get Defaults assigned",
+			features:              defaultTestFeatures,
+			expectedFeaturesState: expectedDefaultFeaturesState,
+		},
+		{
+			name:                  "incorrect env var, feature gets Default assigned",
+			features:              defaultTestFeatures,
+			envVariables:          map[string]string{"TestAlpha": "true"},
+			expectedFeaturesState: expectedDefaultFeaturesState,
+		},
+		{
+			name:         "correct env var changes the feature gets state",
+			features:     defaultTestFeatures,
+			envVariables: map[string]string{"KUBE_FEATURE_TestAlpha": "true"},
+			expectedFeaturesState: func() map[Feature]bool {
+				expectedDefaultFeaturesStateCopy := copyExpectedStateMap(expectedDefaultFeaturesState)
+				expectedDefaultFeaturesStateCopy["TestAlpha"] = true
+				return expectedDefaultFeaturesStateCopy
+			}(),
+			expectedInternalEnabledFeatureState: map[Feature]bool{"TestAlpha": true},
+		},
+		{
+			name:                  "incorrect env var value gets ignored",
+			features:              defaultTestFeatures,
+			envVariables:          map[string]string{"KUBE_FEATURE_TestAlpha": "TrueFalse"},
+			expectedFeaturesState: expectedDefaultFeaturesState,
+		},
+		{
+			name:                  "empty env var value gets ignored",
+			features:              defaultTestFeatures,
+			envVariables:          map[string]string{"KUBE_FEATURE_TestAlpha": ""},
+			expectedFeaturesState: expectedDefaultFeaturesState,
+		},
+		{
+			name: "a feature LockToDefault wins",
+			features: map[Feature]FeatureSpec{
+				"TestAlpha": {
+					Default:       true,
+					LockToDefault: true,
+					PreRelease:    "Alpha",
+				},
+			},
+			envVariables:          map[string]string{"KUBE_FEATURE_TestAlpha": "False"},
+			expectedFeaturesState: map[Feature]bool{"TestAlpha": true},
+		},
+		{
+			name: "setting a feature to LockToDefault changes the internal state",
+			features: map[Feature]FeatureSpec{
+				"TestAlpha": {
+					Default:       true,
+					LockToDefault: true,
+					PreRelease:    "Alpha",
+				},
+			},
+			envVariables:                        map[string]string{"KUBE_FEATURE_TestAlpha": "True"},
+			expectedFeaturesState:               map[Feature]bool{"TestAlpha": true},
+			expectedInternalEnabledFeatureState: map[Feature]bool{"TestAlpha": true},
+		},
+	}
+	for _, scenario := range scenarios {
+		t.Run(scenario.name, func(t *testing.T) {
+			for k, v := range scenario.envVariables {
+				t.Setenv(k, v)
+			}
+			target := newEnvVarFeatureGates(scenario.features)
+
+			for expectedFeature, expectedValue := range scenario.expectedFeaturesState {
+				actualValue := target.Enabled(expectedFeature)
+				require.Equal(t, actualValue, expectedValue, "expected feature=%v, to be=%v, not=%v", expectedFeature, expectedValue, actualValue)
+			}
+
+			enabledInternalMap := target.enabled.Load().(map[Feature]bool)
+			require.Len(t, enabledInternalMap, len(scenario.expectedInternalEnabledFeatureState))
+
+			for expectedFeature, expectedInternalPresence := range scenario.expectedInternalEnabledFeatureState {
+				featureInternalValue, featureSet := enabledInternalMap[expectedFeature]
+				require.Equal(t, expectedInternalPresence, featureSet, "feature %v present = %v, expected = %v", expectedFeature, featureSet, expectedInternalPresence)
+
+				expectedFeatureInternalValue := scenario.expectedFeaturesState[expectedFeature]
+				require.Equal(t, expectedFeatureInternalValue, featureInternalValue)
+			}
+		})
+	}
+}
+
+func TestEnvVarFeatureGatesEnabledPanic(t *testing.T) {
+	target := newEnvVarFeatureGates(nil)
+	require.PanicsWithError(t, fmt.Errorf("feature %q is not registered in FeatureGates %q", "UnknownFeature", target.callSiteName).Error(), func() { target.Enabled("UnknownFeature") })
+}
+
+func TestHasAlreadyReadEnvVar(t *testing.T) {
+	target := newEnvVarFeatureGates(nil)
+	require.False(t, target.hasAlreadyReadEnvVar())
+
+	_ = target.getEnabledMapFromEnvVar()
+	require.True(t, target.hasAlreadyReadEnvVar())
+}

staging/src/k8s.io/client-go/features/features.go

@@ -0,0 +1,149 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package features
+
+import (
+	"errors"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"sync/atomic"
+)
+
+// NOTE: types Feature, FeatureSpec, prerelease (and its values)
+// were duplicated from the component-base repository
+//
+// for more information please refer to https://docs.google.com/document/d/1g9BGCRw-7ucUxO6OtCWbb3lfzUGA_uU9178wLdXAIfs
+
+const (
+	// Values for PreRelease.
+	Alpha = prerelease("ALPHA")
+	Beta  = prerelease("BETA")
+	GA    = prerelease("")
+
+	// Deprecated
+	Deprecated = prerelease("DEPRECATED")
+)
+
+type prerelease string
+
+type Feature string
+
+type FeatureSpec struct {
+	// Default is the default enablement state for the feature
+	Default bool
+	// LockToDefault indicates that the feature is locked to its default and cannot be changed
+	LockToDefault bool
+	// PreRelease indicates the maturity level of the feature
+	PreRelease prerelease
+}
+
+// Gates indicates whether a given feature is enabled or not.
+type Gates interface {
+	// Enabled returns true if the key is enabled.
+	Enabled(key Feature) bool
+}
+
+// Registry represents an external feature gates registry.
+type Registry interface {
+	// Add adds existing feature gates to the provided registry.
+	//
+	// As of today, this method is used by AddFeaturesToExistingFeatureGates and
+	// ReplaceFeatureGates to take control of the features exposed by this library.
+	Add(map[Feature]FeatureSpec) error
+}
+
+// FeatureGates returns the feature gates exposed by this library.
+//
+// By default, only the default features gates will be returned.
+// The default implementation allows controlling the features
+// via environmental variables.
+// For example, if you have a feature named "MyFeature"
+// setting an environmental variable "KUBE_FEATURE_MyFeature"
+// will allow you to configure the state of that feature.
+//
+// Please note that the actual set of the feature gates
+// might be overwritten by calling ReplaceFeatureGates method.
+func FeatureGates() Gates {
+	return featureGates.Load().(*featureGatesWrapper).Gates
+}
+
+// AddFeaturesToExistingFeatureGates adds the default feature gates to the provided registry.
+// Usually this function is combined with ReplaceFeatureGates to take control of the
+// features exposed by this library.
+func AddFeaturesToExistingFeatureGates(registry Registry) error {
+	return registry.Add(defaultKubernetesFeatureGates)
+}
+
+// ReplaceFeatureGates overwrites the default implementation of the feature gates
+// used by this library.
+//
+// Useful for binaries that would like to have full control of the features
+// exposed by this library, such as allowing consumers of a binary
+// to interact with the features via a command line flag.
+//
+// For example:
+//
+//	// first, register client-go's features to your registry.
+//	clientgofeaturegate.AddFeaturesToExistingFeatureGates(utilfeature.DefaultMutableFeatureGate)
+//	// then replace client-go's feature gates implementation with your implementation
+//	clientgofeaturegate.ReplaceFeatureGates(utilfeature.DefaultMutableFeatureGate)
+func ReplaceFeatureGates(newFeatureGates Gates) {
+	if replaceFeatureGatesWithWarningIndicator(newFeatureGates) {
+		utilruntime.HandleError(errors.New("the default feature gates implementation has already been used and now it's being overwritten. This might lead to unexpected behaviour. Check your initialization order"))
+	}
+}
+
+func replaceFeatureGatesWithWarningIndicator(newFeatureGates Gates) bool {
+	shouldProduceWarning := false
+
+	if defaultFeatureGates, ok := FeatureGates().(*envVarFeatureGates); ok {
+		if defaultFeatureGates.hasAlreadyReadEnvVar() {
+			shouldProduceWarning = true
+		}
+	}
+	wrappedFeatureGates := &featureGatesWrapper{newFeatureGates}
+	featureGates.Store(wrappedFeatureGates)
+
+	return shouldProduceWarning
+}
+
+func init() {
+	envVarGates := newEnvVarFeatureGates(defaultKubernetesFeatureGates)
+
+	wrappedFeatureGates := &featureGatesWrapper{envVarGates}
+	featureGates.Store(wrappedFeatureGates)
+}
+
+// featureGatesWrapper a thin wrapper to satisfy featureGates variable (atomic.Value).
+// That is, all calls to Store for a given Value must use values of the same concrete type.
+type featureGatesWrapper struct {
+	Gates
+}
+
+var (
+	// featureGates is a shared global FeatureGates.
+	//
+	// Top-level commands/options setup that needs to modify this feature gates
+	// should use AddFeaturesToExistingFeatureGates followed by ReplaceFeatureGates.
+	featureGates = &atomic.Value{}
+)
+
+// defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys.
+//
+// To add a new feature, define a key for it above and add it here. The features will be
+// available throughout Kubernetes binaries.
+var defaultKubernetesFeatureGates = map[Feature]FeatureSpec{}

staging/src/k8s.io/client-go/features/features_test.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package features
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestAddFeaturesToExistingFeatureGates ensures that
+// the defaultKubernetesFeatureGates are added to a test feature gates registry.
+func TestAddFeaturesToExistingFeatureGates(t *testing.T) {
+	fakeFeatureGates := &fakeRegistry{}
+	require.NoError(t, AddFeaturesToExistingFeatureGates(fakeFeatureGates))
+	require.Equal(t, defaultKubernetesFeatureGates, fakeFeatureGates.specs)
+}
+
+type fakeRegistry struct {
+	specs map[Feature]FeatureSpec
+}
+
+func (f *fakeRegistry) Add(specs map[Feature]FeatureSpec) error {
+	f.specs = specs
+	return nil
+}

staging/src/k8s.io/client-go/features/testing/features_init_test.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"k8s.io/client-go/features"
+)
+
+func TestDriveInitDefaultFeatureGates(t *testing.T) {
+	featureGates := features.FeatureGates()
+	assertFunctionPanicsWithMessage(t, func() { featureGates.Enabled("FakeFeatureGate") }, "features.FeatureGates().Enabled", fmt.Sprintf("feature %q is not registered in FeatureGate", "FakeFeatureGate"))
+
+	fakeFeatureGates := &alwaysEnabledFakeGates{}
+	require.True(t, fakeFeatureGates.Enabled("FakeFeatureGate"))
+
+	features.ReplaceFeatureGates(fakeFeatureGates)
+	featureGates = features.FeatureGates()
+
+	assertFeatureGatesType(t, featureGates)
+	require.True(t, featureGates.Enabled("FakeFeatureGate"))
+}
+
+type alwaysEnabledFakeGates struct{}
+
+func (f *alwaysEnabledFakeGates) Enabled(features.Feature) bool {
+	return true
+}
+
+func assertFeatureGatesType(t *testing.T, fg features.Gates) {
+	_, ok := fg.(*alwaysEnabledFakeGates)
+	if !ok {
+		t.Fatalf("passed features.FeatureGates() is NOT of type *alwaysEnabledFakeGates, it is of type = %T", fg)
+	}
+}
+
+func assertFunctionPanicsWithMessage(t *testing.T, f func(), fName, errMessage string) {
+	didPanic, panicMessage := didFunctionPanic(f)
+	if !didPanic {
+		t.Fatalf("function %q did not panicked", fName)
+	}
+
+	panicError, ok := panicMessage.(error)
+	if !ok || !strings.Contains(panicError.Error(), errMessage) {
+		t.Fatalf("func %q should panic with error message:\t%#v\n\tPanic value:\t%#v\n", fName, errMessage, panicMessage)
+	}
+}
+
+func didFunctionPanic(f func()) (didPanic bool, panicMessage interface{}) {
+	didPanic = true
+
+	defer func() {
+		panicMessage = recover()
+	}()
+
+	f()
+	didPanic = false
+
+	return
+}