node: graduate CPUManagerPolicyOptions to beta

We graduate the `CPUManagerPolicyOptions` feature to beta
in the 1.23 cycle, and we add new experimental feature gates
to guard new options which are planned in the 1.23 and in the
following cycles.

We introduce additional feature gate called `CPUManagerPolicyAlphaOptions` and
`CPUManagerPolicyBetaOptions`. The basic idea is to avoid the
cumbersome process of adding a feature gate for each option, and to have
feature gates which track the maturity level of _groups_ of options.
Besides this change, the graduation process, and the process in general,
for adding new policy options is still unchanged.

The `full-pcpus-only` option added in the 1.22 cycle is intentionally
moved into the beta policy options

For more details:
- KEP: https://github.com/kubernetes/enhancements/pull/2933
- sig-arch discussion:
  https://groups.google.com/u/1/g/kubernetes-sig-architecture/c/Nxsc7pfe5rw

Signed-off-by: Francesco Romani <fromani@redhat.com>

pkg/kubelet/cm/cpumanager/policy_options.go

@@ -19,6 +19,10 @@ package cpumanager
 import (
 	"fmt"
 	"strconv"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	kubefeatures "k8s.io/kubernetes/pkg/features"
 )
 
 const (
@@ -26,6 +30,30 @@ const (
 	FullPCPUsOnlyOption string = "full-pcpus-only"
 )
 
+var (
+	alphaOptions = sets.NewString()
+	betaOptions  = sets.NewString(
+		FullPCPUsOnlyOption,
+	)
+	stableOptions = sets.NewString()
+)
+
+func CheckPolicyOptionAvailable(option string) error {
+	if !alphaOptions.Has(option) && !betaOptions.Has(option) && !stableOptions.Has(option) {
+		return fmt.Errorf("unknown CPU Manager Policy option: %q", option)
+	}
+
+	if alphaOptions.Has(option) && !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CPUManagerPolicyAlphaOptions) {
+		return fmt.Errorf("CPU Manager Policy Alpha-level Options not enabled, but option %q provided", option)
+	}
+
+	if betaOptions.Has(option) && !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CPUManagerPolicyBetaOptions) {
+		return fmt.Errorf("CPU Manager Policy Beta-level Options not enabled, but option %q provided", option)
+	}
+
+	return nil
+}
+
 type StaticPolicyOptions struct {
 	// flag to enable extra allocation restrictions to avoid
 	// different containers to possibly end up on the same core.
@@ -41,6 +69,10 @@ type StaticPolicyOptions struct {
 func NewStaticPolicyOptions(policyOptions map[string]string) (StaticPolicyOptions, error) {
 	opts := StaticPolicyOptions{}
 	for name, value := range policyOptions {
+		if err := CheckPolicyOptionAvailable(name); err != nil {
+			return opts, err
+		}
+
 		switch name {
 		case FullPCPUsOnlyOption:
 			optValue, err := strconv.ParseBool(value)
@@ -49,6 +81,8 @@ func NewStaticPolicyOptions(policyOptions map[string]string) (StaticPolicyOption
 			}
 			opts.FullPhysicalCPUsOnly = optValue
 		default:
+			// this should never be reached, we already detect unknown options,
+			// but we keep it as further safety.
 			return opts, fmt.Errorf("unsupported cpumanager option: %q (%s)", name, value)
 		}
 	}

pkg/kubelet/cm/cpumanager/policy_options_test.go

@@ -0,0 +1,94 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cpumanager
+
+import (
+	"testing"
+
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/component-base/featuregate"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
+	pkgfeatures "k8s.io/kubernetes/pkg/features"
+)
+
+type optionAvailTest struct {
+	option            string
+	featureGate       featuregate.Feature
+	featureGateEnable bool
+	expectedAvailable bool
+}
+
+func TestPolicyDefaultsAvailable(t *testing.T) {
+	testCases := []optionAvailTest{
+		{
+			option:            "this-option-does-not-exist",
+			expectedAvailable: false,
+		},
+		{
+			option:            FullPCPUsOnlyOption,
+			expectedAvailable: true,
+		},
+	}
+	for _, testCase := range testCases {
+		t.Run(testCase.option, func(t *testing.T) {
+			err := CheckPolicyOptionAvailable(testCase.option)
+			isEnabled := (err == nil)
+			if isEnabled != testCase.expectedAvailable {
+				t.Errorf("option %q available got=%v expected=%v", testCase.option, isEnabled, testCase.expectedAvailable)
+			}
+		})
+	}
+}
+
+func TestPolicyBetaOptionsAvailable(t *testing.T) {
+	testCases := []optionAvailTest{
+		{
+			option:            "this-option-does-not-exist",
+			featureGate:       pkgfeatures.CPUManagerPolicyBetaOptions,
+			featureGateEnable: false,
+			expectedAvailable: false,
+		},
+		{
+			option:            "this-option-does-not-exist",
+			featureGate:       pkgfeatures.CPUManagerPolicyBetaOptions,
+			featureGateEnable: true,
+			expectedAvailable: false,
+		},
+		{
+			option:            FullPCPUsOnlyOption,
+			featureGate:       pkgfeatures.CPUManagerPolicyBetaOptions,
+			featureGateEnable: true,
+			expectedAvailable: true,
+		},
+		{
+			option:            FullPCPUsOnlyOption,
+			featureGate:       pkgfeatures.CPUManagerPolicyBetaOptions,
+			featureGateEnable: false,
+			expectedAvailable: false,
+		},
+	}
+	for _, testCase := range testCases {
+		t.Run(testCase.option, func(t *testing.T) {
+			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, testCase.featureGate, testCase.featureGateEnable)()
+			err := CheckPolicyOptionAvailable(testCase.option)
+			isEnabled := (err == nil)
+			if isEnabled != testCase.expectedAvailable {
+				t.Errorf("option %q available got=%v expected=%v", testCase.option, isEnabled, testCase.expectedAvailable)
+			}
+		})
+	}
+}