github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

apiserver: Add API emulation versioning.

Browse Source 
Co-authored-by: Siyuan Zhang <sizhang@google.com>
Co-authored-by: Joe Betz <jpbetz@google.com>
Co-authored-by: Alex Zielenski <zielenski@google.com>

Signed-off-by: Siyuan Zhang <sizhang@google.com>
This commit is contained in:
Siyuan Zhang
2024-01-19 16:07:00 -08:00
parent d0579b6f9c
commit 403301bfdf
86 changed files with 3420 additions and 427 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/kube-apiserver/app/options/options.go
View File

@@ -26,6 +26,8 @@ import (
utilnet "k8s.io/apimachinery/pkg/util/net"
cliflag "k8s.io/component-base/cli/flag"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/component-base/featuregate"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/cluster/ports"
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
@@ -63,10 +65,10 @@ type Extra struct {
MasterCount int
}
// NewServerRunOptions creates a new ServerRunOptions object with default parameters
func NewServerRunOptions() *ServerRunOptions {
// NewServerRunOptions creates and returns ServerRunOptions according to the given featureGate and effectiveVersion of the server binary to run.
func NewServerRunOptions(featureGate featuregate.FeatureGate, effectiveVersion utilversion.EffectiveVersion) *ServerRunOptions {
s := ServerRunOptions{
Options: controlplaneapiserver.NewOptions(),
Options: controlplaneapiserver.NewOptions(featureGate, effectiveVersion),
CloudProvider: kubeoptions.NewCloudProviderOptions(),
Extra: Extra{

16
cmd/kube-apiserver/app/options/options_test.go
View File

@@ -31,9 +31,11 @@ import (
apiserveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/etcd3"
"k8s.io/apiserver/pkg/storage/storagebackend"
utilversion "k8s.io/apiserver/pkg/util/version"
auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics"
kapi "k8s.io/kubernetes/pkg/apis/core"
@@ -46,10 +48,15 @@ import (
func TestAddFlags(t *testing.T) {
fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
s := NewServerRunOptions()
featureGate := featuregate.NewFeatureGate()
effectiveVersion := utilversion.NewEffectiveVersion("1.32")
s := NewServerRunOptions(featureGate, effectiveVersion)
for _, f := range s.Flags().FlagSets {
fs.AddFlagSet(f)
}
featureGate.AddFlag(fs, "")
effectiveVersion.AddFlags(fs, "")
args := []string{
"--enable-admission-plugins=AlwaysDeny",
@@ -121,6 +128,7 @@ func TestAddFlags(t *testing.T) {
"--storage-backend=etcd3",
"--service-cluster-ip-range=192.168.128.0/17",
"--lease-reuse-duration-seconds=100",
"--emulated-version=1.31",
}
fs.Parse(args)
@@ -136,6 +144,8 @@ func TestAddFlags(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
FeatureGate: featureGate,
EffectiveVersion: effectiveVersion,
},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &apiserveroptions.AdmissionOptions{
@@ -337,4 +347,8 @@ func TestAddFlags(t *testing.T) {
if !reflect.DeepEqual(expected, s) {
t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{})))
}
if s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String() != "1.31" {
t.Errorf("Got emulation version %s, wanted %s", s.GenericServerRunOptions.EffectiveVersion.EmulationVersion().String(), "1.31")
}
}

17
cmd/kube-apiserver/app/server.go
View File

@@ -36,6 +36,7 @@ import (
serverstorage "k8s.io/apiserver/pkg/server/storage"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/apiserver/pkg/util/notfoundhandler"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/apiserver/pkg/util/webhook"
clientgoinformers "k8s.io/client-go/informers"
"k8s.io/client-go/rest"
@@ -63,7 +64,10 @@ func init() {
// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
s := options.NewServerRunOptions()
effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
utilversion.ComponentGenericAPIServer, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
s := options.NewServerRunOptions(featureGate, effectiveVersion)
cmd := &cobra.Command{
Use: "kube-apiserver",
Long: `The Kubernetes API server validates and configures data
@@ -83,9 +87,13 @@ cluster's shared state through which all other components interact.`,
verflag.PrintAndExitIfRequested()
fs := cmd.Flags()
if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil {
return err
}
// Activate logging as soon as possible, after that
// show flags with the final logging configuration.
if err := logsapi.ValidateAndApply(s.Logs, utilfeature.DefaultFeatureGate); err != nil {
if err := logsapi.ValidateAndApply(s.Logs, featureGate); err != nil {
return err
}
cliflag.PrintFlags(fs)
@@ -101,7 +109,7 @@ cluster's shared state through which all other components interact.`,
return utilerrors.NewAggregate(errs)
}
// add feature enablement metrics
utilfeature.DefaultMutableFeatureGate.AddMetrics()
featureGate.AddMetrics()
return Run(cmd.Context(), completedOptions)
},
Args: func(cmd *cobra.Command, args []string) error {
@@ -118,6 +126,9 @@ cluster's shared state through which all other components interact.`,
fs := cmd.Flags()
namedFlagSets := s.Flags()
verflag.AddFlags(namedFlagSets.FlagSet("global"))
featureGate.AddFlag(namedFlagSets.FlagSet("global"), "")
effectiveVersion.AddFlags(namedFlagSets.FlagSet("global"), "")
globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags())
options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic"))
for _, f := range namedFlagSets.FlagSets {

21
cmd/kube-apiserver/app/testing/testserver.go
View File

@@ -48,6 +48,7 @@ import (
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/apiserver/pkg/storageversion"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
clientgotransport "k8s.io/client-go/transport"
@@ -98,6 +99,9 @@ type TestServerInstanceOptions struct {
// We specify this as on option to pass a common proxyCA to multiple apiservers to simulate
// an apiserver version skew scenario where all apiservers use the same proxyCA to verify client connections.
ProxyCA *ProxyCA
// Set the BinaryVersion of server effective version.
// Default to 1.31
BinaryVersion string
}
// TestServer return values supplied by kube-test-ApiServer
@@ -177,10 +181,21 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
fs := pflag.NewFlagSet("test", pflag.PanicOnError)
s := options.NewServerRunOptions()
featureGate := utilfeature.DefaultMutableFeatureGate
binaryVersion := utilversion.DefaultKubeEffectiveVersion().BinaryVersion().String()
if instanceOptions.BinaryVersion != "" {
binaryVersion = instanceOptions.BinaryVersion
}
effectiveVersion := utilversion.NewEffectiveVersion(binaryVersion)
_ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.ComponentGenericAPIServer, effectiveVersion, featureGate, true)
s := options.NewServerRunOptions(featureGate, effectiveVersion)
for _, f := range s.Flags().FlagSets {
fs.AddFlagSet(f)
}
featureGate.AddFlag(fs, "")
effectiveVersion.AddFlags(fs, "")
s.SecureServing.Listener, s.SecureServing.BindPort, err = createLocalhostListenerOnFreePort()
if err != nil {
@@ -321,6 +336,10 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
return result, err
}
if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil {
return result, err
}
saSigningKeyFile, err := os.CreateTemp("/tmp", "insecure_test_key")
if err != nil {
t.Fatalf("create temp file failed: %v", err)