MOVE: cmd/kube-apiserver/app/options: split apart controlplane part

cmd/kube-apiserver/app/options/completion.go

@@ -19,141 +19,78 @@ package options
 import (
 	"fmt"
 	"net"
-	"os"
 	"strings"
-	"time"
 
-	serveroptions "k8s.io/apiserver/pkg/server/options"
+	apiserveroptions "k8s.io/apiserver/pkg/server/options"
-	"k8s.io/client-go/util/keyutil"
 	_ "k8s.io/component-base/metrics/prometheus/workqueue"
-	"k8s.io/klog/v2"
 	netutils "k8s.io/utils/net"
 
-	"k8s.io/kubernetes/pkg/controlplane"
+	controlplane "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 	"k8s.io/kubernetes/pkg/kubeapiserver"
-	kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator"
+	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
-	"k8s.io/kubernetes/pkg/serviceaccount"
 )
 
 // completedOptions is a private wrapper that enforces a call of Complete() before Run can be invoked.
 type completedOptions struct {
-	*ServerRunOptions
+	controlplane.CompletedOptions
+	CloudProvider *kubeoptions.CloudProviderOptions
+
+	Extra
 }
 
 type CompletedOptions struct {
-	completedOptions
+	// Embed a private pointer that cannot be instantiated outside of this package.
+	*completedOptions
 }
 
 // Complete set default ServerRunOptions.
 // Should be called after kube-apiserver flags parsed.
 func Complete(opts *ServerRunOptions) (CompletedOptions, error) {
-	// set defaults
+	if opts == nil {
-	if err := opts.GenericServerRunOptions.DefaultAdvertiseAddress(opts.SecureServing.SecureServingOptions); err != nil {
+		return CompletedOptions{completedOptions: &completedOptions{}}, nil
-		return CompletedOptions{}, err
 	}
 
-	// process s.ServiceClusterIPRange from list to Primary and Secondary
+	// process opts.ServiceClusterIPRange from list to Primary and Secondary
 	// we process secondary only if provided by user
 	apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, err := getServiceIPAndRanges(opts.ServiceClusterIPRanges)
 	if err != nil {
 		return CompletedOptions{}, err
 	}
-	opts.PrimaryServiceClusterIPRange = primaryServiceIPRange
+	controlplane, err := opts.Options.Complete([]string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP})
-	opts.SecondaryServiceClusterIPRange = secondaryServiceIPRange
+	if err != nil {
-	opts.APIServerServiceIP = apiServerServiceIP
+		return CompletedOptions{}, err
-
-	if err := opts.SecureServing.MaybeDefaultWithSelfSignedCerts(opts.GenericServerRunOptions.AdvertiseAddress.String(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP}); err != nil {
-		return CompletedOptions{}, fmt.Errorf("error creating self-signed certificates: %v", err)
 	}
 
-	if len(opts.GenericServerRunOptions.ExternalHost) == 0 {
+	completed := completedOptions{
-		if len(opts.GenericServerRunOptions.AdvertiseAddress) > 0 {
+		CompletedOptions: controlplane,
-			opts.GenericServerRunOptions.ExternalHost = opts.GenericServerRunOptions.AdvertiseAddress.String()
+		CloudProvider:    opts.CloudProvider,
-		} else {
+
-			hostname, err := os.Hostname()
+		Extra: opts.Extra,
-			if err != nil {
-				return CompletedOptions{}, fmt.Errorf("error finding host name: %v", err)
-			}
-			opts.GenericServerRunOptions.ExternalHost = hostname
-		}
-		klog.Infof("external host was not specified, using %v", opts.GenericServerRunOptions.ExternalHost)
 	}
 
-	opts.Authentication.ApplyAuthorization(opts.Authorization)
+	completed.PrimaryServiceClusterIPRange = primaryServiceIPRange
+	completed.SecondaryServiceClusterIPRange = secondaryServiceIPRange
+	completed.APIServerServiceIP = apiServerServiceIP
 
-	// Use (ServiceAccountSigningKeyFile != "") as a proxy to the user enabling
+	if completed.Etcd != nil && completed.Etcd.EnableWatchCache {
-	// TokenRequest functionality. This defaulting was convenient, but messed up
-	// a lot of people when they rotated their serving cert with no idea it was
-	// connected to their service account keys. We are taking this opportunity to
-	// remove this problematic defaulting.
-	if opts.ServiceAccountSigningKeyFile == "" {
-		// Default to the private server key for service account token signing
-		if len(opts.Authentication.ServiceAccounts.KeyFiles) == 0 && opts.SecureServing.ServerCert.CertKey.KeyFile != "" {
-			if kubeauthenticator.IsValidServiceAccountKeyFile(opts.SecureServing.ServerCert.CertKey.KeyFile) {
-				opts.Authentication.ServiceAccounts.KeyFiles = []string{opts.SecureServing.ServerCert.CertKey.KeyFile}
-			} else {
-				klog.Warning("No TLS key provided, service account token authentication disabled")
-			}
-		}
-	}
-
-	if opts.ServiceAccountSigningKeyFile != "" && len(opts.Authentication.ServiceAccounts.Issuers) != 0 && opts.Authentication.ServiceAccounts.Issuers[0] != "" {
-		sk, err := keyutil.PrivateKeyFromFile(opts.ServiceAccountSigningKeyFile)
-		if err != nil {
-			return CompletedOptions{}, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err)
-		}
-		if opts.Authentication.ServiceAccounts.MaxExpiration != 0 {
-			lowBound := time.Hour
-			upBound := time.Duration(1<<32) * time.Second
-			if opts.Authentication.ServiceAccounts.MaxExpiration < lowBound ||
-				opts.Authentication.ServiceAccounts.MaxExpiration > upBound {
-				return CompletedOptions{}, fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds")
-			}
-			if opts.Authentication.ServiceAccounts.ExtendExpiration {
-				if opts.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.WarnOnlyBoundTokenExpirationSeconds*time.Second {
-					klog.Warningf("service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, service-account-max-token-expiration must be set longer than %d seconds (currently %s)", serviceaccount.WarnOnlyBoundTokenExpirationSeconds, opts.Authentication.ServiceAccounts.MaxExpiration)
-				}
-				if opts.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.ExpirationExtensionSeconds*time.Second {
-					klog.Warningf("service-account-extend-token-expiration is true, enabling tokens valid up to %d seconds, which is longer than service-account-max-token-expiration set to %s seconds", serviceaccount.ExpirationExtensionSeconds, opts.Authentication.ServiceAccounts.MaxExpiration)
-				}
-			}
-		}
-
-		opts.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(opts.Authentication.ServiceAccounts.Issuers[0], sk)
-		if err != nil {
-			return CompletedOptions{}, fmt.Errorf("failed to build token generator: %v", err)
-		}
-		opts.ServiceAccountTokenMaxExpiration = opts.Authentication.ServiceAccounts.MaxExpiration
-	}
-
-	if opts.Etcd.EnableWatchCache {
 		sizes := kubeapiserver.DefaultWatchCacheSizes()
 		// Ensure that overrides parse correctly.
-		userSpecified, err := serveroptions.ParseWatchCacheSizes(opts.Etcd.WatchCacheSizes)
+		userSpecified, err := apiserveroptions.ParseWatchCacheSizes(completed.Etcd.WatchCacheSizes)
 		if err != nil {
 			return CompletedOptions{}, err
 		}
 		for resource, size := range userSpecified {
 			sizes[resource] = size
 		}
-		opts.Etcd.WatchCacheSizes, err = serveroptions.WriteWatchCacheSizes(sizes)
+		completed.Etcd.WatchCacheSizes, err = apiserveroptions.WriteWatchCacheSizes(sizes)
 		if err != nil {
 			return CompletedOptions{}, err
 		}
 	}
 
-	for key, value := range opts.APIEnablement.RuntimeConfig {
+	return CompletedOptions{
-		if key == "v1" || strings.HasPrefix(key, "v1/") ||
+		completedOptions: &completed,
-			key == "api/v1" || strings.HasPrefix(key, "api/v1/") {
+	}, nil
-			delete(opts.APIEnablement.RuntimeConfig, key)
-			opts.APIEnablement.RuntimeConfig["/v1"] = value
-		}
-		if key == "api/legacy" {
-			delete(opts.APIEnablement.RuntimeConfig, key)
-		}
-	}
-
-	return CompletedOptions{completedOptions: completedOptions{ServerRunOptions: opts}}, nil
 }
 
 func getServiceIPAndRanges(serviceClusterIPRanges string) (net.IP, net.IPNet, net.IPNet, error) {

cmd/kube-apiserver/app/options/options.go

@@ -23,45 +23,29 @@ import (
 	"time"
 
 	utilnet "k8s.io/apimachinery/pkg/util/net"
-	genericoptions "k8s.io/apiserver/pkg/server/options"
-	"k8s.io/apiserver/pkg/storage/storagebackend"
 	cliflag "k8s.io/component-base/cli/flag"
-	"k8s.io/component-base/logs"
-	"k8s.io/component-base/metrics"
 
-	logsapi "k8s.io/component-base/logs/api/v1"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/cluster/ports"
+	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 	"k8s.io/kubernetes/pkg/controlplane/reconcilers"
 	_ "k8s.io/kubernetes/pkg/features" // add the kubernetes feature gates
 	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
 	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
-	"k8s.io/kubernetes/pkg/serviceaccount"
 )
 
 // ServerRunOptions runs a kubernetes api server.
 type ServerRunOptions struct {
-	GenericServerRunOptions *genericoptions.ServerRunOptions
+	*controlplaneapiserver.Options // embedded to avoid noise in existing consumers
-	Etcd                    *genericoptions.EtcdOptions
+	CloudProvider                  *kubeoptions.CloudProviderOptions
-	SecureServing           *genericoptions.SecureServingOptionsWithLoopback
-	Audit                   *genericoptions.AuditOptions
-	Features                *genericoptions.FeatureOptions
-	Admission               *kubeoptions.AdmissionOptions
-	Authentication          *kubeoptions.BuiltInAuthenticationOptions
-	Authorization           *kubeoptions.BuiltInAuthorizationOptions
-	CloudProvider           *kubeoptions.CloudProviderOptions
-	APIEnablement           *genericoptions.APIEnablementOptions
-	EgressSelector          *genericoptions.EgressSelectorOptions
-	Metrics                 *metrics.Options
-	Logs                    *logs.Options
-	Traces                  *genericoptions.TracingOptions
 
+	Extra
+}
+
+type Extra struct {
 	AllowPrivileged           bool
-	EnableLogsHandler         bool
-	EventTTL                  time.Duration
 	KubeletConfig             kubeletclient.KubeletClientConfig
 	KubernetesServiceNodePort int
-	MaxConnectionBytesPerSec  int64
 	// ServiceClusterIPRange is mapped to input provided by user
 	ServiceClusterIPRanges string
 	// PrimaryServiceClusterIPRange and SecondaryServiceClusterIPRange are the results
@@ -73,110 +57,53 @@ type ServerRunOptions struct {
 
 	ServiceNodePortRange utilnet.PortRange
 
-	ProxyClientCertFile string
-	ProxyClientKeyFile  string
-
-	EnableAggregatorRouting             bool
-	AggregatorRejectForwardingRedirects bool
-
-	MasterCount            int
 	EndpointReconcilerType string
-
-	ServiceAccountSigningKeyFile     string
-	ServiceAccountIssuer             serviceaccount.TokenGenerator
-	ServiceAccountTokenMaxExpiration time.Duration
-
-	ShowHiddenMetricsForVersion string
 }
 
 // NewServerRunOptions creates a new ServerRunOptions object with default parameters
 func NewServerRunOptions() *ServerRunOptions {
 	s := ServerRunOptions{
-		GenericServerRunOptions: genericoptions.NewServerRunOptions(),
+		Options:       controlplaneapiserver.NewOptions(),
-		Etcd:                    genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)),
+		CloudProvider: kubeoptions.NewCloudProviderOptions(),
-		SecureServing:           kubeoptions.NewSecureServingOptions(),
-		Audit:                   genericoptions.NewAuditOptions(),
-		Features:                genericoptions.NewFeatureOptions(),
-		Admission:               kubeoptions.NewAdmissionOptions(),
-		Authentication:          kubeoptions.NewBuiltInAuthenticationOptions().WithAll(),
-		Authorization:           kubeoptions.NewBuiltInAuthorizationOptions(),
-		CloudProvider:           kubeoptions.NewCloudProviderOptions(),
-		APIEnablement:           genericoptions.NewAPIEnablementOptions(),
-		EgressSelector:          genericoptions.NewEgressSelectorOptions(),
-		Metrics:                 metrics.NewOptions(),
-		Logs:                    logs.NewOptions(),
-		Traces:                  genericoptions.NewTracingOptions(),
 
-		EnableLogsHandler:      true,
+		Extra: Extra{
-		EventTTL:               1 * time.Hour,
+			EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
-		MasterCount:            1,
+			KubeletConfig: kubeletclient.KubeletClientConfig{
-		EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
+				Port:         ports.KubeletPort,
-		KubeletConfig: kubeletclient.KubeletClientConfig{
+				ReadOnlyPort: ports.KubeletReadOnlyPort,
-			Port:         ports.KubeletPort,
+				PreferredAddressTypes: []string{
-			ReadOnlyPort: ports.KubeletReadOnlyPort,
+					// --override-hostname
-			PreferredAddressTypes: []string{
+					string(api.NodeHostName),
-				// --override-hostname
-				string(api.NodeHostName),
 
-				// internal, preferring DNS if reported
+					// internal, preferring DNS if reported
-				string(api.NodeInternalDNS),
+					string(api.NodeInternalDNS),
-				string(api.NodeInternalIP),
+					string(api.NodeInternalIP),
 
-				// external, preferring DNS if reported
+					// external, preferring DNS if reported
-				string(api.NodeExternalDNS),
+					string(api.NodeExternalDNS),
-				string(api.NodeExternalIP),
+					string(api.NodeExternalIP),
+				},
+				HTTPTimeout: time.Duration(5) * time.Second,
 			},
-			HTTPTimeout: time.Duration(5) * time.Second,
+			ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange,
 		},
-		ServiceNodePortRange:                kubeoptions.DefaultServiceNodePortRange,
-		AggregatorRejectForwardingRedirects: true,
 	}
 
-	// Overwrite the default for storage data format.
-	s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf"
-
 	return &s
 }
 
 // Flags returns flags for a specific APIServer by section name
 func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
-	// Add the generic flags.
+	s.Options.AddFlags(&fss)
-	s.GenericServerRunOptions.AddUniversalFlags(fss.FlagSet("generic"))
-	s.Etcd.AddFlags(fss.FlagSet("etcd"))
-	s.SecureServing.AddFlags(fss.FlagSet("secure serving"))
-	s.Audit.AddFlags(fss.FlagSet("auditing"))
-	s.Features.AddFlags(fss.FlagSet("features"))
-	s.Authentication.AddFlags(fss.FlagSet("authentication"))
-	s.Authorization.AddFlags(fss.FlagSet("authorization"))
 	s.CloudProvider.AddFlags(fss.FlagSet("cloud provider"))
-	s.APIEnablement.AddFlags(fss.FlagSet("API enablement"))
-	s.EgressSelector.AddFlags(fss.FlagSet("egress selector"))
-	s.Admission.AddFlags(fss.FlagSet("admission"))
-	s.Metrics.AddFlags(fss.FlagSet("metrics"))
-	logsapi.AddFlags(s.Logs, fss.FlagSet("logs"))
-	s.Traces.AddFlags(fss.FlagSet("traces"))
 
 	// Note: the weird ""+ in below lines seems to be the only way to get gofmt to
 	// arrange these text blocks sensibly. Grrr.
 	fs := fss.FlagSet("misc")
-	fs.DurationVar(&s.EventTTL, "event-ttl", s.EventTTL,
-		"Amount of time to retain events.")
 
 	fs.BoolVar(&s.AllowPrivileged, "allow-privileged", s.AllowPrivileged,
 		"If true, allow privileged containers. [default=false]")
 
-	fs.BoolVar(&s.EnableLogsHandler, "enable-logs-handler", s.EnableLogsHandler,
-		"If true, install a /logs handler for the apiserver logs.")
-	fs.MarkDeprecated("enable-logs-handler", "This flag will be removed in v1.19")
-
-	fs.Int64Var(&s.MaxConnectionBytesPerSec, "max-connection-bytes-per-sec", s.MaxConnectionBytesPerSec, ""+
-		"If non-zero, throttle each user connection to this number of bytes/sec. "+
-		"Currently only applies to long-running requests.")
-
-	fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount,
-		"The number of apiservers running in the cluster, must be a positive number. (In use when --endpoint-reconciler-type=master-count is enabled.)")
-	fs.MarkDeprecated("apiserver-count", "apiserver-count is deprecated and will be removed in a future version.")
-
 	fs.StringVar(&s.EndpointReconcilerType, "endpoint-reconciler-type", s.EndpointReconcilerType,
 		"Use an endpoint reconciler ("+strings.Join(reconcilers.AllTypes.Names(), ", ")+") master-count is deprecated, and will be removed in a future version.")
 
@@ -219,27 +146,5 @@ func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
 	fs.StringVar(&s.KubeletConfig.TLSClientConfig.CAFile, "kubelet-certificate-authority", s.KubeletConfig.TLSClientConfig.CAFile,
 		"Path to a cert file for the certificate authority.")
 
-	fs.StringVar(&s.ProxyClientCertFile, "proxy-client-cert-file", s.ProxyClientCertFile, ""+
-		"Client certificate used to prove the identity of the aggregator or kube-apiserver "+
-		"when it must call out during a request. This includes proxying requests to a user "+
-		"api-server and calling out to webhook admission plugins. It is expected that this "+
-		"cert includes a signature from the CA in the --requestheader-client-ca-file flag. "+
-		"That CA is published in the 'extension-apiserver-authentication' configmap in "+
-		"the kube-system namespace. Components receiving calls from kube-aggregator should "+
-		"use that CA to perform their half of the mutual TLS verification.")
-	fs.StringVar(&s.ProxyClientKeyFile, "proxy-client-key-file", s.ProxyClientKeyFile, ""+
-		"Private key for the client certificate used to prove the identity of the aggregator or kube-apiserver "+
-		"when it must call out during a request. This includes proxying requests to a user "+
-		"api-server and calling out to webhook admission plugins.")
-
-	fs.BoolVar(&s.EnableAggregatorRouting, "enable-aggregator-routing", s.EnableAggregatorRouting,
-		"Turns on aggregator routing requests to endpoints IP rather than cluster IP.")
-
-	fs.BoolVar(&s.AggregatorRejectForwardingRedirects, "aggregator-reject-forwarding-redirect", s.AggregatorRejectForwardingRedirects,
-		"Aggregator reject forwarding redirect response back to client.")
-
-	fs.StringVar(&s.ServiceAccountSigningKeyFile, "service-account-signing-key-file", s.ServiceAccountSigningKeyFile, ""+
-		"Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key.")
-
 	return fss
 }

cmd/kube-apiserver/app/options/options_test.go

@@ -37,6 +37,7 @@ import (
 	"k8s.io/component-base/logs"
 	"k8s.io/component-base/metrics"
 	kapi "k8s.io/kubernetes/pkg/apis/core"
+	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 	"k8s.io/kubernetes/pkg/controlplane/reconcilers"
 	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
 	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
@@ -125,200 +126,205 @@ func TestAddFlags(t *testing.T) {
 
 	// This is a snapshot of expected options parsed by args.
 	expected := &ServerRunOptions{
-		ServiceNodePortRange:   kubeoptions.DefaultServiceNodePortRange,
+		Options: &controlplaneapiserver.Options{
-		ServiceClusterIPRanges: (&net.IPNet{IP: netutils.ParseIPSloppy("192.168.128.0"), Mask: net.CIDRMask(17, 32)}).String(),
+			MasterCount: 5,
-		MasterCount:            5,
+			GenericServerRunOptions: &apiserveroptions.ServerRunOptions{
-		EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
+				AdvertiseAddress:            netutils.ParseIPSloppy("192.168.10.10"),
-		AllowPrivileged:        false,
+				CorsAllowedOriginList:       []string{"10.10.10.100", "10.10.10.200"},
-		GenericServerRunOptions: &apiserveroptions.ServerRunOptions{
+				MaxRequestsInFlight:         400,
-			AdvertiseAddress:            netutils.ParseIPSloppy("192.168.10.10"),
+				MaxMutatingRequestsInFlight: 200,
-			CorsAllowedOriginList:       []string{"10.10.10.100", "10.10.10.200"},
+				RequestTimeout:              time.Duration(2) * time.Minute,
-			MaxRequestsInFlight:         400,
+				MinRequestTimeout:           1800,
-			MaxMutatingRequestsInFlight: 200,
+				JSONPatchMaxCopyBytes:       int64(3 * 1024 * 1024),
-			RequestTimeout:              time.Duration(2) * time.Minute,
+				MaxRequestBodyBytes:         int64(3 * 1024 * 1024),
-			MinRequestTimeout:           1800,
-			JSONPatchMaxCopyBytes:       int64(3 * 1024 * 1024),
-			MaxRequestBodyBytes:         int64(3 * 1024 * 1024),
-		},
-		Admission: &kubeoptions.AdmissionOptions{
-			GenericAdmission: &apiserveroptions.AdmissionOptions{
-				RecommendedPluginOrder: s.Admission.GenericAdmission.RecommendedPluginOrder,
-				DefaultOffPlugins:      s.Admission.GenericAdmission.DefaultOffPlugins,
-				EnablePlugins:          []string{"AlwaysDeny"},
-				ConfigFile:             "/admission-control-config",
-				Plugins:                s.Admission.GenericAdmission.Plugins,
-				Decorators:             s.Admission.GenericAdmission.Decorators,
 			},
-		},
+			Admission: &kubeoptions.AdmissionOptions{
-		Etcd: &apiserveroptions.EtcdOptions{
+				GenericAdmission: &apiserveroptions.AdmissionOptions{
-			StorageConfig: storagebackend.Config{
+					RecommendedPluginOrder: s.Options.Admission.GenericAdmission.RecommendedPluginOrder,
-				Type: "etcd3",
+					DefaultOffPlugins:      s.Options.Admission.GenericAdmission.DefaultOffPlugins,
-				Transport: storagebackend.TransportConfig{
+					EnablePlugins:          []string{"AlwaysDeny"},
-					ServerList:     nil,
+					ConfigFile:             "/admission-control-config",
-					KeyFile:        "/var/run/kubernetes/etcd.key",
+					Plugins:                s.Options.Admission.GenericAdmission.Plugins,
-					TrustedCAFile:  "/var/run/kubernetes/etcdca.crt",
+					Decorators:             s.Options.Admission.GenericAdmission.Decorators,
-					CertFile:       "/var/run/kubernetes/etcdce.crt",
-					TracerProvider: oteltrace.NewNoopTracerProvider(),
-				},
-				Paging:                true,
-				Prefix:                "/registry",
-				CompactionInterval:    storagebackend.DefaultCompactInterval,
-				CountMetricPollPeriod: time.Minute,
-				DBMetricPollInterval:  storagebackend.DefaultDBMetricPollInterval,
-				HealthcheckTimeout:    storagebackend.DefaultHealthcheckTimeout,
-				ReadycheckTimeout:     storagebackend.DefaultReadinessTimeout,
-				LeaseManagerConfig: etcd3.LeaseManagerConfig{
-					ReuseDurationSeconds: 100,
-					MaxObjectCount:       1000,
 				},
 			},
-			DefaultStorageMediaType: "application/vnd.kubernetes.protobuf",
+			Etcd: &apiserveroptions.EtcdOptions{
-			DeleteCollectionWorkers: 1,
+				StorageConfig: storagebackend.Config{
-			EnableGarbageCollection: true,
+					Type: "etcd3",
-			EnableWatchCache:        true,
+					Transport: storagebackend.TransportConfig{
-			DefaultWatchCacheSize:   100,
+						ServerList:     nil,
-		},
+						KeyFile:        "/var/run/kubernetes/etcd.key",
-		SecureServing: (&apiserveroptions.SecureServingOptions{
+						TrustedCAFile:  "/var/run/kubernetes/etcdca.crt",
-			BindAddress: netutils.ParseIPSloppy("192.168.10.20"),
+						CertFile:       "/var/run/kubernetes/etcdce.crt",
-			BindPort:    6443,
+						TracerProvider: oteltrace.NewNoopTracerProvider(),
-			ServerCert: apiserveroptions.GeneratableKeyCert{
+					},
-				CertDirectory: "/var/run/kubernetes",
+					Paging:                true,
-				PairName:      "apiserver",
+					Prefix:                "/registry",
-			},
+					CompactionInterval:    storagebackend.DefaultCompactInterval,
-			HTTP2MaxStreamsPerConnection: 42,
+					CountMetricPollPeriod: time.Minute,
-			Required:                     true,
+					DBMetricPollInterval:  storagebackend.DefaultDBMetricPollInterval,
-		}).WithLoopback(),
+					HealthcheckTimeout:    storagebackend.DefaultHealthcheckTimeout,
-		EventTTL: 1 * time.Hour,
+					ReadycheckTimeout:     storagebackend.DefaultReadinessTimeout,
-		KubeletConfig: kubeletclient.KubeletClientConfig{
+					LeaseManagerConfig: etcd3.LeaseManagerConfig{
-			Port:         10250,
+						ReuseDurationSeconds: 100,
-			ReadOnlyPort: 10255,
+						MaxObjectCount:       1000,
-			PreferredAddressTypes: []string{
-				string(kapi.NodeHostName),
-				string(kapi.NodeInternalDNS),
-				string(kapi.NodeInternalIP),
-				string(kapi.NodeExternalDNS),
-				string(kapi.NodeExternalIP),
-			},
-			HTTPTimeout: time.Duration(5) * time.Second,
-			TLSClientConfig: kubeletclient.KubeletTLSConfig{
-				CertFile: "/var/run/kubernetes/ceserver.crt",
-				KeyFile:  "/var/run/kubernetes/server.key",
-				CAFile:   "/var/run/kubernetes/caserver.crt",
-			},
-		},
-		Audit: &apiserveroptions.AuditOptions{
-			LogOptions: apiserveroptions.AuditLogOptions{
-				Path:       "/var/log",
-				MaxAge:     11,
-				MaxBackups: 12,
-				MaxSize:    13,
-				Format:     "json",
-				BatchOptions: apiserveroptions.AuditBatchOptions{
-					Mode: "blocking",
-					BatchConfig: auditbuffered.BatchConfig{
-						BufferSize:     46,
-						MaxBatchSize:   47,
-						MaxBatchWait:   48 * time.Second,
-						ThrottleEnable: true,
-						ThrottleQPS:    49.5,
-						ThrottleBurst:  50,
 					},
 				},
-				TruncateOptions: apiserveroptions.AuditTruncateOptions{
+				DefaultStorageMediaType: "application/vnd.kubernetes.protobuf",
-					Enabled: true,
+				DeleteCollectionWorkers: 1,
-					TruncateConfig: audittruncate.Config{
+				EnableGarbageCollection: true,
-						MaxBatchSize: 45,
+				EnableWatchCache:        true,
-						MaxEventSize: 44,
+				DefaultWatchCacheSize:   100,
-					},
+			},
+			SecureServing: (&apiserveroptions.SecureServingOptions{
+				BindAddress: netutils.ParseIPSloppy("192.168.10.20"),
+				BindPort:    6443,
+				ServerCert: apiserveroptions.GeneratableKeyCert{
+					CertDirectory: "/var/run/kubernetes",
+					PairName:      "apiserver",
 				},
-				GroupVersionString: "audit.k8s.io/v1",
+				HTTP2MaxStreamsPerConnection: 42,
-			},
+				Required:                     true,
-			WebhookOptions: apiserveroptions.AuditWebhookOptions{
+			}).WithLoopback(),
-				ConfigFile: "/webhook-config",
+			EventTTL: 1 * time.Hour,
-				BatchOptions: apiserveroptions.AuditBatchOptions{
+			Audit: &apiserveroptions.AuditOptions{
-					Mode: "blocking",
+				LogOptions: apiserveroptions.AuditLogOptions{
-					BatchConfig: auditbuffered.BatchConfig{
+					Path:       "/var/log",
-						BufferSize:     42,
+					MaxAge:     11,
-						MaxBatchSize:   43,
+					MaxBackups: 12,
-						MaxBatchWait:   1 * time.Second,
+					MaxSize:    13,
-						ThrottleEnable: false,
+					Format:     "json",
-						ThrottleQPS:    43.5,
+					BatchOptions: apiserveroptions.AuditBatchOptions{
-						ThrottleBurst:  44,
+						Mode: "blocking",
-						AsyncDelegate:  true,
+						BatchConfig: auditbuffered.BatchConfig{
+							BufferSize:     46,
+							MaxBatchSize:   47,
+							MaxBatchWait:   48 * time.Second,
+							ThrottleEnable: true,
+							ThrottleQPS:    49.5,
+							ThrottleBurst:  50,
+						},
 					},
-				},
+					TruncateOptions: apiserveroptions.AuditTruncateOptions{
-				TruncateOptions: apiserveroptions.AuditTruncateOptions{
+						Enabled: true,
-					Enabled: true,
+						TruncateConfig: audittruncate.Config{
-					TruncateConfig: audittruncate.Config{
+							MaxBatchSize: 45,
-						MaxBatchSize: 43,
+							MaxEventSize: 44,
-						MaxEventSize: 42,
+						},
 					},
+					GroupVersionString: "audit.k8s.io/v1",
 				},
-				InitialBackoff:     2 * time.Second,
+				WebhookOptions: apiserveroptions.AuditWebhookOptions{
-				GroupVersionString: "audit.k8s.io/v1",
+					ConfigFile: "/webhook-config",
+					BatchOptions: apiserveroptions.AuditBatchOptions{
+						Mode: "blocking",
+						BatchConfig: auditbuffered.BatchConfig{
+							BufferSize:     42,
+							MaxBatchSize:   43,
+							MaxBatchWait:   1 * time.Second,
+							ThrottleEnable: false,
+							ThrottleQPS:    43.5,
+							ThrottleBurst:  44,
+							AsyncDelegate:  true,
+						},
+					},
+					TruncateOptions: apiserveroptions.AuditTruncateOptions{
+						Enabled: true,
+						TruncateConfig: audittruncate.Config{
+							MaxBatchSize: 43,
+							MaxEventSize: 42,
+						},
+					},
+					InitialBackoff:     2 * time.Second,
+					GroupVersionString: "audit.k8s.io/v1",
+				},
+				PolicyFile: "/policy",
 			},
-			PolicyFile: "/policy",
+			Features: &apiserveroptions.FeatureOptions{
+				EnableProfiling:           true,
+				EnableContentionProfiling: true,
+			},
+			Authentication: &kubeoptions.BuiltInAuthenticationOptions{
+				Anonymous: &kubeoptions.AnonymousAuthenticationOptions{
+					Allow: false,
+				},
+				ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{
+					ClientCA: "/client-ca",
+				},
+				WebHook: &kubeoptions.WebHookAuthenticationOptions{
+					CacheTTL:     180000000000,
+					ConfigFile:   "/token-webhook-config",
+					Version:      "v1beta1",
+					RetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(),
+				},
+				BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{},
+				OIDC: &kubeoptions.OIDCAuthenticationOptions{
+					UsernameClaim: "sub",
+					SigningAlgs:   []string{"RS256"},
+				},
+				RequestHeader: &apiserveroptions.RequestHeaderAuthenticationOptions{},
+				ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{
+					Lookup:           true,
+					ExtendExpiration: true,
+				},
+				TokenFile:            &kubeoptions.TokenFileAuthenticationOptions{},
+				TokenSuccessCacheTTL: 10 * time.Second,
+				TokenFailureCacheTTL: 0,
+			},
+			Authorization: &kubeoptions.BuiltInAuthorizationOptions{
+				Modes:                       []string{"AlwaysDeny", "RBAC"},
+				PolicyFile:                  "/policy",
+				WebhookConfigFile:           "/webhook-config",
+				WebhookCacheAuthorizedTTL:   180000000000,
+				WebhookCacheUnauthorizedTTL: 60000000000,
+				WebhookVersion:              "v1beta1",
+				WebhookRetryBackoff:         apiserveroptions.DefaultAuthWebhookRetryBackoff(),
+			},
+			APIEnablement: &apiserveroptions.APIEnablementOptions{
+				RuntimeConfig: cliflag.ConfigurationMap{},
+			},
+			EgressSelector: &apiserveroptions.EgressSelectorOptions{
+				ConfigFile: "/var/run/kubernetes/egress-selector/connectivity.yaml",
+			},
+			EnableLogsHandler:       false,
+			EnableAggregatorRouting: true,
+			ProxyClientKeyFile:      "/var/run/kubernetes/proxy.key",
+			ProxyClientCertFile:     "/var/run/kubernetes/proxy.crt",
+			Metrics:                 &metrics.Options{},
+			Logs:                    logs.NewOptions(),
+			Traces: &apiserveroptions.TracingOptions{
+				ConfigFile: "/var/run/kubernetes/tracing_config.yaml",
+			},
+			AggregatorRejectForwardingRedirects: true,
 		},
-		Features: &apiserveroptions.FeatureOptions{
+
-			EnableProfiling:           true,
+		Extra: Extra{
-			EnableContentionProfiling: true,
+			ServiceNodePortRange:   kubeoptions.DefaultServiceNodePortRange,
-		},
+			ServiceClusterIPRanges: (&net.IPNet{IP: netutils.ParseIPSloppy("192.168.128.0"), Mask: net.CIDRMask(17, 32)}).String(),
-		Authentication: &kubeoptions.BuiltInAuthenticationOptions{
+			EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
-			Anonymous: &kubeoptions.AnonymousAuthenticationOptions{
+			AllowPrivileged:        false,
-				Allow: false,
+			KubeletConfig: kubeletclient.KubeletClientConfig{
+				Port:         10250,
+				ReadOnlyPort: 10255,
+				PreferredAddressTypes: []string{
+					string(kapi.NodeHostName),
+					string(kapi.NodeInternalDNS),
+					string(kapi.NodeInternalIP),
+					string(kapi.NodeExternalDNS),
+					string(kapi.NodeExternalIP),
+				},
+				HTTPTimeout: time.Duration(5) * time.Second,
+				TLSClientConfig: kubeletclient.KubeletTLSConfig{
+					CertFile: "/var/run/kubernetes/ceserver.crt",
+					KeyFile:  "/var/run/kubernetes/server.key",
+					CAFile:   "/var/run/kubernetes/caserver.crt",
+				},
 			},
-			ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{
-				ClientCA: "/client-ca",
-			},
-			WebHook: &kubeoptions.WebHookAuthenticationOptions{
-				CacheTTL:     180000000000,
-				ConfigFile:   "/token-webhook-config",
-				Version:      "v1beta1",
-				RetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(),
-			},
-			BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{},
-			OIDC: &kubeoptions.OIDCAuthenticationOptions{
-				UsernameClaim: "sub",
-				SigningAlgs:   []string{"RS256"},
-			},
-			RequestHeader: &apiserveroptions.RequestHeaderAuthenticationOptions{},
-			ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{
-				Lookup:           true,
-				ExtendExpiration: true,
-			},
-			TokenFile:            &kubeoptions.TokenFileAuthenticationOptions{},
-			TokenSuccessCacheTTL: 10 * time.Second,
-			TokenFailureCacheTTL: 0,
-		},
-		Authorization: &kubeoptions.BuiltInAuthorizationOptions{
-			Modes:                       []string{"AlwaysDeny", "RBAC"},
-			PolicyFile:                  "/policy",
-			WebhookConfigFile:           "/webhook-config",
-			WebhookCacheAuthorizedTTL:   180000000000,
-			WebhookCacheUnauthorizedTTL: 60000000000,
-			WebhookVersion:              "v1beta1",
-			WebhookRetryBackoff:         apiserveroptions.DefaultAuthWebhookRetryBackoff(),
 		},
 		CloudProvider: &kubeoptions.CloudProviderOptions{
 			CloudConfigFile: "/cloud-config",
 			CloudProvider:   "azure",
 		},
-		APIEnablement: &apiserveroptions.APIEnablementOptions{
-			RuntimeConfig: cliflag.ConfigurationMap{},
-		},
-		EgressSelector: &apiserveroptions.EgressSelectorOptions{
-			ConfigFile: "/var/run/kubernetes/egress-selector/connectivity.yaml",
-		},
-		EnableLogsHandler:       false,
-		EnableAggregatorRouting: true,
-		ProxyClientKeyFile:      "/var/run/kubernetes/proxy.key",
-		ProxyClientCertFile:     "/var/run/kubernetes/proxy.crt",
-		Metrics:                 &metrics.Options{},
-		Logs:                    logs.NewOptions(),
-		Traces: &apiserveroptions.TracingOptions{
-			ConfigFile: "/var/run/kubernetes/tracing_config.yaml",
-		},
-		AggregatorRejectForwardingRedirects: true,
 	}
 
 	if !reflect.DeepEqual(expected, s) {

cmd/kube-apiserver/app/options/validation.go

@@ -22,11 +22,7 @@ import (
 	"net"
 	"strings"
 
-	apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
-	genericfeatures "k8s.io/apiserver/pkg/features"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
-	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/features"
 	netutils "k8s.io/utils/net"
 )
@@ -106,64 +102,13 @@ func validateServiceNodePort(options *ServerRunOptions) []error {
 	return errs
 }
 
-func validateTokenRequest(options *ServerRunOptions) []error {
-	var errs []error
-
-	enableAttempted := options.ServiceAccountSigningKeyFile != "" ||
-		(len(options.Authentication.ServiceAccounts.Issuers) != 0 && options.Authentication.ServiceAccounts.Issuers[0] != "") ||
-		len(options.Authentication.APIAudiences) != 0
-
-	enableSucceeded := options.ServiceAccountIssuer != nil
-
-	if !enableAttempted {
-		errs = append(errs, errors.New("--service-account-signing-key-file and --service-account-issuer are required flags"))
-	}
-
-	if enableAttempted && !enableSucceeded {
-		errs = append(errs, errors.New("--service-account-signing-key-file, --service-account-issuer, and --api-audiences should be specified together"))
-	}
-
-	return errs
-}
-
-func validateAPIPriorityAndFairness(options *ServerRunOptions) []error {
-	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) && options.GenericServerRunOptions.EnablePriorityAndFairness {
-		// If none of the following runtime config options are specified,
-		// APF is assumed to be turned on. The internal APF controller uses
-		// v1beta3 so it should be enabled.
-		enabledAPIString := options.APIEnablement.RuntimeConfig.String()
-		testConfigs := []string{"flowcontrol.apiserver.k8s.io/v1beta3", "api/beta", "api/all"} // in the order of precedence
-		for _, testConfig := range testConfigs {
-			if strings.Contains(enabledAPIString, fmt.Sprintf("%s=false", testConfig)) {
-				return []error{fmt.Errorf("--runtime-config=%s=false conflicts with --enable-priority-and-fairness=true and --feature-gates=APIPriorityAndFairness=true", testConfig)}
-			}
-			if strings.Contains(enabledAPIString, fmt.Sprintf("%s=true", testConfig)) {
-				return nil
-			}
-		}
-	}
-
-	return nil
-}
-
 // Validate checks ServerRunOptions and return a slice of found errs.
 func (s *ServerRunOptions) Validate() []error {
 	var errs []error
-	if s.MasterCount <= 0 {
+
-		errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount))
+	errs = append(errs, s.Options.Validate()...)
-	}
-	errs = append(errs, s.Etcd.Validate()...)
 	errs = append(errs, validateClusterIPFlags(s)...)
 	errs = append(errs, validateServiceNodePort(s)...)
-	errs = append(errs, validateAPIPriorityAndFairness(s)...)
-	errs = append(errs, s.SecureServing.Validate()...)
-	errs = append(errs, s.Authentication.Validate()...)
-	errs = append(errs, s.Authorization.Validate()...)
-	errs = append(errs, s.Audit.Validate()...)
-	errs = append(errs, s.Admission.Validate()...)
-	errs = append(errs, s.APIEnablement.Validate(legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme)...)
-	errs = append(errs, validateTokenRequest(s)...)
-	errs = append(errs, s.Metrics.Validate()...)
 
 	return errs
 }

cmd/kube-apiserver/app/options/validation_test.go

@@ -18,17 +18,12 @@ package options
 
 import (
 	"net"
-	"strings"
 	"testing"
 
 	utilnet "k8s.io/apimachinery/pkg/util/net"
-	kubeapiserveradmission "k8s.io/apiserver/pkg/admission"
-	genericoptions "k8s.io/apiserver/pkg/server/options"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
-	basemetrics "k8s.io/component-base/metrics"
 	"k8s.io/kubernetes/pkg/features"
-	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
 	netutils "k8s.io/utils/net"
 )
 
@@ -53,9 +48,11 @@ func makeOptionsWithCIDRs(serviceCIDR string, secondaryServiceCIDR string) *Serv
 		}
 	}
 	return &ServerRunOptions{
-		ServiceClusterIPRanges:         value,
+		Extra: Extra{
-		PrimaryServiceClusterIPRange:   primaryCIDR,
+			ServiceClusterIPRanges:         value,
-		SecondaryServiceClusterIPRange: secondaryCIDR,
+			PrimaryServiceClusterIPRange:   primaryCIDR,
+			SecondaryServiceClusterIPRange: secondaryCIDR,
+		},
 	}
 }
 
@@ -203,9 +200,9 @@ func TestValidateServiceNodePort(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			err := validateServiceNodePort(tc.options)
+			errs := validateServiceNodePort(tc.options)
-			if err != nil && !tc.expectErrors {
+			if errs != nil && !tc.expectErrors {
-				t.Errorf("expected no errors, error found %+v", err)
+				t.Errorf("expected no errors, error found %+v", errs)
 			}
 		})
 	}
@@ -217,8 +214,10 @@ func makeOptionsWithPort(kubernetesServiceNodePort int, base int, size int) *Ser
 		Size: size,
 	}
 	return &ServerRunOptions{
-		ServiceNodePortRange:      portRange,
+		Extra: Extra{
-		KubernetesServiceNodePort: kubernetesServiceNodePort,
+			ServiceNodePortRange:      portRange,
+			KubernetesServiceNodePort: kubernetesServiceNodePort,
+		},
 	}
 }
 
@@ -283,144 +282,3 @@ func TestValidateMaxCIDRRange(t *testing.T) {
 		})
 	}
 }
-
-func TestValidateAPIPriorityAndFairness(t *testing.T) {
-	const conflict = "conflicts with --enable-priority-and-fairness=true and --feature-gates=APIPriorityAndFairness=true"
-	tests := []struct {
-		runtimeConfig    string
-		errShouldContain string
-	}{
-		{
-			runtimeConfig:    "api/all=false",
-			errShouldContain: conflict,
-		},
-		{
-			runtimeConfig:    "api/beta=false",
-			errShouldContain: conflict,
-		},
-		{
-			runtimeConfig:    "flowcontrol.apiserver.k8s.io/v1beta1=false",
-			errShouldContain: "",
-		},
-		{
-			runtimeConfig:    "flowcontrol.apiserver.k8s.io/v1beta2=false",
-			errShouldContain: "",
-		},
-		{
-			runtimeConfig:    "flowcontrol.apiserver.k8s.io/v1beta3=false",
-			errShouldContain: conflict,
-		},
-		{
-			runtimeConfig:    "flowcontrol.apiserver.k8s.io/v1beta3=true",
-			errShouldContain: "",
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.runtimeConfig, func(t *testing.T) {
-			options := &ServerRunOptions{
-				GenericServerRunOptions: &genericoptions.ServerRunOptions{
-					EnablePriorityAndFairness: true,
-				},
-				APIEnablement: genericoptions.NewAPIEnablementOptions(),
-			}
-			options.APIEnablement.RuntimeConfig.Set(test.runtimeConfig)
-
-			var errMessageGot string
-			if errs := validateAPIPriorityAndFairness(options); len(errs) > 0 {
-				errMessageGot = errs[0].Error()
-			}
-			if !strings.Contains(errMessageGot, test.errShouldContain) {
-				t.Errorf("Expected error message to contain: %q, but got: %q", test.errShouldContain, errMessageGot)
-			}
-		})
-	}
-}
-
-func TestValidateServerRunOptions(t *testing.T) {
-	cidrOpts := makeOptionsWithCIDRs("10.0.0.0/16", "3000::/64")
-	nodePortOpts := makeOptionsWithPort(-1, 30065, 1)
-
-	testCases := []struct {
-		name         string
-		options      *ServerRunOptions
-		expectErrors bool
-	}{
-		{
-			name:         "validate master count equal 0",
-			expectErrors: true,
-			options: &ServerRunOptions{
-				MasterCount:             0,
-				GenericServerRunOptions: &genericoptions.ServerRunOptions{},
-				Etcd:                    &genericoptions.EtcdOptions{},
-				SecureServing:           &genericoptions.SecureServingOptionsWithLoopback{},
-				Audit:                   &genericoptions.AuditOptions{},
-				Admission: &kubeoptions.AdmissionOptions{
-					GenericAdmission: &genericoptions.AdmissionOptions{
-						EnablePlugins: []string{"foo"},
-						Plugins:       kubeapiserveradmission.NewPlugins(),
-					},
-					PluginNames: []string{"foo"},
-				},
-				Authentication: &kubeoptions.BuiltInAuthenticationOptions{
-					APIAudiences: []string{"bar"},
-					ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{
-						Issuers: []string{"baz"},
-					},
-				},
-				Authorization:                  &kubeoptions.BuiltInAuthorizationOptions{},
-				APIEnablement:                  genericoptions.NewAPIEnablementOptions(),
-				Metrics:                        &basemetrics.Options{},
-				ServiceClusterIPRanges:         cidrOpts.ServiceClusterIPRanges,
-				PrimaryServiceClusterIPRange:   cidrOpts.PrimaryServiceClusterIPRange,
-				SecondaryServiceClusterIPRange: cidrOpts.SecondaryServiceClusterIPRange,
-				ServiceNodePortRange:           nodePortOpts.ServiceNodePortRange,
-				KubernetesServiceNodePort:      nodePortOpts.KubernetesServiceNodePort,
-				ServiceAccountSigningKeyFile:   "",
-			},
-		},
-		{
-			name:         "validate token request enable not attempted",
-			expectErrors: true,
-			options: &ServerRunOptions{
-				MasterCount:             1,
-				GenericServerRunOptions: &genericoptions.ServerRunOptions{},
-				Etcd:                    &genericoptions.EtcdOptions{},
-				SecureServing:           &genericoptions.SecureServingOptionsWithLoopback{},
-				Audit:                   &genericoptions.AuditOptions{},
-				Admission: &kubeoptions.AdmissionOptions{
-					GenericAdmission: &genericoptions.AdmissionOptions{
-						EnablePlugins: []string{""},
-						Plugins:       kubeapiserveradmission.NewPlugins(),
-					},
-					PluginNames: []string{""},
-				},
-				Authentication: &kubeoptions.BuiltInAuthenticationOptions{
-					ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{},
-				},
-				Authorization:                  &kubeoptions.BuiltInAuthorizationOptions{},
-				APIEnablement:                  genericoptions.NewAPIEnablementOptions(),
-				Metrics:                        &basemetrics.Options{},
-				ServiceClusterIPRanges:         cidrOpts.ServiceClusterIPRanges,
-				PrimaryServiceClusterIPRange:   cidrOpts.PrimaryServiceClusterIPRange,
-				SecondaryServiceClusterIPRange: cidrOpts.SecondaryServiceClusterIPRange,
-				ServiceNodePortRange:           nodePortOpts.ServiceNodePortRange,
-				KubernetesServiceNodePort:      nodePortOpts.KubernetesServiceNodePort,
-				ServiceAccountSigningKeyFile:   "",
-			},
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			errs := tc.options.Validate()
-			if len(errs) > 0 && !tc.expectErrors {
-				t.Errorf("expected no errors, errors found %+v", errs)
-			}
-
-			if len(errs) == 0 && tc.expectErrors {
-				t.Errorf("expected errors, no errors found")
-			}
-		})
-	}
-}

pkg/controlplane/apiserver/options/options.go

@@ -0,0 +1,292 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package options contains flags and options for initializing an apiserver
+package options
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"strings"
+	"time"
+
+	genericoptions "k8s.io/apiserver/pkg/server/options"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
+	"k8s.io/client-go/util/keyutil"
+	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/component-base/logs"
+	logsapi "k8s.io/component-base/logs/api/v1"
+	"k8s.io/component-base/metrics"
+	"k8s.io/klog/v2"
+	"k8s.io/utils/integer"
+	netutil "k8s.io/utils/net"
+
+	_ "k8s.io/kubernetes/pkg/features"
+	kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator"
+	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
+	"k8s.io/kubernetes/pkg/serviceaccount"
+)
+
+type Options struct {
+	GenericServerRunOptions *genericoptions.ServerRunOptions
+	Etcd                    *genericoptions.EtcdOptions
+	SecureServing           *genericoptions.SecureServingOptionsWithLoopback
+	Audit                   *genericoptions.AuditOptions
+	Features                *genericoptions.FeatureOptions
+	Admission               *kubeoptions.AdmissionOptions
+	Authentication          *kubeoptions.BuiltInAuthenticationOptions
+	Authorization           *kubeoptions.BuiltInAuthorizationOptions
+	APIEnablement           *genericoptions.APIEnablementOptions
+	EgressSelector          *genericoptions.EgressSelectorOptions
+	Metrics                 *metrics.Options
+	Logs                    *logs.Options
+	Traces                  *genericoptions.TracingOptions
+
+	EnableLogsHandler        bool
+	EventTTL                 time.Duration
+	MaxConnectionBytesPerSec int64
+
+	ProxyClientCertFile string
+	ProxyClientKeyFile  string
+
+	EnableAggregatorRouting             bool
+	AggregatorRejectForwardingRedirects bool
+
+	MasterCount int
+
+	ServiceAccountSigningKeyFile     string
+	ServiceAccountIssuer             serviceaccount.TokenGenerator
+	ServiceAccountTokenMaxExpiration time.Duration
+
+	ShowHiddenMetricsForVersion string
+}
+
+// completedServerRunOptions is a private wrapper that enforces a call of Complete() before Run can be invoked.
+type completedOptions struct {
+	Options
+}
+
+type CompletedOptions struct {
+	// Embed a private pointer that cannot be instantiated outside of this package.
+	*completedOptions
+}
+
+// NewOptions creates a new ServerRunOptions object with default parameters
+func NewOptions() *Options {
+	s := Options{
+		GenericServerRunOptions: genericoptions.NewServerRunOptions(),
+		Etcd:                    genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)),
+		SecureServing:           kubeoptions.NewSecureServingOptions(),
+		Audit:                   genericoptions.NewAuditOptions(),
+		Features:                genericoptions.NewFeatureOptions(),
+		Admission:               kubeoptions.NewAdmissionOptions(),
+		Authentication:          kubeoptions.NewBuiltInAuthenticationOptions().WithAll(),
+		Authorization:           kubeoptions.NewBuiltInAuthorizationOptions(),
+		APIEnablement:           genericoptions.NewAPIEnablementOptions(),
+		EgressSelector:          genericoptions.NewEgressSelectorOptions(),
+		Metrics:                 metrics.NewOptions(),
+		Logs:                    logs.NewOptions(),
+		Traces:                  genericoptions.NewTracingOptions(),
+
+		EnableLogsHandler:                   true,
+		EventTTL:                            1 * time.Hour,
+		MasterCount:                         1,
+		AggregatorRejectForwardingRedirects: true,
+	}
+
+	// Overwrite the default for storage data format.
+	s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf"
+
+	return &s
+}
+
+func (s *Options) AddFlags(fss *cliflag.NamedFlagSets) {
+	// Add the generic flags.
+	s.GenericServerRunOptions.AddUniversalFlags(fss.FlagSet("generic"))
+	s.Etcd.AddFlags(fss.FlagSet("etcd"))
+	s.SecureServing.AddFlags(fss.FlagSet("secure serving"))
+	s.Audit.AddFlags(fss.FlagSet("auditing"))
+	s.Features.AddFlags(fss.FlagSet("features"))
+	s.Authentication.AddFlags(fss.FlagSet("authentication"))
+	s.Authorization.AddFlags(fss.FlagSet("authorization"))
+	s.APIEnablement.AddFlags(fss.FlagSet("API enablement"))
+	s.EgressSelector.AddFlags(fss.FlagSet("egress selector"))
+	s.Admission.AddFlags(fss.FlagSet("admission"))
+	s.Metrics.AddFlags(fss.FlagSet("metrics"))
+	logsapi.AddFlags(s.Logs, fss.FlagSet("logs"))
+	s.Traces.AddFlags(fss.FlagSet("traces"))
+
+	// Note: the weird ""+ in below lines seems to be the only way to get gofmt to
+	// arrange these text blocks sensibly. Grrr.
+	fs := fss.FlagSet("misc")
+	fs.DurationVar(&s.EventTTL, "event-ttl", s.EventTTL,
+		"Amount of time to retain events.")
+
+	fs.BoolVar(&s.EnableLogsHandler, "enable-logs-handler", s.EnableLogsHandler,
+		"If true, install a /logs handler for the apiserver logs.")
+	fs.MarkDeprecated("enable-logs-handler", "This flag will be removed in v1.19")
+
+	fs.Int64Var(&s.MaxConnectionBytesPerSec, "max-connection-bytes-per-sec", s.MaxConnectionBytesPerSec, ""+
+		"If non-zero, throttle each user connection to this number of bytes/sec. "+
+		"Currently only applies to long-running requests.")
+
+	fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount,
+		"The number of apiservers running in the cluster, must be a positive number. (In use when --endpoint-reconciler-type=master-count is enabled.)")
+	fs.MarkDeprecated("apiserver-count", "apiserver-count is deprecated and will be removed in a future version.")
+
+	fs.StringVar(&s.ProxyClientCertFile, "proxy-client-cert-file", s.ProxyClientCertFile, ""+
+		"Client certificate used to prove the identity of the aggregator or kube-apiserver "+
+		"when it must call out during a request. This includes proxying requests to a user "+
+		"api-server and calling out to webhook admission plugins. It is expected that this "+
+		"cert includes a signature from the CA in the --requestheader-client-ca-file flag. "+
+		"That CA is published in the 'extension-apiserver-authentication' configmap in "+
+		"the kube-system namespace. Components receiving calls from kube-aggregator should "+
+		"use that CA to perform their half of the mutual TLS verification.")
+	fs.StringVar(&s.ProxyClientKeyFile, "proxy-client-key-file", s.ProxyClientKeyFile, ""+
+		"Private key for the client certificate used to prove the identity of the aggregator or kube-apiserver "+
+		"when it must call out during a request. This includes proxying requests to a user "+
+		"api-server and calling out to webhook admission plugins.")
+
+	fs.BoolVar(&s.EnableAggregatorRouting, "enable-aggregator-routing", s.EnableAggregatorRouting,
+		"Turns on aggregator routing requests to endpoints IP rather than cluster IP.")
+
+	fs.BoolVar(&s.AggregatorRejectForwardingRedirects, "aggregator-reject-forwarding-redirect", s.AggregatorRejectForwardingRedirects,
+		"Aggregator reject forwarding redirect response back to client.")
+
+	fs.StringVar(&s.ServiceAccountSigningKeyFile, "service-account-signing-key-file", s.ServiceAccountSigningKeyFile, ""+
+		"Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key.")
+}
+
+func (o *Options) Complete(alternateDNS []string, alternateIPs []net.IP) (CompletedOptions, error) {
+	if o == nil {
+		return CompletedOptions{completedOptions: &completedOptions{}}, nil
+	}
+
+	completed := completedOptions{
+		Options: *o,
+	}
+
+	// set defaults
+	if err := completed.GenericServerRunOptions.DefaultAdvertiseAddress(completed.SecureServing.SecureServingOptions); err != nil {
+		return CompletedOptions{}, err
+	}
+
+	if err := completed.SecureServing.MaybeDefaultWithSelfSignedCerts(completed.GenericServerRunOptions.AdvertiseAddress.String(), alternateDNS, alternateIPs); err != nil {
+		return CompletedOptions{}, fmt.Errorf("error creating self-signed certificates: %v", err)
+	}
+
+	if len(completed.GenericServerRunOptions.ExternalHost) == 0 {
+		if len(completed.GenericServerRunOptions.AdvertiseAddress) > 0 {
+			completed.GenericServerRunOptions.ExternalHost = completed.GenericServerRunOptions.AdvertiseAddress.String()
+		} else {
+			hostname, err := os.Hostname()
+			if err != nil {
+				return CompletedOptions{}, fmt.Errorf("error finding host name: %v", err)
+			}
+			completed.GenericServerRunOptions.ExternalHost = hostname
+		}
+		klog.Infof("external host was not specified, using %v", completed.GenericServerRunOptions.ExternalHost)
+	}
+
+	completed.Authentication.ApplyAuthorization(completed.Authorization)
+
+	// Use (ServiceAccountSigningKeyFile != "") as a proxy to the user enabling
+	// TokenRequest functionality. This defaulting was convenient, but messed up
+	// a lot of people when they rotated their serving cert with no idea it was
+	// connected to their service account keys. We are taking this opportunity to
+	// remove this problematic defaulting.
+	if completed.ServiceAccountSigningKeyFile == "" {
+		// Default to the private server key for service account token signing
+		if len(completed.Authentication.ServiceAccounts.KeyFiles) == 0 && completed.SecureServing.ServerCert.CertKey.KeyFile != "" {
+			if kubeauthenticator.IsValidServiceAccountKeyFile(completed.SecureServing.ServerCert.CertKey.KeyFile) {
+				completed.Authentication.ServiceAccounts.KeyFiles = []string{completed.SecureServing.ServerCert.CertKey.KeyFile}
+			} else {
+				klog.Warning("No TLS key provided, service account token authentication disabled")
+			}
+		}
+	}
+
+	if completed.ServiceAccountSigningKeyFile != "" && len(completed.Authentication.ServiceAccounts.Issuers) != 0 && completed.Authentication.ServiceAccounts.Issuers[0] != "" {
+		sk, err := keyutil.PrivateKeyFromFile(completed.ServiceAccountSigningKeyFile)
+		if err != nil {
+			return CompletedOptions{}, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err)
+		}
+		if completed.Authentication.ServiceAccounts.MaxExpiration != 0 {
+			lowBound := time.Hour
+			upBound := time.Duration(1<<32) * time.Second
+			if completed.Authentication.ServiceAccounts.MaxExpiration < lowBound ||
+				completed.Authentication.ServiceAccounts.MaxExpiration > upBound {
+				return CompletedOptions{}, fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds")
+			}
+			if completed.Authentication.ServiceAccounts.ExtendExpiration {
+				if completed.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.WarnOnlyBoundTokenExpirationSeconds*time.Second {
+					klog.Warningf("service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, service-account-max-token-expiration must be set longer than %d seconds (currently %s)", serviceaccount.WarnOnlyBoundTokenExpirationSeconds, completed.Authentication.ServiceAccounts.MaxExpiration)
+				}
+				if completed.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.ExpirationExtensionSeconds*time.Second {
+					klog.Warningf("service-account-extend-token-expiration is true, enabling tokens valid up to %d seconds, which is longer than service-account-max-token-expiration set to %s seconds", serviceaccount.ExpirationExtensionSeconds, completed.Authentication.ServiceAccounts.MaxExpiration)
+				}
+			}
+		}
+
+		completed.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(completed.Authentication.ServiceAccounts.Issuers[0], sk)
+		if err != nil {
+			return CompletedOptions{}, fmt.Errorf("failed to build token generator: %v", err)
+		}
+		completed.ServiceAccountTokenMaxExpiration = completed.Authentication.ServiceAccounts.MaxExpiration
+	}
+
+	for key, value := range completed.APIEnablement.RuntimeConfig {
+		if key == "v1" || strings.HasPrefix(key, "v1/") ||
+			key == "api/v1" || strings.HasPrefix(key, "api/v1/") {
+			delete(completed.APIEnablement.RuntimeConfig, key)
+			completed.APIEnablement.RuntimeConfig["/v1"] = value
+		}
+		if key == "api/legacy" {
+			delete(completed.APIEnablement.RuntimeConfig, key)
+		}
+	}
+
+	return CompletedOptions{
+		completedOptions: &completed,
+	}, nil
+}
+
+// ServiceIPRange checks if the serviceClusterIPRange flag is nil, raising a warning if so and
+// setting service ip range to the default value in kubeoptions.DefaultServiceIPCIDR
+// for now until the default is removed per the deprecation timeline guidelines.
+// Returns service ip range, api server service IP, and an error
+func ServiceIPRange(passedServiceClusterIPRange net.IPNet) (net.IPNet, net.IP, error) {
+	serviceClusterIPRange := passedServiceClusterIPRange
+	if passedServiceClusterIPRange.IP == nil {
+		klog.Warningf("No CIDR for service cluster IPs specified. Default value which was %s is deprecated and will be removed in future releases. Please specify it using --service-cluster-ip-range on kube-apiserver.", kubeoptions.DefaultServiceIPCIDR.String())
+		serviceClusterIPRange = kubeoptions.DefaultServiceIPCIDR
+	}
+
+	size := integer.Int64Min(netutil.RangeSize(&serviceClusterIPRange), 1<<16)
+	if size < 8 {
+		return net.IPNet{}, net.IP{}, fmt.Errorf("the service cluster IP range must be at least %d IP addresses", 8)
+	}
+
+	// Select the first valid IP from ServiceClusterIPRange to use as the GenericAPIServer service IP.
+	apiServerServiceIP, err := netutil.GetIndexedIP(&serviceClusterIPRange, 1)
+	if err != nil {
+		return net.IPNet{}, net.IP{}, err
+	}
+	klog.V(4).Infof("Setting service IP to %q (read-write).", apiServerServiceIP)
+
+	return serviceClusterIPRange, apiServerServiceIP, nil
+}

pkg/controlplane/apiserver/options/options_test.go

@@ -0,0 +1,291 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/spf13/pflag"
+	oteltrace "go.opentelemetry.io/otel/trace"
+
+	"k8s.io/apiserver/pkg/admission"
+	apiserveroptions "k8s.io/apiserver/pkg/server/options"
+	"k8s.io/apiserver/pkg/storage/etcd3"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
+	auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
+	audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
+	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/component-base/logs"
+	"k8s.io/component-base/metrics"
+	netutils "k8s.io/utils/net"
+
+	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
+)
+
+func TestAddFlags(t *testing.T) {
+	fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
+	s := NewOptions()
+	var fss cliflag.NamedFlagSets
+	s.AddFlags(&fss)
+	for _, f := range fss.FlagSets {
+		fs.AddFlagSet(f)
+	}
+
+	args := []string{
+		"--enable-admission-plugins=AlwaysDeny",
+		"--admission-control-config-file=/admission-control-config",
+		"--advertise-address=192.168.10.10",
+		"--anonymous-auth=false",
+		"--apiserver-count=5",
+		"--audit-log-maxage=11",
+		"--audit-log-maxbackup=12",
+		"--audit-log-maxsize=13",
+		"--audit-log-path=/var/log",
+		"--audit-log-mode=blocking",
+		"--audit-log-batch-buffer-size=46",
+		"--audit-log-batch-max-size=47",
+		"--audit-log-batch-max-wait=48s",
+		"--audit-log-batch-throttle-enable=true",
+		"--audit-log-batch-throttle-qps=49.5",
+		"--audit-log-batch-throttle-burst=50",
+		"--audit-log-truncate-enabled=true",
+		"--audit-log-truncate-max-batch-size=45",
+		"--audit-log-truncate-max-event-size=44",
+		"--audit-log-version=audit.k8s.io/v1",
+		"--audit-policy-file=/policy",
+		"--audit-webhook-config-file=/webhook-config",
+		"--audit-webhook-mode=blocking",
+		"--audit-webhook-batch-buffer-size=42",
+		"--audit-webhook-batch-max-size=43",
+		"--audit-webhook-batch-max-wait=1s",
+		"--audit-webhook-batch-throttle-enable=false",
+		"--audit-webhook-batch-throttle-qps=43.5",
+		"--audit-webhook-batch-throttle-burst=44",
+		"--audit-webhook-truncate-enabled=true",
+		"--audit-webhook-truncate-max-batch-size=43",
+		"--audit-webhook-truncate-max-event-size=42",
+		"--audit-webhook-initial-backoff=2s",
+		"--audit-webhook-version=audit.k8s.io/v1",
+		"--authentication-token-webhook-cache-ttl=3m",
+		"--authentication-token-webhook-config-file=/token-webhook-config",
+		"--authorization-mode=AlwaysDeny,RBAC",
+		"--authorization-policy-file=/policy",
+		"--authorization-webhook-cache-authorized-ttl=3m",
+		"--authorization-webhook-cache-unauthorized-ttl=1m",
+		"--authorization-webhook-config-file=/webhook-config",
+		"--bind-address=192.168.10.20",
+		"--client-ca-file=/client-ca",
+		"--cors-allowed-origins=10.10.10.100,10.10.10.200",
+		"--contention-profiling=true",
+		"--egress-selector-config-file=/var/run/kubernetes/egress-selector/connectivity.yaml",
+		"--enable-aggregator-routing=true",
+		"--enable-priority-and-fairness=false",
+		"--enable-logs-handler=false",
+		"--etcd-keyfile=/var/run/kubernetes/etcd.key",
+		"--etcd-certfile=/var/run/kubernetes/etcdce.crt",
+		"--etcd-cafile=/var/run/kubernetes/etcdca.crt",
+		"--http2-max-streams-per-connection=42",
+		"--tracing-config-file=/var/run/kubernetes/tracing_config.yaml",
+		"--proxy-client-cert-file=/var/run/kubernetes/proxy.crt",
+		"--proxy-client-key-file=/var/run/kubernetes/proxy.key",
+		"--request-timeout=2m",
+		"--storage-backend=etcd3",
+		"--lease-reuse-duration-seconds=100",
+	}
+	fs.Parse(args)
+
+	// This is a snapshot of expected options parsed by args.
+	expected := &Options{
+		MasterCount: 5,
+		GenericServerRunOptions: &apiserveroptions.ServerRunOptions{
+			AdvertiseAddress:            netutils.ParseIPSloppy("192.168.10.10"),
+			CorsAllowedOriginList:       []string{"10.10.10.100", "10.10.10.200"},
+			MaxRequestsInFlight:         400,
+			MaxMutatingRequestsInFlight: 200,
+			RequestTimeout:              time.Duration(2) * time.Minute,
+			MinRequestTimeout:           1800,
+			JSONPatchMaxCopyBytes:       int64(3 * 1024 * 1024),
+			MaxRequestBodyBytes:         int64(3 * 1024 * 1024),
+		},
+		Admission: &kubeoptions.AdmissionOptions{
+			GenericAdmission: &apiserveroptions.AdmissionOptions{
+				RecommendedPluginOrder: s.Admission.GenericAdmission.RecommendedPluginOrder,
+				DefaultOffPlugins:      s.Admission.GenericAdmission.DefaultOffPlugins,
+				EnablePlugins:          []string{"AlwaysDeny"},
+				ConfigFile:             "/admission-control-config",
+				Plugins:                s.Admission.GenericAdmission.Plugins,
+				Decorators:             s.Admission.GenericAdmission.Decorators,
+			},
+		},
+		Etcd: &apiserveroptions.EtcdOptions{
+			StorageConfig: storagebackend.Config{
+				Type: "etcd3",
+				Transport: storagebackend.TransportConfig{
+					ServerList:     nil,
+					KeyFile:        "/var/run/kubernetes/etcd.key",
+					TrustedCAFile:  "/var/run/kubernetes/etcdca.crt",
+					CertFile:       "/var/run/kubernetes/etcdce.crt",
+					TracerProvider: oteltrace.NewNoopTracerProvider(),
+				},
+				Paging:                true,
+				Prefix:                "/registry",
+				CompactionInterval:    storagebackend.DefaultCompactInterval,
+				CountMetricPollPeriod: time.Minute,
+				DBMetricPollInterval:  storagebackend.DefaultDBMetricPollInterval,
+				HealthcheckTimeout:    storagebackend.DefaultHealthcheckTimeout,
+				ReadycheckTimeout:     storagebackend.DefaultReadinessTimeout,
+				LeaseManagerConfig: etcd3.LeaseManagerConfig{
+					ReuseDurationSeconds: 100,
+					MaxObjectCount:       1000,
+				},
+			},
+			DefaultStorageMediaType: "application/vnd.kubernetes.protobuf",
+			DeleteCollectionWorkers: 1,
+			EnableGarbageCollection: true,
+			EnableWatchCache:        true,
+			DefaultWatchCacheSize:   100,
+		},
+		SecureServing: (&apiserveroptions.SecureServingOptions{
+			BindAddress: netutils.ParseIPSloppy("192.168.10.20"),
+			BindPort:    6443,
+			ServerCert: apiserveroptions.GeneratableKeyCert{
+				CertDirectory: "/var/run/kubernetes",
+				PairName:      "apiserver",
+			},
+			HTTP2MaxStreamsPerConnection: 42,
+			Required:                     true,
+		}).WithLoopback(),
+		EventTTL: 1 * time.Hour,
+		Audit: &apiserveroptions.AuditOptions{
+			LogOptions: apiserveroptions.AuditLogOptions{
+				Path:       "/var/log",
+				MaxAge:     11,
+				MaxBackups: 12,
+				MaxSize:    13,
+				Format:     "json",
+				BatchOptions: apiserveroptions.AuditBatchOptions{
+					Mode: "blocking",
+					BatchConfig: auditbuffered.BatchConfig{
+						BufferSize:     46,
+						MaxBatchSize:   47,
+						MaxBatchWait:   48 * time.Second,
+						ThrottleEnable: true,
+						ThrottleQPS:    49.5,
+						ThrottleBurst:  50,
+					},
+				},
+				TruncateOptions: apiserveroptions.AuditTruncateOptions{
+					Enabled: true,
+					TruncateConfig: audittruncate.Config{
+						MaxBatchSize: 45,
+						MaxEventSize: 44,
+					},
+				},
+				GroupVersionString: "audit.k8s.io/v1",
+			},
+			WebhookOptions: apiserveroptions.AuditWebhookOptions{
+				ConfigFile: "/webhook-config",
+				BatchOptions: apiserveroptions.AuditBatchOptions{
+					Mode: "blocking",
+					BatchConfig: auditbuffered.BatchConfig{
+						BufferSize:     42,
+						MaxBatchSize:   43,
+						MaxBatchWait:   1 * time.Second,
+						ThrottleEnable: false,
+						ThrottleQPS:    43.5,
+						ThrottleBurst:  44,
+						AsyncDelegate:  true,
+					},
+				},
+				TruncateOptions: apiserveroptions.AuditTruncateOptions{
+					Enabled: true,
+					TruncateConfig: audittruncate.Config{
+						MaxBatchSize: 43,
+						MaxEventSize: 42,
+					},
+				},
+				InitialBackoff:     2 * time.Second,
+				GroupVersionString: "audit.k8s.io/v1",
+			},
+			PolicyFile: "/policy",
+		},
+		Features: &apiserveroptions.FeatureOptions{
+			EnableProfiling:           true,
+			EnableContentionProfiling: true,
+		},
+		Authentication: &kubeoptions.BuiltInAuthenticationOptions{
+			Anonymous: &kubeoptions.AnonymousAuthenticationOptions{
+				Allow: false,
+			},
+			ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{
+				ClientCA: "/client-ca",
+			},
+			WebHook: &kubeoptions.WebHookAuthenticationOptions{
+				CacheTTL:     180000000000,
+				ConfigFile:   "/token-webhook-config",
+				Version:      "v1beta1",
+				RetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(),
+			},
+			BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{},
+			OIDC: &kubeoptions.OIDCAuthenticationOptions{
+				UsernameClaim: "sub",
+				SigningAlgs:   []string{"RS256"},
+			},
+			RequestHeader: &apiserveroptions.RequestHeaderAuthenticationOptions{},
+			ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{
+				Lookup:           true,
+				ExtendExpiration: true,
+			},
+			TokenFile:            &kubeoptions.TokenFileAuthenticationOptions{},
+			TokenSuccessCacheTTL: 10 * time.Second,
+			TokenFailureCacheTTL: 0,
+		},
+		Authorization: &kubeoptions.BuiltInAuthorizationOptions{
+			Modes:                       []string{"AlwaysDeny", "RBAC"},
+			PolicyFile:                  "/policy",
+			WebhookConfigFile:           "/webhook-config",
+			WebhookCacheAuthorizedTTL:   180000000000,
+			WebhookCacheUnauthorizedTTL: 60000000000,
+			WebhookVersion:              "v1beta1",
+			WebhookRetryBackoff:         apiserveroptions.DefaultAuthWebhookRetryBackoff(),
+		},
+		APIEnablement: &apiserveroptions.APIEnablementOptions{
+			RuntimeConfig: cliflag.ConfigurationMap{},
+		},
+		EgressSelector: &apiserveroptions.EgressSelectorOptions{
+			ConfigFile: "/var/run/kubernetes/egress-selector/connectivity.yaml",
+		},
+		EnableLogsHandler:       false,
+		EnableAggregatorRouting: true,
+		ProxyClientKeyFile:      "/var/run/kubernetes/proxy.key",
+		ProxyClientCertFile:     "/var/run/kubernetes/proxy.crt",
+		Metrics:                 &metrics.Options{},
+		Logs:                    logs.NewOptions(),
+		Traces: &apiserveroptions.TracingOptions{
+			ConfigFile: "/var/run/kubernetes/tracing_config.yaml",
+		},
+		AggregatorRejectForwardingRedirects: true,
+	}
+
+	if !reflect.DeepEqual(expected, s) {
+		t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{})))
+	}
+}

pkg/controlplane/apiserver/options/validation.go

@@ -0,0 +1,90 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
+	genericfeatures "k8s.io/apiserver/pkg/features"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
+
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+)
+
+func validateTokenRequest(options *Options) []error {
+	var errs []error
+
+	enableAttempted := options.ServiceAccountSigningKeyFile != "" ||
+		(len(options.Authentication.ServiceAccounts.Issuers) != 0 && options.Authentication.ServiceAccounts.Issuers[0] != "") ||
+		len(options.Authentication.APIAudiences) != 0
+
+	enableSucceeded := options.ServiceAccountIssuer != nil
+
+	if !enableAttempted {
+		errs = append(errs, errors.New("--service-account-signing-key-file and --service-account-issuer are required flags"))
+	}
+
+	if enableAttempted && !enableSucceeded {
+		errs = append(errs, errors.New("--service-account-signing-key-file, --service-account-issuer, and --api-audiences should be specified together"))
+	}
+
+	return errs
+}
+
+func validateAPIPriorityAndFairness(options *Options) []error {
+	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) && options.GenericServerRunOptions.EnablePriorityAndFairness {
+		// If none of the following runtime config options are specified,
+		// APF is assumed to be turned on. The internal APF controller uses
+		// v1beta3 so it should be enabled.
+		enabledAPIString := options.APIEnablement.RuntimeConfig.String()
+		testConfigs := []string{"flowcontrol.apiserver.k8s.io/v1beta3", "api/beta", "api/all"} // in the order of precedence
+		for _, testConfig := range testConfigs {
+			if strings.Contains(enabledAPIString, fmt.Sprintf("%s=false", testConfig)) {
+				return []error{fmt.Errorf("--runtime-config=%s=false conflicts with --enable-priority-and-fairness=true and --feature-gates=APIPriorityAndFairness=true", testConfig)}
+			}
+			if strings.Contains(enabledAPIString, fmt.Sprintf("%s=true", testConfig)) {
+				return nil
+			}
+		}
+	}
+
+	return nil
+}
+
+// Validate checks Options and return a slice of found errs.
+func (s *Options) Validate() []error {
+	var errs []error
+	if s.MasterCount <= 0 {
+		errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount))
+	}
+	errs = append(errs, s.Etcd.Validate()...)
+	errs = append(errs, validateAPIPriorityAndFairness(s)...)
+	errs = append(errs, s.SecureServing.Validate()...)
+	errs = append(errs, s.Authentication.Validate()...)
+	errs = append(errs, s.Authorization.Validate()...)
+	errs = append(errs, s.Audit.Validate()...)
+	errs = append(errs, s.Admission.Validate()...)
+	errs = append(errs, s.APIEnablement.Validate(legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme)...)
+	errs = append(errs, validateTokenRequest(s)...)
+	errs = append(errs, s.Metrics.Validate()...)
+
+	return errs
+}

pkg/controlplane/apiserver/options/validation_test.go

@@ -0,0 +1,154 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"strings"
+	"testing"
+
+	kubeapiserveradmission "k8s.io/apiserver/pkg/admission"
+	genericoptions "k8s.io/apiserver/pkg/server/options"
+	basemetrics "k8s.io/component-base/metrics"
+
+	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
+)
+
+func TestValidateAPIPriorityAndFairness(t *testing.T) {
+	const conflict = "conflicts with --enable-priority-and-fairness=true and --feature-gates=APIPriorityAndFairness=true"
+	tests := []struct {
+		runtimeConfig    string
+		errShouldContain string
+	}{
+		{
+			runtimeConfig:    "api/all=false",
+			errShouldContain: conflict,
+		},
+		{
+			runtimeConfig:    "api/beta=false",
+			errShouldContain: conflict,
+		},
+		{
+			runtimeConfig:    "flowcontrol.apiserver.k8s.io/v1beta1=false",
+			errShouldContain: "",
+		},
+		{
+			runtimeConfig:    "flowcontrol.apiserver.k8s.io/v1beta2=false",
+			errShouldContain: "",
+		},
+		{
+			runtimeConfig:    "flowcontrol.apiserver.k8s.io/v1beta3=false",
+			errShouldContain: conflict,
+		},
+		{
+			runtimeConfig:    "flowcontrol.apiserver.k8s.io/v1beta3=true",
+			errShouldContain: "",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.runtimeConfig, func(t *testing.T) {
+			options := &Options{
+				GenericServerRunOptions: &genericoptions.ServerRunOptions{
+					EnablePriorityAndFairness: true,
+				},
+				APIEnablement: genericoptions.NewAPIEnablementOptions(),
+			}
+			options.APIEnablement.RuntimeConfig.Set(test.runtimeConfig)
+
+			var errMessageGot string
+			if errs := validateAPIPriorityAndFairness(options); len(errs) > 0 {
+				errMessageGot = errs[0].Error()
+			}
+			if !strings.Contains(errMessageGot, test.errShouldContain) {
+				t.Errorf("Expected error message to contain: %q, but got: %q", test.errShouldContain, errMessageGot)
+			}
+		})
+	}
+}
+
+func TestValidateOptions(t *testing.T) {
+	testCases := []struct {
+		name         string
+		options      *Options
+		expectErrors bool
+	}{
+		{
+			name:         "validate master count equal 0",
+			expectErrors: true,
+			options: &Options{
+				MasterCount:             0,
+				GenericServerRunOptions: &genericoptions.ServerRunOptions{},
+				Etcd:                    &genericoptions.EtcdOptions{},
+				SecureServing:           &genericoptions.SecureServingOptionsWithLoopback{},
+				Audit:                   &genericoptions.AuditOptions{},
+				Admission: &kubeoptions.AdmissionOptions{
+					GenericAdmission: &genericoptions.AdmissionOptions{
+						EnablePlugins: []string{"foo"},
+						Plugins:       kubeapiserveradmission.NewPlugins(),
+					},
+					PluginNames: []string{"foo"},
+				},
+				Authentication: &kubeoptions.BuiltInAuthenticationOptions{
+					APIAudiences: []string{"bar"},
+					ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{
+						Issuers: []string{"baz"},
+					},
+				},
+				APIEnablement:                genericoptions.NewAPIEnablementOptions(),
+				Metrics:                      &basemetrics.Options{},
+				ServiceAccountSigningKeyFile: "",
+			},
+		},
+		{
+			name:         "validate token request enable not attempted",
+			expectErrors: true,
+			options: &Options{
+				MasterCount:             1,
+				GenericServerRunOptions: &genericoptions.ServerRunOptions{},
+				Etcd:                    &genericoptions.EtcdOptions{},
+				SecureServing:           &genericoptions.SecureServingOptionsWithLoopback{},
+				Audit:                   &genericoptions.AuditOptions{},
+				Admission: &kubeoptions.AdmissionOptions{
+					GenericAdmission: &genericoptions.AdmissionOptions{
+						EnablePlugins: []string{""},
+						Plugins:       kubeapiserveradmission.NewPlugins(),
+					},
+					PluginNames: []string{""},
+				},
+				Authentication: &kubeoptions.BuiltInAuthenticationOptions{
+					ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{},
+				},
+				APIEnablement:                genericoptions.NewAPIEnablementOptions(),
+				Metrics:                      &basemetrics.Options{},
+				ServiceAccountSigningKeyFile: "",
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			errs := tc.options.Validate()
+			if len(errs) > 0 && !tc.expectErrors {
+				t.Errorf("expected no errors, errors found %+v", errs)
+			}
+
+			if len(errs) == 0 && tc.expectErrors {
+				t.Errorf("expected errors, no errors found")
+			}
+		})
+	}
+}

pkg/controlplane/services.go

@@ -1,54 +0,0 @@
-/*
-Copyright 2016 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package controlplane
-
-import (
-	"fmt"
-	"net"
-
-	"k8s.io/klog/v2"
-	"k8s.io/utils/integer"
-	utilnet "k8s.io/utils/net"
-
-	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
-)
-
-// ServiceIPRange checks if the serviceClusterIPRange flag is nil, raising a warning if so and
-// setting service ip range to the default value in kubeoptions.DefaultServiceIPCIDR
-// for now until the default is removed per the deprecation timeline guidelines.
-// Returns service ip range, api server service IP, and an error
-func ServiceIPRange(passedServiceClusterIPRange net.IPNet) (net.IPNet, net.IP, error) {
-	serviceClusterIPRange := passedServiceClusterIPRange
-	if passedServiceClusterIPRange.IP == nil {
-		klog.Warningf("No CIDR for service cluster IPs specified. Default value which was %s is deprecated and will be removed in future releases. Please specify it using --service-cluster-ip-range on kube-apiserver.", kubeoptions.DefaultServiceIPCIDR.String())
-		serviceClusterIPRange = kubeoptions.DefaultServiceIPCIDR
-	}
-
-	size := integer.Int64Min(utilnet.RangeSize(&serviceClusterIPRange), 1<<16)
-	if size < 8 {
-		return net.IPNet{}, net.IP{}, fmt.Errorf("the service cluster IP range must be at least %d IP addresses", 8)
-	}
-
-	// Select the first valid IP from ServiceClusterIPRange to use as the GenericAPIServer service IP.
-	apiServerServiceIP, err := utilnet.GetIndexedIP(&serviceClusterIPRange, 1)
-	if err != nil {
-		return net.IPNet{}, net.IP{}, err
-	}
-	klog.V(4).Infof("Setting service IP to %q (read-write).", apiServerServiceIP)
-
-	return serviceClusterIPRange, apiServerServiceIP, nil
-}