Merge pull request #118633 from sttts/sttts-controlplane-split

kube-apiserver: split apart generic control plane options

cmd/kube-apiserver/app/aggregator.go

@@ -47,13 +47,13 @@ import (
 	apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1"
 	informers "k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1"
 	"k8s.io/kube-aggregator/pkg/controllers/autoregister"
-	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
+	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 	"k8s.io/kubernetes/pkg/controlplane/controller/crdregistration"
 )
 
 func createAggregatorConfig(
 	kubeAPIServerConfig genericapiserver.Config,
-	commandOptions *options.ServerRunOptions,
+	commandOptions controlplaneapiserver.CompletedOptions,
 	externalInformers kubeexternalinformers.SharedInformerFactory,
 	serviceResolver aggregatorapiserver.ServiceResolver,
 	proxyTransport *http.Transport,

cmd/kube-apiserver/app/config.go

@@ -77,14 +77,14 @@ func NewConfig(opts options.CompletedOptions) (*Config, error) {
 	}
 	c.ControlPlane = controlPlane
 
-	apiExtensions, err := apiserver.CreateAPIExtensionsConfig(*controlPlane.GenericConfig, controlPlane.ExtraConfig.VersionedInformers, pluginInitializer, opts.ServerRunOptions, opts.MasterCount,
+	apiExtensions, err := apiserver.CreateAPIExtensionsConfig(*controlPlane.GenericConfig, controlPlane.ExtraConfig.VersionedInformers, pluginInitializer, opts.CompletedOptions, opts.MasterCount,
 		serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(controlPlane.ExtraConfig.ProxyTransport, controlPlane.GenericConfig.EgressSelector, controlPlane.GenericConfig.LoopbackClientConfig, controlPlane.GenericConfig.TracerProvider))
 	if err != nil {
 		return nil, err
 	}
 	c.ApiExtensions = apiExtensions
 
-	aggregator, err := createAggregatorConfig(*controlPlane.GenericConfig, opts.ServerRunOptions, controlPlane.ExtraConfig.VersionedInformers, serviceResolver, controlPlane.ExtraConfig.ProxyTransport, pluginInitializer)
+	aggregator, err := createAggregatorConfig(*controlPlane.GenericConfig, opts.CompletedOptions, controlPlane.ExtraConfig.VersionedInformers, serviceResolver, controlPlane.ExtraConfig.ProxyTransport, pluginInitializer)
 	if err != nil {
 		return nil, err
 	}

cmd/kube-apiserver/app/options/completion.go

@@ -19,141 +19,78 @@ package options
 import (
 	"fmt"
 	"net"
-	"os"
 	"strings"
-	"time"
 
-	serveroptions "k8s.io/apiserver/pkg/server/options"
-	"k8s.io/client-go/util/keyutil"
+	apiserveroptions "k8s.io/apiserver/pkg/server/options"
 	_ "k8s.io/component-base/metrics/prometheus/workqueue"
-	"k8s.io/klog/v2"
 	netutils "k8s.io/utils/net"
 
-	"k8s.io/kubernetes/pkg/controlplane"
+	controlplane "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 	"k8s.io/kubernetes/pkg/kubeapiserver"
-	kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator"
-	"k8s.io/kubernetes/pkg/serviceaccount"
+	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
 )
 
 // completedOptions is a private wrapper that enforces a call of Complete() before Run can be invoked.
 type completedOptions struct {
-	*ServerRunOptions
+	controlplane.CompletedOptions
+	CloudProvider *kubeoptions.CloudProviderOptions
+
+	Extra
 }
 
 type CompletedOptions struct {
-	completedOptions
+	// Embed a private pointer that cannot be instantiated outside of this package.
+	*completedOptions
 }
 
 // Complete set default ServerRunOptions.
 // Should be called after kube-apiserver flags parsed.
-func Complete(opts *ServerRunOptions) (CompletedOptions, error) {
-	// set defaults
-	if err := opts.GenericServerRunOptions.DefaultAdvertiseAddress(opts.SecureServing.SecureServingOptions); err != nil {
-		return CompletedOptions{}, err
+func (opts *ServerRunOptions) Complete() (CompletedOptions, error) {
+	if opts == nil {
+		return CompletedOptions{completedOptions: &completedOptions{}}, nil
 	}
 
-	// process s.ServiceClusterIPRange from list to Primary and Secondary
+	// process opts.ServiceClusterIPRange from list to Primary and Secondary
 	// we process secondary only if provided by user
 	apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, err := getServiceIPAndRanges(opts.ServiceClusterIPRanges)
 	if err != nil {
 		return CompletedOptions{}, err
 	}
-	opts.PrimaryServiceClusterIPRange = primaryServiceIPRange
-	opts.SecondaryServiceClusterIPRange = secondaryServiceIPRange
-	opts.APIServerServiceIP = apiServerServiceIP
-
-	if err := opts.SecureServing.MaybeDefaultWithSelfSignedCerts(opts.GenericServerRunOptions.AdvertiseAddress.String(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP}); err != nil {
-		return CompletedOptions{}, fmt.Errorf("error creating self-signed certificates: %v", err)
-	}
-
-	if len(opts.GenericServerRunOptions.ExternalHost) == 0 {
-		if len(opts.GenericServerRunOptions.AdvertiseAddress) > 0 {
-			opts.GenericServerRunOptions.ExternalHost = opts.GenericServerRunOptions.AdvertiseAddress.String()
-		} else {
-			hostname, err := os.Hostname()
+	controlplane, err := opts.Options.Complete([]string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP})
 	if err != nil {
-				return CompletedOptions{}, fmt.Errorf("error finding host name: %v", err)
-			}
-			opts.GenericServerRunOptions.ExternalHost = hostname
-		}
-		klog.Infof("external host was not specified, using %v", opts.GenericServerRunOptions.ExternalHost)
+		return CompletedOptions{}, err
 	}
 
-	opts.Authentication.ApplyAuthorization(opts.Authorization)
+	completed := completedOptions{
+		CompletedOptions: controlplane,
+		CloudProvider:    opts.CloudProvider,
 
-	// Use (ServiceAccountSigningKeyFile != "") as a proxy to the user enabling
-	// TokenRequest functionality. This defaulting was convenient, but messed up
-	// a lot of people when they rotated their serving cert with no idea it was
-	// connected to their service account keys. We are taking this opportunity to
-	// remove this problematic defaulting.
-	if opts.ServiceAccountSigningKeyFile == "" {
-		// Default to the private server key for service account token signing
-		if len(opts.Authentication.ServiceAccounts.KeyFiles) == 0 && opts.SecureServing.ServerCert.CertKey.KeyFile != "" {
-			if kubeauthenticator.IsValidServiceAccountKeyFile(opts.SecureServing.ServerCert.CertKey.KeyFile) {
-				opts.Authentication.ServiceAccounts.KeyFiles = []string{opts.SecureServing.ServerCert.CertKey.KeyFile}
-			} else {
-				klog.Warning("No TLS key provided, service account token authentication disabled")
-			}
-		}
+		Extra: opts.Extra,
 	}
 
-	if opts.ServiceAccountSigningKeyFile != "" && len(opts.Authentication.ServiceAccounts.Issuers) != 0 && opts.Authentication.ServiceAccounts.Issuers[0] != "" {
-		sk, err := keyutil.PrivateKeyFromFile(opts.ServiceAccountSigningKeyFile)
-		if err != nil {
-			return CompletedOptions{}, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err)
-		}
-		if opts.Authentication.ServiceAccounts.MaxExpiration != 0 {
-			lowBound := time.Hour
-			upBound := time.Duration(1<<32) * time.Second
-			if opts.Authentication.ServiceAccounts.MaxExpiration < lowBound ||
-				opts.Authentication.ServiceAccounts.MaxExpiration > upBound {
-				return CompletedOptions{}, fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds")
-			}
-			if opts.Authentication.ServiceAccounts.ExtendExpiration {
-				if opts.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.WarnOnlyBoundTokenExpirationSeconds*time.Second {
-					klog.Warningf("service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, service-account-max-token-expiration must be set longer than %d seconds (currently %s)", serviceaccount.WarnOnlyBoundTokenExpirationSeconds, opts.Authentication.ServiceAccounts.MaxExpiration)
-				}
-				if opts.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.ExpirationExtensionSeconds*time.Second {
-					klog.Warningf("service-account-extend-token-expiration is true, enabling tokens valid up to %d seconds, which is longer than service-account-max-token-expiration set to %s seconds", serviceaccount.ExpirationExtensionSeconds, opts.Authentication.ServiceAccounts.MaxExpiration)
-				}
-			}
-		}
+	completed.PrimaryServiceClusterIPRange = primaryServiceIPRange
+	completed.SecondaryServiceClusterIPRange = secondaryServiceIPRange
+	completed.APIServerServiceIP = apiServerServiceIP
 
-		opts.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(opts.Authentication.ServiceAccounts.Issuers[0], sk)
-		if err != nil {
-			return CompletedOptions{}, fmt.Errorf("failed to build token generator: %v", err)
-		}
-		opts.ServiceAccountTokenMaxExpiration = opts.Authentication.ServiceAccounts.MaxExpiration
-	}
-
-	if opts.Etcd.EnableWatchCache {
+	if completed.Etcd != nil && completed.Etcd.EnableWatchCache {
 		sizes := kubeapiserver.DefaultWatchCacheSizes()
 		// Ensure that overrides parse correctly.
-		userSpecified, err := serveroptions.ParseWatchCacheSizes(opts.Etcd.WatchCacheSizes)
+		userSpecified, err := apiserveroptions.ParseWatchCacheSizes(completed.Etcd.WatchCacheSizes)
 		if err != nil {
 			return CompletedOptions{}, err
 		}
 		for resource, size := range userSpecified {
 			sizes[resource] = size
 		}
-		opts.Etcd.WatchCacheSizes, err = serveroptions.WriteWatchCacheSizes(sizes)
+		completed.Etcd.WatchCacheSizes, err = apiserveroptions.WriteWatchCacheSizes(sizes)
 		if err != nil {
 			return CompletedOptions{}, err
 		}
 	}
 
-	for key, value := range opts.APIEnablement.RuntimeConfig {
-		if key == "v1" || strings.HasPrefix(key, "v1/") ||
-			key == "api/v1" || strings.HasPrefix(key, "api/v1/") {
-			delete(opts.APIEnablement.RuntimeConfig, key)
-			opts.APIEnablement.RuntimeConfig["/v1"] = value
-		}
-		if key == "api/legacy" {
-			delete(opts.APIEnablement.RuntimeConfig, key)
-		}
-	}
-
-	return CompletedOptions{completedOptions: completedOptions{ServerRunOptions: opts}}, nil
+	return CompletedOptions{
+		completedOptions: &completed,
+	}, nil
 }
 
 func getServiceIPAndRanges(serviceClusterIPRanges string) (net.IP, net.IPNet, net.IPNet, error) {

cmd/kube-apiserver/app/options/options.go

@@ -23,45 +23,29 @@ import (
 	"time"
 
 	utilnet "k8s.io/apimachinery/pkg/util/net"
-	genericoptions "k8s.io/apiserver/pkg/server/options"
-	"k8s.io/apiserver/pkg/storage/storagebackend"
 	cliflag "k8s.io/component-base/cli/flag"
-	"k8s.io/component-base/logs"
-	"k8s.io/component-base/metrics"
 
-	logsapi "k8s.io/component-base/logs/api/v1"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/cluster/ports"
+	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 	"k8s.io/kubernetes/pkg/controlplane/reconcilers"
 	_ "k8s.io/kubernetes/pkg/features" // add the kubernetes feature gates
 	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
 	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
-	"k8s.io/kubernetes/pkg/serviceaccount"
 )
 
 // ServerRunOptions runs a kubernetes api server.
 type ServerRunOptions struct {
-	GenericServerRunOptions *genericoptions.ServerRunOptions
-	Etcd                    *genericoptions.EtcdOptions
-	SecureServing           *genericoptions.SecureServingOptionsWithLoopback
-	Audit                   *genericoptions.AuditOptions
-	Features                *genericoptions.FeatureOptions
-	Admission               *kubeoptions.AdmissionOptions
-	Authentication          *kubeoptions.BuiltInAuthenticationOptions
-	Authorization           *kubeoptions.BuiltInAuthorizationOptions
+	*controlplaneapiserver.Options // embedded to avoid noise in existing consumers
 	CloudProvider                  *kubeoptions.CloudProviderOptions
-	APIEnablement           *genericoptions.APIEnablementOptions
-	EgressSelector          *genericoptions.EgressSelectorOptions
-	Metrics                 *metrics.Options
-	Logs                    *logs.Options
-	Traces                  *genericoptions.TracingOptions
 
+	Extra
+}
+
+type Extra struct {
 	AllowPrivileged           bool
-	EnableLogsHandler         bool
-	EventTTL                  time.Duration
 	KubeletConfig             kubeletclient.KubeletClientConfig
 	KubernetesServiceNodePort int
-	MaxConnectionBytesPerSec  int64
 	// ServiceClusterIPRange is mapped to input provided by user
 	ServiceClusterIPRanges string
 	// PrimaryServiceClusterIPRange and SecondaryServiceClusterIPRange are the results
@@ -73,43 +57,16 @@ type ServerRunOptions struct {
 
 	ServiceNodePortRange utilnet.PortRange
 
-	ProxyClientCertFile string
-	ProxyClientKeyFile  string
-
-	EnableAggregatorRouting             bool
-	AggregatorRejectForwardingRedirects bool
-
-	MasterCount            int
 	EndpointReconcilerType string
-
-	ServiceAccountSigningKeyFile     string
-	ServiceAccountIssuer             serviceaccount.TokenGenerator
-	ServiceAccountTokenMaxExpiration time.Duration
-
-	ShowHiddenMetricsForVersion string
 }
 
 // NewServerRunOptions creates a new ServerRunOptions object with default parameters
 func NewServerRunOptions() *ServerRunOptions {
 	s := ServerRunOptions{
-		GenericServerRunOptions: genericoptions.NewServerRunOptions(),
-		Etcd:                    genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)),
-		SecureServing:           kubeoptions.NewSecureServingOptions(),
-		Audit:                   genericoptions.NewAuditOptions(),
-		Features:                genericoptions.NewFeatureOptions(),
-		Admission:               kubeoptions.NewAdmissionOptions(),
-		Authentication:          kubeoptions.NewBuiltInAuthenticationOptions().WithAll(),
-		Authorization:           kubeoptions.NewBuiltInAuthorizationOptions(),
+		Options:       controlplaneapiserver.NewOptions(),
 		CloudProvider: kubeoptions.NewCloudProviderOptions(),
-		APIEnablement:           genericoptions.NewAPIEnablementOptions(),
-		EgressSelector:          genericoptions.NewEgressSelectorOptions(),
-		Metrics:                 metrics.NewOptions(),
-		Logs:                    logs.NewOptions(),
-		Traces:                  genericoptions.NewTracingOptions(),
 
-		EnableLogsHandler:      true,
-		EventTTL:               1 * time.Hour,
-		MasterCount:            1,
+		Extra: Extra{
 			EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
 			KubeletConfig: kubeletclient.KubeletClientConfig{
 				Port:         ports.KubeletPort,
@@ -129,54 +86,24 @@ func NewServerRunOptions() *ServerRunOptions {
 				HTTPTimeout: time.Duration(5) * time.Second,
 			},
 			ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange,
-		AggregatorRejectForwardingRedirects: true,
+		},
 	}
 
-	// Overwrite the default for storage data format.
-	s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf"
-
 	return &s
 }
 
 // Flags returns flags for a specific APIServer by section name
 func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
-	// Add the generic flags.
-	s.GenericServerRunOptions.AddUniversalFlags(fss.FlagSet("generic"))
-	s.Etcd.AddFlags(fss.FlagSet("etcd"))
-	s.SecureServing.AddFlags(fss.FlagSet("secure serving"))
-	s.Audit.AddFlags(fss.FlagSet("auditing"))
-	s.Features.AddFlags(fss.FlagSet("features"))
-	s.Authentication.AddFlags(fss.FlagSet("authentication"))
-	s.Authorization.AddFlags(fss.FlagSet("authorization"))
+	s.Options.AddFlags(&fss)
 	s.CloudProvider.AddFlags(fss.FlagSet("cloud provider"))
-	s.APIEnablement.AddFlags(fss.FlagSet("API enablement"))
-	s.EgressSelector.AddFlags(fss.FlagSet("egress selector"))
-	s.Admission.AddFlags(fss.FlagSet("admission"))
-	s.Metrics.AddFlags(fss.FlagSet("metrics"))
-	logsapi.AddFlags(s.Logs, fss.FlagSet("logs"))
-	s.Traces.AddFlags(fss.FlagSet("traces"))
 
 	// Note: the weird ""+ in below lines seems to be the only way to get gofmt to
 	// arrange these text blocks sensibly. Grrr.
 	fs := fss.FlagSet("misc")
-	fs.DurationVar(&s.EventTTL, "event-ttl", s.EventTTL,
-		"Amount of time to retain events.")
 
 	fs.BoolVar(&s.AllowPrivileged, "allow-privileged", s.AllowPrivileged,
 		"If true, allow privileged containers. [default=false]")
 
-	fs.BoolVar(&s.EnableLogsHandler, "enable-logs-handler", s.EnableLogsHandler,
-		"If true, install a /logs handler for the apiserver logs.")
-	fs.MarkDeprecated("enable-logs-handler", "This flag will be removed in v1.19")
-
-	fs.Int64Var(&s.MaxConnectionBytesPerSec, "max-connection-bytes-per-sec", s.MaxConnectionBytesPerSec, ""+
-		"If non-zero, throttle each user connection to this number of bytes/sec. "+
-		"Currently only applies to long-running requests.")
-
-	fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount,
-		"The number of apiservers running in the cluster, must be a positive number. (In use when --endpoint-reconciler-type=master-count is enabled.)")
-	fs.MarkDeprecated("apiserver-count", "apiserver-count is deprecated and will be removed in a future version.")
-
 	fs.StringVar(&s.EndpointReconcilerType, "endpoint-reconciler-type", s.EndpointReconcilerType,
 		"Use an endpoint reconciler ("+strings.Join(reconcilers.AllTypes.Names(), ", ")+") master-count is deprecated, and will be removed in a future version.")
 
@@ -219,27 +146,5 @@ func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
 	fs.StringVar(&s.KubeletConfig.TLSClientConfig.CAFile, "kubelet-certificate-authority", s.KubeletConfig.TLSClientConfig.CAFile,
 		"Path to a cert file for the certificate authority.")
 
-	fs.StringVar(&s.ProxyClientCertFile, "proxy-client-cert-file", s.ProxyClientCertFile, ""+
-		"Client certificate used to prove the identity of the aggregator or kube-apiserver "+
-		"when it must call out during a request. This includes proxying requests to a user "+
-		"api-server and calling out to webhook admission plugins. It is expected that this "+
-		"cert includes a signature from the CA in the --requestheader-client-ca-file flag. "+
-		"That CA is published in the 'extension-apiserver-authentication' configmap in "+
-		"the kube-system namespace. Components receiving calls from kube-aggregator should "+
-		"use that CA to perform their half of the mutual TLS verification.")
-	fs.StringVar(&s.ProxyClientKeyFile, "proxy-client-key-file", s.ProxyClientKeyFile, ""+
-		"Private key for the client certificate used to prove the identity of the aggregator or kube-apiserver "+
-		"when it must call out during a request. This includes proxying requests to a user "+
-		"api-server and calling out to webhook admission plugins.")
-
-	fs.BoolVar(&s.EnableAggregatorRouting, "enable-aggregator-routing", s.EnableAggregatorRouting,
-		"Turns on aggregator routing requests to endpoints IP rather than cluster IP.")
-
-	fs.BoolVar(&s.AggregatorRejectForwardingRedirects, "aggregator-reject-forwarding-redirect", s.AggregatorRejectForwardingRedirects,
-		"Aggregator reject forwarding redirect response back to client.")
-
-	fs.StringVar(&s.ServiceAccountSigningKeyFile, "service-account-signing-key-file", s.ServiceAccountSigningKeyFile, ""+
-		"Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key.")
-
 	return fss
 }

cmd/kube-apiserver/app/options/options_test.go

@@ -37,6 +37,7 @@ import (
 	"k8s.io/component-base/logs"
 	"k8s.io/component-base/metrics"
 	kapi "k8s.io/kubernetes/pkg/apis/core"
+	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 	"k8s.io/kubernetes/pkg/controlplane/reconcilers"
 	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
 	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
@@ -44,7 +45,7 @@ import (
 )
 
 func TestAddFlags(t *testing.T) {
-	fs := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError)
+	fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
 	s := NewServerRunOptions()
 	for _, f := range s.Flags().FlagSets {
 		fs.AddFlagSet(f)
@@ -125,11 +126,8 @@ func TestAddFlags(t *testing.T) {
 
 	// This is a snapshot of expected options parsed by args.
 	expected := &ServerRunOptions{
-		ServiceNodePortRange:   kubeoptions.DefaultServiceNodePortRange,
-		ServiceClusterIPRanges: (&net.IPNet{IP: netutils.ParseIPSloppy("192.168.128.0"), Mask: net.CIDRMask(17, 32)}).String(),
+		Options: &controlplaneapiserver.Options{
 			MasterCount: 5,
-		EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
-		AllowPrivileged:        false,
 			GenericServerRunOptions: &apiserveroptions.ServerRunOptions{
 				AdvertiseAddress:            netutils.ParseIPSloppy("192.168.10.10"),
 				CorsAllowedOriginList:       []string{"10.10.10.100", "10.10.10.200"},
@@ -142,12 +140,12 @@ func TestAddFlags(t *testing.T) {
 			},
 			Admission: &kubeoptions.AdmissionOptions{
 				GenericAdmission: &apiserveroptions.AdmissionOptions{
-				RecommendedPluginOrder: s.Admission.GenericAdmission.RecommendedPluginOrder,
-				DefaultOffPlugins:      s.Admission.GenericAdmission.DefaultOffPlugins,
+					RecommendedPluginOrder: s.Options.Admission.GenericAdmission.RecommendedPluginOrder,
+					DefaultOffPlugins:      s.Options.Admission.GenericAdmission.DefaultOffPlugins,
 					EnablePlugins:          []string{"AlwaysDeny"},
 					ConfigFile:             "/admission-control-config",
-				Plugins:                s.Admission.GenericAdmission.Plugins,
-				Decorators:             s.Admission.GenericAdmission.Decorators,
+					Plugins:                s.Options.Admission.GenericAdmission.Plugins,
+					Decorators:             s.Options.Admission.GenericAdmission.Decorators,
 				},
 			},
 			Etcd: &apiserveroptions.EtcdOptions{
@@ -189,23 +187,6 @@ func TestAddFlags(t *testing.T) {
 				Required:                     true,
 			}).WithLoopback(),
 			EventTTL: 1 * time.Hour,
-		KubeletConfig: kubeletclient.KubeletClientConfig{
-			Port:         10250,
-			ReadOnlyPort: 10255,
-			PreferredAddressTypes: []string{
-				string(kapi.NodeHostName),
-				string(kapi.NodeInternalDNS),
-				string(kapi.NodeInternalIP),
-				string(kapi.NodeExternalDNS),
-				string(kapi.NodeExternalIP),
-			},
-			HTTPTimeout: time.Duration(5) * time.Second,
-			TLSClientConfig: kubeletclient.KubeletTLSConfig{
-				CertFile: "/var/run/kubernetes/ceserver.crt",
-				KeyFile:  "/var/run/kubernetes/server.key",
-				CAFile:   "/var/run/kubernetes/caserver.crt",
-			},
-		},
 			Audit: &apiserveroptions.AuditOptions{
 				LogOptions: apiserveroptions.AuditLogOptions{
 					Path:       "/var/log",
@@ -299,10 +280,6 @@ func TestAddFlags(t *testing.T) {
 				WebhookVersion:              "v1beta1",
 				WebhookRetryBackoff:         apiserveroptions.DefaultAuthWebhookRetryBackoff(),
 			},
-		CloudProvider: &kubeoptions.CloudProviderOptions{
-			CloudConfigFile: "/cloud-config",
-			CloudProvider:   "azure",
-		},
 			APIEnablement: &apiserveroptions.APIEnablementOptions{
 				RuntimeConfig: cliflag.ConfigurationMap{},
 			},
@@ -319,6 +296,35 @@ func TestAddFlags(t *testing.T) {
 				ConfigFile: "/var/run/kubernetes/tracing_config.yaml",
 			},
 			AggregatorRejectForwardingRedirects: true,
+		},
+
+		Extra: Extra{
+			ServiceNodePortRange:   kubeoptions.DefaultServiceNodePortRange,
+			ServiceClusterIPRanges: (&net.IPNet{IP: netutils.ParseIPSloppy("192.168.128.0"), Mask: net.CIDRMask(17, 32)}).String(),
+			EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
+			AllowPrivileged:        false,
+			KubeletConfig: kubeletclient.KubeletClientConfig{
+				Port:         10250,
+				ReadOnlyPort: 10255,
+				PreferredAddressTypes: []string{
+					string(kapi.NodeHostName),
+					string(kapi.NodeInternalDNS),
+					string(kapi.NodeInternalIP),
+					string(kapi.NodeExternalDNS),
+					string(kapi.NodeExternalIP),
+				},
+				HTTPTimeout: time.Duration(5) * time.Second,
+				TLSClientConfig: kubeletclient.KubeletTLSConfig{
+					CertFile: "/var/run/kubernetes/ceserver.crt",
+					KeyFile:  "/var/run/kubernetes/server.key",
+					CAFile:   "/var/run/kubernetes/caserver.crt",
+				},
+			},
+		},
+		CloudProvider: &kubeoptions.CloudProviderOptions{
+			CloudConfigFile: "/cloud-config",
+			CloudProvider:   "azure",
+		},
 	}
 
 	if !reflect.DeepEqual(expected, s) {

cmd/kube-apiserver/app/options/validation.go

@@ -22,18 +22,14 @@ import (
 	"net"
 	"strings"
 
-	apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
-	genericfeatures "k8s.io/apiserver/pkg/features"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
-	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/features"
 	netutils "k8s.io/utils/net"
 )
 
 // TODO: Longer term we should read this from some config store, rather than a flag.
 // validateClusterIPFlags is expected to be called after Complete()
-func validateClusterIPFlags(options *ServerRunOptions) []error {
+func validateClusterIPFlags(options Extra) []error {
 	var errs []error
 	// maxCIDRBits is used to define the maximum CIDR size for the cluster ip(s)
 	maxCIDRBits := 20
@@ -93,7 +89,7 @@ func validateMaxCIDRRange(cidr net.IPNet, maxCIDRBits int, cidrFlag string) erro
 	return nil
 }
 
-func validateServiceNodePort(options *ServerRunOptions) []error {
+func validateServiceNodePort(options Extra) []error {
 	var errs []error
 
 	if options.KubernetesServiceNodePort < 0 || options.KubernetesServiceNodePort > 65535 {
@@ -106,64 +102,13 @@ func validateServiceNodePort(options *ServerRunOptions) []error {
 	return errs
 }
 
-func validateTokenRequest(options *ServerRunOptions) []error {
-	var errs []error
-
-	enableAttempted := options.ServiceAccountSigningKeyFile != "" ||
-		(len(options.Authentication.ServiceAccounts.Issuers) != 0 && options.Authentication.ServiceAccounts.Issuers[0] != "") ||
-		len(options.Authentication.APIAudiences) != 0
-
-	enableSucceeded := options.ServiceAccountIssuer != nil
-
-	if !enableAttempted {
-		errs = append(errs, errors.New("--service-account-signing-key-file and --service-account-issuer are required flags"))
-	}
-
-	if enableAttempted && !enableSucceeded {
-		errs = append(errs, errors.New("--service-account-signing-key-file, --service-account-issuer, and --api-audiences should be specified together"))
-	}
-
-	return errs
-}
-
-func validateAPIPriorityAndFairness(options *ServerRunOptions) []error {
-	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) && options.GenericServerRunOptions.EnablePriorityAndFairness {
-		// If none of the following runtime config options are specified,
-		// APF is assumed to be turned on. The internal APF controller uses
-		// v1beta3 so it should be enabled.
-		enabledAPIString := options.APIEnablement.RuntimeConfig.String()
-		testConfigs := []string{"flowcontrol.apiserver.k8s.io/v1beta3", "api/beta", "api/all"} // in the order of precedence
-		for _, testConfig := range testConfigs {
-			if strings.Contains(enabledAPIString, fmt.Sprintf("%s=false", testConfig)) {
-				return []error{fmt.Errorf("--runtime-config=%s=false conflicts with --enable-priority-and-fairness=true and --feature-gates=APIPriorityAndFairness=true", testConfig)}
-			}
-			if strings.Contains(enabledAPIString, fmt.Sprintf("%s=true", testConfig)) {
-				return nil
-			}
-		}
-	}
-
-	return nil
-}
-
 // Validate checks ServerRunOptions and return a slice of found errs.
-func (s *ServerRunOptions) Validate() []error {
+func (s CompletedOptions) Validate() []error {
 	var errs []error
-	if s.MasterCount <= 0 {
-		errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount))
-	}
-	errs = append(errs, s.Etcd.Validate()...)
-	errs = append(errs, validateClusterIPFlags(s)...)
-	errs = append(errs, validateServiceNodePort(s)...)
-	errs = append(errs, validateAPIPriorityAndFairness(s)...)
-	errs = append(errs, s.SecureServing.Validate()...)
-	errs = append(errs, s.Authentication.Validate()...)
-	errs = append(errs, s.Authorization.Validate()...)
-	errs = append(errs, s.Audit.Validate()...)
-	errs = append(errs, s.Admission.Validate()...)
-	errs = append(errs, s.APIEnablement.Validate(legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme)...)
-	errs = append(errs, validateTokenRequest(s)...)
-	errs = append(errs, s.Metrics.Validate()...)
+
+	errs = append(errs, s.CompletedOptions.Validate()...)
+	errs = append(errs, validateClusterIPFlags(s.Extra)...)
+	errs = append(errs, validateServiceNodePort(s.Extra)...)
 
 	return errs
 }

cmd/kube-apiserver/app/options/validation_test.go

@@ -18,17 +18,12 @@ package options
 
 import (
 	"net"
-	"strings"
 	"testing"
 
 	utilnet "k8s.io/apimachinery/pkg/util/net"
-	kubeapiserveradmission "k8s.io/apiserver/pkg/admission"
-	genericoptions "k8s.io/apiserver/pkg/server/options"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
-	basemetrics "k8s.io/component-base/metrics"
 	"k8s.io/kubernetes/pkg/features"
-	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
 	netutils "k8s.io/utils/net"
 )
 
@@ -53,9 +48,11 @@ func makeOptionsWithCIDRs(serviceCIDR string, secondaryServiceCIDR string) *Serv
 		}
 	}
 	return &ServerRunOptions{
+		Extra: Extra{
 			ServiceClusterIPRanges:         value,
 			PrimaryServiceClusterIPRange:   primaryCIDR,
 			SecondaryServiceClusterIPRange: secondaryCIDR,
+		},
 	}
 }
 
@@ -146,7 +143,7 @@ func TestClusterServiceIPRange(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MultiCIDRServiceAllocator, tc.gate)()
 
-			errs := validateClusterIPFlags(tc.options)
+			errs := validateClusterIPFlags(tc.options.Extra)
 			if len(errs) > 0 && !tc.expectErrors {
 				t.Errorf("expected no errors, errors found %+v", errs)
 			}
@@ -203,9 +200,9 @@ func TestValidateServiceNodePort(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			err := validateServiceNodePort(tc.options)
-			if err != nil && !tc.expectErrors {
-				t.Errorf("expected no errors, error found %+v", err)
+			errs := validateServiceNodePort(tc.options.Extra)
+			if errs != nil && !tc.expectErrors {
+				t.Errorf("expected no errors, error found %+v", errs)
 			}
 		})
 	}
@@ -217,8 +214,10 @@ func makeOptionsWithPort(kubernetesServiceNodePort int, base int, size int) *Ser
 		Size: size,
 	}
 	return &ServerRunOptions{
+		Extra: Extra{
 			ServiceNodePortRange:      portRange,
 			KubernetesServiceNodePort: kubernetesServiceNodePort,
+		},
 	}
 }
 
@@ -283,144 +282,3 @@ func TestValidateMaxCIDRRange(t *testing.T) {
 		})
 	}
 }
-
-func TestValidateAPIPriorityAndFairness(t *testing.T) {
-	const conflict = "conflicts with --enable-priority-and-fairness=true and --feature-gates=APIPriorityAndFairness=true"
-	tests := []struct {
-		runtimeConfig    string
-		errShouldContain string
-	}{
-		{
-			runtimeConfig:    "api/all=false",
-			errShouldContain: conflict,
-		},
-		{
-			runtimeConfig:    "api/beta=false",
-			errShouldContain: conflict,
-		},
-		{
-			runtimeConfig:    "flowcontrol.apiserver.k8s.io/v1beta1=false",
-			errShouldContain: "",
-		},
-		{
-			runtimeConfig:    "flowcontrol.apiserver.k8s.io/v1beta2=false",
-			errShouldContain: "",
-		},
-		{
-			runtimeConfig:    "flowcontrol.apiserver.k8s.io/v1beta3=false",
-			errShouldContain: conflict,
-		},
-		{
-			runtimeConfig:    "flowcontrol.apiserver.k8s.io/v1beta3=true",
-			errShouldContain: "",
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.runtimeConfig, func(t *testing.T) {
-			options := &ServerRunOptions{
-				GenericServerRunOptions: &genericoptions.ServerRunOptions{
-					EnablePriorityAndFairness: true,
-				},
-				APIEnablement: genericoptions.NewAPIEnablementOptions(),
-			}
-			options.APIEnablement.RuntimeConfig.Set(test.runtimeConfig)
-
-			var errMessageGot string
-			if errs := validateAPIPriorityAndFairness(options); len(errs) > 0 {
-				errMessageGot = errs[0].Error()
-			}
-			if !strings.Contains(errMessageGot, test.errShouldContain) {
-				t.Errorf("Expected error message to contain: %q, but got: %q", test.errShouldContain, errMessageGot)
-			}
-		})
-	}
-}
-
-func TestValidateServerRunOptions(t *testing.T) {
-	cidrOpts := makeOptionsWithCIDRs("10.0.0.0/16", "3000::/64")
-	nodePortOpts := makeOptionsWithPort(-1, 30065, 1)
-
-	testCases := []struct {
-		name         string
-		options      *ServerRunOptions
-		expectErrors bool
-	}{
-		{
-			name:         "validate master count equal 0",
-			expectErrors: true,
-			options: &ServerRunOptions{
-				MasterCount:             0,
-				GenericServerRunOptions: &genericoptions.ServerRunOptions{},
-				Etcd:                    &genericoptions.EtcdOptions{},
-				SecureServing:           &genericoptions.SecureServingOptionsWithLoopback{},
-				Audit:                   &genericoptions.AuditOptions{},
-				Admission: &kubeoptions.AdmissionOptions{
-					GenericAdmission: &genericoptions.AdmissionOptions{
-						EnablePlugins: []string{"foo"},
-						Plugins:       kubeapiserveradmission.NewPlugins(),
-					},
-					PluginNames: []string{"foo"},
-				},
-				Authentication: &kubeoptions.BuiltInAuthenticationOptions{
-					APIAudiences: []string{"bar"},
-					ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{
-						Issuers: []string{"baz"},
-					},
-				},
-				Authorization:                  &kubeoptions.BuiltInAuthorizationOptions{},
-				APIEnablement:                  genericoptions.NewAPIEnablementOptions(),
-				Metrics:                        &basemetrics.Options{},
-				ServiceClusterIPRanges:         cidrOpts.ServiceClusterIPRanges,
-				PrimaryServiceClusterIPRange:   cidrOpts.PrimaryServiceClusterIPRange,
-				SecondaryServiceClusterIPRange: cidrOpts.SecondaryServiceClusterIPRange,
-				ServiceNodePortRange:           nodePortOpts.ServiceNodePortRange,
-				KubernetesServiceNodePort:      nodePortOpts.KubernetesServiceNodePort,
-				ServiceAccountSigningKeyFile:   "",
-			},
-		},
-		{
-			name:         "validate token request enable not attempted",
-			expectErrors: true,
-			options: &ServerRunOptions{
-				MasterCount:             1,
-				GenericServerRunOptions: &genericoptions.ServerRunOptions{},
-				Etcd:                    &genericoptions.EtcdOptions{},
-				SecureServing:           &genericoptions.SecureServingOptionsWithLoopback{},
-				Audit:                   &genericoptions.AuditOptions{},
-				Admission: &kubeoptions.AdmissionOptions{
-					GenericAdmission: &genericoptions.AdmissionOptions{
-						EnablePlugins: []string{""},
-						Plugins:       kubeapiserveradmission.NewPlugins(),
-					},
-					PluginNames: []string{""},
-				},
-				Authentication: &kubeoptions.BuiltInAuthenticationOptions{
-					ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{},
-				},
-				Authorization:                  &kubeoptions.BuiltInAuthorizationOptions{},
-				APIEnablement:                  genericoptions.NewAPIEnablementOptions(),
-				Metrics:                        &basemetrics.Options{},
-				ServiceClusterIPRanges:         cidrOpts.ServiceClusterIPRanges,
-				PrimaryServiceClusterIPRange:   cidrOpts.PrimaryServiceClusterIPRange,
-				SecondaryServiceClusterIPRange: cidrOpts.SecondaryServiceClusterIPRange,
-				ServiceNodePortRange:           nodePortOpts.ServiceNodePortRange,
-				KubernetesServiceNodePort:      nodePortOpts.KubernetesServiceNodePort,
-				ServiceAccountSigningKeyFile:   "",
-			},
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			errs := tc.options.Validate()
-			if len(errs) > 0 && !tc.expectErrors {
-				t.Errorf("expected no errors, errors found %+v", errs)
-			}
-
-			if len(errs) == 0 && tc.expectErrors {
-				t.Errorf("expected errors, no errors found")
-			}
-		})
-	}
-}

cmd/kube-apiserver/app/server.go

@@ -105,7 +105,7 @@ cluster's shared state through which all other components interact.`,
 			cliflag.PrintFlags(fs)
 
 			// set default options
-			completedOptions, err := options.Complete(s)
+			completedOptions, err := s.Complete()
 			if err != nil {
 				return err
 			}
@@ -217,7 +217,7 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
 	proxyTransport := CreateProxyTransport()
 
 	genericConfig, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig(
-		opts.ServerRunOptions,
+		opts.CompletedOptions,
 		[]*runtime.Scheme{legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme},
 		generatedopenapi.GetOpenAPIDefinitions,
 	)

cmd/kube-apiserver/app/testing/testserver.go

@@ -236,7 +236,7 @@ func StartTestServer(t Logger, instanceOptions *TestServerInstanceOptions, custo
 	s.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"}
 	s.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()}
 
-	completedOptions, err := options.Complete(s)
+	completedOptions, err := s.Complete()
 	if err != nil {
 		return result, fmt.Errorf("failed to set default ServerRunOptions: %v", err)
 	}

pkg/controlplane/apiserver/.import-restrictions

@@ -1,12 +0,0 @@
-rules:
-  - selectorRegexp: k8s[.]io/kubernetes
-    allowedPrefixes:
-      - ''
-    forbiddenPrefixes:
-      # prevent pkg/ from depending on cmd/
-      #   note: pkg/kubemark overrides this
-      # - k8s.io/kubernetes/cmd # temporarily disabled until options are split and moved
-      # use sigs.k8s.io/yaml instead
-      - github.com/ghodss/yaml
-      # prevent kubernetes from opening sctp sockets (ref: https://github.com/kubernetes/kubernetes/pull/87926#discussion_r376642015)
-      - github.com/ishidawataru/sctp

pkg/controlplane/apiserver/apiextensions.go

@@ -30,14 +30,14 @@ import (
 	"k8s.io/apiserver/pkg/util/webhook"
 	"k8s.io/client-go/informers"
 
-	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
+	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 )
 
 func CreateAPIExtensionsConfig(
 	kubeAPIServerConfig server.Config,
 	kubeInformers informers.SharedInformerFactory,
 	pluginInitializers []admission.PluginInitializer,
-	commandOptions *options.ServerRunOptions,
+	commandOptions controlplaneapiserver.CompletedOptions,
 	masterCount int,
 	serviceResolver webhook.ServiceResolver,
 	authResolverWrapper webhook.AuthenticationInfoResolverWrapper,

pkg/controlplane/apiserver/config.go

@@ -40,9 +40,9 @@ import (
 	"k8s.io/component-base/version"
 	openapicommon "k8s.io/kube-openapi/pkg/common"
 
-	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/controlplane"
+	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 	"k8s.io/kubernetes/pkg/kubeapiserver"
 	"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
 	rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest"
@@ -50,7 +50,7 @@ import (
 
 // BuildGenericConfig takes the master server options and produces the genericapiserver.Config associated with it
 func BuildGenericConfig(
-	s *options.ServerRunOptions,
+	s controlplaneapiserver.CompletedOptions,
 	schemes []*runtime.Scheme,
 	getOpenAPIDefinitions func(ref openapicommon.ReferenceCallback) map[string]openapicommon.OpenAPIDefinition,
 ) (
@@ -167,7 +167,7 @@ func BuildGenericConfig(
 }
 
 // BuildAuthorizer constructs the authorizer
-func BuildAuthorizer(s *options.ServerRunOptions, EgressSelector *egressselector.EgressSelector, versionedInformers clientgoinformers.SharedInformerFactory) (authorizer.Authorizer, authorizer.RuleResolver, error) {
+func BuildAuthorizer(s controlplaneapiserver.CompletedOptions, EgressSelector *egressselector.EgressSelector, versionedInformers clientgoinformers.SharedInformerFactory) (authorizer.Authorizer, authorizer.RuleResolver, error) {
 	authorizationConfig := s.Authorization.ToAuthorizationConfig(versionedInformers)
 
 	if EgressSelector != nil {
@@ -182,7 +182,7 @@ func BuildAuthorizer(s *options.ServerRunOptions, EgressSelector *egressselector
 }
 
 // BuildPriorityAndFairness constructs the guts of the API Priority and Fairness filter
-func BuildPriorityAndFairness(s *options.ServerRunOptions, extclient clientgoclientset.Interface, versionedInformer clientgoinformers.SharedInformerFactory) (utilflowcontrol.Interface, error) {
+func BuildPriorityAndFairness(s controlplaneapiserver.CompletedOptions, extclient clientgoclientset.Interface, versionedInformer clientgoinformers.SharedInformerFactory) (utilflowcontrol.Interface, error) {
 	if s.GenericServerRunOptions.MaxRequestsInFlight+s.GenericServerRunOptions.MaxMutatingRequestsInFlight <= 0 {
 		return nil, fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", s.GenericServerRunOptions.MaxRequestsInFlight, s.GenericServerRunOptions.MaxMutatingRequestsInFlight)
 	}

pkg/controlplane/apiserver/options/options.go

@@ -0,0 +1,292 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package options contains flags and options for initializing an apiserver
+package options
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"strings"
+	"time"
+
+	genericoptions "k8s.io/apiserver/pkg/server/options"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
+	"k8s.io/client-go/util/keyutil"
+	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/component-base/logs"
+	logsapi "k8s.io/component-base/logs/api/v1"
+	"k8s.io/component-base/metrics"
+	"k8s.io/klog/v2"
+	"k8s.io/utils/integer"
+	netutil "k8s.io/utils/net"
+
+	_ "k8s.io/kubernetes/pkg/features"
+	kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator"
+	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
+	"k8s.io/kubernetes/pkg/serviceaccount"
+)
+
+type Options struct {
+	GenericServerRunOptions *genericoptions.ServerRunOptions
+	Etcd                    *genericoptions.EtcdOptions
+	SecureServing           *genericoptions.SecureServingOptionsWithLoopback
+	Audit                   *genericoptions.AuditOptions
+	Features                *genericoptions.FeatureOptions
+	Admission               *kubeoptions.AdmissionOptions
+	Authentication          *kubeoptions.BuiltInAuthenticationOptions
+	Authorization           *kubeoptions.BuiltInAuthorizationOptions
+	APIEnablement           *genericoptions.APIEnablementOptions
+	EgressSelector          *genericoptions.EgressSelectorOptions
+	Metrics                 *metrics.Options
+	Logs                    *logs.Options
+	Traces                  *genericoptions.TracingOptions
+
+	EnableLogsHandler        bool
+	EventTTL                 time.Duration
+	MaxConnectionBytesPerSec int64
+
+	ProxyClientCertFile string
+	ProxyClientKeyFile  string
+
+	EnableAggregatorRouting             bool
+	AggregatorRejectForwardingRedirects bool
+
+	MasterCount int
+
+	ServiceAccountSigningKeyFile     string
+	ServiceAccountIssuer             serviceaccount.TokenGenerator
+	ServiceAccountTokenMaxExpiration time.Duration
+
+	ShowHiddenMetricsForVersion string
+}
+
+// completedServerRunOptions is a private wrapper that enforces a call of Complete() before Run can be invoked.
+type completedOptions struct {
+	Options
+}
+
+type CompletedOptions struct {
+	// Embed a private pointer that cannot be instantiated outside of this package.
+	*completedOptions
+}
+
+// NewOptions creates a new ServerRunOptions object with default parameters
+func NewOptions() *Options {
+	s := Options{
+		GenericServerRunOptions: genericoptions.NewServerRunOptions(),
+		Etcd:                    genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)),
+		SecureServing:           kubeoptions.NewSecureServingOptions(),
+		Audit:                   genericoptions.NewAuditOptions(),
+		Features:                genericoptions.NewFeatureOptions(),
+		Admission:               kubeoptions.NewAdmissionOptions(),
+		Authentication:          kubeoptions.NewBuiltInAuthenticationOptions().WithAll(),
+		Authorization:           kubeoptions.NewBuiltInAuthorizationOptions(),
+		APIEnablement:           genericoptions.NewAPIEnablementOptions(),
+		EgressSelector:          genericoptions.NewEgressSelectorOptions(),
+		Metrics:                 metrics.NewOptions(),
+		Logs:                    logs.NewOptions(),
+		Traces:                  genericoptions.NewTracingOptions(),
+
+		EnableLogsHandler:                   true,
+		EventTTL:                            1 * time.Hour,
+		MasterCount:                         1,
+		AggregatorRejectForwardingRedirects: true,
+	}
+
+	// Overwrite the default for storage data format.
+	s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf"
+
+	return &s
+}
+
+func (s *Options) AddFlags(fss *cliflag.NamedFlagSets) {
+	// Add the generic flags.
+	s.GenericServerRunOptions.AddUniversalFlags(fss.FlagSet("generic"))
+	s.Etcd.AddFlags(fss.FlagSet("etcd"))
+	s.SecureServing.AddFlags(fss.FlagSet("secure serving"))
+	s.Audit.AddFlags(fss.FlagSet("auditing"))
+	s.Features.AddFlags(fss.FlagSet("features"))
+	s.Authentication.AddFlags(fss.FlagSet("authentication"))
+	s.Authorization.AddFlags(fss.FlagSet("authorization"))
+	s.APIEnablement.AddFlags(fss.FlagSet("API enablement"))
+	s.EgressSelector.AddFlags(fss.FlagSet("egress selector"))
+	s.Admission.AddFlags(fss.FlagSet("admission"))
+	s.Metrics.AddFlags(fss.FlagSet("metrics"))
+	logsapi.AddFlags(s.Logs, fss.FlagSet("logs"))
+	s.Traces.AddFlags(fss.FlagSet("traces"))
+
+	// Note: the weird ""+ in below lines seems to be the only way to get gofmt to
+	// arrange these text blocks sensibly. Grrr.
+	fs := fss.FlagSet("misc")
+	fs.DurationVar(&s.EventTTL, "event-ttl", s.EventTTL,
+		"Amount of time to retain events.")
+
+	fs.BoolVar(&s.EnableLogsHandler, "enable-logs-handler", s.EnableLogsHandler,
+		"If true, install a /logs handler for the apiserver logs.")
+	fs.MarkDeprecated("enable-logs-handler", "This flag will be removed in v1.19")
+
+	fs.Int64Var(&s.MaxConnectionBytesPerSec, "max-connection-bytes-per-sec", s.MaxConnectionBytesPerSec, ""+
+		"If non-zero, throttle each user connection to this number of bytes/sec. "+
+		"Currently only applies to long-running requests.")
+
+	fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount,
+		"The number of apiservers running in the cluster, must be a positive number. (In use when --endpoint-reconciler-type=master-count is enabled.)")
+	fs.MarkDeprecated("apiserver-count", "apiserver-count is deprecated and will be removed in a future version.")
+
+	fs.StringVar(&s.ProxyClientCertFile, "proxy-client-cert-file", s.ProxyClientCertFile, ""+
+		"Client certificate used to prove the identity of the aggregator or kube-apiserver "+
+		"when it must call out during a request. This includes proxying requests to a user "+
+		"api-server and calling out to webhook admission plugins. It is expected that this "+
+		"cert includes a signature from the CA in the --requestheader-client-ca-file flag. "+
+		"That CA is published in the 'extension-apiserver-authentication' configmap in "+
+		"the kube-system namespace. Components receiving calls from kube-aggregator should "+
+		"use that CA to perform their half of the mutual TLS verification.")
+	fs.StringVar(&s.ProxyClientKeyFile, "proxy-client-key-file", s.ProxyClientKeyFile, ""+
+		"Private key for the client certificate used to prove the identity of the aggregator or kube-apiserver "+
+		"when it must call out during a request. This includes proxying requests to a user "+
+		"api-server and calling out to webhook admission plugins.")
+
+	fs.BoolVar(&s.EnableAggregatorRouting, "enable-aggregator-routing", s.EnableAggregatorRouting,
+		"Turns on aggregator routing requests to endpoints IP rather than cluster IP.")
+
+	fs.BoolVar(&s.AggregatorRejectForwardingRedirects, "aggregator-reject-forwarding-redirect", s.AggregatorRejectForwardingRedirects,
+		"Aggregator reject forwarding redirect response back to client.")
+
+	fs.StringVar(&s.ServiceAccountSigningKeyFile, "service-account-signing-key-file", s.ServiceAccountSigningKeyFile, ""+
+		"Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key.")
+}
+
+func (o *Options) Complete(alternateDNS []string, alternateIPs []net.IP) (CompletedOptions, error) {
+	if o == nil {
+		return CompletedOptions{completedOptions: &completedOptions{}}, nil
+	}
+
+	completed := completedOptions{
+		Options: *o,
+	}
+
+	// set defaults
+	if err := completed.GenericServerRunOptions.DefaultAdvertiseAddress(completed.SecureServing.SecureServingOptions); err != nil {
+		return CompletedOptions{}, err
+	}
+
+	if err := completed.SecureServing.MaybeDefaultWithSelfSignedCerts(completed.GenericServerRunOptions.AdvertiseAddress.String(), alternateDNS, alternateIPs); err != nil {
+		return CompletedOptions{}, fmt.Errorf("error creating self-signed certificates: %v", err)
+	}
+
+	if len(completed.GenericServerRunOptions.ExternalHost) == 0 {
+		if len(completed.GenericServerRunOptions.AdvertiseAddress) > 0 {
+			completed.GenericServerRunOptions.ExternalHost = completed.GenericServerRunOptions.AdvertiseAddress.String()
+		} else {
+			hostname, err := os.Hostname()
+			if err != nil {
+				return CompletedOptions{}, fmt.Errorf("error finding host name: %v", err)
+			}
+			completed.GenericServerRunOptions.ExternalHost = hostname
+		}
+		klog.Infof("external host was not specified, using %v", completed.GenericServerRunOptions.ExternalHost)
+	}
+
+	completed.Authentication.ApplyAuthorization(completed.Authorization)
+
+	// Use (ServiceAccountSigningKeyFile != "") as a proxy to the user enabling
+	// TokenRequest functionality. This defaulting was convenient, but messed up
+	// a lot of people when they rotated their serving cert with no idea it was
+	// connected to their service account keys. We are taking this opportunity to
+	// remove this problematic defaulting.
+	if completed.ServiceAccountSigningKeyFile == "" {
+		// Default to the private server key for service account token signing
+		if len(completed.Authentication.ServiceAccounts.KeyFiles) == 0 && completed.SecureServing.ServerCert.CertKey.KeyFile != "" {
+			if kubeauthenticator.IsValidServiceAccountKeyFile(completed.SecureServing.ServerCert.CertKey.KeyFile) {
+				completed.Authentication.ServiceAccounts.KeyFiles = []string{completed.SecureServing.ServerCert.CertKey.KeyFile}
+			} else {
+				klog.Warning("No TLS key provided, service account token authentication disabled")
+			}
+		}
+	}
+
+	if completed.ServiceAccountSigningKeyFile != "" && len(completed.Authentication.ServiceAccounts.Issuers) != 0 && completed.Authentication.ServiceAccounts.Issuers[0] != "" {
+		sk, err := keyutil.PrivateKeyFromFile(completed.ServiceAccountSigningKeyFile)
+		if err != nil {
+			return CompletedOptions{}, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err)
+		}
+		if completed.Authentication.ServiceAccounts.MaxExpiration != 0 {
+			lowBound := time.Hour
+			upBound := time.Duration(1<<32) * time.Second
+			if completed.Authentication.ServiceAccounts.MaxExpiration < lowBound ||
+				completed.Authentication.ServiceAccounts.MaxExpiration > upBound {
+				return CompletedOptions{}, fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds")
+			}
+			if completed.Authentication.ServiceAccounts.ExtendExpiration {
+				if completed.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.WarnOnlyBoundTokenExpirationSeconds*time.Second {
+					klog.Warningf("service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, service-account-max-token-expiration must be set longer than %d seconds (currently %s)", serviceaccount.WarnOnlyBoundTokenExpirationSeconds, completed.Authentication.ServiceAccounts.MaxExpiration)
+				}
+				if completed.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.ExpirationExtensionSeconds*time.Second {
+					klog.Warningf("service-account-extend-token-expiration is true, enabling tokens valid up to %d seconds, which is longer than service-account-max-token-expiration set to %s seconds", serviceaccount.ExpirationExtensionSeconds, completed.Authentication.ServiceAccounts.MaxExpiration)
+				}
+			}
+		}
+
+		completed.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(completed.Authentication.ServiceAccounts.Issuers[0], sk)
+		if err != nil {
+			return CompletedOptions{}, fmt.Errorf("failed to build token generator: %v", err)
+		}
+		completed.ServiceAccountTokenMaxExpiration = completed.Authentication.ServiceAccounts.MaxExpiration
+	}
+
+	for key, value := range completed.APIEnablement.RuntimeConfig {
+		if key == "v1" || strings.HasPrefix(key, "v1/") ||
+			key == "api/v1" || strings.HasPrefix(key, "api/v1/") {
+			delete(completed.APIEnablement.RuntimeConfig, key)
+			completed.APIEnablement.RuntimeConfig["/v1"] = value
+		}
+		if key == "api/legacy" {
+			delete(completed.APIEnablement.RuntimeConfig, key)
+		}
+	}
+
+	return CompletedOptions{
+		completedOptions: &completed,
+	}, nil
+}
+
+// ServiceIPRange checks if the serviceClusterIPRange flag is nil, raising a warning if so and
+// setting service ip range to the default value in kubeoptions.DefaultServiceIPCIDR
+// for now until the default is removed per the deprecation timeline guidelines.
+// Returns service ip range, api server service IP, and an error
+func ServiceIPRange(passedServiceClusterIPRange net.IPNet) (net.IPNet, net.IP, error) {
+	serviceClusterIPRange := passedServiceClusterIPRange
+	if passedServiceClusterIPRange.IP == nil {
+		klog.Warningf("No CIDR for service cluster IPs specified. Default value which was %s is deprecated and will be removed in future releases. Please specify it using --service-cluster-ip-range on kube-apiserver.", kubeoptions.DefaultServiceIPCIDR.String())
+		serviceClusterIPRange = kubeoptions.DefaultServiceIPCIDR
+	}
+
+	size := integer.Int64Min(netutil.RangeSize(&serviceClusterIPRange), 1<<16)
+	if size < 8 {
+		return net.IPNet{}, net.IP{}, fmt.Errorf("the service cluster IP range must be at least %d IP addresses", 8)
+	}
+
+	// Select the first valid IP from ServiceClusterIPRange to use as the GenericAPIServer service IP.
+	apiServerServiceIP, err := netutil.GetIndexedIP(&serviceClusterIPRange, 1)
+	if err != nil {
+		return net.IPNet{}, net.IP{}, err
+	}
+	klog.V(4).Infof("Setting service IP to %q (read-write).", apiServerServiceIP)
+
+	return serviceClusterIPRange, apiServerServiceIP, nil
+}

pkg/controlplane/apiserver/options/options_test.go

@@ -0,0 +1,291 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/spf13/pflag"
+	oteltrace "go.opentelemetry.io/otel/trace"
+
+	"k8s.io/apiserver/pkg/admission"
+	apiserveroptions "k8s.io/apiserver/pkg/server/options"
+	"k8s.io/apiserver/pkg/storage/etcd3"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
+	auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
+	audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
+	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/component-base/logs"
+	"k8s.io/component-base/metrics"
+	netutils "k8s.io/utils/net"
+
+	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
+)
+
+func TestAddFlags(t *testing.T) {
+	fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
+	s := NewOptions()
+	var fss cliflag.NamedFlagSets
+	s.AddFlags(&fss)
+	for _, f := range fss.FlagSets {
+		fs.AddFlagSet(f)
+	}
+
+	args := []string{
+		"--enable-admission-plugins=AlwaysDeny",
+		"--admission-control-config-file=/admission-control-config",
+		"--advertise-address=192.168.10.10",
+		"--anonymous-auth=false",
+		"--apiserver-count=5",
+		"--audit-log-maxage=11",
+		"--audit-log-maxbackup=12",
+		"--audit-log-maxsize=13",
+		"--audit-log-path=/var/log",
+		"--audit-log-mode=blocking",
+		"--audit-log-batch-buffer-size=46",
+		"--audit-log-batch-max-size=47",
+		"--audit-log-batch-max-wait=48s",
+		"--audit-log-batch-throttle-enable=true",
+		"--audit-log-batch-throttle-qps=49.5",
+		"--audit-log-batch-throttle-burst=50",
+		"--audit-log-truncate-enabled=true",
+		"--audit-log-truncate-max-batch-size=45",
+		"--audit-log-truncate-max-event-size=44",
+		"--audit-log-version=audit.k8s.io/v1",
+		"--audit-policy-file=/policy",
+		"--audit-webhook-config-file=/webhook-config",
+		"--audit-webhook-mode=blocking",
+		"--audit-webhook-batch-buffer-size=42",
+		"--audit-webhook-batch-max-size=43",
+		"--audit-webhook-batch-max-wait=1s",
+		"--audit-webhook-batch-throttle-enable=false",
+		"--audit-webhook-batch-throttle-qps=43.5",
+		"--audit-webhook-batch-throttle-burst=44",
+		"--audit-webhook-truncate-enabled=true",
+		"--audit-webhook-truncate-max-batch-size=43",
+		"--audit-webhook-truncate-max-event-size=42",
+		"--audit-webhook-initial-backoff=2s",
+		"--audit-webhook-version=audit.k8s.io/v1",
+		"--authentication-token-webhook-cache-ttl=3m",
+		"--authentication-token-webhook-config-file=/token-webhook-config",
+		"--authorization-mode=AlwaysDeny,RBAC",
+		"--authorization-policy-file=/policy",
+		"--authorization-webhook-cache-authorized-ttl=3m",
+		"--authorization-webhook-cache-unauthorized-ttl=1m",
+		"--authorization-webhook-config-file=/webhook-config",
+		"--bind-address=192.168.10.20",
+		"--client-ca-file=/client-ca",
+		"--cors-allowed-origins=10.10.10.100,10.10.10.200",
+		"--contention-profiling=true",
+		"--egress-selector-config-file=/var/run/kubernetes/egress-selector/connectivity.yaml",
+		"--enable-aggregator-routing=true",
+		"--enable-priority-and-fairness=false",
+		"--enable-logs-handler=false",
+		"--etcd-keyfile=/var/run/kubernetes/etcd.key",
+		"--etcd-certfile=/var/run/kubernetes/etcdce.crt",
+		"--etcd-cafile=/var/run/kubernetes/etcdca.crt",
+		"--http2-max-streams-per-connection=42",
+		"--tracing-config-file=/var/run/kubernetes/tracing_config.yaml",
+		"--proxy-client-cert-file=/var/run/kubernetes/proxy.crt",
+		"--proxy-client-key-file=/var/run/kubernetes/proxy.key",
+		"--request-timeout=2m",
+		"--storage-backend=etcd3",
+		"--lease-reuse-duration-seconds=100",
+	}
+	fs.Parse(args)
+
+	// This is a snapshot of expected options parsed by args.
+	expected := &Options{
+		MasterCount: 5,
+		GenericServerRunOptions: &apiserveroptions.ServerRunOptions{
+			AdvertiseAddress:            netutils.ParseIPSloppy("192.168.10.10"),
+			CorsAllowedOriginList:       []string{"10.10.10.100", "10.10.10.200"},
+			MaxRequestsInFlight:         400,
+			MaxMutatingRequestsInFlight: 200,
+			RequestTimeout:              time.Duration(2) * time.Minute,
+			MinRequestTimeout:           1800,
+			JSONPatchMaxCopyBytes:       int64(3 * 1024 * 1024),
+			MaxRequestBodyBytes:         int64(3 * 1024 * 1024),
+		},
+		Admission: &kubeoptions.AdmissionOptions{
+			GenericAdmission: &apiserveroptions.AdmissionOptions{
+				RecommendedPluginOrder: s.Admission.GenericAdmission.RecommendedPluginOrder,
+				DefaultOffPlugins:      s.Admission.GenericAdmission.DefaultOffPlugins,
+				EnablePlugins:          []string{"AlwaysDeny"},
+				ConfigFile:             "/admission-control-config",
+				Plugins:                s.Admission.GenericAdmission.Plugins,
+				Decorators:             s.Admission.GenericAdmission.Decorators,
+			},
+		},
+		Etcd: &apiserveroptions.EtcdOptions{
+			StorageConfig: storagebackend.Config{
+				Type: "etcd3",
+				Transport: storagebackend.TransportConfig{
+					ServerList:     nil,
+					KeyFile:        "/var/run/kubernetes/etcd.key",
+					TrustedCAFile:  "/var/run/kubernetes/etcdca.crt",
+					CertFile:       "/var/run/kubernetes/etcdce.crt",
+					TracerProvider: oteltrace.NewNoopTracerProvider(),
+				},
+				Paging:                true,
+				Prefix:                "/registry",
+				CompactionInterval:    storagebackend.DefaultCompactInterval,
+				CountMetricPollPeriod: time.Minute,
+				DBMetricPollInterval:  storagebackend.DefaultDBMetricPollInterval,
+				HealthcheckTimeout:    storagebackend.DefaultHealthcheckTimeout,
+				ReadycheckTimeout:     storagebackend.DefaultReadinessTimeout,
+				LeaseManagerConfig: etcd3.LeaseManagerConfig{
+					ReuseDurationSeconds: 100,
+					MaxObjectCount:       1000,
+				},
+			},
+			DefaultStorageMediaType: "application/vnd.kubernetes.protobuf",
+			DeleteCollectionWorkers: 1,
+			EnableGarbageCollection: true,
+			EnableWatchCache:        true,
+			DefaultWatchCacheSize:   100,
+		},
+		SecureServing: (&apiserveroptions.SecureServingOptions{
+			BindAddress: netutils.ParseIPSloppy("192.168.10.20"),
+			BindPort:    6443,
+			ServerCert: apiserveroptions.GeneratableKeyCert{
+				CertDirectory: "/var/run/kubernetes",
+				PairName:      "apiserver",
+			},
+			HTTP2MaxStreamsPerConnection: 42,
+			Required:                     true,
+		}).WithLoopback(),
+		EventTTL: 1 * time.Hour,
+		Audit: &apiserveroptions.AuditOptions{
+			LogOptions: apiserveroptions.AuditLogOptions{
+				Path:       "/var/log",
+				MaxAge:     11,
+				MaxBackups: 12,
+				MaxSize:    13,
+				Format:     "json",
+				BatchOptions: apiserveroptions.AuditBatchOptions{
+					Mode: "blocking",
+					BatchConfig: auditbuffered.BatchConfig{
+						BufferSize:     46,
+						MaxBatchSize:   47,
+						MaxBatchWait:   48 * time.Second,
+						ThrottleEnable: true,
+						ThrottleQPS:    49.5,
+						ThrottleBurst:  50,
+					},
+				},
+				TruncateOptions: apiserveroptions.AuditTruncateOptions{
+					Enabled: true,
+					TruncateConfig: audittruncate.Config{
+						MaxBatchSize: 45,
+						MaxEventSize: 44,
+					},
+				},
+				GroupVersionString: "audit.k8s.io/v1",
+			},
+			WebhookOptions: apiserveroptions.AuditWebhookOptions{
+				ConfigFile: "/webhook-config",
+				BatchOptions: apiserveroptions.AuditBatchOptions{
+					Mode: "blocking",
+					BatchConfig: auditbuffered.BatchConfig{
+						BufferSize:     42,
+						MaxBatchSize:   43,
+						MaxBatchWait:   1 * time.Second,
+						ThrottleEnable: false,
+						ThrottleQPS:    43.5,
+						ThrottleBurst:  44,
+						AsyncDelegate:  true,
+					},
+				},
+				TruncateOptions: apiserveroptions.AuditTruncateOptions{
+					Enabled: true,
+					TruncateConfig: audittruncate.Config{
+						MaxBatchSize: 43,
+						MaxEventSize: 42,
+					},
+				},
+				InitialBackoff:     2 * time.Second,
+				GroupVersionString: "audit.k8s.io/v1",
+			},
+			PolicyFile: "/policy",
+		},
+		Features: &apiserveroptions.FeatureOptions{
+			EnableProfiling:           true,
+			EnableContentionProfiling: true,
+		},
+		Authentication: &kubeoptions.BuiltInAuthenticationOptions{
+			Anonymous: &kubeoptions.AnonymousAuthenticationOptions{
+				Allow: false,
+			},
+			ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{
+				ClientCA: "/client-ca",
+			},
+			WebHook: &kubeoptions.WebHookAuthenticationOptions{
+				CacheTTL:     180000000000,
+				ConfigFile:   "/token-webhook-config",
+				Version:      "v1beta1",
+				RetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(),
+			},
+			BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{},
+			OIDC: &kubeoptions.OIDCAuthenticationOptions{
+				UsernameClaim: "sub",
+				SigningAlgs:   []string{"RS256"},
+			},
+			RequestHeader: &apiserveroptions.RequestHeaderAuthenticationOptions{},
+			ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{
+				Lookup:           true,
+				ExtendExpiration: true,
+			},
+			TokenFile:            &kubeoptions.TokenFileAuthenticationOptions{},
+			TokenSuccessCacheTTL: 10 * time.Second,
+			TokenFailureCacheTTL: 0,
+		},
+		Authorization: &kubeoptions.BuiltInAuthorizationOptions{
+			Modes:                       []string{"AlwaysDeny", "RBAC"},
+			PolicyFile:                  "/policy",
+			WebhookConfigFile:           "/webhook-config",
+			WebhookCacheAuthorizedTTL:   180000000000,
+			WebhookCacheUnauthorizedTTL: 60000000000,
+			WebhookVersion:              "v1beta1",
+			WebhookRetryBackoff:         apiserveroptions.DefaultAuthWebhookRetryBackoff(),
+		},
+		APIEnablement: &apiserveroptions.APIEnablementOptions{
+			RuntimeConfig: cliflag.ConfigurationMap{},
+		},
+		EgressSelector: &apiserveroptions.EgressSelectorOptions{
+			ConfigFile: "/var/run/kubernetes/egress-selector/connectivity.yaml",
+		},
+		EnableLogsHandler:       false,
+		EnableAggregatorRouting: true,
+		ProxyClientKeyFile:      "/var/run/kubernetes/proxy.key",
+		ProxyClientCertFile:     "/var/run/kubernetes/proxy.crt",
+		Metrics:                 &metrics.Options{},
+		Logs:                    logs.NewOptions(),
+		Traces: &apiserveroptions.TracingOptions{
+			ConfigFile: "/var/run/kubernetes/tracing_config.yaml",
+		},
+		AggregatorRejectForwardingRedirects: true,
+	}
+
+	if !reflect.DeepEqual(expected, s) {
+		t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{})))
+	}
+}

pkg/controlplane/apiserver/options/validation.go

@@ -0,0 +1,90 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
+	genericfeatures "k8s.io/apiserver/pkg/features"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
+
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+)
+
+func validateTokenRequest(options *Options) []error {
+	var errs []error
+
+	enableAttempted := options.ServiceAccountSigningKeyFile != "" ||
+		(len(options.Authentication.ServiceAccounts.Issuers) != 0 && options.Authentication.ServiceAccounts.Issuers[0] != "") ||
+		len(options.Authentication.APIAudiences) != 0
+
+	enableSucceeded := options.ServiceAccountIssuer != nil
+
+	if !enableAttempted {
+		errs = append(errs, errors.New("--service-account-signing-key-file and --service-account-issuer are required flags"))
+	}
+
+	if enableAttempted && !enableSucceeded {
+		errs = append(errs, errors.New("--service-account-signing-key-file, --service-account-issuer, and --api-audiences should be specified together"))
+	}
+
+	return errs
+}
+
+func validateAPIPriorityAndFairness(options *Options) []error {
+	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) && options.GenericServerRunOptions.EnablePriorityAndFairness {
+		// If none of the following runtime config options are specified,
+		// APF is assumed to be turned on. The internal APF controller uses
+		// v1beta3 so it should be enabled.
+		enabledAPIString := options.APIEnablement.RuntimeConfig.String()
+		testConfigs := []string{"flowcontrol.apiserver.k8s.io/v1beta3", "api/beta", "api/all"} // in the order of precedence
+		for _, testConfig := range testConfigs {
+			if strings.Contains(enabledAPIString, fmt.Sprintf("%s=false", testConfig)) {
+				return []error{fmt.Errorf("--runtime-config=%s=false conflicts with --enable-priority-and-fairness=true and --feature-gates=APIPriorityAndFairness=true", testConfig)}
+			}
+			if strings.Contains(enabledAPIString, fmt.Sprintf("%s=true", testConfig)) {
+				return nil
+			}
+		}
+	}
+
+	return nil
+}
+
+// Validate checks Options and return a slice of found errs.
+func (s *Options) Validate() []error {
+	var errs []error
+	if s.MasterCount <= 0 {
+		errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount))
+	}
+	errs = append(errs, s.Etcd.Validate()...)
+	errs = append(errs, validateAPIPriorityAndFairness(s)...)
+	errs = append(errs, s.SecureServing.Validate()...)
+	errs = append(errs, s.Authentication.Validate()...)
+	errs = append(errs, s.Authorization.Validate()...)
+	errs = append(errs, s.Audit.Validate()...)
+	errs = append(errs, s.Admission.Validate()...)
+	errs = append(errs, s.APIEnablement.Validate(legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme)...)
+	errs = append(errs, validateTokenRequest(s)...)
+	errs = append(errs, s.Metrics.Validate()...)
+
+	return errs
+}

pkg/controlplane/apiserver/options/validation_test.go

@@ -0,0 +1,154 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"strings"
+	"testing"
+
+	kubeapiserveradmission "k8s.io/apiserver/pkg/admission"
+	genericoptions "k8s.io/apiserver/pkg/server/options"
+	basemetrics "k8s.io/component-base/metrics"
+
+	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
+)
+
+func TestValidateAPIPriorityAndFairness(t *testing.T) {
+	const conflict = "conflicts with --enable-priority-and-fairness=true and --feature-gates=APIPriorityAndFairness=true"
+	tests := []struct {
+		runtimeConfig    string
+		errShouldContain string
+	}{
+		{
+			runtimeConfig:    "api/all=false",
+			errShouldContain: conflict,
+		},
+		{
+			runtimeConfig:    "api/beta=false",
+			errShouldContain: conflict,
+		},
+		{
+			runtimeConfig:    "flowcontrol.apiserver.k8s.io/v1beta1=false",
+			errShouldContain: "",
+		},
+		{
+			runtimeConfig:    "flowcontrol.apiserver.k8s.io/v1beta2=false",
+			errShouldContain: "",
+		},
+		{
+			runtimeConfig:    "flowcontrol.apiserver.k8s.io/v1beta3=false",
+			errShouldContain: conflict,
+		},
+		{
+			runtimeConfig:    "flowcontrol.apiserver.k8s.io/v1beta3=true",
+			errShouldContain: "",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.runtimeConfig, func(t *testing.T) {
+			options := &Options{
+				GenericServerRunOptions: &genericoptions.ServerRunOptions{
+					EnablePriorityAndFairness: true,
+				},
+				APIEnablement: genericoptions.NewAPIEnablementOptions(),
+			}
+			options.APIEnablement.RuntimeConfig.Set(test.runtimeConfig)
+
+			var errMessageGot string
+			if errs := validateAPIPriorityAndFairness(options); len(errs) > 0 {
+				errMessageGot = errs[0].Error()
+			}
+			if !strings.Contains(errMessageGot, test.errShouldContain) {
+				t.Errorf("Expected error message to contain: %q, but got: %q", test.errShouldContain, errMessageGot)
+			}
+		})
+	}
+}
+
+func TestValidateOptions(t *testing.T) {
+	testCases := []struct {
+		name         string
+		options      *Options
+		expectErrors bool
+	}{
+		{
+			name:         "validate master count equal 0",
+			expectErrors: true,
+			options: &Options{
+				MasterCount:             0,
+				GenericServerRunOptions: &genericoptions.ServerRunOptions{},
+				Etcd:                    &genericoptions.EtcdOptions{},
+				SecureServing:           &genericoptions.SecureServingOptionsWithLoopback{},
+				Audit:                   &genericoptions.AuditOptions{},
+				Admission: &kubeoptions.AdmissionOptions{
+					GenericAdmission: &genericoptions.AdmissionOptions{
+						EnablePlugins: []string{"foo"},
+						Plugins:       kubeapiserveradmission.NewPlugins(),
+					},
+					PluginNames: []string{"foo"},
+				},
+				Authentication: &kubeoptions.BuiltInAuthenticationOptions{
+					APIAudiences: []string{"bar"},
+					ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{
+						Issuers: []string{"baz"},
+					},
+				},
+				APIEnablement:                genericoptions.NewAPIEnablementOptions(),
+				Metrics:                      &basemetrics.Options{},
+				ServiceAccountSigningKeyFile: "",
+			},
+		},
+		{
+			name:         "validate token request enable not attempted",
+			expectErrors: true,
+			options: &Options{
+				MasterCount:             1,
+				GenericServerRunOptions: &genericoptions.ServerRunOptions{},
+				Etcd:                    &genericoptions.EtcdOptions{},
+				SecureServing:           &genericoptions.SecureServingOptionsWithLoopback{},
+				Audit:                   &genericoptions.AuditOptions{},
+				Admission: &kubeoptions.AdmissionOptions{
+					GenericAdmission: &genericoptions.AdmissionOptions{
+						EnablePlugins: []string{""},
+						Plugins:       kubeapiserveradmission.NewPlugins(),
+					},
+					PluginNames: []string{""},
+				},
+				Authentication: &kubeoptions.BuiltInAuthenticationOptions{
+					ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{},
+				},
+				APIEnablement:                genericoptions.NewAPIEnablementOptions(),
+				Metrics:                      &basemetrics.Options{},
+				ServiceAccountSigningKeyFile: "",
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			errs := tc.options.Validate()
+			if len(errs) > 0 && !tc.expectErrors {
+				t.Errorf("expected no errors, errors found %+v", errs)
+			}
+
+			if len(errs) == 0 && tc.expectErrors {
+				t.Errorf("expected errors, no errors found")
+			}
+		})
+	}
+}

pkg/controlplane/instance.go

@@ -77,6 +77,7 @@ import (
 	flowcontrolv1beta1 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta1"
 	flowcontrolv1beta2 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta2"
 	flowcontrolv1beta3 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta3"
+	"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 	"k8s.io/kubernetes/pkg/controlplane/controller/apiserverleasegc"
 	"k8s.io/kubernetes/pkg/controlplane/controller/clusterauthenticationtrust"
 	"k8s.io/kubernetes/pkg/controlplane/controller/legacytokentracking"
@@ -295,7 +296,7 @@ func (c *Config) Complete() CompletedConfig {
 		&c.ExtraConfig,
 	}
 
-	serviceIPRange, apiServerServiceIP, err := ServiceIPRange(cfg.ExtraConfig.ServiceIPRange)
+	serviceIPRange, apiServerServiceIP, err := options.ServiceIPRange(cfg.ExtraConfig.ServiceIPRange)
 	if err != nil {
 		klog.Fatalf("Error determining service IP ranges: %v", err)
 	}

pkg/controlplane/services.go

@@ -1,54 +0,0 @@
-/*
-Copyright 2016 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package controlplane
-
-import (
-	"fmt"
-	"net"
-
-	"k8s.io/klog/v2"
-	"k8s.io/utils/integer"
-	utilnet "k8s.io/utils/net"
-
-	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
-)
-
-// ServiceIPRange checks if the serviceClusterIPRange flag is nil, raising a warning if so and
-// setting service ip range to the default value in kubeoptions.DefaultServiceIPCIDR
-// for now until the default is removed per the deprecation timeline guidelines.
-// Returns service ip range, api server service IP, and an error
-func ServiceIPRange(passedServiceClusterIPRange net.IPNet) (net.IPNet, net.IP, error) {
-	serviceClusterIPRange := passedServiceClusterIPRange
-	if passedServiceClusterIPRange.IP == nil {
-		klog.Warningf("No CIDR for service cluster IPs specified. Default value which was %s is deprecated and will be removed in future releases. Please specify it using --service-cluster-ip-range on kube-apiserver.", kubeoptions.DefaultServiceIPCIDR.String())
-		serviceClusterIPRange = kubeoptions.DefaultServiceIPCIDR
-	}
-
-	size := integer.Int64Min(utilnet.RangeSize(&serviceClusterIPRange), 1<<16)
-	if size < 8 {
-		return net.IPNet{}, net.IP{}, fmt.Errorf("the service cluster IP range must be at least %d IP addresses", 8)
-	}
-
-	// Select the first valid IP from ServiceClusterIPRange to use as the GenericAPIServer service IP.
-	apiServerServiceIP, err := utilnet.GetIndexedIP(&serviceClusterIPRange, 1)
-	if err != nil {
-		return net.IPNet{}, net.IP{}, err
-	}
-	klog.V(4).Infof("Setting service IP to %q (read-write).", apiServerServiceIP)
-
-	return serviceClusterIPRange, apiServerServiceIP, nil
-}

test/e2e_node/services/apiserver.go

@@ -96,7 +96,7 @@ func (a *APIServer) Start() error {
 	errCh := make(chan error)
 	go func() {
 		defer close(errCh)
-		completedOptions, err := options.Complete(o)
+		completedOptions, err := o.Complete()
 		if err != nil {
 			errCh <- fmt.Errorf("set apiserver default options error: %w", err)
 			return

test/integration/etcd/server.go

@@ -88,23 +88,23 @@ func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRu
 		t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err)
 	}
 
-	kubeAPIServerOptions := options.NewServerRunOptions()
-	kubeAPIServerOptions.SecureServing.Listener = listener
-	kubeAPIServerOptions.SecureServing.ServerCert.CertDirectory = certDir
-	kubeAPIServerOptions.ServiceAccountSigningKeyFile = saSigningKeyFile.Name()
-	kubeAPIServerOptions.Etcd.StorageConfig.Transport.ServerList = []string{framework.GetEtcdURL()}
-	kubeAPIServerOptions.Etcd.DefaultStorageMediaType = runtime.ContentTypeJSON // force json we can easily interpret the result in etcd
-	kubeAPIServerOptions.ServiceClusterIPRanges = defaultServiceClusterIPRange.String()
-	kubeAPIServerOptions.Authentication.APIAudiences = []string{"https://foo.bar.example.com"}
-	kubeAPIServerOptions.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"}
-	kubeAPIServerOptions.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()}
-	kubeAPIServerOptions.Authorization.Modes = []string{"RBAC"}
-	kubeAPIServerOptions.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
-	kubeAPIServerOptions.APIEnablement.RuntimeConfig["api/all"] = "true"
+	opts := options.NewServerRunOptions()
+	opts.Options.SecureServing.Listener = listener
+	opts.Options.SecureServing.ServerCert.CertDirectory = certDir
+	opts.Options.ServiceAccountSigningKeyFile = saSigningKeyFile.Name()
+	opts.Options.Etcd.StorageConfig.Transport.ServerList = []string{framework.GetEtcdURL()}
+	opts.Options.Etcd.DefaultStorageMediaType = runtime.ContentTypeJSON // force json we can easily interpret the result in etcd
+	opts.ServiceClusterIPRanges = defaultServiceClusterIPRange.String()
+	opts.Options.Authentication.APIAudiences = []string{"https://foo.bar.example.com"}
+	opts.Options.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"}
+	opts.Options.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()}
+	opts.Options.Authorization.Modes = []string{"RBAC"}
+	opts.Options.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
+	opts.Options.APIEnablement.RuntimeConfig["api/all"] = "true"
 	for _, f := range configFuncs {
-		f(kubeAPIServerOptions)
+		f(opts)
 	}
-	completedOptions, err := options.Complete(kubeAPIServerOptions)
+	completedOptions, err := opts.Complete()
 	if err != nil {
 		t.Fatal(err)
 	}

test/integration/framework/test_server.go

@@ -127,30 +127,30 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) (
 		t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err)
 	}
 
-	kubeAPIServerOptions := options.NewServerRunOptions()
-	kubeAPIServerOptions.SecureServing.Listener = listener
-	kubeAPIServerOptions.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1")
-	kubeAPIServerOptions.SecureServing.ServerCert.CertDirectory = certDir
-	kubeAPIServerOptions.ServiceAccountSigningKeyFile = saSigningKeyFile.Name()
-	kubeAPIServerOptions.Etcd.StorageConfig.Prefix = path.Join("/", uuid.New().String(), "registry")
-	kubeAPIServerOptions.Etcd.StorageConfig.Transport.ServerList = []string{GetEtcdURL()}
-	kubeAPIServerOptions.ServiceClusterIPRanges = defaultServiceClusterIPRange.String()
-	kubeAPIServerOptions.Authentication.RequestHeader.UsernameHeaders = []string{"X-Remote-User"}
-	kubeAPIServerOptions.Authentication.RequestHeader.GroupHeaders = []string{"X-Remote-Group"}
-	kubeAPIServerOptions.Authentication.RequestHeader.ExtraHeaderPrefixes = []string{"X-Remote-Extra-"}
-	kubeAPIServerOptions.Authentication.RequestHeader.AllowedNames = []string{"kube-aggregator"}
-	kubeAPIServerOptions.Authentication.RequestHeader.ClientCAFile = proxyCACertFile.Name()
-	kubeAPIServerOptions.Authentication.APIAudiences = []string{"https://foo.bar.example.com"}
-	kubeAPIServerOptions.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"}
-	kubeAPIServerOptions.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()}
-	kubeAPIServerOptions.Authentication.ClientCert.ClientCA = clientCACertFile.Name()
-	kubeAPIServerOptions.Authorization.Modes = []string{"Node", "RBAC"}
+	opts := options.NewServerRunOptions()
+	opts.SecureServing.Listener = listener
+	opts.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1")
+	opts.SecureServing.ServerCert.CertDirectory = certDir
+	opts.ServiceAccountSigningKeyFile = saSigningKeyFile.Name()
+	opts.Etcd.StorageConfig.Prefix = path.Join("/", uuid.New().String(), "registry")
+	opts.Etcd.StorageConfig.Transport.ServerList = []string{GetEtcdURL()}
+	opts.ServiceClusterIPRanges = defaultServiceClusterIPRange.String()
+	opts.Authentication.RequestHeader.UsernameHeaders = []string{"X-Remote-User"}
+	opts.Authentication.RequestHeader.GroupHeaders = []string{"X-Remote-Group"}
+	opts.Authentication.RequestHeader.ExtraHeaderPrefixes = []string{"X-Remote-Extra-"}
+	opts.Authentication.RequestHeader.AllowedNames = []string{"kube-aggregator"}
+	opts.Authentication.RequestHeader.ClientCAFile = proxyCACertFile.Name()
+	opts.Authentication.APIAudiences = []string{"https://foo.bar.example.com"}
+	opts.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"}
+	opts.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()}
+	opts.Authentication.ClientCert.ClientCA = clientCACertFile.Name()
+	opts.Authorization.Modes = []string{"Node", "RBAC"}
 
 	if setup.ModifyServerRunOptions != nil {
-		setup.ModifyServerRunOptions(kubeAPIServerOptions)
+		setup.ModifyServerRunOptions(opts)
 	}
 
-	completedOptions, err := options.Complete(kubeAPIServerOptions)
+	completedOptions, err := opts.Complete()
 	if err != nil {
 		t.Fatal(err)
 	}