Merge pull request #124638 from sttts/sttts-kube-apiserver-config-split-up

kube-apiserver: split up config creation into generic and non-generic part
2024-05-01 08:51:39 -07:00
parent e342ab05bb d8c5f71403
commit 84a6ed4862
4 changed files with 191 additions and 146 deletions
--- a/cmd/kube-apiserver/app/config.go
+++ b/cmd/kube-apiserver/app/config.go
@@ -18,11 +18,16 @@ package app

 import (
 	apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/util/webhook"
 	aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
+	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
+
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/controlplane"
-	"k8s.io/kubernetes/pkg/controlplane/apiserver"
+	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver"
+	generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
 )

 type Config struct {
@@ -71,13 +76,23 @@ func NewConfig(opts options.CompletedOptions) (*Config, error) {
 		Options: opts,
 	}

-	kubeAPIs, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(opts)
+	genericConfig, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig(
+		opts.CompletedOptions,
+		[]*runtime.Scheme{legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme},
+		controlplane.DefaultAPIResourceConfigSource(),
+		generatedopenapi.GetOpenAPIDefinitions,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	kubeAPIs, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(opts, genericConfig, versionedInformers, storageFactory)
 	if err != nil {
 		return nil, err
 	}
 	c.KubeAPIs = kubeAPIs

-	apiExtensions, err := apiserver.CreateAPIExtensionsConfig(*kubeAPIs.ControlPlane.Generic, kubeAPIs.ControlPlane.VersionedInformers, pluginInitializer, opts.CompletedOptions, opts.MasterCount,
+	apiExtensions, err := controlplaneapiserver.CreateAPIExtensionsConfig(*kubeAPIs.ControlPlane.Generic, kubeAPIs.ControlPlane.VersionedInformers, pluginInitializer, opts.CompletedOptions, opts.MasterCount,
 		serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(kubeAPIs.ControlPlane.ProxyTransport, kubeAPIs.ControlPlane.Generic.EgressSelector, kubeAPIs.ControlPlane.Generic.LoopbackClientConfig, kubeAPIs.ControlPlane.Generic.TracerProvider))
 	if err != nil {
 		return nil, err
--- a/cmd/kube-apiserver/app/server.go
+++ b/cmd/kube-apiserver/app/server.go
@@ -21,31 +21,24 @@ package app

 import (
 	"context"
-	"crypto/tls"
 	"fmt"
-	"net/http"
 	"net/url"
 	"os"

 	"github.com/spf13/cobra"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
-	extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
-	"k8s.io/apimachinery/pkg/runtime"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
-	utilnet "k8s.io/apimachinery/pkg/util/net"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apiserver/pkg/admission"
 	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/egressselector"
+	serverstorage "k8s.io/apiserver/pkg/server/storage"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/apiserver/pkg/util/notfoundhandler"
 	"k8s.io/apiserver/pkg/util/webhook"
-	"k8s.io/client-go/dynamic"
 	clientgoinformers "k8s.io/client-go/informers"
-	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
-	"k8s.io/client-go/util/keyutil"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/cli/globalflag"
 	"k8s.io/component-base/logs"
@@ -56,19 +49,12 @@ import (
 	"k8s.io/component-base/version/verflag"
 	"k8s.io/klog/v2"
 	aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
-	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
-	controlplaneadmission "k8s.io/kubernetes/pkg/controlplane/apiserver/admission"
-
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
-	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/capabilities"
 	"k8s.io/kubernetes/pkg/controlplane"
 	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver"
 	"k8s.io/kubernetes/pkg/controlplane/reconcilers"
-	"k8s.io/kubernetes/pkg/features"
-	generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
 	kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
-	"k8s.io/kubernetes/pkg/serviceaccount"
 )

 func init() {
@@ -196,49 +182,22 @@ func CreateServerChain(config CompletedConfig) (*aggregatorapiserver.APIAggregat
 	return aggregatorServer, nil
 }

-// CreateProxyTransport creates the dialer infrastructure to connect to the nodes.
-func CreateProxyTransport() *http.Transport {
-	var proxyDialerFn utilnet.DialFunc
-	// Proxying to pods and services is IP-based... don't expect to be able to verify the hostname
-	proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
-	proxyTransport := utilnet.SetTransportDefaults(&http.Transport{
-		DialContext:     proxyDialerFn,
-		TLSClientConfig: proxyTLSClientConfig,
-	})
-	return proxyTransport
-}
-
 // CreateKubeAPIServerConfig creates all the resources for running the API server, but runs none of them
-func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
+func CreateKubeAPIServerConfig(
+	opts options.CompletedOptions,
+	genericConfig *genericapiserver.Config,
+	versionedInformers clientgoinformers.SharedInformerFactory,
+	storageFactory *serverstorage.DefaultStorageFactory,
+) (
 	*controlplane.Config,
 	aggregatorapiserver.ServiceResolver,
 	[]admission.PluginInitializer,
 	error,
 ) {
-	proxyTransport := CreateProxyTransport()
+	// global stuff
+	capabilities.Setup(opts.AllowPrivileged, opts.MaxConnectionBytesPerSec)

-	genericConfig, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig(
-		opts.CompletedOptions,
-		[]*runtime.Scheme{legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme},
-		controlplane.DefaultAPIResourceConfigSource(),
-		generatedopenapi.GetOpenAPIDefinitions,
-	)
-	if err != nil {
-		return nil, nil, nil, fmt.Errorf("failed to create generic config: %w", err)
-	}
-
-	// generic controlplane admission initializers
-	controlPlaneAdmissionConfig := &controlplaneadmission.Config{
-		ExternalInformers:    versionedInformers,
-		LoopbackClientConfig: genericConfig.LoopbackClientConfig,
-	}
-	serviceResolver := buildServiceResolver(opts.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers)
-	pluginInitializers, err := controlPlaneAdmissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver, genericConfig.TracerProvider)
-	if err != nil {
-		return nil, nil, nil, fmt.Errorf("failed to create admission plugin initializer: %w", err)
-	}
-
-	// additional kube admission initializers
+	// additional admission initializers
 	kubeAdmissionConfig := &kubeapiserveradmission.Config{
 		CloudConfigFile: opts.CloudProvider.CloudConfigFile,
 	}
@@ -246,31 +205,15 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("failed to create admission plugin initializer: %w", err)
 	}
-	pluginInitializers = append(pluginInitializers, kubeInitializers...)

-	capabilities.Setup(opts.AllowPrivileged, opts.MaxConnectionBytesPerSec)
-
-	opts.Metrics.Apply()
-	serviceaccount.RegisterMetrics()
+	serviceResolver := buildServiceResolver(opts.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers)
+	controlplaneConfig, admissionInitializers, err := controlplaneapiserver.CreateConfig(opts.CompletedOptions, genericConfig, versionedInformers, storageFactory, serviceResolver, kubeInitializers)
+	if err != nil {
+		return nil, nil, nil, err
+	}

 	config := &controlplane.Config{
-		ControlPlane: controlplaneapiserver.Config{
-			Generic: genericConfig,
-			Extra: controlplaneapiserver.Extra{
-				APIResourceConfigSource: storageFactory.APIResourceConfigSource,
-				StorageFactory:          storageFactory,
-				EventTTL:                opts.EventTTL,
-				EnableLogsSupport:       opts.EnableLogsHandler,
-				ProxyTransport:          proxyTransport,
-				SystemNamespaces:        opts.SystemNamespaces,
-
-				ServiceAccountIssuer:        opts.ServiceAccountIssuer,
-				ServiceAccountMaxExpiration: opts.ServiceAccountTokenMaxExpiration,
-				ExtendExpiration:            opts.Authentication.ServiceAccounts.ExtendExpiration,
-
-				VersionedInformers: versionedInformers,
-			},
-		},
+		ControlPlane: *controlplaneConfig,
 		Extra: controlplane.Extra{
 			KubeletClientConfig: opts.KubeletConfig,

@@ -288,59 +231,6 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
 		},
 	}

-	if utilfeature.DefaultFeatureGate.Enabled(features.UnknownVersionInteroperabilityProxy) {
-		config.ControlPlane.PeerEndpointLeaseReconciler, err = controlplaneapiserver.CreatePeerEndpointLeaseReconciler(*genericConfig, storageFactory)
-		if err != nil {
-			return nil, nil, nil, err
-		}
-		// build peer proxy config only if peer ca file exists
-		if opts.PeerCAFile != "" {
-			config.ControlPlane.PeerProxy, err = controlplaneapiserver.BuildPeerProxy(versionedInformers, genericConfig.StorageVersionManager, opts.ProxyClientCertFile,
-				opts.ProxyClientKeyFile, opts.PeerCAFile, opts.PeerAdvertiseAddress, genericConfig.APIServerID, config.ControlPlane.Extra.PeerEndpointLeaseReconciler, config.ControlPlane.Generic.Serializer)
-			if err != nil {
-				return nil, nil, nil, err
-			}
-		}
-	}
-
-	clientCAProvider, err := opts.Authentication.ClientCert.GetClientCAContentProvider()
-	if err != nil {
-		return nil, nil, nil, err
-	}
-	config.ControlPlane.ClusterAuthenticationInfo.ClientCA = clientCAProvider
-
-	requestHeaderConfig, err := opts.Authentication.RequestHeader.ToAuthenticationRequestHeaderConfig()
-	if err != nil {
-		return nil, nil, nil, err
-	}
-	if requestHeaderConfig != nil {
-		config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderCA = requestHeaderConfig.CAContentProvider
-		config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderAllowedNames = requestHeaderConfig.AllowedClientNames
-		config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderExtraHeaderPrefixes = requestHeaderConfig.ExtraHeaderPrefixes
-		config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderGroupHeaders = requestHeaderConfig.GroupHeaders
-		config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderUsernameHeaders = requestHeaderConfig.UsernameHeaders
-	}
-
-	// setup admission
-	clientgoExternalClient, err := clientset.NewForConfig(genericConfig.LoopbackClientConfig)
-	if err != nil {
-		return nil, nil, nil, fmt.Errorf("failed to create real client-go external client: %w", err)
-	}
-	dynamicExternalClient, err := dynamic.NewForConfig(genericConfig.LoopbackClientConfig)
-	if err != nil {
-		return nil, nil, nil, fmt.Errorf("failed to create real dynamic external client: %w", err)
-	}
-	err = opts.Admission.ApplyTo(
-		genericConfig,
-		versionedInformers,
-		clientgoExternalClient,
-		dynamicExternalClient,
-		utilfeature.DefaultFeatureGate,
-		pluginInitializers...)
-	if err != nil {
-		return nil, nil, nil, fmt.Errorf("failed to apply admission: %w", err)
-	}
-
 	if config.ControlPlane.Generic.EgressSelector != nil {
 		// Use the config.ControlPlane.Generic.EgressSelector lookup to find the dialer to connect to the kubelet
 		config.Extra.KubeletClientConfig.Lookup = config.ControlPlane.Generic.EgressSelector.Lookup
@@ -351,25 +241,12 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
 		if err != nil {
 			return nil, nil, nil, err
 		}
-		c := proxyTransport.Clone()
+		c := config.ControlPlane.Extra.ProxyTransport.Clone()
 		c.DialContext = dialer
 		config.ControlPlane.ProxyTransport = c
 	}

-	// Load and set the public keys.
-	var pubKeys []interface{}
-	for _, f := range opts.Authentication.ServiceAccounts.KeyFiles {
-		keys, err := keyutil.PublicKeysFromFile(f)
-		if err != nil {
-			return nil, nil, nil, fmt.Errorf("failed to parse key file %q: %v", f, err)
-		}
-		pubKeys = append(pubKeys, keys...)
-	}
-	config.ControlPlane.ServiceAccountIssuerURL = opts.Authentication.ServiceAccounts.Issuers[0]
-	config.ControlPlane.ServiceAccountJWKSURI = opts.Authentication.ServiceAccounts.JWKSURI
-	config.ControlPlane.ServiceAccountPublicKeys = pubKeys
-
-	return config, serviceResolver, pluginInitializers, nil
+	return config, serviceResolver, admissionInitializers, nil
 }

 var testServiceResolver webhook.ServiceResolver
--- a/pkg/controlplane/apiserver/config.go
+++ b/pkg/controlplane/apiserver/config.go
@@ -18,6 +18,7 @@ package apiserver

 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"net/http"
 	"time"
@@ -25,8 +26,10 @@ import (
 	oteltrace "go.opentelemetry.io/otel/trace"

 	"k8s.io/apimachinery/pkg/runtime"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
 	openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
@@ -39,14 +42,19 @@ import (
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/apiserver/pkg/util/openapi"
 	utilpeerproxy "k8s.io/apiserver/pkg/util/peerproxy"
+	"k8s.io/client-go/dynamic"
 	clientgoinformers "k8s.io/client-go/informers"
 	clientgoclientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/util/keyutil"
 	"k8s.io/component-base/version"
+	aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
 	openapicommon "k8s.io/kube-openapi/pkg/common"

 	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	controlplaneadmission "k8s.io/kubernetes/pkg/controlplane/apiserver/admission"
 	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 	"k8s.io/kubernetes/pkg/controlplane/controller/clusterauthenticationtrust"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubeapiserver"
 	"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
 	rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest"
@@ -254,3 +262,131 @@ func BuildAuthorizer(ctx context.Context, s controlplaneapiserver.CompletedOptio

 	return authorizer, ruleResolver, enablesRBAC, err
 }
+
+// CreateConfig takes the generic controlplane apiserver options and
+// creates a config for the generic Kube APIs out of it.
+func CreateConfig(
+	opts controlplaneapiserver.CompletedOptions,
+	genericConfig *genericapiserver.Config,
+	versionedInformers clientgoinformers.SharedInformerFactory,
+	storageFactory *serverstorage.DefaultStorageFactory,
+	serviceResolver aggregatorapiserver.ServiceResolver,
+	additionalInitializers []admission.PluginInitializer,
+) (
+	*Config,
+	[]admission.PluginInitializer,
+	error,
+) {
+	proxyTransport := CreateProxyTransport()
+
+	opts.Metrics.Apply()
+	serviceaccount.RegisterMetrics()
+
+	config := &Config{
+		Generic: genericConfig,
+		Extra: Extra{
+			APIResourceConfigSource: storageFactory.APIResourceConfigSource,
+			StorageFactory:          storageFactory,
+			EventTTL:                opts.EventTTL,
+			EnableLogsSupport:       opts.EnableLogsHandler,
+			ProxyTransport:          proxyTransport,
+			SystemNamespaces:        opts.SystemNamespaces,
+
+			ServiceAccountIssuer:        opts.ServiceAccountIssuer,
+			ServiceAccountMaxExpiration: opts.ServiceAccountTokenMaxExpiration,
+			ExtendExpiration:            opts.Authentication.ServiceAccounts.ExtendExpiration,
+
+			VersionedInformers: versionedInformers,
+		},
+	}
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.UnknownVersionInteroperabilityProxy) {
+		var err error
+		config.PeerEndpointLeaseReconciler, err = CreatePeerEndpointLeaseReconciler(*genericConfig, storageFactory)
+		if err != nil {
+			return nil, nil, err
+		}
+		// build peer proxy config only if peer ca file exists
+		if opts.PeerCAFile != "" {
+			config.PeerProxy, err = BuildPeerProxy(versionedInformers, genericConfig.StorageVersionManager, opts.ProxyClientCertFile,
+				opts.ProxyClientKeyFile, opts.PeerCAFile, opts.PeerAdvertiseAddress, genericConfig.APIServerID, config.Extra.PeerEndpointLeaseReconciler, config.Generic.Serializer)
+			if err != nil {
+				return nil, nil, err
+			}
+		}
+	}
+
+	clientCAProvider, err := opts.Authentication.ClientCert.GetClientCAContentProvider()
+	if err != nil {
+		return nil, nil, err
+	}
+	config.ClusterAuthenticationInfo.ClientCA = clientCAProvider
+
+	requestHeaderConfig, err := opts.Authentication.RequestHeader.ToAuthenticationRequestHeaderConfig()
+	if err != nil {
+		return nil, nil, err
+	}
+	if requestHeaderConfig != nil {
+		config.ClusterAuthenticationInfo.RequestHeaderCA = requestHeaderConfig.CAContentProvider
+		config.ClusterAuthenticationInfo.RequestHeaderAllowedNames = requestHeaderConfig.AllowedClientNames
+		config.ClusterAuthenticationInfo.RequestHeaderExtraHeaderPrefixes = requestHeaderConfig.ExtraHeaderPrefixes
+		config.ClusterAuthenticationInfo.RequestHeaderGroupHeaders = requestHeaderConfig.GroupHeaders
+		config.ClusterAuthenticationInfo.RequestHeaderUsernameHeaders = requestHeaderConfig.UsernameHeaders
+	}
+
+	// setup admission
+	genericAdmissionConfig := controlplaneadmission.Config{
+		ExternalInformers:    versionedInformers,
+		LoopbackClientConfig: genericConfig.LoopbackClientConfig,
+	}
+	genericInitializers, err := genericAdmissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver, genericConfig.TracerProvider)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to create admission plugin initializer: %w", err)
+	}
+	clientgoExternalClient, err := clientgoclientset.NewForConfig(genericConfig.LoopbackClientConfig)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to create real client-go external client: %w", err)
+	}
+	dynamicExternalClient, err := dynamic.NewForConfig(genericConfig.LoopbackClientConfig)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to create real dynamic external client: %w", err)
+	}
+	err = opts.Admission.ApplyTo(
+		genericConfig,
+		versionedInformers,
+		clientgoExternalClient,
+		dynamicExternalClient,
+		utilfeature.DefaultFeatureGate,
+		append(genericInitializers, additionalInitializers...)...,
+	)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to apply admission: %w", err)
+	}
+
+	// Load and set the public keys.
+	var pubKeys []interface{}
+	for _, f := range opts.Authentication.ServiceAccounts.KeyFiles {
+		keys, err := keyutil.PublicKeysFromFile(f)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to parse key file %q: %w", f, err)
+		}
+		pubKeys = append(pubKeys, keys...)
+	}
+	config.ServiceAccountIssuerURL = opts.Authentication.ServiceAccounts.Issuers[0]
+	config.ServiceAccountJWKSURI = opts.Authentication.ServiceAccounts.JWKSURI
+	config.ServiceAccountPublicKeys = pubKeys
+
+	return config, genericInitializers, nil
+}
+
+// CreateProxyTransport creates the dialer infrastructure to connect to the nodes.
+func CreateProxyTransport() *http.Transport {
+	var proxyDialerFn utilnet.DialFunc
+	// Proxying to pods and services is IP-based... don't expect to be able to verify the hostname
+	proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
+	proxyTransport := utilnet.SetTransportDefaults(&http.Transport{
+		DialContext:     proxyDialerFn,
+		TLSClientConfig: proxyTLSClientConfig,
+	})
+	return proxyTransport
+}
--- a/test/integration/framework/test_server.go
+++ b/test/integration/framework/test_server.go
@@ -28,7 +28,9 @@ import (

 	"github.com/google/uuid"

+	apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
 	genericapiserver "k8s.io/apiserver/pkg/server"
@@ -36,11 +38,16 @@ import (
 	client "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/util/cert"
+	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
+	netutils "k8s.io/utils/net"
+
 	"k8s.io/kubernetes/cmd/kube-apiserver/app"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/controlplane"
+	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver"
+	generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
 	"k8s.io/kubernetes/test/utils"
-	netutils "k8s.io/utils/net"
 )

 // This key is for testing purposes only and is not considered secure.
@@ -160,7 +167,17 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) (
 		t.Fatalf("failed to validate ServerRunOptions: %v", utilerrors.NewAggregate(errs))
 	}

-	kubeAPIServerConfig, _, _, err := app.CreateKubeAPIServerConfig(completedOptions)
+	genericConfig, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig(
+		completedOptions.CompletedOptions,
+		[]*runtime.Scheme{legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme},
+		controlplane.DefaultAPIResourceConfigSource(),
+		generatedopenapi.GetOpenAPIDefinitions,
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	kubeAPIServerConfig, _, _, err := app.CreateKubeAPIServerConfig(completedOptions, genericConfig, versionedInformers, storageFactory)
 	if err != nil {
 		t.Fatal(err)
 	}