github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #124551 from sttts/sttts-kube-apiserver-config-split

Browse Source 
pkg/controlplane: split up config into generic controlplane and kube-related part
This commit is contained in:
Kubernetes Prow Robot
2024-04-26 14:16:12 -07:00
committed by GitHub
parent 4b2d1b6a20 96a144c977
commit 563885e4bd
26 changed files with 369 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
cmd/kube-apiserver/app/config.go
View File

@@ -29,7 +29,7 @@ type Config struct {
Options options.CompletedOptions
Aggregator *aggregatorapiserver.Config
ControlPlane *controlplane.Config
KubeAPIs *controlplane.Config
ApiExtensions *apiextensionsapiserver.Config
ExtraConfig
@@ -42,7 +42,7 @@ type completedConfig struct {
Options options.CompletedOptions
Aggregator aggregatorapiserver.CompletedConfig
ControlPlane controlplane.CompletedConfig
KubeAPIs controlplane.CompletedConfig
ApiExtensions apiextensionsapiserver.CompletedConfig
ExtraConfig
@@ -58,7 +58,7 @@ func (c *Config) Complete() (CompletedConfig, error) {
Options: c.Options,
Aggregator: c.Aggregator.Complete(),
ControlPlane: c.ControlPlane.Complete(),
KubeAPIs: c.KubeAPIs.Complete(),
ApiExtensions: c.ApiExtensions.Complete(),
ExtraConfig: c.ExtraConfig,
@@ -71,20 +71,20 @@ func NewConfig(opts options.CompletedOptions) (*Config, error) {
Options: opts,
}
controlPlane, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(opts)
kubeAPIs, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(opts)
if err != nil {
return nil, err
}
c.ControlPlane = controlPlane
c.KubeAPIs = kubeAPIs
apiExtensions, err := apiserver.CreateAPIExtensionsConfig(*controlPlane.GenericConfig, controlPlane.ExtraConfig.VersionedInformers, pluginInitializer, opts.CompletedOptions, opts.MasterCount,
serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(controlPlane.ExtraConfig.ProxyTransport, controlPlane.GenericConfig.EgressSelector, controlPlane.GenericConfig.LoopbackClientConfig, controlPlane.GenericConfig.TracerProvider))
apiExtensions, err := apiserver.CreateAPIExtensionsConfig(*kubeAPIs.ControlPlane.Generic, kubeAPIs.ControlPlane.VersionedInformers, pluginInitializer, opts.CompletedOptions, opts.MasterCount,
serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(kubeAPIs.ControlPlane.ProxyTransport, kubeAPIs.ControlPlane.Generic.EgressSelector, kubeAPIs.ControlPlane.Generic.LoopbackClientConfig, kubeAPIs.ControlPlane.Generic.TracerProvider))
if err != nil {
return nil, err
}
c.ApiExtensions = apiExtensions
aggregator, err := createAggregatorConfig(*controlPlane.GenericConfig, opts.CompletedOptions, controlPlane.ExtraConfig.VersionedInformers, serviceResolver, controlPlane.ExtraConfig.ProxyTransport, controlPlane.ExtraConfig.PeerProxy, pluginInitializer)
aggregator, err := createAggregatorConfig(*kubeAPIs.ControlPlane.Generic, opts.CompletedOptions, kubeAPIs.ControlPlane.VersionedInformers, serviceResolver, kubeAPIs.ControlPlane.ProxyTransport, kubeAPIs.Extra.PeerProxy, pluginInitializer)
if err != nil {
return nil, err
}

63
cmd/kube-apiserver/app/server.go
View File

@@ -172,14 +172,14 @@ func Run(opts options.CompletedOptions, stopCh <-chan struct{}) error {
// CreateServerChain creates the apiservers connected via delegation.
func CreateServerChain(config CompletedConfig) (*aggregatorapiserver.APIAggregator, error) {
notFoundHandler := notfoundhandler.New(config.ControlPlane.GenericConfig.Serializer, genericapifilters.NoMuxAndDiscoveryIncompleteKey)
notFoundHandler := notfoundhandler.New(config.KubeAPIs.ControlPlane.Generic.Serializer, genericapifilters.NoMuxAndDiscoveryIncompleteKey)
apiExtensionsServer, err := config.ApiExtensions.New(genericapiserver.NewEmptyDelegateWithCustomHandler(notFoundHandler))
if err != nil {
return nil, err
}
crdAPIEnabled := config.ApiExtensions.GenericConfig.MergedResourceConfig.ResourceEnabled(apiextensionsv1.SchemeGroupVersion.WithResource("customresourcedefinitions"))
kubeAPIServer, err := config.ControlPlane.New(apiExtensionsServer.GenericAPIServer)
kubeAPIServer, err := config.KubeAPIs.New(apiExtensionsServer.GenericAPIServer)
if err != nil {
return nil, err
}
@@ -218,6 +218,7 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
genericConfig, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig(
opts.CompletedOptions,
[]*runtime.Scheme{legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme},
controlplane.DefaultAPIResourceConfigSource(),
generatedopenapi.GetOpenAPIDefinitions,
)
if err != nil {
@@ -230,15 +231,25 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
serviceaccount.RegisterMetrics()
config := &controlplane.Config{
GenericConfig: genericConfig,
ExtraConfig: controlplane.ExtraConfig{
ControlPlane: controlplaneapiserver.Config{
Generic: genericConfig,
Extra: controlplaneapiserver.Extra{
APIResourceConfigSource: storageFactory.APIResourceConfigSource,
StorageFactory: storageFactory,
EventTTL: opts.EventTTL,
KubeletClientConfig: opts.KubeletConfig,
EnableLogsSupport: opts.EnableLogsHandler,
ProxyTransport: proxyTransport,
ServiceAccountIssuer: opts.ServiceAccountIssuer,
ServiceAccountMaxExpiration: opts.ServiceAccountTokenMaxExpiration,
ExtendExpiration: opts.Authentication.ServiceAccounts.ExtendExpiration,
VersionedInformers: versionedInformers,
},
},
Extra: controlplane.Extra{
KubeletClientConfig: opts.KubeletConfig,
ServiceIPRange: opts.PrimaryServiceClusterIPRange,
APIServerServiceIP: opts.APIServerServiceIP,
SecondaryServiceIPRange: opts.SecondaryServiceClusterIPRange,
@@ -250,24 +261,18 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
EndpointReconcilerType: reconcilers.Type(opts.EndpointReconcilerType),
MasterCount: opts.MasterCount,
ServiceAccountIssuer: opts.ServiceAccountIssuer,
ServiceAccountMaxExpiration: opts.ServiceAccountTokenMaxExpiration,
ExtendExpiration: opts.Authentication.ServiceAccounts.ExtendExpiration,
VersionedInformers: versionedInformers,
},
}
if utilfeature.DefaultFeatureGate.Enabled(features.UnknownVersionInteroperabilityProxy) {
config.ExtraConfig.PeerEndpointLeaseReconciler, err = controlplaneapiserver.CreatePeerEndpointLeaseReconciler(*genericConfig, storageFactory)
config.Extra.PeerEndpointLeaseReconciler, err = controlplane.CreatePeerEndpointLeaseReconciler(*genericConfig, storageFactory)
if err != nil {
return nil, nil, nil, err
}
// build peer proxy config only if peer ca file exists
if opts.PeerCAFile != "" {
config.ExtraConfig.PeerProxy, err = controlplaneapiserver.BuildPeerProxy(versionedInformers, genericConfig.StorageVersionManager, opts.ProxyClientCertFile,
opts.ProxyClientKeyFile, opts.PeerCAFile, opts.PeerAdvertiseAddress, genericConfig.APIServerID, config.ExtraConfig.PeerEndpointLeaseReconciler, config.GenericConfig.Serializer)
config.Extra.PeerProxy, err = controlplane.BuildPeerProxy(versionedInformers, genericConfig.StorageVersionManager, opts.ProxyClientCertFile,
opts.ProxyClientKeyFile, opts.PeerCAFile, opts.PeerAdvertiseAddress, genericConfig.APIServerID, config.Extra.PeerEndpointLeaseReconciler, config.ControlPlane.Generic.Serializer)
if err != nil {
return nil, nil, nil, err
}
@@ -278,18 +283,18 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
if err != nil {
return nil, nil, nil, err
}
config.ExtraConfig.ClusterAuthenticationInfo.ClientCA = clientCAProvider
config.ControlPlane.ClusterAuthenticationInfo.ClientCA = clientCAProvider
requestHeaderConfig, err := opts.Authentication.RequestHeader.ToAuthenticationRequestHeaderConfig()
if err != nil {
return nil, nil, nil, err
}
if requestHeaderConfig != nil {
config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderCA = requestHeaderConfig.CAContentProvider
config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderAllowedNames = requestHeaderConfig.AllowedClientNames
config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderExtraHeaderPrefixes = requestHeaderConfig.ExtraHeaderPrefixes
config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderGroupHeaders = requestHeaderConfig.GroupHeaders
config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderUsernameHeaders = requestHeaderConfig.UsernameHeaders
config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderCA = requestHeaderConfig.CAContentProvider
config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderAllowedNames = requestHeaderConfig.AllowedClientNames
config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderExtraHeaderPrefixes = requestHeaderConfig.ExtraHeaderPrefixes
config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderGroupHeaders = requestHeaderConfig.GroupHeaders
config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderUsernameHeaders = requestHeaderConfig.UsernameHeaders
}
// setup admission
@@ -322,19 +327,19 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
return nil, nil, nil, fmt.Errorf("failed to apply admission: %w", err)
}
if config.GenericConfig.EgressSelector != nil {
// Use the config.GenericConfig.EgressSelector lookup to find the dialer to connect to the kubelet
config.ExtraConfig.KubeletClientConfig.Lookup = config.GenericConfig.EgressSelector.Lookup
if config.ControlPlane.Generic.EgressSelector != nil {
// Use the config.ControlPlane.Generic.EgressSelector lookup to find the dialer to connect to the kubelet
config.Extra.KubeletClientConfig.Lookup = config.ControlPlane.Generic.EgressSelector.Lookup
// Use the config.GenericConfig.EgressSelector lookup as the transport used by the "proxy" subresources.
// Use the config.ControlPlane.Generic.EgressSelector lookup as the transport used by the "proxy" subresources.
networkContext := egressselector.Cluster.AsNetworkContext()
dialer, err := config.GenericConfig.EgressSelector.Lookup(networkContext)
dialer, err := config.ControlPlane.Generic.EgressSelector.Lookup(networkContext)
if err != nil {
return nil, nil, nil, err
}
c := proxyTransport.Clone()
c.DialContext = dialer
config.ExtraConfig.ProxyTransport = c
config.ControlPlane.ProxyTransport = c
}
// Load and set the public keys.
@@ -346,9 +351,9 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
}
pubKeys = append(pubKeys, keys...)
}
config.ExtraConfig.ServiceAccountIssuerURL = opts.Authentication.ServiceAccounts.Issuers[0]
config.ExtraConfig.ServiceAccountJWKSURI = opts.Authentication.ServiceAccounts.JWKSURI
config.ExtraConfig.ServiceAccountPublicKeys = pubKeys
config.ControlPlane.ServiceAccountIssuerURL = opts.Authentication.ServiceAccounts.Issuers[0]
config.ControlPlane.ServiceAccountJWKSURI = opts.Authentication.ServiceAccounts.JWKSURI
config.ControlPlane.ServiceAccountPublicKeys = pubKeys
return config, serviceResolver, pluginInitializers, nil
}

44
pkg/controlplane/apiserver/completion.go Normal file
View File

@@ -0,0 +1,44 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package apiserver
import (
"k8s.io/apiserver/pkg/endpoints/discovery"
genericapiserver "k8s.io/apiserver/pkg/server"
)
type completedConfig struct {
Generic genericapiserver.CompletedConfig
*Extra
}
// CompletedConfig embeds a private pointer that cannot be instantiated outside of this package
type CompletedConfig struct {
*completedConfig
}
func (c *Config) Complete() CompletedConfig {
cfg := completedConfig{
c.Generic.Complete(c.VersionedInformers),
&c.Extra,
}
discoveryAddresses := discovery.DefaultAddresses{DefaultAddress: cfg.Generic.ExternalAddress}
cfg.Generic.DiscoveryAddresses = discoveryAddresses
return CompletedConfig{&cfg}
}

91
pkg/controlplane/apiserver/config.go
View File

@@ -19,6 +19,7 @@ package apiserver
import (
"context"
"fmt"
"net/http"
"time"
oteltrace "go.opentelemetry.io/otel/trace"
@@ -30,45 +31,68 @@ import (
"k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
genericfeatures "k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/reconcilers"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/egressselector"
"k8s.io/apiserver/pkg/server/filters"
serverstorage "k8s.io/apiserver/pkg/server/storage"
"k8s.io/apiserver/pkg/storageversion"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/apiserver/pkg/util/openapi"
utilpeerproxy "k8s.io/apiserver/pkg/util/peerproxy"
clientgoinformers "k8s.io/client-go/informers"
clientgoclientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/transport"
"k8s.io/component-base/version"
"k8s.io/klog/v2"
openapicommon "k8s.io/kube-openapi/pkg/common"
"k8s.io/kubernetes/pkg/api/legacyscheme"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/controlplane"
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
"k8s.io/kubernetes/pkg/controlplane/controller/clusterauthenticationtrust"
"k8s.io/kubernetes/pkg/kubeapiserver"
"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest"
"k8s.io/kubernetes/pkg/serviceaccount"
)
// Config defines configuration for the master
type Config struct {
Generic *genericapiserver.Config
Extra
}
type Extra struct {
ClusterAuthenticationInfo clusterauthenticationtrust.ClusterAuthenticationInfo
APIResourceConfigSource serverstorage.APIResourceConfigSource
StorageFactory serverstorage.StorageFactory
EventTTL time.Duration
EnableLogsSupport bool
ProxyTransport *http.Transport
ServiceAccountIssuer serviceaccount.TokenGenerator
ServiceAccountMaxExpiration time.Duration
ExtendExpiration bool
// ServiceAccountIssuerDiscovery
ServiceAccountIssuerURL string
ServiceAccountJWKSURI string
ServiceAccountPublicKeys []interface{}
VersionedInformers clientgoinformers.SharedInformerFactory
}
// BuildGenericConfig takes the master server options and produces the genericapiserver.Config associated with it
func BuildGenericConfig(
s controlplaneapiserver.CompletedOptions,
schemes []*runtime.Scheme,
resourceConfig *serverstorage.ResourceConfig,
getOpenAPIDefinitions func(ref openapicommon.ReferenceCallback) map[string]openapicommon.OpenAPIDefinition,
) (
genericConfig *genericapiserver.Config,
versionedInformers clientgoinformers.SharedInformerFactory,
storageFactory *serverstorage.DefaultStorageFactory,
lastErr error,
) {
genericConfig = genericapiserver.NewConfig(legacyscheme.Codecs)
genericConfig.MergedResourceConfig = controlplane.DefaultAPIResourceConfigSource()
genericConfig.MergedResourceConfig = resourceConfig
if lastErr = s.GenericServerRunOptions.ApplyTo(genericConfig); lastErr != nil {
return
@@ -98,7 +122,7 @@ func BuildGenericConfig(
if lastErr = s.Features.ApplyTo(genericConfig, clientgoExternalClient, versionedInformers); lastErr != nil {
return
}
if lastErr = s.APIEnablement.ApplyTo(genericConfig, controlplane.DefaultAPIResourceConfigSource(), legacyscheme.Scheme); lastErr != nil {
if lastErr = s.APIEnablement.ApplyTo(genericConfig, resourceConfig, legacyscheme.Scheme); lastErr != nil {
return
}
if lastErr = s.EgressSelector.ApplyTo(genericConfig); lastErr != nil {
@@ -212,50 +236,3 @@ func BuildAuthorizer(ctx context.Context, s controlplaneapiserver.CompletedOptio
return authorizer, ruleResolver, enablesRBAC, err
}
// CreatePeerEndpointLeaseReconciler creates a apiserver endpoint lease reconciliation loop
// The peer endpoint leases are used to find network locations of apiservers for peer proxy
func CreatePeerEndpointLeaseReconciler(c genericapiserver.Config, storageFactory serverstorage.StorageFactory) (reconcilers.PeerEndpointLeaseReconciler, error) {
ttl := controlplane.DefaultEndpointReconcilerTTL
config, err := storageFactory.NewConfig(api.Resource("apiServerPeerIPInfo"))
if err != nil {
return nil, fmt.Errorf("error creating storage factory config: %w", err)
}
reconciler, err := reconcilers.NewPeerEndpointLeaseReconciler(config, "/peerserverleases/", ttl)
return reconciler, err
}
func BuildPeerProxy(versionedInformer clientgoinformers.SharedInformerFactory, svm storageversion.Manager,
proxyClientCertFile string, proxyClientKeyFile string, peerCAFile string, peerAdvertiseAddress reconcilers.PeerAdvertiseAddress,
apiServerID string, reconciler reconcilers.PeerEndpointLeaseReconciler, serializer runtime.NegotiatedSerializer) (utilpeerproxy.Interface, error) {
if proxyClientCertFile == "" {
return nil, fmt.Errorf("error building peer proxy handler, proxy-cert-file not specified")
}
if proxyClientKeyFile == "" {
return nil, fmt.Errorf("error building peer proxy handler, proxy-key-file not specified")
}
// create proxy client config
clientConfig := &transport.Config{
TLS: transport.TLSConfig{
Insecure: false,
CertFile: proxyClientCertFile,
KeyFile: proxyClientKeyFile,
CAFile: peerCAFile,
ServerName: "kubernetes.default.svc",
}}
// build proxy transport
proxyRoundTripper, transportBuildingError := transport.New(clientConfig)
if transportBuildingError != nil {
klog.Error(transportBuildingError.Error())
return nil, transportBuildingError
}
return utilpeerproxy.NewPeerProxyHandler(
versionedInformer,
svm,
proxyRoundTripper,
apiServerID,
reconciler,
serializer,
), nil
}

6
pkg/controlplane/apiserver/config_test.go
View File

@@ -17,6 +17,9 @@ limitations under the License.
package apiserver
import (
"net"
"testing"
extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
@@ -26,8 +29,6 @@ import (
"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
netutils "k8s.io/utils/net"
"net"
"testing"
)
func TestBuildGenericConfig(t *testing.T) {
@@ -52,6 +53,7 @@ func TestBuildGenericConfig(t *testing.T) {
genericConfig, _, storageFactory, err := BuildGenericConfig(
completedOptions,
[]*runtime.Scheme{legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme},
nil,
generatedopenapi.GetOpenAPIDefinitions,
)
if err != nil {

273
pkg/controlplane/instance.go
View File

@@ -19,7 +19,6 @@ package controlplane
import (
"fmt"
"net"
"net/http"
"os"
"reflect"
"strconv"
@@ -55,6 +54,7 @@ import (
storageapiv1beta1 "k8s.io/api/storage/v1beta1"
svmv1alpha1 "k8s.io/api/storagemigration/v1alpha1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kruntime "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/runtime"
@@ -67,12 +67,14 @@ import (
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
serverstorage "k8s.io/apiserver/pkg/server/storage"
"k8s.io/apiserver/pkg/storageversion"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilpeerproxy "k8s.io/apiserver/pkg/util/peerproxy"
"k8s.io/client-go/informers"
clientgoinformers "k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
discoveryclient "k8s.io/client-go/kubernetes/typed/discovery/v1"
"k8s.io/client-go/transport"
"k8s.io/component-helpers/apimachinery/lease"
"k8s.io/klog/v2"
api "k8s.io/kubernetes/pkg/apis/core"
@@ -80,6 +82,7 @@ import (
flowcontrolv1beta1 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta1"
flowcontrolv1beta2 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta2"
flowcontrolv1beta3 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta3"
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver"
"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
"k8s.io/kubernetes/pkg/controlplane/controller/apiserverleasegc"
"k8s.io/kubernetes/pkg/controlplane/controller/clusterauthenticationtrust"
@@ -144,36 +147,25 @@ var (
// IdentityLeaseDurationSeconds is the duration of kube-apiserver lease in seconds
// IdentityLeaseDurationSeconds is exposed so integration tests can tune this value.
IdentityLeaseDurationSeconds = 3600
// IdentityLeaseRenewIntervalSeconds is the interval of kube-apiserver renewing its lease in seconds
// IdentityLeaseRenewIntervalSeconds is exposed so integration tests can tune this value.
// IdentityLeaseRenewIntervalPeriod is the interval of kube-apiserver renewing its lease in seconds
// IdentityLeaseRenewIntervalPeriod is exposed so integration tests can tune this value.
IdentityLeaseRenewIntervalPeriod = 10 * time.Second
)
// ExtraConfig defines extra configuration for the master
type ExtraConfig struct {
ClusterAuthenticationInfo clusterauthenticationtrust.ClusterAuthenticationInfo
APIResourceConfigSource serverstorage.APIResourceConfigSource
StorageFactory serverstorage.StorageFactory
// Extra defines extra configuration for kube-apiserver
type Extra struct {
EndpointReconcilerConfig EndpointReconcilerConfig
EventTTL time.Duration
KubeletClientConfig kubeletclient.KubeletClientConfig
EnableLogsSupport bool
ProxyTransport *http.Transport
// PeerProxy, if not nil, sets proxy transport between kube-apiserver peers for requests
// that can not be served locally
PeerProxy utilpeerproxy.Interface
// PeerEndpointLeaseReconciler updates the peer endpoint leases
PeerEndpointLeaseReconciler peerreconcilers.PeerEndpointLeaseReconciler
// PeerCAFile is the ca bundle used by this kube-apiserver to verify peer apiservers'
// serving certs when routing a request to the peer in the case the request can not be served
// locally due to version skew.
PeerCAFile string
// PeerAdvertiseAddress is the IP for this kube-apiserver which is used by peer apiservers to route a request
// to this apiserver. This happens in cases where the peer is not able to serve the request due to
// version skew. If unset, AdvertiseAddress/BindAddress will be used.
@@ -219,17 +211,6 @@ type ExtraConfig struct {
// Selects which reconciler to use
EndpointReconcilerType reconcilers.Type
ServiceAccountIssuer serviceaccount.TokenGenerator
ServiceAccountMaxExpiration time.Duration
ExtendExpiration bool
// ServiceAccountIssuerDiscovery
ServiceAccountIssuerURL string
ServiceAccountJWKSURI string
ServiceAccountPublicKeys []interface{}
VersionedInformers informers.SharedInformerFactory
// RepairServicesInterval interval used by the repair loops for
// the Services NodePort and ClusterIP resources
RepairServicesInterval time.Duration
@@ -237,13 +218,13 @@ type ExtraConfig struct {
// Config defines configuration for the master
type Config struct {
GenericConfig *genericapiserver.Config
ExtraConfig ExtraConfig
ControlPlane controlplaneapiserver.Config
Extra
}
type completedConfig struct {
GenericConfig genericapiserver.CompletedConfig
ExtraConfig *ExtraConfig
ControlPlane controlplaneapiserver.CompletedConfig
*Extra
}
// CompletedConfig embeds a private pointer that cannot be instantiated outside of this package
@@ -266,11 +247,11 @@ type Instance struct {
}
func (c *Config) createMasterCountReconciler() reconcilers.EndpointReconciler {
endpointClient := corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig)
endpointSliceClient := discoveryclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig)
endpointClient := corev1client.NewForConfigOrDie(c.ControlPlane.Generic.LoopbackClientConfig)
endpointSliceClient := discoveryclient.NewForConfigOrDie(c.ControlPlane.Generic.LoopbackClientConfig)
endpointsAdapter := reconcilers.NewEndpointsAdapter(endpointClient, endpointSliceClient)
return reconcilers.NewMasterCountEndpointReconciler(c.ExtraConfig.MasterCount, endpointsAdapter)
return reconcilers.NewMasterCountEndpointReconciler(c.Extra.MasterCount, endpointsAdapter)
}
func (c *Config) createNoneReconciler() reconcilers.EndpointReconciler {
@@ -278,12 +259,12 @@ func (c *Config) createNoneReconciler() reconcilers.EndpointReconciler {
}
func (c *Config) createLeaseReconciler() reconcilers.EndpointReconciler {
endpointClient := corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig)
endpointSliceClient := discoveryclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig)
endpointClient := corev1client.NewForConfigOrDie(c.ControlPlane.Generic.LoopbackClientConfig)
endpointSliceClient := discoveryclient.NewForConfigOrDie(c.ControlPlane.Generic.LoopbackClientConfig)
endpointsAdapter := reconcilers.NewEndpointsAdapter(endpointClient, endpointSliceClient)
ttl := c.ExtraConfig.MasterEndpointReconcileTTL
config, err := c.ExtraConfig.StorageFactory.NewConfig(api.Resource("apiServerIPInfo"))
ttl := c.Extra.MasterEndpointReconcileTTL
config, err := c.ControlPlane.StorageFactory.NewConfig(api.Resource("apiServerIPInfo"))
if err != nil {
klog.Fatalf("Error creating storage factory config: %v", err)
}
@@ -296,8 +277,8 @@ func (c *Config) createLeaseReconciler() reconcilers.EndpointReconciler {
}
func (c *Config) createEndpointReconciler() reconcilers.EndpointReconciler {
klog.Infof("Using reconciler: %v", c.ExtraConfig.EndpointReconcilerType)
switch c.ExtraConfig.EndpointReconcilerType {
klog.Infof("Using reconciler: %v", c.Extra.EndpointReconcilerType)
switch c.Extra.EndpointReconcilerType {
// there are numerous test dependencies that depend on a default controller
case reconcilers.MasterCountReconcilerType:
return c.createMasterCountReconciler()
@@ -306,7 +287,7 @@ func (c *Config) createEndpointReconciler() reconcilers.EndpointReconciler {
case reconcilers.NoneEndpointReconcilerType:
return c.createNoneReconciler()
default:
klog.Fatalf("Reconciler not implemented: %v", c.ExtraConfig.EndpointReconcilerType)
klog.Fatalf("Reconciler not implemented: %v", c.Extra.EndpointReconcilerType)
}
return nil
}
@@ -314,49 +295,50 @@ func (c *Config) createEndpointReconciler() reconcilers.EndpointReconciler {
// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
func (c *Config) Complete() CompletedConfig {
cfg := completedConfig{
c.GenericConfig.Complete(c.ExtraConfig.VersionedInformers),
&c.ExtraConfig,
c.ControlPlane.Complete(),
&c.Extra,
}
serviceIPRange, apiServerServiceIP, err := options.ServiceIPRange(cfg.ExtraConfig.ServiceIPRange)
serviceIPRange, apiServerServiceIP, err := options.ServiceIPRange(cfg.Extra.ServiceIPRange)
if err != nil {
klog.Fatalf("Error determining service IP ranges: %v", err)
}
if cfg.ExtraConfig.ServiceIPRange.IP == nil {
cfg.ExtraConfig.ServiceIPRange = serviceIPRange
if cfg.Extra.ServiceIPRange.IP == nil {
cfg.Extra.ServiceIPRange = serviceIPRange
}
if cfg.ExtraConfig.APIServerServiceIP == nil {
cfg.ExtraConfig.APIServerServiceIP = apiServerServiceIP
if cfg.Extra.APIServerServiceIP == nil {
cfg.Extra.APIServerServiceIP = apiServerServiceIP
}
discoveryAddresses := discovery.DefaultAddresses{DefaultAddress: cfg.GenericConfig.ExternalAddress}
// override the default discovery addresses in the generic controlplane adding service IP support
discoveryAddresses := discovery.DefaultAddresses{DefaultAddress: cfg.ControlPlane.Generic.ExternalAddress}
discoveryAddresses.CIDRRules = append(discoveryAddresses.CIDRRules,
discovery.CIDRRule{IPRange: cfg.ExtraConfig.ServiceIPRange, Address: net.JoinHostPort(cfg.ExtraConfig.APIServerServiceIP.String(), strconv.Itoa(cfg.ExtraConfig.APIServerServicePort))})
cfg.GenericConfig.DiscoveryAddresses = discoveryAddresses
discovery.CIDRRule{IPRange: cfg.Extra.ServiceIPRange, Address: net.JoinHostPort(cfg.Extra.APIServerServiceIP.String(), strconv.Itoa(cfg.Extra.APIServerServicePort))})
cfg.ControlPlane.Generic.DiscoveryAddresses = discoveryAddresses
if cfg.ExtraConfig.ServiceNodePortRange.Size == 0 {
if cfg.Extra.ServiceNodePortRange.Size == 0 {
// TODO: Currently no way to specify an empty range (do we need to allow this?)
// We should probably allow this for clouds that don't require NodePort to do load-balancing (GCE)
// but then that breaks the strict nestedness of ServiceType.
// Review post-v1
cfg.ExtraConfig.ServiceNodePortRange = kubeoptions.DefaultServiceNodePortRange
klog.Infof("Node port range unspecified. Defaulting to %v.", cfg.ExtraConfig.ServiceNodePortRange)
cfg.Extra.ServiceNodePortRange = kubeoptions.DefaultServiceNodePortRange
klog.Infof("Node port range unspecified. Defaulting to %v.", cfg.Extra.ServiceNodePortRange)
}
if cfg.ExtraConfig.EndpointReconcilerConfig.Interval == 0 {
cfg.ExtraConfig.EndpointReconcilerConfig.Interval = DefaultEndpointReconcilerInterval
if cfg.Extra.EndpointReconcilerConfig.Interval == 0 {
cfg.Extra.EndpointReconcilerConfig.Interval = DefaultEndpointReconcilerInterval
}
if cfg.ExtraConfig.MasterEndpointReconcileTTL == 0 {
cfg.ExtraConfig.MasterEndpointReconcileTTL = DefaultEndpointReconcilerTTL
if cfg.Extra.MasterEndpointReconcileTTL == 0 {
cfg.Extra.MasterEndpointReconcileTTL = DefaultEndpointReconcilerTTL
}
if cfg.ExtraConfig.EndpointReconcilerConfig.Reconciler == nil {
cfg.ExtraConfig.EndpointReconcilerConfig.Reconciler = c.createEndpointReconciler()
if cfg.Extra.EndpointReconcilerConfig.Reconciler == nil {
cfg.Extra.EndpointReconcilerConfig.Reconciler = c.createEndpointReconciler()
}
if cfg.ExtraConfig.RepairServicesInterval == 0 {
cfg.ExtraConfig.RepairServicesInterval = repairLoopInterval
if cfg.Extra.RepairServicesInterval == 0 {
cfg.Extra.RepairServicesInterval = repairLoopInterval
}
return CompletedConfig{&cfg}
@@ -366,27 +348,27 @@ func (c *Config) Complete() CompletedConfig {
// Certain config fields will be set to a default value if unset.
// Certain config fields must be specified, including:
// KubeletClientConfig
func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*Instance, error) {
if reflect.DeepEqual(c.ExtraConfig.KubeletClientConfig, kubeletclient.KubeletClientConfig{}) {
func (c CompletedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*Instance, error) {
if reflect.DeepEqual(c.Extra.KubeletClientConfig, kubeletclient.KubeletClientConfig{}) {
return nil, fmt.Errorf("Master.New() called with empty config.KubeletClientConfig")
}
s, err := c.GenericConfig.New("kube-apiserver", delegationTarget)
s, err := c.ControlPlane.Generic.New("kube-apiserver", delegationTarget)
if err != nil {
return nil, err
}
if c.ExtraConfig.EnableLogsSupport {
if c.ControlPlane.Extra.EnableLogsSupport {
routes.Logs{}.Install(s.Handler.GoRestfulContainer)
}
// Metadata and keys are expected to only change across restarts at present,
// so we just marshal immediately and serve the cached JSON bytes.
md, err := serviceaccount.NewOpenIDMetadata(
c.ExtraConfig.ServiceAccountIssuerURL,
c.ExtraConfig.ServiceAccountJWKSURI,
c.GenericConfig.ExternalAddress,
c.ExtraConfig.ServiceAccountPublicKeys,
c.ControlPlane.Extra.ServiceAccountIssuerURL,
c.ControlPlane.Extra.ServiceAccountJWKSURI,
c.ControlPlane.Generic.ExternalAddress,
c.ControlPlane.Extra.ServiceAccountPublicKeys,
)
if err != nil {
// If there was an error, skip installing the endpoints and log the
@@ -396,7 +378,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
msg := fmt.Sprintf("Could not construct pre-rendered responses for"+
" ServiceAccountIssuerDiscovery endpoints. Endpoints will not be"+
" enabled. Error: %v", err)
if c.ExtraConfig.ServiceAccountIssuerURL != "" {
if c.ControlPlane.Extra.ServiceAccountIssuerURL != "" {
// The user likely expects this feature to be enabled if issuer URL is
// set and the feature gate is enabled. In the future, if there is no
// longer a feature gate and issuer URL is not set, the user may not
@@ -413,37 +395,37 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
m := &Instance{
GenericAPIServer: s,
ClusterAuthenticationInfo: c.ExtraConfig.ClusterAuthenticationInfo,
ClusterAuthenticationInfo: c.ControlPlane.Extra.ClusterAuthenticationInfo,
}
clientset, err := kubernetes.NewForConfig(c.GenericConfig.LoopbackClientConfig)
client, err := kubernetes.NewForConfig(c.ControlPlane.Generic.LoopbackClientConfig)
if err != nil {
return nil, err
}
// TODO: update to a version that caches success but will recheck on failure, unlike memcache discovery
discoveryClientForAdmissionRegistration := clientset.Discovery()
discoveryClientForAdmissionRegistration := client.Discovery()
legacyRESTStorageProvider, err := corerest.New(corerest.Config{
GenericConfig: corerest.GenericConfig{
StorageFactory: c.ExtraConfig.StorageFactory,
EventTTL: c.ExtraConfig.EventTTL,
LoopbackClientConfig: c.GenericConfig.LoopbackClientConfig,
ServiceAccountIssuer: c.ExtraConfig.ServiceAccountIssuer,
ExtendExpiration: c.ExtraConfig.ExtendExpiration,
ServiceAccountMaxExpiration: c.ExtraConfig.ServiceAccountMaxExpiration,
APIAudiences: c.GenericConfig.Authentication.APIAudiences,
Informers: c.ExtraConfig.VersionedInformers,
StorageFactory: c.ControlPlane.Extra.StorageFactory,
EventTTL: c.ControlPlane.Extra.EventTTL,
LoopbackClientConfig: c.ControlPlane.Generic.LoopbackClientConfig,
ServiceAccountIssuer: c.ControlPlane.Extra.ServiceAccountIssuer,
ExtendExpiration: c.ControlPlane.Extra.ExtendExpiration,
ServiceAccountMaxExpiration: c.ControlPlane.Extra.ServiceAccountMaxExpiration,
APIAudiences: c.ControlPlane.Generic.Authentication.APIAudiences,
Informers: c.ControlPlane.Extra.VersionedInformers,
},
Proxy: corerest.ProxyConfig{
Transport: c.ExtraConfig.ProxyTransport,
KubeletClientConfig: c.ExtraConfig.KubeletClientConfig,
Transport: c.ControlPlane.Extra.ProxyTransport,
KubeletClientConfig: c.Extra.KubeletClientConfig,
},
Services: corerest.ServicesConfig{
ClusterIPRange: c.ExtraConfig.ServiceIPRange,
SecondaryClusterIPRange: c.ExtraConfig.SecondaryServiceIPRange,
NodePortRange: c.ExtraConfig.ServiceNodePortRange,
IPRepairInterval: c.ExtraConfig.RepairServicesInterval,
ClusterIPRange: c.Extra.ServiceIPRange,
SecondaryClusterIPRange: c.Extra.SecondaryServiceIPRange,
NodePortRange: c.Extra.ServiceNodePortRange,
IPRepairInterval: c.Extra.RepairServicesInterval,
},
})
if err != nil {
@@ -460,8 +442,8 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
restStorageProviders := []RESTStorageProvider{
legacyRESTStorageProvider,
apiserverinternalrest.StorageProvider{},
authenticationrest.RESTStorageProvider{Authenticator: c.GenericConfig.Authentication.Authenticator, APIAudiences: c.GenericConfig.Authentication.APIAudiences},
authorizationrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer, RuleResolver: c.GenericConfig.RuleResolver},
authenticationrest.RESTStorageProvider{Authenticator: c.ControlPlane.Generic.Authentication.Authenticator, APIAudiences: c.ControlPlane.Generic.Authentication.APIAudiences},
authorizationrest.RESTStorageProvider{Authorizer: c.ControlPlane.Generic.Authorization.Authorizer, RuleResolver: c.ControlPlane.Generic.RuleResolver},
autoscalingrest.RESTStorageProvider{},
batchrest.RESTStorageProvider{},
certificatesrest.RESTStorageProvider{},
@@ -470,47 +452,47 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
networkingrest.RESTStorageProvider{},
noderest.RESTStorageProvider{},
policyrest.RESTStorageProvider{},
rbacrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer},
rbacrest.RESTStorageProvider{Authorizer: c.ControlPlane.Generic.Authorization.Authorizer},
schedulingrest.RESTStorageProvider{},
storagerest.RESTStorageProvider{},
svmrest.RESTStorageProvider{},
flowcontrolrest.RESTStorageProvider{InformerFactory: c.GenericConfig.SharedInformerFactory},
flowcontrolrest.RESTStorageProvider{InformerFactory: c.ControlPlane.Generic.SharedInformerFactory},
// keep apps after extensions so legacy clients resolve the extensions versions of shared resource names.
// See https://github.com/kubernetes/kubernetes/issues/42392
appsrest.StorageProvider{},
admissionregistrationrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer, DiscoveryClient: discoveryClientForAdmissionRegistration},
eventsrest.RESTStorageProvider{TTL: c.ExtraConfig.EventTTL},
admissionregistrationrest.RESTStorageProvider{Authorizer: c.ControlPlane.Generic.Authorization.Authorizer, DiscoveryClient: discoveryClientForAdmissionRegistration},
eventsrest.RESTStorageProvider{TTL: c.ControlPlane.EventTTL},
resourcerest.RESTStorageProvider{},
}
if err := m.InstallAPIs(c.ExtraConfig.APIResourceConfigSource, c.GenericConfig.RESTOptionsGetter, restStorageProviders...); err != nil {
if err := m.InstallAPIs(c.ControlPlane.Extra.APIResourceConfigSource, c.ControlPlane.Generic.RESTOptionsGetter, restStorageProviders...); err != nil {
return nil, err
}
m.GenericAPIServer.AddPostStartHookOrDie("start-system-namespaces-controller", func(hookContext genericapiserver.PostStartHookContext) error {
go systemnamespaces.NewController(clientset, c.ExtraConfig.VersionedInformers.Core().V1().Namespaces()).Run(hookContext.StopCh)
go systemnamespaces.NewController(client, c.ControlPlane.Extra.VersionedInformers.Core().V1().Namespaces()).Run(hookContext.StopCh)
return nil
})
_, publicServicePort, err := c.GenericConfig.SecureServing.HostPort()
_, publicServicePort, err := c.ControlPlane.Generic.SecureServing.HostPort()
if err != nil {
return nil, fmt.Errorf("failed to get listener address: %w", err)
}
kubernetesServiceCtrl := kubernetesservice.New(kubernetesservice.Config{
PublicIP: c.GenericConfig.PublicAddress,
PublicIP: c.ControlPlane.Generic.PublicAddress,
EndpointReconciler: c.ExtraConfig.EndpointReconcilerConfig.Reconciler,
EndpointInterval: c.ExtraConfig.EndpointReconcilerConfig.Interval,
EndpointReconciler: c.Extra.EndpointReconcilerConfig.Reconciler,
EndpointInterval: c.Extra.EndpointReconcilerConfig.Interval,
ServiceIP: c.ExtraConfig.APIServerServiceIP,
ServicePort: c.ExtraConfig.APIServerServicePort,
ServiceIP: c.Extra.APIServerServiceIP,
ServicePort: c.Extra.APIServerServicePort,
PublicServicePort: publicServicePort,
KubernetesServiceNodePort: c.ExtraConfig.KubernetesServiceNodePort,
}, clientset, c.ExtraConfig.VersionedInformers.Core().V1().Services())
m.GenericAPIServer.AddPostStartHookOrDie("bootstrap-controller", func(hookContext genericapiserver.PostStartHookContext) error {
KubernetesServiceNodePort: c.Extra.KubernetesServiceNodePort,
}, client, c.ControlPlane.Extra.VersionedInformers.Core().V1().Services())
s.AddPostStartHookOrDie("bootstrap-controller", func(hookContext genericapiserver.PostStartHookContext) error {
kubernetesServiceCtrl.Start(hookContext.StopCh)
return nil
})
m.GenericAPIServer.AddPreShutdownHookOrDie("stop-kubernetes-service-controller", func() error {
s.AddPreShutdownHookOrDie("stop-kubernetes-service-controller", func() error {
kubernetesServiceCtrl.Stop()
return nil
})
@@ -518,9 +500,9 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
if utilfeature.DefaultFeatureGate.Enabled(features.MultiCIDRServiceAllocator) {
m.GenericAPIServer.AddPostStartHookOrDie("start-kubernetes-service-cidr-controller", func(hookContext genericapiserver.PostStartHookContext) error {
controller := defaultservicecidr.NewController(
c.ExtraConfig.ServiceIPRange,
c.ExtraConfig.SecondaryServiceIPRange,
clientset,
c.Extra.ServiceIPRange,
c.Extra.SecondaryServiceIPRange,
client,
)
// The default serviceCIDR must exist before the apiserver is healthy
// otherwise the allocators for Services will not work.
@@ -530,13 +512,13 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
}
if utilfeature.DefaultFeatureGate.Enabled(features.UnknownVersionInteroperabilityProxy) {
peeraddress := getPeerAddress(c.ExtraConfig.PeerAdvertiseAddress, c.GenericConfig.PublicAddress, publicServicePort)
peeraddress := getPeerAddress(c.Extra.PeerAdvertiseAddress, c.ControlPlane.Generic.PublicAddress, publicServicePort)
peerEndpointCtrl := peerreconcilers.New(
c.GenericConfig.APIServerID,
c.ControlPlane.Generic.APIServerID,
peeraddress,
c.ExtraConfig.PeerEndpointLeaseReconciler,
c.ExtraConfig.EndpointReconcilerConfig.Interval,
clientset)
c.Extra.PeerEndpointLeaseReconciler,
c.Extra.EndpointReconcilerConfig.Interval,
client)
if err != nil {
return nil, fmt.Errorf("failed to create peer endpoint lease controller: %w", err)
}
@@ -551,16 +533,16 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
return nil
})
// Add PostStartHooks for Unknown Version Proxy filter.
if c.ExtraConfig.PeerProxy != nil {
if c.Extra.PeerProxy != nil {
m.GenericAPIServer.AddPostStartHookOrDie("unknown-version-proxy-filter", func(context genericapiserver.PostStartHookContext) error {
err := c.ExtraConfig.PeerProxy.WaitForCacheSync(context.StopCh)
err := c.Extra.PeerProxy.WaitForCacheSync(context.StopCh)
return err
})
}
}
m.GenericAPIServer.AddPostStartHookOrDie("start-cluster-authentication-info-controller", func(hookContext genericapiserver.PostStartHookContext) error {
controller := clusterauthenticationtrust.NewClusterAuthenticationTrustController(m.ClusterAuthenticationInfo, clientset)
controller := clusterauthenticationtrust.NewClusterAuthenticationTrustController(m.ClusterAuthenticationInfo, client)
// generate a context from stopCh. This is to avoid modifying files which are relying on apiserver
// TODO: See if we can pass ctx to the current method
@@ -601,11 +583,11 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
leaseName := m.GenericAPIServer.APIServerID
holderIdentity := m.GenericAPIServer.APIServerID + "_" + string(uuid.NewUUID())
peeraddress := getPeerAddress(c.ExtraConfig.PeerAdvertiseAddress, c.GenericConfig.PublicAddress, publicServicePort)
peeraddress := getPeerAddress(c.Extra.PeerAdvertiseAddress, c.ControlPlane.Generic.PublicAddress, publicServicePort)
// must replace ':,[]' in [ip:port] to be able to store this as a valid label value
controller := lease.NewController(
clock.RealClock{},
clientset,
client,
holderIdentity,
int32(IdentityLeaseDurationSeconds),
nil,
@@ -620,7 +602,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
// TODO: move this into generic apiserver and make the lease identity value configurable
m.GenericAPIServer.AddPostStartHookOrDie("start-kube-apiserver-identity-lease-garbage-collector", func(hookContext genericapiserver.PostStartHookContext) error {
go apiserverleasegc.NewAPIServerLeaseGC(
clientset,
client,
IdentityLeaseGCPeriod,
metav1.NamespaceSystem,
KubeAPIServerIdentityLeaseLabelSelector,
@@ -630,7 +612,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
}
m.GenericAPIServer.AddPostStartHookOrDie("start-legacy-token-tracking-controller", func(hookContext genericapiserver.PostStartHookContext) error {
go legacytokentracking.NewController(clientset).Run(hookContext.StopCh)
go legacytokentracking.NewController(client).Run(hookContext.StopCh)
return nil
})
@@ -804,6 +786,53 @@ func DefaultAPIResourceConfigSource() *serverstorage.ResourceConfig {
return ret
}
// CreatePeerEndpointLeaseReconciler creates a apiserver endpoint lease reconciliation loop
// The peer endpoint leases are used to find network locations of apiservers for peer proxy
func CreatePeerEndpointLeaseReconciler(c genericapiserver.Config, storageFactory serverstorage.StorageFactory) (peerreconcilers.PeerEndpointLeaseReconciler, error) {
ttl := DefaultEndpointReconcilerTTL
config, err := storageFactory.NewConfig(api.Resource("apiServerPeerIPInfo"))
if err != nil {
return nil, fmt.Errorf("error creating storage factory config: %w", err)
}
reconciler, err := peerreconcilers.NewPeerEndpointLeaseReconciler(config, "/peerserverleases/", ttl)
return reconciler, err
}
func BuildPeerProxy(versionedInformer clientgoinformers.SharedInformerFactory, svm storageversion.Manager,
proxyClientCertFile string, proxyClientKeyFile string, peerCAFile string, peerAdvertiseAddress peerreconcilers.PeerAdvertiseAddress,
apiServerID string, reconciler peerreconcilers.PeerEndpointLeaseReconciler, serializer kruntime.NegotiatedSerializer) (utilpeerproxy.Interface, error) {
if proxyClientCertFile == "" {
return nil, fmt.Errorf("error building peer proxy handler, proxy-cert-file not specified")
}
if proxyClientKeyFile == "" {
return nil, fmt.Errorf("error building peer proxy handler, proxy-key-file not specified")
}
// create proxy client config
clientConfig := &transport.Config{
TLS: transport.TLSConfig{
Insecure: false,
CertFile: proxyClientCertFile,
KeyFile: proxyClientKeyFile,
CAFile: peerCAFile,
ServerName: "kubernetes.default.svc",
}}
// build proxy transport
proxyRoundTripper, transportBuildingError := transport.New(clientConfig)
if transportBuildingError != nil {
klog.Error(transportBuildingError.Error())
return nil, transportBuildingError
}
return utilpeerproxy.NewPeerProxyHandler(
versionedInformer,
svm,
proxyRoundTripper,
apiServerID,
reconciler,
serializer,
), nil
}
// utility function to get the apiserver address that is used by peer apiservers to proxy
// requests to this apiserver in case the peer is incapable of serving the request
func getPeerAddress(peerAdvertiseAddress peerreconcilers.PeerAdvertiseAddress, publicAddress net.IP, publicServicePort int) string {

64
pkg/controlplane/instance_test.go
View File

@@ -28,6 +28,8 @@ import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
autoscalingapiv2beta1 "k8s.io/api/autoscaling/v2beta1"
autoscalingapiv2beta2 "k8s.io/api/autoscaling/v2beta2"
batchapiv1beta1 "k8s.io/api/batch/v1beta1"
@@ -57,8 +59,11 @@ import (
restclient "k8s.io/client-go/rest"
kubeversion "k8s.io/component-base/version"
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
netutils "k8s.io/utils/net"
"k8s.io/kubernetes/pkg/api/legacyscheme"
flowcontrolv1bet3 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta3"
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver"
"k8s.io/kubernetes/pkg/controlplane/reconcilers"
"k8s.io/kubernetes/pkg/controlplane/storageversionhashdata"
generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
@@ -67,9 +72,6 @@ import (
certificatesrest "k8s.io/kubernetes/pkg/registry/certificates/rest"
corerest "k8s.io/kubernetes/pkg/registry/core/rest"
"k8s.io/kubernetes/pkg/registry/registrytest"
netutils "k8s.io/utils/net"
"github.com/stretchr/testify/assert"
)
// setUp is a convenience function for setting up for (most) tests.
@@ -77,9 +79,13 @@ func setUp(t *testing.T) (*etcd3testing.EtcdTestServer, Config, *assert.Assertio
server, storageConfig := etcd3testing.NewUnsecuredEtcd3TestClientServer(t)
config := &Config{
GenericConfig: genericapiserver.NewConfig(legacyscheme.Codecs),
ExtraConfig: ExtraConfig{
ControlPlane: controlplaneapiserver.Config{
Generic: genericapiserver.NewConfig(legacyscheme.Codecs),
Extra: controlplaneapiserver.Extra{
APIResourceConfigSource: DefaultAPIResourceConfigSource(),
},
},
Extra: Extra{
APIServerServicePort: 443,
MasterCount: 1,
EndpointReconcilerType: reconcilers.MasterCountReconcilerType,
@@ -88,42 +94,42 @@ func setUp(t *testing.T) (*etcd3testing.EtcdTestServer, Config, *assert.Assertio
}
storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
storageConfig.StorageObjectCountTracker = config.GenericConfig.StorageObjectCountTracker
storageConfig.StorageObjectCountTracker = config.ControlPlane.Generic.StorageObjectCountTracker
resourceEncoding := resourceconfig.MergeResourceEncodingConfigs(storageFactoryConfig.DefaultResourceEncoding, storageFactoryConfig.ResourceEncodingOverrides)
storageFactory := serverstorage.NewDefaultStorageFactory(*storageConfig, "application/vnd.kubernetes.protobuf", storageFactoryConfig.Serializer, resourceEncoding, DefaultAPIResourceConfigSource(), nil)
etcdOptions := options.NewEtcdOptions(storageConfig)
// unit tests don't need watch cache and it leaks lots of goroutines with etcd testing functions during unit tests
etcdOptions.EnableWatchCache = false
err := etcdOptions.ApplyWithStorageFactoryTo(storageFactory, config.GenericConfig)
err := etcdOptions.ApplyWithStorageFactoryTo(storageFactory, config.ControlPlane.Generic)
if err != nil {
t.Fatal(err)
}
kubeVersion := kubeversion.Get()
config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
config.GenericConfig.Version = &kubeVersion
config.ExtraConfig.StorageFactory = storageFactory
config.GenericConfig.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}}
config.GenericConfig.PublicAddress = netutils.ParseIPSloppy("192.168.10.4")
config.GenericConfig.LegacyAPIGroupPrefixes = sets.NewString("/api")
config.ExtraConfig.KubeletClientConfig = kubeletclient.KubeletClientConfig{Port: 10250}
config.ExtraConfig.ProxyTransport = utilnet.SetTransportDefaults(&http.Transport{
config.ControlPlane.Generic.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
config.ControlPlane.Generic.Version = &kubeVersion
config.ControlPlane.StorageFactory = storageFactory
config.ControlPlane.Generic.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}}
config.ControlPlane.Generic.PublicAddress = netutils.ParseIPSloppy("192.168.10.4")
config.ControlPlane.Generic.LegacyAPIGroupPrefixes = sets.NewString("/api")
config.Extra.KubeletClientConfig = kubeletclient.KubeletClientConfig{Port: 10250}
config.ControlPlane.ProxyTransport = utilnet.SetTransportDefaults(&http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { return nil, nil },
TLSClientConfig: &tls.Config{},
})
// set fake SecureServingInfo because the listener port is needed for the kubernetes service
config.GenericConfig.SecureServing = &genericapiserver.SecureServingInfo{Listener: fakeLocalhost443Listener{}}
config.ControlPlane.Generic.SecureServing = &genericapiserver.SecureServingInfo{Listener: fakeLocalhost443Listener{}}
getOpenAPIDefinitions := openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(generatedopenapi.GetOpenAPIDefinitions)
namer := openapinamer.NewDefinitionNamer(legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme)
config.GenericConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(getOpenAPIDefinitions, namer)
config.ControlPlane.Generic.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(getOpenAPIDefinitions, namer)
clientset, err := kubernetes.NewForConfig(config.GenericConfig.LoopbackClientConfig)
clientset, err := kubernetes.NewForConfig(config.ControlPlane.Generic.LoopbackClientConfig)
if err != nil {
t.Fatalf("unable to create client set due to %v", err)
}
config.ExtraConfig.VersionedInformers = informers.NewSharedInformerFactory(clientset, config.GenericConfig.LoopbackClientConfig.Timeout)
config.ControlPlane.VersionedInformers = informers.NewSharedInformerFactory(clientset, config.ControlPlane.Generic.LoopbackClientConfig.Timeout)
return server, *config, assert.New(t)
}
@@ -154,25 +160,25 @@ func TestLegacyRestStorageStrategies(t *testing.T) {
storageProvider, err := corerest.New(corerest.Config{
GenericConfig: corerest.GenericConfig{
StorageFactory: apiserverCfg.ExtraConfig.StorageFactory,
EventTTL: apiserverCfg.ExtraConfig.EventTTL,
LoopbackClientConfig: apiserverCfg.GenericConfig.LoopbackClientConfig,
Informers: apiserverCfg.ExtraConfig.VersionedInformers,
StorageFactory: apiserverCfg.ControlPlane.Extra.StorageFactory,
EventTTL: apiserverCfg.ControlPlane.Extra.EventTTL,
LoopbackClientConfig: apiserverCfg.ControlPlane.Generic.LoopbackClientConfig,
Informers: apiserverCfg.ControlPlane.Extra.VersionedInformers,
},
Proxy: corerest.ProxyConfig{
Transport: apiserverCfg.ExtraConfig.ProxyTransport,
KubeletClientConfig: apiserverCfg.ExtraConfig.KubeletClientConfig,
Transport: apiserverCfg.ControlPlane.Extra.ProxyTransport,
KubeletClientConfig: apiserverCfg.Extra.KubeletClientConfig,
},
Services: corerest.ServicesConfig{
ClusterIPRange: apiserverCfg.ExtraConfig.ServiceIPRange,
NodePortRange: apiserverCfg.ExtraConfig.ServiceNodePortRange,
ClusterIPRange: apiserverCfg.Extra.ServiceIPRange,
NodePortRange: apiserverCfg.Extra.ServiceNodePortRange,
},
})
if err != nil {
t.Fatalf("unexpected error from REST storage: %v", err)
}
apiGroupInfo, err := storageProvider.NewRESTStorage(serverstorage.NewResourceConfig(), apiserverCfg.GenericConfig.RESTOptionsGetter)
apiGroupInfo, err := storageProvider.NewRESTStorage(serverstorage.NewResourceConfig(), apiserverCfg.ControlPlane.Generic.RESTOptionsGetter)
if err != nil {
t.Errorf("failed to create legacy REST storage: %v", err)
}
@@ -188,7 +194,7 @@ func TestCertificatesRestStorageStrategies(t *testing.T) {
defer etcdserver.Terminate(t)
certStorageProvider := certificatesrest.RESTStorageProvider{}
apiGroupInfo, err := certStorageProvider.NewRESTStorage(apiserverCfg.ExtraConfig.APIResourceConfigSource, apiserverCfg.GenericConfig.RESTOptionsGetter)
apiGroupInfo, err := certStorageProvider.NewRESTStorage(apiserverCfg.ControlPlane.APIResourceConfigSource, apiserverCfg.ControlPlane.Generic.RESTOptionsGetter)
if err != nil {
t.Fatalf("unexpected error from REST storage: %v", err)
}

2
test/integration/apiserver/apiserver_test.go
View File

@@ -84,7 +84,7 @@ func setupWithResources(t *testing.T, groupVersions []schema.GroupVersion, resou
resourceConfig := controlplane.DefaultAPIResourceConfigSource()
resourceConfig.EnableVersions(groupVersions...)
resourceConfig.EnableResources(resources...)
config.ExtraConfig.APIResourceConfigSource = resourceConfig
config.ControlPlane.Extra.APIResourceConfigSource = resourceConfig
}
},
})

2
test/integration/apiserver/flowcontrol/concurrency_util_test.go
View File

@@ -155,7 +155,7 @@ func TestConcurrencyIsolation(t *testing.T) {
},
ModifyServerConfig: func(config *controlplane.Config) {
// Wrap default authorizer with one that delays requests from noxu clients
config.GenericConfig.Authorization.Authorizer = &noxuDelayingAuthorizer{config.GenericConfig.Authorization.Authorizer}
config.ControlPlane.Generic.Authorization.Authorizer = &noxuDelayingAuthorizer{config.ControlPlane.Generic.Authorization.Authorizer}
},
})
defer closeFn()

4
test/integration/apiserver/openapi/openapi_enum_test.go
View File

@@ -77,8 +77,8 @@ func TestEnablingOpenAPIEnumTypes(t *testing.T) {
_, kubeConfig, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
ModifyServerConfig: func(config *controlplane.Config) {
config.GenericConfig.OpenAPIConfig = framework.DefaultOpenAPIConfig()
config.GenericConfig.OpenAPIConfig.GetDefinitions = getDefinitionsFn
config.ControlPlane.Generic.OpenAPIConfig = framework.DefaultOpenAPIConfig()
config.ControlPlane.Generic.OpenAPIConfig.GetDefinitions = getDefinitionsFn
},
})
defer tearDownFn()

4
test/integration/apiserver/watchcache_test.go
View File

@@ -61,7 +61,7 @@ func multiEtcdSetup(ctx context.Context, t *testing.T) (clientset.Interface, fra
},
ModifyServerConfig: func(config *controlplane.Config) {
// Switch off endpoints reconciler to avoid unnecessary operations.
config.ExtraConfig.EndpointReconcilerType = reconcilers.NoneEndpointReconcilerType
config.Extra.EndpointReconcilerType = reconcilers.NoneEndpointReconcilerType
},
})
@@ -170,7 +170,7 @@ func BenchmarkListFromWatchCache(b *testing.B) {
c, _, tearDownFn := framework.StartTestServer(tCtx, b, framework.TestServerSetup{
ModifyServerConfig: func(config *controlplane.Config) {
// Switch off endpoints reconciler to avoid unnecessary operations.
config.ExtraConfig.EndpointReconcilerType = reconcilers.NoneEndpointReconcilerType
config.Extra.EndpointReconcilerType = reconcilers.NoneEndpointReconcilerType
},
})
defer tearDownFn()

18
test/integration/auth/accessreview_test.go
View File

@@ -62,9 +62,9 @@ func TestSubjectAccessReview(t *testing.T) {
clientset, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
ModifyServerConfig: func(config *controlplane.Config) {
// Unset BearerToken to disable BearerToken authenticator.
config.GenericConfig.LoopbackClientConfig.BearerToken = ""
config.GenericConfig.Authentication.Authenticator = authenticator.RequestFunc(alwaysAlice)
config.GenericConfig.Authorization.Authorizer = sarAuthorizer{}
config.ControlPlane.Generic.LoopbackClientConfig.BearerToken = ""
config.ControlPlane.Generic.Authentication.Authenticator = authenticator.RequestFunc(alwaysAlice)
config.ControlPlane.Generic.Authorization.Authorizer = sarAuthorizer{}
},
})
defer tearDownFn()
@@ -172,9 +172,9 @@ func TestSelfSubjectAccessReview(t *testing.T) {
clientset, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
ModifyServerConfig: func(config *controlplane.Config) {
// Unset BearerToken to disable BearerToken authenticator.
config.GenericConfig.LoopbackClientConfig.BearerToken = ""
config.GenericConfig.Authentication.Authenticator = authenticator.RequestFunc(authenticatorFunc)
config.GenericConfig.Authorization.Authorizer = sarAuthorizer{}
config.ControlPlane.Generic.LoopbackClientConfig.BearerToken = ""
config.ControlPlane.Generic.Authentication.Authenticator = authenticator.RequestFunc(authenticatorFunc)
config.ControlPlane.Generic.Authorization.Authorizer = sarAuthorizer{}
},
})
defer tearDownFn()
@@ -256,9 +256,9 @@ func TestLocalSubjectAccessReview(t *testing.T) {
clientset, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
ModifyServerConfig: func(config *controlplane.Config) {
// Unset BearerToken to disable BearerToken authenticator.
config.GenericConfig.LoopbackClientConfig.BearerToken = ""
config.GenericConfig.Authentication.Authenticator = authenticator.RequestFunc(alwaysAlice)
config.GenericConfig.Authorization.Authorizer = sarAuthorizer{}
config.ControlPlane.Generic.LoopbackClientConfig.BearerToken = ""
config.ControlPlane.Generic.Authentication.Authenticator = authenticator.RequestFunc(alwaysAlice)
config.ControlPlane.Generic.Authorization.Authorizer = sarAuthorizer{}
},
})
defer tearDownFn()

8
test/integration/auth/auth_test.go
View File

@@ -813,7 +813,7 @@ func TestImpersonateIsForbidden(t *testing.T) {
},
ModifyServerConfig: func(config *controlplane.Config) {
// Prepend an impersonation authorizer with specific opinions about alice and bob
config.GenericConfig.Authorization.Authorizer = unionauthz.New(impersonateAuthorizer{}, config.GenericConfig.Authorization.Authorizer)
config.ControlPlane.Generic.Authorization.Authorizer = unionauthz.New(impersonateAuthorizer{}, config.ControlPlane.Generic.Authorization.Authorizer)
},
})
defer tearDownFn()
@@ -1118,7 +1118,7 @@ func TestAuthorizationAttributeDetermination(t *testing.T) {
opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
},
ModifyServerConfig: func(config *controlplane.Config) {
config.GenericConfig.Authorization.Authorizer = unionauthz.New(config.GenericConfig.Authorization.Authorizer, trackingAuthorizer)
config.ControlPlane.Generic.Authorization.Authorizer = unionauthz.New(config.ControlPlane.Generic.Authorization.Authorizer, trackingAuthorizer)
},
})
defer tearDownFn()
@@ -1458,9 +1458,9 @@ func testWebhookTokenAuthenticator(customDialer bool, t *testing.T) {
opts.Authorization.PolicyFile = "testdata/allowalice.jsonl"
},
ModifyServerConfig: func(config *controlplane.Config) {
config.GenericConfig.Authentication.Authenticator = group.NewAuthenticatedGroupAdder(authenticator)
config.ControlPlane.Generic.Authentication.Authenticator = group.NewAuthenticatedGroupAdder(authenticator)
// Disable checking API audiences that is set by testserver by default.
config.GenericConfig.Authentication.APIAudiences = nil
config.ControlPlane.Generic.Authentication.APIAudiences = nil
},
})
defer tearDownFn()

2
test/integration/auth/bootstraptoken_test.go
View File

@@ -128,7 +128,7 @@ func TestBootstrapTokenAuth(t *testing.T) {
opts.Authorization.Modes = []string{"AlwaysAllow"}
},
ModifyServerConfig: func(config *controlplane.Config) {
config.GenericConfig.Authentication.Authenticator = authenticator
config.ControlPlane.Generic.Authentication.Authenticator = authenticator
},
})
defer tearDownFn()

6
test/integration/auth/rbac_test.go
View File

@@ -81,7 +81,7 @@ type testRESTOptionsGetter struct {
}
func (getter *testRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
storageConfig, err := getter.config.ExtraConfig.StorageFactory.NewConfig(resource)
storageConfig, err := getter.config.ControlPlane.Extra.StorageFactory.NewConfig(resource)
if err != nil {
return generic.RESTOptions{}, fmt.Errorf("failed to get storage: %v", err)
}
@@ -556,11 +556,11 @@ func TestRBAC(t *testing.T) {
},
ModifyServerConfig: func(config *controlplane.Config) {
// Append our custom test authenticator
config.GenericConfig.Authentication.Authenticator = unionauthn.New(config.GenericConfig.Authentication.Authenticator, authenticator)
config.ControlPlane.Generic.Authentication.Authenticator = unionauthn.New(config.ControlPlane.Generic.Authentication.Authenticator, authenticator)
// Append our custom test authorizer
var rbacAuthz authorizer.Authorizer
rbacAuthz, tearDownAuthorizerFn = newRBACAuthorizer(t, config)
config.GenericConfig.Authorization.Authorizer = unionauthz.New(config.GenericConfig.Authorization.Authorizer, rbacAuthz)
config.ControlPlane.Generic.Authorization.Authorizer = unionauthz.New(config.ControlPlane.Generic.Authorization.Authorizer, rbacAuthz)
},
})
defer tearDownFn()

8
test/integration/auth/selfsubjectreview_test.go
View File

@@ -101,8 +101,8 @@ func TestGetsSelfAttributes(t *testing.T) {
},
ModifyServerConfig: func(config *controlplane.Config) {
// Unset BearerToken to disable BearerToken authenticator.
config.GenericConfig.LoopbackClientConfig.BearerToken = ""
config.GenericConfig.Authentication.Authenticator = authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) {
config.ControlPlane.Generic.LoopbackClientConfig.BearerToken = ""
config.ControlPlane.Generic.Authentication.Authenticator = authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) {
respMu.RLock()
defer respMu.RUnlock()
return &authenticator.Response{User: response}, true, nil
@@ -215,8 +215,8 @@ func TestGetsSelfAttributesError(t *testing.T) {
},
ModifyServerConfig: func(config *controlplane.Config) {
// Unset BearerToken to disable BearerToken authenticator.
config.GenericConfig.LoopbackClientConfig.BearerToken = ""
config.GenericConfig.Authentication.Authenticator = authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) {
config.ControlPlane.Generic.LoopbackClientConfig.BearerToken = ""
config.ControlPlane.Generic.Authentication.Authenticator = authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) {
if toggle.Load().(bool) {
return &authenticator.Response{
User: &user.DefaultInfo{

6
test/integration/auth/svcaccttoken_test.go
View File

@@ -104,10 +104,10 @@ func TestServiceAccountTokenCreate(t *testing.T) {
},
ModifyServerConfig: func(config *controlplane.Config) {
// extract token generator
tokenGenerator = config.ExtraConfig.ServiceAccountIssuer
tokenGenerator = config.ControlPlane.Extra.ServiceAccountIssuer
config.ExtraConfig.ServiceAccountMaxExpiration = maxExpirationDuration
config.ExtraConfig.ExtendExpiration = true
config.ControlPlane.Extra.ServiceAccountMaxExpiration = maxExpirationDuration
config.ControlPlane.Extra.ExtendExpiration = true
},
})
defer tearDownFn()

2
test/integration/daemonset/daemonset_test.go
View File

@@ -1178,7 +1178,7 @@ func TestUpdateStatusDespitePodCreationFailure(t *testing.T) {
limitedPodNumber := 2
ctx, closeFn, dc, informers, clientset := setupWithServerSetup(t, framework.TestServerSetup{
ModifyServerConfig: func(config *controlplane.Config) {
config.GenericConfig.AdmissionControl = &fakePodFailAdmission{
config.ControlPlane.Generic.AdmissionControl = &fakePodFailAdmission{
limitedPodNumber: limitedPodNumber,
}
},

4
test/integration/defaulttolerationseconds/defaulttolerationseconds_test.go
View File

@@ -32,8 +32,8 @@ func TestAdmission(t *testing.T) {
tCtx := ktesting.Init(t)
client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
ModifyServerConfig: func(cfg *controlplane.Config) {
cfg.GenericConfig.EnableProfiling = true
cfg.GenericConfig.AdmissionControl = defaulttolerationseconds.NewDefaultTolerationSeconds()
cfg.ControlPlane.Generic.EnableProfiling = true
cfg.ControlPlane.Generic.AdmissionControl = defaulttolerationseconds.NewDefaultTolerationSeconds()
},
})
defer tearDownFn()

6
test/integration/examples/webhook_test.go
View File

@@ -47,11 +47,11 @@ func TestWebhookLoopback(t *testing.T) {
},
ModifyServerConfig: func(config *controlplane.Config) {
// Avoid resolvable kubernetes service
config.ExtraConfig.EndpointReconcilerType = reconcilers.NoneEndpointReconcilerType
config.Extra.EndpointReconcilerType = reconcilers.NoneEndpointReconcilerType
// Hook into audit to watch requests
config.GenericConfig.AuditBackend = auditSinkFunc(func(events ...*auditinternal.Event) {})
config.GenericConfig.AuditPolicyRuleEvaluator = auditPolicyRuleEvaluator(func(attrs authorizer.Attributes) audit.RequestAuditConfig {
config.ControlPlane.Generic.AuditBackend = auditSinkFunc(func(events ...*auditinternal.Event) {})
config.ControlPlane.Generic.AuditPolicyRuleEvaluator = auditPolicyRuleEvaluator(func(attrs authorizer.Attributes) audit.RequestAuditConfig {
if attrs.GetPath() == webhookPath {
if attrs.GetUser().GetName() != "system:apiserver" {
t.Errorf("expected user %q, got %q", "system:apiserver", attrs.GetUser().GetName())

2
test/integration/framework/test_server.go
View File

@@ -182,7 +182,7 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) (
}()
// Adjust the loopback config for external use (external server name and CA)
kubeAPIServerClientConfig := rest.CopyConfig(kubeAPIServerConfig.GenericConfig.LoopbackClientConfig)
kubeAPIServerClientConfig := rest.CopyConfig(kubeAPIServerConfig.ControlPlane.Generic.LoopbackClientConfig)
kubeAPIServerClientConfig.CAFile = path.Join(certDir, "apiserver.crt")
kubeAPIServerClientConfig.CAData = nil
kubeAPIServerClientConfig.ServerName = ""

2
test/integration/network/services_test.go
View File

@@ -50,7 +50,7 @@ func TestServicesFinalizersRepairLoop(t *testing.T) {
opts.ServiceClusterIPRanges = serviceCIDR
},
ModifyServerConfig: func(cfg *controlplane.Config) {
cfg.ExtraConfig.RepairServicesInterval = interval
cfg.Extra.RepairServicesInterval = interval
},
})
defer tearDownFn()

7
test/integration/openshift/openshift_test.go
View File

@@ -21,18 +21,21 @@ import (
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/kubernetes/pkg/controlplane"
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver"
)
// This test references methods that OpenShift uses to customize the apiserver on startup, that
// are not referenced directly by an instance.
func TestApiserverExportsSymbols(t *testing.T) {
_ = &controlplane.Config{
GenericConfig: &genericapiserver.Config{
ControlPlane: controlplaneapiserver.Config{
Generic: &genericapiserver.Config{
EnableMetrics: true,
},
ExtraConfig: controlplane.ExtraConfig{
Extra: controlplaneapiserver.Extra{
EnableLogsSupport: false,
},
},
}
_ = &controlplane.Instance{
GenericAPIServer: &genericapiserver.GenericAPIServer{},

2
test/integration/serviceaccount/service_account_test.go
View File

@@ -376,7 +376,7 @@ func startServiceAccountTestServerAndWaitForCaches(ctx context.Context, t *testi
return authorizer.DecisionNoOpinion, fmt.Sprintf("User %s is denied (ns=%s, readonly=%v, resource=%s)", username, ns, attrs.IsReadOnly(), attrs.GetResource()), nil
})
config.GenericConfig.Authorization.Authorizer = unionauthz.New(config.GenericConfig.Authorization.Authorizer, authorizer)
config.ControlPlane.Generic.Authorization.Authorizer = unionauthz.New(config.ControlPlane.Generic.Authorization.Authorizer, authorizer)
},
})

2
test/integration/statefulset/statefulset_test.go
View File

@@ -381,7 +381,7 @@ func TestStatefulSetStatusWithPodFail(t *testing.T) {
limitedPodNumber := 2
c, config, closeFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
ModifyServerConfig: func(config *controlplane.Config) {
config.GenericConfig.AdmissionControl = &fakePodFailAdmission{
config.ControlPlane.Generic.AdmissionControl = &fakePodFailAdmission{
limitedPodNumber: limitedPodNumber,
}
},

2
test/integration/util/util.go
View File

@@ -518,7 +518,7 @@ func InitTestAPIServer(t *testing.T, nsPrefix string, admission admission.Interf
},
ModifyServerConfig: func(config *controlplane.Config) {
if admission != nil {
config.GenericConfig.AdmissionControl = admission
config.ControlPlane.Generic.AdmissionControl = admission
}
},
})