279 lines
10 KiB
Go
279 lines
10 KiB
Go
/*
|
|
Copyright 2023 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package apiserver
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"time"
|
|
|
|
oteltrace "go.opentelemetry.io/otel/trace"
|
|
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/util/sets"
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
|
"k8s.io/apiserver/pkg/authorization/authorizer"
|
|
"k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
|
|
openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
|
|
genericfeatures "k8s.io/apiserver/pkg/features"
|
|
"k8s.io/apiserver/pkg/reconcilers"
|
|
genericapiserver "k8s.io/apiserver/pkg/server"
|
|
"k8s.io/apiserver/pkg/server/egressselector"
|
|
"k8s.io/apiserver/pkg/server/filters"
|
|
serverstorage "k8s.io/apiserver/pkg/server/storage"
|
|
"k8s.io/apiserver/pkg/storageversion"
|
|
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
|
"k8s.io/apiserver/pkg/util/openapi"
|
|
utilpeerproxy "k8s.io/apiserver/pkg/util/peerproxy"
|
|
clientgoinformers "k8s.io/client-go/informers"
|
|
clientgoclientset "k8s.io/client-go/kubernetes"
|
|
"k8s.io/client-go/transport"
|
|
"k8s.io/component-base/version"
|
|
"k8s.io/klog/v2"
|
|
openapicommon "k8s.io/kube-openapi/pkg/common"
|
|
|
|
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
|
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
|
|
"k8s.io/kubernetes/pkg/controlplane/controller/clusterauthenticationtrust"
|
|
"k8s.io/kubernetes/pkg/kubeapiserver"
|
|
"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
|
|
rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest"
|
|
"k8s.io/kubernetes/pkg/serviceaccount"
|
|
)
|
|
|
|
// Config defines configuration for the master
|
|
type Config struct {
|
|
Generic *genericapiserver.Config
|
|
Extra
|
|
}
|
|
|
|
type Extra struct {
|
|
ClusterAuthenticationInfo clusterauthenticationtrust.ClusterAuthenticationInfo
|
|
|
|
APIResourceConfigSource serverstorage.APIResourceConfigSource
|
|
StorageFactory serverstorage.StorageFactory
|
|
EventTTL time.Duration
|
|
|
|
EnableLogsSupport bool
|
|
ProxyTransport *http.Transport
|
|
|
|
ServiceAccountIssuer serviceaccount.TokenGenerator
|
|
ServiceAccountMaxExpiration time.Duration
|
|
ExtendExpiration bool
|
|
|
|
// ServiceAccountIssuerDiscovery
|
|
ServiceAccountIssuerURL string
|
|
ServiceAccountJWKSURI string
|
|
ServiceAccountPublicKeys []interface{}
|
|
|
|
VersionedInformers clientgoinformers.SharedInformerFactory
|
|
}
|
|
|
|
// BuildGenericConfig takes the master server options and produces the genericapiserver.Config associated with it
|
|
func BuildGenericConfig(
|
|
s controlplaneapiserver.CompletedOptions,
|
|
schemes []*runtime.Scheme,
|
|
resourceConfig *serverstorage.ResourceConfig,
|
|
getOpenAPIDefinitions func(ref openapicommon.ReferenceCallback) map[string]openapicommon.OpenAPIDefinition,
|
|
) (
|
|
genericConfig *genericapiserver.Config,
|
|
versionedInformers clientgoinformers.SharedInformerFactory,
|
|
storageFactory *serverstorage.DefaultStorageFactory,
|
|
lastErr error,
|
|
) {
|
|
genericConfig = genericapiserver.NewConfig(legacyscheme.Codecs)
|
|
genericConfig.MergedResourceConfig = resourceConfig
|
|
|
|
if lastErr = s.GenericServerRunOptions.ApplyTo(genericConfig); lastErr != nil {
|
|
return
|
|
}
|
|
|
|
if lastErr = s.SecureServing.ApplyTo(&genericConfig.SecureServing, &genericConfig.LoopbackClientConfig); lastErr != nil {
|
|
return
|
|
}
|
|
|
|
// Use protobufs for self-communication.
|
|
// Since not every generic apiserver has to support protobufs, we
|
|
// cannot default to it in generic apiserver and need to explicitly
|
|
// set it in kube-apiserver.
|
|
genericConfig.LoopbackClientConfig.ContentConfig.ContentType = "application/vnd.kubernetes.protobuf"
|
|
// Disable compression for self-communication, since we are going to be
|
|
// on a fast local network
|
|
genericConfig.LoopbackClientConfig.DisableCompression = true
|
|
|
|
kubeClientConfig := genericConfig.LoopbackClientConfig
|
|
clientgoExternalClient, err := clientgoclientset.NewForConfig(kubeClientConfig)
|
|
if err != nil {
|
|
lastErr = fmt.Errorf("failed to create real external clientset: %v", err)
|
|
return
|
|
}
|
|
versionedInformers = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute)
|
|
|
|
if lastErr = s.Features.ApplyTo(genericConfig, clientgoExternalClient, versionedInformers); lastErr != nil {
|
|
return
|
|
}
|
|
if lastErr = s.APIEnablement.ApplyTo(genericConfig, resourceConfig, legacyscheme.Scheme); lastErr != nil {
|
|
return
|
|
}
|
|
if lastErr = s.EgressSelector.ApplyTo(genericConfig); lastErr != nil {
|
|
return
|
|
}
|
|
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) {
|
|
if lastErr = s.Traces.ApplyTo(genericConfig.EgressSelector, genericConfig); lastErr != nil {
|
|
return
|
|
}
|
|
}
|
|
// wrap the definitions to revert any changes from disabled features
|
|
getOpenAPIDefinitions = openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(getOpenAPIDefinitions)
|
|
namer := openapinamer.NewDefinitionNamer(schemes...)
|
|
genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(getOpenAPIDefinitions, namer)
|
|
genericConfig.OpenAPIConfig.Info.Title = "Kubernetes"
|
|
genericConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(getOpenAPIDefinitions, namer)
|
|
genericConfig.OpenAPIV3Config.Info.Title = "Kubernetes"
|
|
|
|
genericConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck(
|
|
sets.NewString("watch", "proxy"),
|
|
sets.NewString("attach", "exec", "proxy", "log", "portforward"),
|
|
)
|
|
|
|
kubeVersion := version.Get()
|
|
genericConfig.Version = &kubeVersion
|
|
|
|
if genericConfig.EgressSelector != nil {
|
|
s.Etcd.StorageConfig.Transport.EgressLookup = genericConfig.EgressSelector.Lookup
|
|
}
|
|
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) {
|
|
s.Etcd.StorageConfig.Transport.TracerProvider = genericConfig.TracerProvider
|
|
} else {
|
|
s.Etcd.StorageConfig.Transport.TracerProvider = oteltrace.NewNoopTracerProvider()
|
|
}
|
|
|
|
storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
|
|
storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig
|
|
storageFactory, lastErr = storageFactoryConfig.Complete(s.Etcd).New()
|
|
if lastErr != nil {
|
|
return
|
|
}
|
|
// storageFactory.StorageConfig is copied from etcdOptions.StorageConfig,
|
|
// the StorageObjectCountTracker is still nil. Here we copy from genericConfig.
|
|
storageFactory.StorageConfig.StorageObjectCountTracker = genericConfig.StorageObjectCountTracker
|
|
if lastErr = s.Etcd.ApplyWithStorageFactoryTo(storageFactory, genericConfig); lastErr != nil {
|
|
return
|
|
}
|
|
|
|
ctx := wait.ContextForChannel(genericConfig.DrainedNotify())
|
|
|
|
// Authentication.ApplyTo requires already applied OpenAPIConfig and EgressSelector if present
|
|
if lastErr = s.Authentication.ApplyTo(ctx, &genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, genericConfig.OpenAPIV3Config, clientgoExternalClient, versionedInformers, genericConfig.APIServerID); lastErr != nil {
|
|
return
|
|
}
|
|
|
|
var enablesRBAC bool
|
|
genericConfig.Authorization.Authorizer, genericConfig.RuleResolver, enablesRBAC, err = BuildAuthorizer(
|
|
ctx,
|
|
s,
|
|
genericConfig.EgressSelector,
|
|
genericConfig.APIServerID,
|
|
versionedInformers,
|
|
)
|
|
if err != nil {
|
|
lastErr = fmt.Errorf("invalid authorization config: %v", err)
|
|
return
|
|
}
|
|
if s.Authorization != nil && !enablesRBAC {
|
|
genericConfig.DisabledPostStartHooks.Insert(rbacrest.PostStartHookName)
|
|
}
|
|
|
|
lastErr = s.Audit.ApplyTo(genericConfig)
|
|
if lastErr != nil {
|
|
return
|
|
}
|
|
|
|
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
|
|
genericConfig.AggregatedDiscoveryGroupManager = aggregated.NewResourceManager("apis")
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// BuildAuthorizer constructs the authorizer. If authorization is not set in s, it returns nil, nil, false, nil
|
|
func BuildAuthorizer(ctx context.Context, s controlplaneapiserver.CompletedOptions, egressSelector *egressselector.EgressSelector, apiserverID string, versionedInformers clientgoinformers.SharedInformerFactory) (authorizer.Authorizer, authorizer.RuleResolver, bool, error) {
|
|
authorizationConfig, err := s.Authorization.ToAuthorizationConfig(versionedInformers)
|
|
if err != nil {
|
|
return nil, nil, false, err
|
|
}
|
|
if authorizationConfig == nil {
|
|
return nil, nil, false, nil
|
|
}
|
|
|
|
if egressSelector != nil {
|
|
egressDialer, err := egressSelector.Lookup(egressselector.ControlPlane.AsNetworkContext())
|
|
if err != nil {
|
|
return nil, nil, false, err
|
|
}
|
|
authorizationConfig.CustomDial = egressDialer
|
|
}
|
|
|
|
enablesRBAC := false
|
|
for _, a := range authorizationConfig.AuthorizationConfiguration.Authorizers {
|
|
if string(a.Type) == modes.ModeRBAC {
|
|
enablesRBAC = true
|
|
break
|
|
}
|
|
}
|
|
|
|
authorizer, ruleResolver, err := authorizationConfig.New(ctx, apiserverID)
|
|
|
|
return authorizer, ruleResolver, enablesRBAC, err
|
|
}
|
|
|
|
func BuildPeerProxy(versionedInformer clientgoinformers.SharedInformerFactory, svm storageversion.Manager,
|
|
proxyClientCertFile string, proxyClientKeyFile string, peerCAFile string, peerAdvertiseAddress reconcilers.PeerAdvertiseAddress,
|
|
apiServerID string, reconciler reconcilers.PeerEndpointLeaseReconciler, serializer runtime.NegotiatedSerializer) (utilpeerproxy.Interface, error) {
|
|
if proxyClientCertFile == "" {
|
|
return nil, fmt.Errorf("error building peer proxy handler, proxy-cert-file not specified")
|
|
}
|
|
if proxyClientKeyFile == "" {
|
|
return nil, fmt.Errorf("error building peer proxy handler, proxy-key-file not specified")
|
|
}
|
|
// create proxy client config
|
|
clientConfig := &transport.Config{
|
|
TLS: transport.TLSConfig{
|
|
Insecure: false,
|
|
CertFile: proxyClientCertFile,
|
|
KeyFile: proxyClientKeyFile,
|
|
CAFile: peerCAFile,
|
|
ServerName: "kubernetes.default.svc",
|
|
}}
|
|
|
|
// build proxy transport
|
|
proxyRoundTripper, transportBuildingError := transport.New(clientConfig)
|
|
if transportBuildingError != nil {
|
|
klog.Error(transportBuildingError.Error())
|
|
return nil, transportBuildingError
|
|
}
|
|
return utilpeerproxy.NewPeerProxyHandler(
|
|
versionedInformer,
|
|
svm,
|
|
proxyRoundTripper,
|
|
apiServerID,
|
|
reconciler,
|
|
serializer,
|
|
), nil
|
|
}
|