3c2a0c84c5
Automatic merge from submit-queue (batch tested with PRs 60054, 60202, 60219, 58090, 60275). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Fixes for HTTP/2 max streams per connection setting **What this PR does / why we need it**: This PR makes two changes. One is to introduce a parameter for the HTTP/2 setting that an api-server sends to its clients telling them how many streams they may have concurrently open in an HTTP/2 connection. If left at its default value of zero, this means to use the default in golang's HTTP/2 code (which is currently 250; see https://github.com/golang/net/blob/master/http2/server.go). The other change is to make the recommended options for an aggregated api-server set this limit to 1000. The limit of 250 is annoyingly low for the use case of many controllers watching objects of Kinds served by an aggregated api-server reached through the main api-server (in its mode as a proxy for the aggregated api-server, in which it uses a single HTTP/2 connection for all calls proxied to that aggregated api-server). **Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*: Fixes #60042 **Special notes for your reviewer**: **Release note**: ```release-note Introduced `--http2-max-streams-per-connection` command line flag on api-servers and set default to 1000 for aggregated API servers. ```
255 lines
9.1 KiB
Go
255 lines
9.1 KiB
Go
/*
|
|
Copyright 2014 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package options
|
|
|
|
import (
|
|
"net"
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/spf13/pflag"
|
|
|
|
"k8s.io/apimachinery/pkg/util/diff"
|
|
apiserveroptions "k8s.io/apiserver/pkg/server/options"
|
|
genericoptions "k8s.io/apiserver/pkg/server/options"
|
|
"k8s.io/apiserver/pkg/storage/storagebackend"
|
|
utilflag "k8s.io/apiserver/pkg/util/flag"
|
|
auditwebhook "k8s.io/apiserver/plugin/pkg/audit/webhook"
|
|
restclient "k8s.io/client-go/rest"
|
|
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
|
kapi "k8s.io/kubernetes/pkg/apis/core"
|
|
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
|
|
kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
|
|
"k8s.io/kubernetes/pkg/master/reconcilers"
|
|
)
|
|
|
|
func TestAddFlags(t *testing.T) {
|
|
f := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError)
|
|
s := NewServerRunOptions()
|
|
s.AddFlags(f)
|
|
|
|
args := []string{
|
|
"--enable-admission-plugins=AlwaysDeny",
|
|
"--admission-control-config-file=/admission-control-config",
|
|
"--advertise-address=192.168.10.10",
|
|
"--allow-privileged=false",
|
|
"--anonymous-auth=false",
|
|
"--apiserver-count=5",
|
|
"--audit-log-maxage=11",
|
|
"--audit-log-maxbackup=12",
|
|
"--audit-log-maxsize=13",
|
|
"--audit-log-path=/var/log",
|
|
"--audit-policy-file=/policy",
|
|
"--audit-webhook-config-file=/webhook-config",
|
|
"--audit-webhook-mode=blocking",
|
|
"--audit-webhook-batch-buffer-size=42",
|
|
"--audit-webhook-batch-max-size=43",
|
|
"--audit-webhook-batch-max-wait=1s",
|
|
"--audit-webhook-batch-throttle-qps=43.5",
|
|
"--audit-webhook-batch-throttle-burst=44",
|
|
"--audit-webhook-batch-initial-backoff=2s",
|
|
"--authentication-token-webhook-cache-ttl=3m",
|
|
"--authentication-token-webhook-config-file=/token-webhook-config",
|
|
"--authorization-mode=AlwaysDeny",
|
|
"--authorization-policy-file=/policy",
|
|
"--authorization-webhook-cache-authorized-ttl=3m",
|
|
"--authorization-webhook-cache-unauthorized-ttl=1m",
|
|
"--authorization-webhook-config-file=/webhook-config",
|
|
"--bind-address=192.168.10.20",
|
|
"--client-ca-file=/client-ca",
|
|
"--cloud-config=/cloud-config",
|
|
"--cloud-provider=azure",
|
|
"--cors-allowed-origins=10.10.10.100,10.10.10.200",
|
|
"--contention-profiling=true",
|
|
"--enable-aggregator-routing=true",
|
|
"--enable-logs-handler=false",
|
|
"--enable-swagger-ui=true",
|
|
"--endpoint-reconciler-type=" + string(reconcilers.MasterCountReconcilerType),
|
|
"--etcd-quorum-read=false",
|
|
"--etcd-keyfile=/var/run/kubernetes/etcd.key",
|
|
"--etcd-certfile=/var/run/kubernetes/etcdce.crt",
|
|
"--etcd-cafile=/var/run/kubernetes/etcdca.crt",
|
|
"--http2-max-streams-per-connection=42",
|
|
"--kubelet-https=true",
|
|
"--kubelet-read-only-port=10255",
|
|
"--kubelet-timeout=5s",
|
|
"--kubelet-client-certificate=/var/run/kubernetes/ceserver.crt",
|
|
"--kubelet-client-key=/var/run/kubernetes/server.key",
|
|
"--kubelet-certificate-authority=/var/run/kubernetes/caserver.crt",
|
|
"--proxy-client-cert-file=/var/run/kubernetes/proxy.crt",
|
|
"--proxy-client-key-file=/var/run/kubernetes/proxy.key",
|
|
"--request-timeout=2m",
|
|
"--storage-backend=etcd2",
|
|
}
|
|
f.Parse(args)
|
|
|
|
// This is a snapshot of expected options parsed by args.
|
|
expected := &ServerRunOptions{
|
|
ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange,
|
|
MasterCount: 5,
|
|
EndpointReconcilerType: string(reconcilers.MasterCountReconcilerType),
|
|
AllowPrivileged: false,
|
|
GenericServerRunOptions: &apiserveroptions.ServerRunOptions{
|
|
AdvertiseAddress: net.ParseIP("192.168.10.10"),
|
|
CorsAllowedOriginList: []string{"10.10.10.100", "10.10.10.200"},
|
|
MaxRequestsInFlight: 400,
|
|
MaxMutatingRequestsInFlight: 200,
|
|
RequestTimeout: time.Duration(2) * time.Minute,
|
|
MinRequestTimeout: 1800,
|
|
},
|
|
Admission: &kubeoptions.AdmissionOptions{
|
|
GenericAdmission: &apiserveroptions.AdmissionOptions{
|
|
RecommendedPluginOrder: s.Admission.GenericAdmission.RecommendedPluginOrder,
|
|
DefaultOffPlugins: s.Admission.GenericAdmission.DefaultOffPlugins,
|
|
EnablePlugins: []string{"AlwaysDeny"},
|
|
ConfigFile: "/admission-control-config",
|
|
Plugins: s.Admission.GenericAdmission.Plugins,
|
|
},
|
|
},
|
|
Etcd: &apiserveroptions.EtcdOptions{
|
|
StorageConfig: storagebackend.Config{
|
|
Type: "etcd2",
|
|
ServerList: nil,
|
|
Prefix: "/registry",
|
|
DeserializationCacheSize: 0,
|
|
Quorum: false,
|
|
KeyFile: "/var/run/kubernetes/etcd.key",
|
|
CAFile: "/var/run/kubernetes/etcdca.crt",
|
|
CertFile: "/var/run/kubernetes/etcdce.crt",
|
|
CompactionInterval: storagebackend.DefaultCompactInterval,
|
|
CountMetricPollPeriod: time.Minute,
|
|
},
|
|
DefaultStorageMediaType: "application/vnd.kubernetes.protobuf",
|
|
DeleteCollectionWorkers: 1,
|
|
EnableGarbageCollection: true,
|
|
EnableWatchCache: true,
|
|
DefaultWatchCacheSize: 100,
|
|
},
|
|
SecureServing: genericoptions.WithLoopback(&apiserveroptions.SecureServingOptions{
|
|
BindAddress: net.ParseIP("192.168.10.20"),
|
|
BindPort: 6443,
|
|
ServerCert: apiserveroptions.GeneratableKeyCert{
|
|
CertDirectory: "/var/run/kubernetes",
|
|
PairName: "apiserver",
|
|
},
|
|
HTTP2MaxStreamsPerConnection: 42,
|
|
}),
|
|
InsecureServing: &kubeoptions.InsecureServingOptions{
|
|
BindAddress: net.ParseIP("127.0.0.1"),
|
|
BindPort: 8080,
|
|
},
|
|
EventTTL: 1 * time.Hour,
|
|
KubeletConfig: kubeletclient.KubeletClientConfig{
|
|
Port: 10250,
|
|
ReadOnlyPort: 10255,
|
|
PreferredAddressTypes: []string{
|
|
string(kapi.NodeHostName),
|
|
string(kapi.NodeInternalDNS),
|
|
string(kapi.NodeInternalIP),
|
|
string(kapi.NodeExternalDNS),
|
|
string(kapi.NodeExternalIP),
|
|
},
|
|
EnableHttps: true,
|
|
HTTPTimeout: time.Duration(5) * time.Second,
|
|
TLSClientConfig: restclient.TLSClientConfig{
|
|
CertFile: "/var/run/kubernetes/ceserver.crt",
|
|
KeyFile: "/var/run/kubernetes/server.key",
|
|
CAFile: "/var/run/kubernetes/caserver.crt",
|
|
},
|
|
},
|
|
Audit: &apiserveroptions.AuditOptions{
|
|
LogOptions: apiserveroptions.AuditLogOptions{
|
|
Path: "/var/log",
|
|
MaxAge: 11,
|
|
MaxBackups: 12,
|
|
MaxSize: 13,
|
|
Format: "json",
|
|
},
|
|
WebhookOptions: apiserveroptions.AuditWebhookOptions{
|
|
Mode: "blocking",
|
|
ConfigFile: "/webhook-config",
|
|
BatchConfig: auditwebhook.BatchBackendConfig{
|
|
BufferSize: 42,
|
|
MaxBatchSize: 43,
|
|
MaxBatchWait: 1 * time.Second,
|
|
ThrottleQPS: 43.5,
|
|
ThrottleBurst: 44,
|
|
InitialBackoff: 2 * time.Second,
|
|
},
|
|
},
|
|
PolicyFile: "/policy",
|
|
},
|
|
Features: &apiserveroptions.FeatureOptions{
|
|
EnableSwaggerUI: true,
|
|
EnableProfiling: true,
|
|
EnableContentionProfiling: true,
|
|
},
|
|
Authentication: &kubeoptions.BuiltInAuthenticationOptions{
|
|
Anonymous: &kubeoptions.AnonymousAuthenticationOptions{
|
|
Allow: false,
|
|
},
|
|
ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{
|
|
ClientCA: "/client-ca",
|
|
},
|
|
WebHook: &kubeoptions.WebHookAuthenticationOptions{
|
|
CacheTTL: 180000000000,
|
|
ConfigFile: "/token-webhook-config",
|
|
},
|
|
BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{},
|
|
OIDC: &kubeoptions.OIDCAuthenticationOptions{
|
|
UsernameClaim: "sub",
|
|
SigningAlgs: []string{"RS256"},
|
|
},
|
|
PasswordFile: &kubeoptions.PasswordFileAuthenticationOptions{},
|
|
RequestHeader: &apiserveroptions.RequestHeaderAuthenticationOptions{},
|
|
ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{
|
|
Lookup: true,
|
|
},
|
|
TokenFile: &kubeoptions.TokenFileAuthenticationOptions{},
|
|
TokenSuccessCacheTTL: 10 * time.Second,
|
|
TokenFailureCacheTTL: 0,
|
|
},
|
|
Authorization: &kubeoptions.BuiltInAuthorizationOptions{
|
|
Mode: "AlwaysDeny",
|
|
PolicyFile: "/policy",
|
|
WebhookConfigFile: "/webhook-config",
|
|
WebhookCacheAuthorizedTTL: 180000000000,
|
|
WebhookCacheUnauthorizedTTL: 60000000000,
|
|
},
|
|
CloudProvider: &kubeoptions.CloudProviderOptions{
|
|
CloudConfigFile: "/cloud-config",
|
|
CloudProvider: "azure",
|
|
},
|
|
StorageSerialization: &kubeoptions.StorageSerializationOptions{
|
|
StorageVersions: legacyscheme.Registry.AllPreferredGroupVersions(),
|
|
DefaultStorageVersions: legacyscheme.Registry.AllPreferredGroupVersions(),
|
|
},
|
|
APIEnablement: &apiserveroptions.APIEnablementOptions{
|
|
RuntimeConfig: utilflag.ConfigurationMap{},
|
|
},
|
|
EnableLogsHandler: false,
|
|
EnableAggregatorRouting: true,
|
|
ProxyClientKeyFile: "/var/run/kubernetes/proxy.key",
|
|
ProxyClientCertFile: "/var/run/kubernetes/proxy.crt",
|
|
}
|
|
|
|
if !reflect.DeepEqual(expected, s) {
|
|
t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", diff.ObjectReflectDiff(expected, s))
|
|
}
|
|
}
|