From a9af8206cbcea7848d700fd0b50dede94eab4239 Mon Sep 17 00:00:00 2001 From: deads2k Date: Wed, 9 Nov 2016 09:54:53 -0500 Subject: [PATCH] split generic etcdoption out of main struct --- cmd/kube-apiserver/app/BUILD | 1 - cmd/kube-apiserver/app/options/options.go | 2 +- cmd/kube-apiserver/app/server.go | 18 +++++---- examples/apiserver/apiserver.go | 7 +++- federation/cmd/federation-apiserver/app/BUILD | 1 - .../app/options/options.go | 2 +- .../cmd/federation-apiserver/app/server.go | 13 ++++--- pkg/genericapiserver/options/etcd_options.go | 39 ++++++++++++++++++- .../options/server_run_options.go | 18 ++------- pkg/genericapiserver/validation/BUILD | 5 +-- .../validation/etcd_validation.go | 28 ------------- test/e2e_node/services/apiserver.go | 2 +- 12 files changed, 66 insertions(+), 70 deletions(-) delete mode 100644 pkg/genericapiserver/validation/etcd_validation.go diff --git a/cmd/kube-apiserver/app/BUILD b/cmd/kube-apiserver/app/BUILD index 2faa23f89dd..080350d652d 100644 --- a/cmd/kube-apiserver/app/BUILD +++ b/cmd/kube-apiserver/app/BUILD @@ -36,7 +36,6 @@ go_library( "//pkg/generated/openapi:go_default_library", "//pkg/genericapiserver:go_default_library", "//pkg/genericapiserver/authorizer:go_default_library", - "//pkg/genericapiserver/validation:go_default_library", "//pkg/master:go_default_library", "//pkg/registry/cachesize:go_default_library", "//pkg/runtime/schema:go_default_library", diff --git a/cmd/kube-apiserver/app/options/options.go b/cmd/kube-apiserver/app/options/options.go index 56afa61ed91..cdfbffe540c 100644 --- a/cmd/kube-apiserver/app/options/options.go +++ b/cmd/kube-apiserver/app/options/options.go @@ -70,7 +70,7 @@ func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) { // Add the generic flags. s.GenericServerRunOptions.AddUniversalFlags(fs) //Add etcd specific flags. - s.GenericServerRunOptions.AddEtcdStorageFlags(fs) + s.GenericServerRunOptions.Etcd.AddEtcdStorageFlags(fs) // Note: the weird ""+ in below lines seems to be the only way to get gofmt to // arrange these text blocks sensibly. Grrr. diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 703a3632391..a3936e17689 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -50,11 +50,11 @@ import ( generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" "k8s.io/kubernetes/pkg/genericapiserver" "k8s.io/kubernetes/pkg/genericapiserver/authorizer" - genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation" "k8s.io/kubernetes/pkg/master" "k8s.io/kubernetes/pkg/registry/cachesize" "k8s.io/kubernetes/pkg/runtime/schema" "k8s.io/kubernetes/pkg/serviceaccount" + utilerrors "k8s.io/kubernetes/pkg/util/errors" utilnet "k8s.io/kubernetes/pkg/util/net" "k8s.io/kubernetes/pkg/util/wait" "k8s.io/kubernetes/pkg/version" @@ -80,7 +80,9 @@ cluster's shared state through which all other components interact.`, // Run runs the specified APIServer. This should never exit. func Run(s *options.ServerRunOptions) error { - genericvalidation.VerifyEtcdServersList(s.GenericServerRunOptions) + if errs := s.GenericServerRunOptions.Etcd.Validate(); len(errs) > 0 { + return utilerrors.NewAggregate(errs) + } genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions) genericConfig := genericapiserver.NewConfig(). // create the new config ApplyOptions(s.GenericServerRunOptions). // apply the options selected @@ -142,7 +144,7 @@ func Run(s *options.ServerRunOptions) error { // Proxying to pods and services is IP-based... don't expect to be able to verify the hostname proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} - if s.GenericServerRunOptions.StorageConfig.DeserializationCacheSize == 0 { + if s.GenericServerRunOptions.Etcd.StorageConfig.DeserializationCacheSize == 0 { // When size of cache is not explicitly set, estimate its size based on // target memory usage. glog.V(2).Infof("Initalizing deserialization cache size based on %dMB limit", s.GenericServerRunOptions.TargetRAMMB) @@ -158,9 +160,9 @@ func Run(s *options.ServerRunOptions) error { // size to compute its size. We may even go further and measure // collective sizes of the objects in the cache. clusterSize := s.GenericServerRunOptions.TargetRAMMB / 60 - s.GenericServerRunOptions.StorageConfig.DeserializationCacheSize = 25 * clusterSize - if s.GenericServerRunOptions.StorageConfig.DeserializationCacheSize < 1000 { - s.GenericServerRunOptions.StorageConfig.DeserializationCacheSize = 1000 + s.GenericServerRunOptions.Etcd.StorageConfig.DeserializationCacheSize = 25 * clusterSize + if s.GenericServerRunOptions.Etcd.StorageConfig.DeserializationCacheSize < 1000 { + s.GenericServerRunOptions.Etcd.StorageConfig.DeserializationCacheSize = 1000 } } @@ -169,7 +171,7 @@ func Run(s *options.ServerRunOptions) error { glog.Fatalf("error generating storage version map: %s", err) } storageFactory, err := genericapiserver.BuildDefaultStorageFactory( - s.GenericServerRunOptions.StorageConfig, s.GenericServerRunOptions.DefaultStorageMediaType, api.Codecs, + s.GenericServerRunOptions.Etcd.StorageConfig, s.GenericServerRunOptions.DefaultStorageMediaType, api.Codecs, genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion, // FIXME: this GroupVersionResource override should be configurable []schema.GroupVersionResource{batch.Resource("cronjobs").WithVersion("v2alpha1")}, @@ -179,7 +181,7 @@ func Run(s *options.ServerRunOptions) error { } storageFactory.AddCohabitatingResources(batch.Resource("jobs"), extensions.Resource("jobs")) storageFactory.AddCohabitatingResources(autoscaling.Resource("horizontalpodautoscalers"), extensions.Resource("horizontalpodautoscalers")) - for _, override := range s.GenericServerRunOptions.EtcdServersOverrides { + for _, override := range s.GenericServerRunOptions.Etcd.EtcdServersOverrides { tokens := strings.Split(override, "#") if len(tokens) != 2 { glog.Errorf("invalid value of etcd server overrides: %s", override) diff --git a/examples/apiserver/apiserver.go b/examples/apiserver/apiserver.go index 3f4f2f82675..316f2744af2 100644 --- a/examples/apiserver/apiserver.go +++ b/examples/apiserver/apiserver.go @@ -32,6 +32,7 @@ import ( "k8s.io/kubernetes/pkg/registry/generic" "k8s.io/kubernetes/pkg/runtime/schema" "k8s.io/kubernetes/pkg/storage/storagebackend" + utilerrors "k8s.io/kubernetes/pkg/util/errors" // Install the testgroup API _ "k8s.io/kubernetes/cmd/libs/go2idl/client-gen/test_apis/testgroup/install" @@ -64,9 +65,11 @@ func Run(serverOptions *genericoptions.ServerRunOptions, stopCh <-chan struct{}) // Set ServiceClusterIPRange _, serviceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24") serverOptions.ServiceClusterIPRange = *serviceClusterIPRange - serverOptions.StorageConfig.ServerList = []string{"http://127.0.0.1:2379"} + serverOptions.Etcd.StorageConfig.ServerList = []string{"http://127.0.0.1:2379"} genericvalidation.ValidateRunOptions(serverOptions) - genericvalidation.VerifyEtcdServersList(serverOptions) + if errs := serverOptions.Etcd.Validate(); len(errs) > 0 { + return utilerrors.NewAggregate(errs) + } config := genericapiserver.NewConfig().ApplyOptions(serverOptions).Complete() if err := config.MaybeGenerateServingCerts(); err != nil { // this wasn't treated as fatal for this process before diff --git a/federation/cmd/federation-apiserver/app/BUILD b/federation/cmd/federation-apiserver/app/BUILD index 4e5a16af2ca..8830677025a 100644 --- a/federation/cmd/federation-apiserver/app/BUILD +++ b/federation/cmd/federation-apiserver/app/BUILD @@ -43,7 +43,6 @@ go_library( "//pkg/generated/openapi:go_default_library", "//pkg/genericapiserver:go_default_library", "//pkg/genericapiserver/authorizer:go_default_library", - "//pkg/genericapiserver/validation:go_default_library", "//pkg/registry/cachesize:go_default_library", "//pkg/registry/core/configmap/etcd:go_default_library", "//pkg/registry/core/event/etcd:go_default_library", diff --git a/federation/cmd/federation-apiserver/app/options/options.go b/federation/cmd/federation-apiserver/app/options/options.go index b1bfafb5071..b1d8af4ac0a 100644 --- a/federation/cmd/federation-apiserver/app/options/options.go +++ b/federation/cmd/federation-apiserver/app/options/options.go @@ -45,7 +45,7 @@ func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) { // Add the generic flags. s.GenericServerRunOptions.AddUniversalFlags(fs) //Add etcd specific flags. - s.GenericServerRunOptions.AddEtcdStorageFlags(fs) + s.GenericServerRunOptions.Etcd.AddEtcdStorageFlags(fs) fs.DurationVar(&s.EventTTL, "event-ttl", s.EventTTL, "Amount of time to retain events. Default is 1h.") diff --git a/federation/cmd/federation-apiserver/app/server.go b/federation/cmd/federation-apiserver/app/server.go index a6f8a1ad569..f18a9d6e2ac 100644 --- a/federation/cmd/federation-apiserver/app/server.go +++ b/federation/cmd/federation-apiserver/app/server.go @@ -38,7 +38,6 @@ import ( "k8s.io/kubernetes/pkg/generated/openapi" "k8s.io/kubernetes/pkg/genericapiserver" "k8s.io/kubernetes/pkg/genericapiserver/authorizer" - genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation" "k8s.io/kubernetes/pkg/registry/cachesize" "k8s.io/kubernetes/pkg/registry/generic" "k8s.io/kubernetes/pkg/registry/generic/registry" @@ -67,7 +66,9 @@ cluster's shared state through which all other components interact.`, // Run runs the specified APIServer. This should never exit. func Run(s *options.ServerRunOptions) error { - genericvalidation.VerifyEtcdServersList(s.GenericServerRunOptions) + if errs := s.GenericServerRunOptions.Etcd.Validate(); len(errs) > 0 { + glog.Fatal(errs) + } genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions) genericConfig := genericapiserver.NewConfig(). // create the new config ApplyOptions(s.GenericServerRunOptions). // apply the options selected @@ -80,23 +81,23 @@ func Run(s *options.ServerRunOptions) error { // TODO: register cluster federation resources here. resourceConfig := genericapiserver.NewResourceConfig() - if s.GenericServerRunOptions.StorageConfig.DeserializationCacheSize == 0 { + if s.GenericServerRunOptions.Etcd.StorageConfig.DeserializationCacheSize == 0 { // When size of cache is not explicitly set, set it to 50000 - s.GenericServerRunOptions.StorageConfig.DeserializationCacheSize = 50000 + s.GenericServerRunOptions.Etcd.StorageConfig.DeserializationCacheSize = 50000 } storageGroupsToEncodingVersion, err := s.GenericServerRunOptions.StorageGroupsToEncodingVersion() if err != nil { glog.Fatalf("error generating storage version map: %s", err) } storageFactory, err := genericapiserver.BuildDefaultStorageFactory( - s.GenericServerRunOptions.StorageConfig, s.GenericServerRunOptions.DefaultStorageMediaType, api.Codecs, + s.GenericServerRunOptions.Etcd.StorageConfig, s.GenericServerRunOptions.DefaultStorageMediaType, api.Codecs, genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion, []schema.GroupVersionResource{}, resourceConfig, s.GenericServerRunOptions.RuntimeConfig) if err != nil { glog.Fatalf("error in initializing storage factory: %s", err) } - for _, override := range s.GenericServerRunOptions.EtcdServersOverrides { + for _, override := range s.GenericServerRunOptions.Etcd.EtcdServersOverrides { tokens := strings.Split(override, "#") if len(tokens) != 2 { glog.Errorf("invalid value of etcd server overrides: %s", override) diff --git a/pkg/genericapiserver/options/etcd_options.go b/pkg/genericapiserver/options/etcd_options.go index c991414b371..53ab03baa89 100644 --- a/pkg/genericapiserver/options/etcd_options.go +++ b/pkg/genericapiserver/options/etcd_options.go @@ -17,20 +17,55 @@ limitations under the License. package options import ( + "fmt" + "github.com/spf13/pflag" + + "k8s.io/kubernetes/pkg/storage/storagebackend" ) const ( DefaultEtcdPathPrefix = "/registry" ) -// AddEtcdFlags adds flags related to etcd storage for a specific APIServer to the specified FlagSet -func (s *ServerRunOptions) AddEtcdStorageFlags(fs *pflag.FlagSet) { +type EtcdOptions struct { + StorageConfig storagebackend.Config + EtcdServersOverrides []string +} + +func NewDefaultEtcdOptions() *EtcdOptions { + return &EtcdOptions{ + StorageConfig: storagebackend.Config{ + Prefix: DefaultEtcdPathPrefix, + // Default cache size to 0 - if unset, its size will be set based on target + // memory usage. + DeserializationCacheSize: 0, + }, + } +} + +func (s *EtcdOptions) Validate() []error { + allErrors := []error{} + if len(s.StorageConfig.ServerList) == 0 { + allErrors = append(allErrors, fmt.Errorf("--etcd-servers must be specified")) + } + + return allErrors +} + +// AddEtcdFlags adds flags related to etcd storage for a specific APIServer to the specified FlagSet +func (s *EtcdOptions) AddEtcdStorageFlags(fs *pflag.FlagSet) { fs.StringSliceVar(&s.EtcdServersOverrides, "etcd-servers-overrides", s.EtcdServersOverrides, ""+ "Per-resource etcd servers overrides, comma separated. The individual override "+ "format: group/resource#servers, where servers are http://ip:port, semicolon separated.") + fs.StringVar(&s.StorageConfig.Type, "storage-backend", s.StorageConfig.Type, + "The storage backend for persistence. Options: 'etcd2' (default), 'etcd3'.") + + fs.IntVar(&s.StorageConfig.DeserializationCacheSize, "deserialization-cache-size", s.StorageConfig.DeserializationCacheSize, + "Number of deserialized json objects to cache in memory.") + fs.StringSliceVar(&s.StorageConfig.ServerList, "etcd-servers", s.StorageConfig.ServerList, "List of etcd servers to connect with (scheme://ip:port), comma separated.") diff --git a/pkg/genericapiserver/options/server_run_options.go b/pkg/genericapiserver/options/server_run_options.go index 8505993aa1f..e021107e3fd 100644 --- a/pkg/genericapiserver/options/server_run_options.go +++ b/pkg/genericapiserver/options/server_run_options.go @@ -29,7 +29,6 @@ import ( clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/client/restclient" "k8s.io/kubernetes/pkg/runtime/schema" - "k8s.io/kubernetes/pkg/storage/storagebackend" "k8s.io/kubernetes/pkg/util/config" utilnet "k8s.io/kubernetes/pkg/util/net" @@ -55,6 +54,8 @@ var AuthorizationModeChoices = []string{ModeAlwaysAllow, ModeAlwaysDeny, ModeABA // ServerRunOptions contains the options while running a generic api server. type ServerRunOptions struct { + Etcd *EtcdOptions + AdmissionControl string AdmissionControlConfigFile string AdvertiseAddress net.IP @@ -86,8 +87,6 @@ type ServerRunOptions struct { EnableContentionProfiling bool EnableSwaggerUI bool EnableWatchCache bool - EtcdServersOverrides []string - StorageConfig storagebackend.Config ExternalHost string InsecureBindAddress net.IP InsecurePort int @@ -157,12 +156,7 @@ func NewServerRunOptions() *ServerRunOptions { } func (o *ServerRunOptions) WithEtcdOptions() *ServerRunOptions { - o.StorageConfig = storagebackend.Config{ - Prefix: DefaultEtcdPathPrefix, - // Default cache size to 0 - if unset, its size will be set based on target - // memory usage. - DeserializationCacheSize: 0, - } + o.Etcd = NewDefaultEtcdOptions() return o } @@ -472,12 +466,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { fs.Var(&s.ServiceNodePortRange, "service-node-ports", "DEPRECATED: see --service-node-port-range instead") fs.MarkDeprecated("service-node-ports", "see --service-node-port-range instead") - fs.StringVar(&s.StorageConfig.Type, "storage-backend", s.StorageConfig.Type, - "The storage backend for persistence. Options: 'etcd2' (default), 'etcd3'.") - - fs.IntVar(&s.StorageConfig.DeserializationCacheSize, "deserialization-cache-size", s.StorageConfig.DeserializationCacheSize, - "Number of deserialized json objects to cache in memory.") - deprecatedStorageVersion := "" fs.StringVar(&deprecatedStorageVersion, "storage-version", deprecatedStorageVersion, "DEPRECATED: the version to store the legacy v1 resources with. Defaults to server preferred.") diff --git a/pkg/genericapiserver/validation/BUILD b/pkg/genericapiserver/validation/BUILD index 2842b5f9603..d4be4ef0abc 100644 --- a/pkg/genericapiserver/validation/BUILD +++ b/pkg/genericapiserver/validation/BUILD @@ -12,10 +12,7 @@ load( go_library( name = "go_default_library", - srcs = [ - "etcd_validation.go", - "universal_validation.go", - ], + srcs = ["universal_validation.go"], tags = ["automanaged"], deps = [ "//pkg/genericapiserver/options:go_default_library", diff --git a/pkg/genericapiserver/validation/etcd_validation.go b/pkg/genericapiserver/validation/etcd_validation.go deleted file mode 100644 index 54e4a4fdfb8..00000000000 --- a/pkg/genericapiserver/validation/etcd_validation.go +++ /dev/null @@ -1,28 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package validation - -import ( - "github.com/golang/glog" - "k8s.io/kubernetes/pkg/genericapiserver/options" -) - -func VerifyEtcdServersList(options *options.ServerRunOptions) { - if len(options.StorageConfig.ServerList) == 0 { - glog.Fatalf("--etcd-servers must be specified") - } -} diff --git a/test/e2e_node/services/apiserver.go b/test/e2e_node/services/apiserver.go index 6b5b66bab26..27a4adc5027 100644 --- a/test/e2e_node/services/apiserver.go +++ b/test/e2e_node/services/apiserver.go @@ -41,7 +41,7 @@ func NewAPIServer() *APIServer { // Start starts the apiserver, returns when apiserver is ready. func (a *APIServer) Start() error { config := options.NewServerRunOptions() - config.GenericServerRunOptions.StorageConfig.ServerList = []string{getEtcdClientURL()} + config.GenericServerRunOptions.Etcd.StorageConfig.ServerList = []string{getEtcdClientURL()} _, ipnet, err := net.ParseCIDR(clusterIPRange) if err != nil { return err