diff --git a/api/swagger-spec/resourceListing.json b/api/swagger-spec/resourceListing.json index abf8cddff97..33fb4c36002 100644 --- a/api/swagger-spec/resourceListing.json +++ b/api/swagger-spec/resourceListing.json @@ -5,6 +5,10 @@ "path": "/api/v1", "description": "API at /api/v1" }, + { + "path": "/version", + "description": "git code version from which this is built" + }, { "path": "/api", "description": "get available API versions" @@ -20,10 +24,6 @@ { "path": "/apis", "description": "get available API versions" - }, - { - "path": "/version", - "description": "git code version from which this is built" } ], "apiVersion": "", diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 33ffe811c07..d9633a6ec1c 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -44,6 +44,7 @@ import ( "k8s.io/kubernetes/pkg/capabilities" client "k8s.io/kubernetes/pkg/client/unversioned" "k8s.io/kubernetes/pkg/cloudprovider" + "k8s.io/kubernetes/pkg/genericapiserver" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" "k8s.io/kubernetes/pkg/master" "k8s.io/kubernetes/pkg/master/ports" @@ -130,7 +131,7 @@ func NewAPIServer() *APIServer { EventTTL: 1 * time.Hour, AuthorizationMode: "AlwaysAllow", AdmissionControl: "AlwaysAdmit", - EtcdPathPrefix: master.DefaultEtcdPathPrefix, + EtcdPathPrefix: genericapiserver.DefaultEtcdPathPrefix, EnableLogsSupport: true, MasterServiceNamespace: api.NamespaceDefault, CertDirectory: "/var/run/kubernetes", @@ -314,7 +315,7 @@ func generateStorageVersionMap(legacyVersion string, storageVersions string) map } // parse the value of --etcd-servers-overrides and update given storageDestinations. -func updateEtcdOverrides(overrides []string, storageVersions map[string]string, prefix string, storageDestinations *master.StorageDestinations, newEtcdFn newEtcdFunc) { +func updateEtcdOverrides(overrides []string, storageVersions map[string]string, prefix string, storageDestinations *genericapiserver.StorageDestinations, newEtcdFn newEtcdFunc) { if len(overrides) == 0 { return } @@ -448,7 +449,7 @@ func (s *APIServer) Run(_ []string) error { return err } - storageDestinations := master.NewStorageDestinations() + storageDestinations := genericapiserver.NewStorageDestinations() storageVersions := generateStorageVersionMap(s.DeprecatedStorageVersion, s.StorageVersions) if _, found := storageVersions[legacyV1Group.GroupVersion.Group]; !found { @@ -539,36 +540,39 @@ func (s *APIServer) Run(_ []string) error { } config := &master.Config{ - StorageDestinations: storageDestinations, - StorageVersions: storageVersions, - EventTTL: s.EventTTL, - KubeletClient: kubeletClient, - ServiceClusterIPRange: &n, - EnableCoreControllers: true, - EnableLogsSupport: s.EnableLogsSupport, - EnableUISupport: true, - EnableSwaggerSupport: true, - EnableProfiling: s.EnableProfiling, - EnableWatchCache: s.EnableWatchCache, - EnableIndex: true, - APIPrefix: s.APIPrefix, - APIGroupPrefix: s.APIGroupPrefix, - CorsAllowedOriginList: s.CorsAllowedOriginList, - ReadWritePort: s.SecurePort, - PublicAddress: s.AdvertiseAddress, - Authenticator: authenticator, - SupportsBasicAuth: len(s.BasicAuthFile) > 0, - Authorizer: authorizer, - AdmissionControl: admissionController, - APIGroupVersionOverrides: apiGroupVersionOverrides, - MasterServiceNamespace: s.MasterServiceNamespace, - ExternalHost: s.ExternalHost, - MinRequestTimeout: s.MinRequestTimeout, - ProxyDialer: proxyDialerFn, - ProxyTLSClientConfig: proxyTLSClientConfig, - Tunneler: tunneler, - ServiceNodePortRange: s.ServiceNodePortRange, - KubernetesServiceNodePort: s.KubernetesServiceNodePort, + Config: &genericapiserver.Config{ + StorageDestinations: storageDestinations, + StorageVersions: storageVersions, + ServiceClusterIPRange: &n, + EnableLogsSupport: s.EnableLogsSupport, + EnableUISupport: true, + EnableSwaggerSupport: true, + EnableProfiling: s.EnableProfiling, + EnableWatchCache: s.EnableWatchCache, + EnableIndex: true, + APIPrefix: s.APIPrefix, + APIGroupPrefix: s.APIGroupPrefix, + CorsAllowedOriginList: s.CorsAllowedOriginList, + ReadWritePort: s.SecurePort, + PublicAddress: s.AdvertiseAddress, + Authenticator: authenticator, + SupportsBasicAuth: len(s.BasicAuthFile) > 0, + Authorizer: authorizer, + AdmissionControl: admissionController, + APIGroupVersionOverrides: apiGroupVersionOverrides, + MasterServiceNamespace: s.MasterServiceNamespace, + ExternalHost: s.ExternalHost, + MinRequestTimeout: s.MinRequestTimeout, + ProxyDialer: proxyDialerFn, + ProxyTLSClientConfig: proxyTLSClientConfig, + ServiceNodePortRange: s.ServiceNodePortRange, + KubernetesServiceNodePort: s.KubernetesServiceNodePort, + }, + EnableCoreControllers: true, + EventTTL: s.EventTTL, + KubeletClient: kubeletClient, + + Tunneler: tunneler, } m := master.New(config) @@ -682,7 +686,7 @@ func (s *APIServer) getRuntimeConfigValue(apiKey string, defaultValue bool) bool } // Parses the given runtime-config and formats it into map[string]ApiGroupVersionOverride -func (s *APIServer) parseRuntimeConfig() (map[string]master.APIGroupVersionOverride, error) { +func (s *APIServer) parseRuntimeConfig() (map[string]genericapiserver.APIGroupVersionOverride, error) { // "api/all=false" allows users to selectively enable specific api versions. disableAllAPIs := false allAPIFlagValue, ok := s.RuntimeConfig["api/all"] @@ -703,9 +707,9 @@ func (s *APIServer) parseRuntimeConfig() (map[string]master.APIGroupVersionOverr disableV1 := disableAllAPIs v1GroupVersion := "api/v1" disableV1 = !s.getRuntimeConfigValue(v1GroupVersion, !disableV1) - apiGroupVersionOverrides := map[string]master.APIGroupVersionOverride{} + apiGroupVersionOverrides := map[string]genericapiserver.APIGroupVersionOverride{} if disableV1 { - apiGroupVersionOverrides[v1GroupVersion] = master.APIGroupVersionOverride{ + apiGroupVersionOverrides[v1GroupVersion] = genericapiserver.APIGroupVersionOverride{ Disable: true, } } @@ -717,7 +721,7 @@ func (s *APIServer) parseRuntimeConfig() (map[string]master.APIGroupVersionOverr // TODO: Make this a loop over all group/versions when there are more of them. disableExtensions = !s.getRuntimeConfigValue(extensionsGroupVersion, !disableExtensions) if disableExtensions { - apiGroupVersionOverrides[extensionsGroupVersion] = master.APIGroupVersionOverride{ + apiGroupVersionOverrides[extensionsGroupVersion] = genericapiserver.APIGroupVersionOverride{ Disable: true, } } diff --git a/cmd/kube-apiserver/app/server_test.go b/cmd/kube-apiserver/app/server_test.go index f258708f650..ec17fc904f7 100644 --- a/cmd/kube-apiserver/app/server_test.go +++ b/cmd/kube-apiserver/app/server_test.go @@ -25,7 +25,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/meta" "k8s.io/kubernetes/pkg/apis/extensions" - "k8s.io/kubernetes/pkg/master" + "k8s.io/kubernetes/pkg/genericapiserver" "k8s.io/kubernetes/pkg/storage" ) @@ -138,7 +138,7 @@ func TestUpdateEtcdOverrides(t *testing.T) { } return nil, nil } - storageDestinations := master.NewStorageDestinations() + storageDestinations := genericapiserver.NewStorageDestinations() override := test.apigroup + "/" + test.resource + "#" + strings.Join(test.servers, ";") updateEtcdOverrides([]string{override}, storageVersions, "", &storageDestinations, newEtcd) apigroup, ok := storageDestinations.APIGroups[test.apigroup] @@ -160,12 +160,12 @@ func TestUpdateEtcdOverrides(t *testing.T) { func TestParseRuntimeConfig(t *testing.T) { testCases := []struct { runtimeConfig map[string]string - apiGroupVersionOverrides map[string]master.APIGroupVersionOverride + apiGroupVersionOverrides map[string]genericapiserver.APIGroupVersionOverride err bool }{ { runtimeConfig: map[string]string{}, - apiGroupVersionOverrides: map[string]master.APIGroupVersionOverride{}, + apiGroupVersionOverrides: map[string]genericapiserver.APIGroupVersionOverride{}, err: false, }, { @@ -173,7 +173,7 @@ func TestParseRuntimeConfig(t *testing.T) { runtimeConfig: map[string]string{ "api/v1/pods": "false", }, - apiGroupVersionOverrides: map[string]master.APIGroupVersionOverride{}, + apiGroupVersionOverrides: map[string]genericapiserver.APIGroupVersionOverride{}, err: true, }, { @@ -181,7 +181,7 @@ func TestParseRuntimeConfig(t *testing.T) { runtimeConfig: map[string]string{ "api/v1": "false", }, - apiGroupVersionOverrides: map[string]master.APIGroupVersionOverride{ + apiGroupVersionOverrides: map[string]genericapiserver.APIGroupVersionOverride{ "api/v1": { Disable: true, }, @@ -193,7 +193,7 @@ func TestParseRuntimeConfig(t *testing.T) { runtimeConfig: map[string]string{ "extensions/v1beta1": "false", }, - apiGroupVersionOverrides: map[string]master.APIGroupVersionOverride{ + apiGroupVersionOverrides: map[string]genericapiserver.APIGroupVersionOverride{ "extensions/v1beta1": { Disable: true, }, @@ -205,7 +205,7 @@ func TestParseRuntimeConfig(t *testing.T) { runtimeConfig: map[string]string{ "extensions/v1beta1/deployments": "false", }, - apiGroupVersionOverrides: map[string]master.APIGroupVersionOverride{ + apiGroupVersionOverrides: map[string]genericapiserver.APIGroupVersionOverride{ "extensions/v1beta1": { ResourceOverrides: map[string]bool{ "deployments": false, @@ -220,7 +220,7 @@ func TestParseRuntimeConfig(t *testing.T) { "extensions/v1beta1/deployments": "true", "extensions/v1beta1/jobs": "false", }, - apiGroupVersionOverrides: map[string]master.APIGroupVersionOverride{ + apiGroupVersionOverrides: map[string]genericapiserver.APIGroupVersionOverride{ "extensions/v1beta1": { ResourceOverrides: map[string]bool{ "deployments": true, diff --git a/docs/admin/kube-apiserver.md b/docs/admin/kube-apiserver.md index abc48f8130f..2d5b996a332 100644 --- a/docs/admin/kube-apiserver.md +++ b/docs/admin/kube-apiserver.md @@ -105,7 +105,7 @@ kube-apiserver --watch-cache[=true]: Enable watch caching in the apiserver ``` -###### Auto generated by spf13/cobra on 9-Dec-2015 +###### Auto generated by spf13/cobra on 15-Dec-2015 diff --git a/pkg/genericapiserver/doc.go b/pkg/genericapiserver/doc.go new file mode 100644 index 00000000000..12238d568eb --- /dev/null +++ b/pkg/genericapiserver/doc.go @@ -0,0 +1,23 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package genericapiserver contains code to setup a generic kubernetes-like API server. +// This does not contain any kubernetes API specific code. +// Note that this is a work in progress. We are pulling out generic code (specifically from +// pkg/master and pkg/apiserver) here. +// We plan to move this package into a separate repo on github once it is done. +// For more details: https://github.com/kubernetes/kubernetes/issues/2742 +package genericapiserver diff --git a/pkg/genericapiserver/genericapiserver.go b/pkg/genericapiserver/genericapiserver.go new file mode 100644 index 00000000000..b2e03bfc755 --- /dev/null +++ b/pkg/genericapiserver/genericapiserver.go @@ -0,0 +1,534 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package genericapiserver + +import ( + "crypto/tls" + "net" + "net/http" + "net/http/pprof" + "strconv" + "strings" + "time" + + "k8s.io/kubernetes/pkg/admission" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/rest" + "k8s.io/kubernetes/pkg/apiserver" + "k8s.io/kubernetes/pkg/auth/authenticator" + "k8s.io/kubernetes/pkg/auth/authorizer" + "k8s.io/kubernetes/pkg/auth/handlers" + "k8s.io/kubernetes/pkg/registry/generic" + genericetcd "k8s.io/kubernetes/pkg/registry/generic/etcd" + ipallocator "k8s.io/kubernetes/pkg/registry/service/ipallocator" + "k8s.io/kubernetes/pkg/storage" + "k8s.io/kubernetes/pkg/ui" + "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/sets" + + "github.com/emicklei/go-restful" + "github.com/emicklei/go-restful/swagger" + "github.com/golang/glog" + "golang.org/x/net/context" +) + +const ( + DefaultEtcdPathPrefix = "/registry" +) + +// StorageDestinations is a mapping from API group & resource to +// the underlying storage interfaces. +type StorageDestinations struct { + APIGroups map[string]*StorageDestinationsForAPIGroup +} + +type StorageDestinationsForAPIGroup struct { + Default storage.Interface + Overrides map[string]storage.Interface +} + +func NewStorageDestinations() StorageDestinations { + return StorageDestinations{ + APIGroups: map[string]*StorageDestinationsForAPIGroup{}, + } +} + +func (s *StorageDestinations) AddAPIGroup(group string, defaultStorage storage.Interface) { + s.APIGroups[group] = &StorageDestinationsForAPIGroup{ + Default: defaultStorage, + Overrides: map[string]storage.Interface{}, + } +} + +func (s *StorageDestinations) AddStorageOverride(group, resource string, override storage.Interface) { + if _, ok := s.APIGroups[group]; !ok { + s.AddAPIGroup(group, nil) + } + if s.APIGroups[group].Overrides == nil { + s.APIGroups[group].Overrides = map[string]storage.Interface{} + } + s.APIGroups[group].Overrides[resource] = override +} + +func (s *StorageDestinations) Get(group, resource string) storage.Interface { + apigroup, ok := s.APIGroups[group] + if !ok { + glog.Errorf("No storage defined for API group: '%s'", apigroup) + return nil + } + if apigroup.Overrides != nil { + if client, exists := apigroup.Overrides[resource]; exists { + return client + } + } + return apigroup.Default +} + +// Get all backends for all registered storage destinations. +// Used for getting all instances for health validations. +func (s *StorageDestinations) Backends() []string { + backends := sets.String{} + for _, group := range s.APIGroups { + if group.Default != nil { + for _, backend := range group.Default.Backends(context.TODO()) { + backends.Insert(backend) + } + } + if group.Overrides != nil { + for _, storage := range group.Overrides { + for _, backend := range storage.Backends(context.TODO()) { + backends.Insert(backend) + } + } + } + } + return backends.List() +} + +// Specifies the overrides for various API group versions. +// This can be used to enable/disable entire group versions or specific resources. +type APIGroupVersionOverride struct { + // Whether to enable or disable this group version. + Disable bool + // List of overrides for individual resources in this group version. + ResourceOverrides map[string]bool +} + +// Config is a structure used to configure a GenericAPIServer. +type Config struct { + StorageDestinations StorageDestinations + // StorageVersions is a map between groups and their storage versions + StorageVersions map[string]string + // allow downstream consumers to disable the core controller loops + EnableLogsSupport bool + EnableUISupport bool + // allow downstream consumers to disable swagger + EnableSwaggerSupport bool + // Allows api group versions or specific resources to be conditionally enabled/disabled. + APIGroupVersionOverrides map[string]APIGroupVersionOverride + // allow downstream consumers to disable the index route + EnableIndex bool + EnableProfiling bool + EnableWatchCache bool + APIPrefix string + APIGroupPrefix string + CorsAllowedOriginList []string + Authenticator authenticator.Request + // TODO(roberthbailey): Remove once the server no longer supports http basic auth. + SupportsBasicAuth bool + Authorizer authorizer.Authorizer + AdmissionControl admission.Interface + MasterServiceNamespace string + + // Map requests to contexts. Exported so downstream consumers can provider their own mappers + RequestContextMapper api.RequestContextMapper + + // If specified, all web services will be registered into this container + RestfulContainer *restful.Container + + // If specified, requests will be allocated a random timeout between this value, and twice this value. + // Note that it is up to the request handlers to ignore or honor this timeout. In seconds. + MinRequestTimeout int + + // Number of masters running; all masters must be started with the + // same value for this field. (Numbers > 1 currently untested.) + MasterCount int + + // The port on PublicAddress where a read-write server will be installed. + // Defaults to 6443 if not set. + ReadWritePort int + + // ExternalHost is the host name to use for external (public internet) facing URLs (e.g. Swagger) + ExternalHost string + + // PublicAddress is the IP address where members of the cluster (kubelet, + // kube-proxy, services, etc.) can reach the GenericAPIServer. + // If nil or 0.0.0.0, the host's default interface will be used. + PublicAddress net.IP + + // Control the interval that pod, node IP, and node heath status caches + // expire. + CacheTimeout time.Duration + + // The range of IPs to be assigned to services with type=ClusterIP or greater + ServiceClusterIPRange *net.IPNet + + // The IP address for the GenericAPIServer service (must be inside ServiceClusterIPRange + ServiceReadWriteIP net.IP + + // The range of ports to be assigned to services with type=NodePort or greater + ServiceNodePortRange util.PortRange + + // Used to customize default proxy dial/tls options + ProxyDialer apiserver.ProxyDialerFunc + ProxyTLSClientConfig *tls.Config + + // Additional ports to be exposed on the GenericAPIServer service + // extraServicePorts is injectable in the event that more ports + // (other than the default 443/tcp) are exposed on the GenericAPIServer + // and those ports need to be load balanced by the GenericAPIServer + // service because this pkg is linked by out-of-tree projects + // like openshift which want to use the GenericAPIServer but also do + // more stuff. + ExtraServicePorts []api.ServicePort + // Additional ports to be exposed on the GenericAPIServer endpoints + // Port names should align with ports defined in ExtraServicePorts + ExtraEndpointPorts []api.EndpointPort + + KubernetesServiceNodePort int +} + +// GenericAPIServer contains state for a Kubernetes cluster api server. +type GenericAPIServer struct { + // "Inputs", Copied from Config + ServiceClusterIPRange *net.IPNet + ServiceNodePortRange util.PortRange + cacheTimeout time.Duration + MinRequestTimeout time.Duration + + mux apiserver.Mux + MuxHelper *apiserver.MuxHelper + HandlerContainer *restful.Container + RootWebService *restful.WebService + enableLogsSupport bool + enableUISupport bool + enableSwaggerSupport bool + enableProfiling bool + enableWatchCache bool + ApiPrefix string + ApiGroupPrefix string + corsAllowedOriginList []string + authenticator authenticator.Request + authorizer authorizer.Authorizer + AdmissionControl admission.Interface + MasterCount int + ApiGroupVersionOverrides map[string]APIGroupVersionOverride + RequestContextMapper api.RequestContextMapper + + // External host is the name that should be used in external (public internet) URLs for this GenericAPIServer + externalHost string + // ClusterIP is the IP address of the GenericAPIServer within the cluster. + ClusterIP net.IP + PublicReadWritePort int + ServiceReadWriteIP net.IP + ServiceReadWritePort int + masterServices *util.Runner + ExtraServicePorts []api.ServicePort + ExtraEndpointPorts []api.EndpointPort + + // storage contains the RESTful endpoints exposed by this GenericAPIServer + storage map[string]rest.Storage + + // "Outputs" + Handler http.Handler + InsecureHandler http.Handler + + // Used for custom proxy dialing, and proxy TLS options + ProxyTransport http.RoundTripper + + KubernetesServiceNodePort int +} + +func (s *GenericAPIServer) StorageDecorator() generic.StorageDecorator { + if s.enableWatchCache { + return genericetcd.StorageWithCacher + } + return generic.UndecoratedStorage +} + +// setDefaults fills in any fields not set that are required to have valid data. +func setDefaults(c *Config) { + if c.ServiceClusterIPRange == nil { + defaultNet := "10.0.0.0/24" + glog.Warningf("Network range for service cluster IPs is unspecified. Defaulting to %v.", defaultNet) + _, serviceClusterIPRange, err := net.ParseCIDR(defaultNet) + if err != nil { + glog.Fatalf("Unable to parse CIDR: %v", err) + } + if size := ipallocator.RangeSize(serviceClusterIPRange); size < 8 { + glog.Fatalf("The service cluster IP range must be at least %d IP addresses", 8) + } + c.ServiceClusterIPRange = serviceClusterIPRange + } + if c.ServiceReadWriteIP == nil { + // Select the first valid IP from ServiceClusterIPRange to use as the GenericAPIServer service IP. + serviceReadWriteIP, err := ipallocator.GetIndexedIP(c.ServiceClusterIPRange, 1) + if err != nil { + glog.Fatalf("Failed to generate service read-write IP for GenericAPIServer service: %v", err) + } + glog.V(4).Infof("Setting GenericAPIServer service IP to %q (read-write).", serviceReadWriteIP) + c.ServiceReadWriteIP = serviceReadWriteIP + } + if c.ServiceNodePortRange.Size == 0 { + // TODO: Currently no way to specify an empty range (do we need to allow this?) + // We should probably allow this for clouds that don't require NodePort to do load-balancing (GCE) + // but then that breaks the strict nestedness of ServiceType. + // Review post-v1 + defaultServiceNodePortRange := util.PortRange{Base: 30000, Size: 2768} + c.ServiceNodePortRange = defaultServiceNodePortRange + glog.Infof("Node port range unspecified. Defaulting to %v.", c.ServiceNodePortRange) + } + if c.MasterCount == 0 { + // Clearly, there will be at least one GenericAPIServer. + c.MasterCount = 1 + } + if c.ReadWritePort == 0 { + c.ReadWritePort = 6443 + } + if c.CacheTimeout == 0 { + c.CacheTimeout = 5 * time.Second + } + if c.RequestContextMapper == nil { + c.RequestContextMapper = api.NewRequestContextMapper() + } +} + +// New returns a new instance of GenericAPIServer from the given config. +// Certain config fields will be set to a default value if unset, +// including: +// ServiceClusterIPRange +// ServiceNodePortRange +// MasterCount +// ReadWritePort +// PublicAddress +// Public fields: +// Handler -- The returned GenericAPIServer has a field TopHandler which is an +// http.Handler which handles all the endpoints provided by the GenericAPIServer, +// including the API, the UI, and miscellaneous debugging endpoints. All +// these are subject to authorization and authentication. +// InsecureHandler -- an http.Handler which handles all the same +// endpoints as Handler, but no authorization and authentication is done. +// Public methods: +// HandleWithAuth -- Allows caller to add an http.Handler for an endpoint +// that uses the same authentication and authorization (if any is configured) +// as the GenericAPIServer's built-in endpoints. +// If the caller wants to add additional endpoints not using the GenericAPIServer's +// auth, then the caller should create a handler for those endpoints, which delegates the +// any unhandled paths to "Handler". +func New(c *Config) *GenericAPIServer { + setDefaults(c) + + s := &GenericAPIServer{ + ServiceClusterIPRange: c.ServiceClusterIPRange, + ServiceNodePortRange: c.ServiceNodePortRange, + RootWebService: new(restful.WebService), + enableLogsSupport: c.EnableLogsSupport, + enableUISupport: c.EnableUISupport, + enableSwaggerSupport: c.EnableSwaggerSupport, + enableProfiling: c.EnableProfiling, + enableWatchCache: c.EnableWatchCache, + ApiPrefix: c.APIPrefix, + ApiGroupPrefix: c.APIGroupPrefix, + corsAllowedOriginList: c.CorsAllowedOriginList, + authenticator: c.Authenticator, + authorizer: c.Authorizer, + AdmissionControl: c.AdmissionControl, + ApiGroupVersionOverrides: c.APIGroupVersionOverrides, + RequestContextMapper: c.RequestContextMapper, + + cacheTimeout: c.CacheTimeout, + MinRequestTimeout: time.Duration(c.MinRequestTimeout) * time.Second, + + MasterCount: c.MasterCount, + externalHost: c.ExternalHost, + ClusterIP: c.PublicAddress, + PublicReadWritePort: c.ReadWritePort, + ServiceReadWriteIP: c.ServiceReadWriteIP, + // TODO: ServiceReadWritePort should be passed in as an argument, it may not always be 443 + ServiceReadWritePort: 443, + ExtraServicePorts: c.ExtraServicePorts, + ExtraEndpointPorts: c.ExtraEndpointPorts, + + KubernetesServiceNodePort: c.KubernetesServiceNodePort, + } + + var handlerContainer *restful.Container + if c.RestfulContainer != nil { + s.mux = c.RestfulContainer.ServeMux + handlerContainer = c.RestfulContainer + } else { + mux := http.NewServeMux() + s.mux = mux + handlerContainer = NewHandlerContainer(mux) + } + s.HandlerContainer = handlerContainer + // Use CurlyRouter to be able to use regular expressions in paths. Regular expressions are required in paths for example for proxy (where the path is proxy/{kind}/{name}/{*}) + s.HandlerContainer.Router(restful.CurlyRouter{}) + s.MuxHelper = &apiserver.MuxHelper{Mux: s.mux, RegisteredPaths: []string{}} + + s.init(c) + + return s +} + +func (s *GenericAPIServer) NewRequestInfoResolver() *apiserver.RequestInfoResolver { + return &apiserver.RequestInfoResolver{ + sets.NewString(strings.Trim(s.ApiPrefix, "/"), strings.Trim(s.ApiGroupPrefix, "/")), // all possible API prefixes + sets.NewString(strings.Trim(s.ApiPrefix, "/")), // APIPrefixes that won't have groups (legacy) + } +} + +// HandleWithAuth adds an http.Handler for pattern to an http.ServeMux +// Applies the same authentication and authorization (if any is configured) +// to the request is used for the GenericAPIServer's built-in endpoints. +func (s *GenericAPIServer) HandleWithAuth(pattern string, handler http.Handler) { + // TODO: Add a way for plugged-in endpoints to translate their + // URLs into attributes that an Authorizer can understand, and have + // sensible policy defaults for plugged-in endpoints. This will be different + // for generic endpoints versus REST object endpoints. + // TODO: convert to go-restful + s.MuxHelper.Handle(pattern, handler) +} + +// HandleFuncWithAuth adds an http.Handler for pattern to an http.ServeMux +// Applies the same authentication and authorization (if any is configured) +// to the request is used for the GenericAPIServer's built-in endpoints. +func (s *GenericAPIServer) HandleFuncWithAuth(pattern string, handler func(http.ResponseWriter, *http.Request)) { + // TODO: convert to go-restful + s.MuxHelper.HandleFunc(pattern, handler) +} + +func NewHandlerContainer(mux *http.ServeMux) *restful.Container { + container := restful.NewContainer() + container.ServeMux = mux + apiserver.InstallRecoverHandler(container) + return container +} + +// init initializes GenericAPIServer. +func (s *GenericAPIServer) init(c *Config) { + + if c.ProxyDialer != nil || c.ProxyTLSClientConfig != nil { + s.ProxyTransport = util.SetTransportDefaults(&http.Transport{ + Dial: c.ProxyDialer, + TLSClientConfig: c.ProxyTLSClientConfig, + }) + } + + // Register root handler. + // We do not register this using restful Webservice since we do not want to surface this in api docs. + // Allow GenericAPIServer to be embedded in contexts which already have something registered at the root + if c.EnableIndex { + s.mux.HandleFunc("/", apiserver.IndexHandler(s.HandlerContainer, s.MuxHelper)) + } + + if c.EnableLogsSupport { + apiserver.InstallLogsSupport(s.MuxHelper) + } + if c.EnableUISupport { + ui.InstallSupport(s.MuxHelper, s.enableSwaggerSupport) + } + + if c.EnableProfiling { + s.mux.HandleFunc("/debug/pprof/", pprof.Index) + s.mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + s.mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + } + + handler := http.Handler(s.mux.(*http.ServeMux)) + + // TODO: handle CORS and auth using go-restful + // See github.com/emicklei/go-restful/blob/GenericAPIServer/examples/restful-CORS-filter.go, and + // github.com/emicklei/go-restful/blob/GenericAPIServer/examples/restful-basic-authentication.go + + if len(c.CorsAllowedOriginList) > 0 { + allowedOriginRegexps, err := util.CompileRegexps(c.CorsAllowedOriginList) + if err != nil { + glog.Fatalf("Invalid CORS allowed origin, --cors-allowed-origins flag was set to %v - %v", strings.Join(c.CorsAllowedOriginList, ","), err) + } + handler = apiserver.CORS(handler, allowedOriginRegexps, nil, nil, "true") + } + + s.InsecureHandler = handler + + attributeGetter := apiserver.NewRequestAttributeGetter(s.RequestContextMapper, s.NewRequestInfoResolver()) + handler = apiserver.WithAuthorizationCheck(handler, attributeGetter, s.authorizer) + + // Install Authenticator + if c.Authenticator != nil { + authenticatedHandler, err := handlers.NewRequestAuthenticator(s.RequestContextMapper, c.Authenticator, handlers.Unauthorized(c.SupportsBasicAuth), handler) + if err != nil { + glog.Fatalf("Could not initialize authenticator: %v", err) + } + handler = authenticatedHandler + } + + // TODO: Make this optional? Consumers of GenericAPIServer depend on this currently. + s.Handler = handler + + // After all wrapping is done, put a context filter around both handlers + if handler, err := api.NewRequestContextFilter(s.RequestContextMapper, s.Handler); err != nil { + glog.Fatalf("Could not initialize request context filter: %v", err) + } else { + s.Handler = handler + } + + if handler, err := api.NewRequestContextFilter(s.RequestContextMapper, s.InsecureHandler); err != nil { + glog.Fatalf("Could not initialize request context filter: %v", err) + } else { + s.InsecureHandler = handler + } +} + +// InstallSwaggerAPI installs the /swaggerapi/ endpoint to allow schema discovery +// and traversal. It is optional to allow consumers of the Kubernetes GenericAPIServer to +// register their own web services into the Kubernetes mux prior to initialization +// of swagger, so that other resource types show up in the documentation. +func (s *GenericAPIServer) InstallSwaggerAPI() { + hostAndPort := s.externalHost + protocol := "https://" + + // TODO: this is kind of messed up, we should just pipe in the full URL from the outside, rather + // than guessing at it. + if len(s.externalHost) == 0 && s.ClusterIP != nil { + host := s.ClusterIP.String() + if s.PublicReadWritePort != 0 { + hostAndPort = net.JoinHostPort(host, strconv.Itoa(s.PublicReadWritePort)) + } + } + webServicesUrl := protocol + hostAndPort + + // Enable swagger UI and discovery API + swaggerConfig := swagger.Config{ + WebServicesUrl: webServicesUrl, + WebServices: s.HandlerContainer.RegisteredWebServices(), + ApiPath: "/swaggerapi/", + SwaggerPath: "/swaggerui/", + SwaggerFilePath: "/swagger-ui/", + } + swagger.RegisterSwaggerService(swaggerConfig, s.HandlerContainer) +} diff --git a/pkg/genericapiserver/genericapiserver_test.go b/pkg/genericapiserver/genericapiserver_test.go new file mode 100644 index 00000000000..b49c18a236c --- /dev/null +++ b/pkg/genericapiserver/genericapiserver_test.go @@ -0,0 +1,154 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package genericapiserver + +import ( + "crypto/tls" + "fmt" + "net" + "net/http" + "testing" + + "k8s.io/kubernetes/pkg/apiserver" + etcdtesting "k8s.io/kubernetes/pkg/storage/etcd/testing" + "k8s.io/kubernetes/pkg/util" + + "github.com/stretchr/testify/assert" +) + +// setUp is a convience function for setting up for (most) tests. +func setUp(t *testing.T) (GenericAPIServer, *etcdtesting.EtcdTestServer, Config, *assert.Assertions) { + etcdServer := etcdtesting.NewEtcdTestClientServer(t) + + genericapiserver := GenericAPIServer{} + config := Config{} + config.PublicAddress = net.ParseIP("192.168.10.4") + + return genericapiserver, etcdServer, config, assert.New(t) +} + +// TestNew verifies that the New function returns a GenericAPIServer +// using the configuration properly. +func TestNew(t *testing.T) { + _, etcdserver, config, assert := setUp(t) + defer etcdserver.Terminate(t) + + config.ProxyDialer = func(network, addr string) (net.Conn, error) { return nil, nil } + config.ProxyTLSClientConfig = &tls.Config{} + + s := New(&config) + + // Verify many of the variables match their config counterparts + assert.Equal(s.enableLogsSupport, config.EnableLogsSupport) + assert.Equal(s.enableUISupport, config.EnableUISupport) + assert.Equal(s.enableSwaggerSupport, config.EnableSwaggerSupport) + assert.Equal(s.enableProfiling, config.EnableProfiling) + assert.Equal(s.ApiPrefix, config.APIPrefix) + assert.Equal(s.ApiGroupPrefix, config.APIGroupPrefix) + assert.Equal(s.corsAllowedOriginList, config.CorsAllowedOriginList) + assert.Equal(s.authenticator, config.Authenticator) + assert.Equal(s.authorizer, config.Authorizer) + assert.Equal(s.AdmissionControl, config.AdmissionControl) + assert.Equal(s.ApiGroupVersionOverrides, config.APIGroupVersionOverrides) + assert.Equal(s.RequestContextMapper, config.RequestContextMapper) + assert.Equal(s.cacheTimeout, config.CacheTimeout) + assert.Equal(s.externalHost, config.ExternalHost) + assert.Equal(s.ClusterIP, config.PublicAddress) + assert.Equal(s.PublicReadWritePort, config.ReadWritePort) + assert.Equal(s.ServiceReadWriteIP, config.ServiceReadWriteIP) + + // These functions should point to the same memory location + serverDialer, _ := util.Dialer(s.ProxyTransport) + serverDialerFunc := fmt.Sprintf("%p", serverDialer) + configDialerFunc := fmt.Sprintf("%p", config.ProxyDialer) + assert.Equal(serverDialerFunc, configDialerFunc) + + assert.Equal(s.ProxyTransport.(*http.Transport).TLSClientConfig, config.ProxyTLSClientConfig) +} + +// TestNewHandlerContainer verifies that NewHandlerContainer uses the +// mux provided +func TestNewHandlerContainer(t *testing.T) { + assert := assert.New(t) + mux := http.NewServeMux() + container := NewHandlerContainer(mux) + assert.Equal(mux, container.ServeMux, "ServerMux's do not match") +} + +// TestHandleWithAuth verifies HandleWithAuth adds the path +// to the MuxHelper.RegisteredPaths. +func TestHandleWithAuth(t *testing.T) { + server, etcdserver, _, assert := setUp(t) + defer etcdserver.Terminate(t) + + mh := apiserver.MuxHelper{Mux: http.NewServeMux()} + server.MuxHelper = &mh + handler := func(r http.ResponseWriter, w *http.Request) { w.Write(nil) } + server.HandleWithAuth("/test", http.HandlerFunc(handler)) + + assert.Contains(server.MuxHelper.RegisteredPaths, "/test", "Path not found in MuxHelper") +} + +// TestHandleFuncWithAuth verifies HandleFuncWithAuth adds the path +// to the MuxHelper.RegisteredPaths. +func TestHandleFuncWithAuth(t *testing.T) { + server, etcdserver, _, assert := setUp(t) + defer etcdserver.Terminate(t) + + mh := apiserver.MuxHelper{Mux: http.NewServeMux()} + server.MuxHelper = &mh + handler := func(r http.ResponseWriter, w *http.Request) { w.Write(nil) } + server.HandleFuncWithAuth("/test", handler) + + assert.Contains(server.MuxHelper.RegisteredPaths, "/test", "Path not found in MuxHelper") +} + +// TestInstallSwaggerAPI verifies that the swagger api is added +// at the proper endpoint. +func TestInstallSwaggerAPI(t *testing.T) { + server, etcdserver, _, assert := setUp(t) + defer etcdserver.Terminate(t) + + mux := http.NewServeMux() + server.HandlerContainer = NewHandlerContainer(mux) + + // Ensure swagger isn't installed without the call + ws := server.HandlerContainer.RegisteredWebServices() + if !assert.Equal(len(ws), 0) { + for x := range ws { + assert.NotEqual("/swaggerapi", ws[x].RootPath(), "SwaggerAPI was installed without a call to InstallSwaggerAPI()") + } + } + + // Install swagger and test + server.InstallSwaggerAPI() + ws = server.HandlerContainer.RegisteredWebServices() + if assert.NotEqual(0, len(ws), "SwaggerAPI not installed.") { + assert.Equal("/swaggerapi/", ws[0].RootPath(), "SwaggerAPI did not install to the proper path. %s != /swaggerapi", ws[0].RootPath()) + } + + // Empty externalHost verification + mux = http.NewServeMux() + server.HandlerContainer = NewHandlerContainer(mux) + server.externalHost = "" + server.ClusterIP = net.IPv4(10, 10, 10, 10) + server.PublicReadWritePort = 1010 + server.InstallSwaggerAPI() + if assert.NotEqual(0, len(ws), "SwaggerAPI not installed.") { + assert.Equal("/swaggerapi/", ws[0].RootPath(), "SwaggerAPI did not install to the proper path. %s != /swaggerapi", ws[0].RootPath()) + } +} diff --git a/pkg/master/master.go b/pkg/master/master.go index f1408669118..3433ef569e6 100644 --- a/pkg/master/master.go +++ b/pkg/master/master.go @@ -17,18 +17,15 @@ limitations under the License. package master import ( - "crypto/tls" "fmt" "net" "net/http" - "net/http/pprof" "net/url" "strconv" "strings" "sync" "time" - "k8s.io/kubernetes/pkg/admission" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/latest" "k8s.io/kubernetes/pkg/api/rest" @@ -37,9 +34,7 @@ import ( "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/apis/extensions" "k8s.io/kubernetes/pkg/apiserver" - "k8s.io/kubernetes/pkg/auth/authenticator" - "k8s.io/kubernetes/pkg/auth/authorizer" - "k8s.io/kubernetes/pkg/auth/handlers" + "k8s.io/kubernetes/pkg/genericapiserver" "k8s.io/kubernetes/pkg/healthz" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" "k8s.io/kubernetes/pkg/master/ports" @@ -51,7 +46,6 @@ import ( eventetcd "k8s.io/kubernetes/pkg/registry/event/etcd" expcontrolleretcd "k8s.io/kubernetes/pkg/registry/experimental/controller/etcd" "k8s.io/kubernetes/pkg/registry/generic" - genericetcd "k8s.io/kubernetes/pkg/registry/generic/etcd" ingressetcd "k8s.io/kubernetes/pkg/registry/ingress/etcd" jobetcd "k8s.io/kubernetes/pkg/registry/job/etcd" limitrangeetcd "k8s.io/kubernetes/pkg/registry/limitrange/etcd" @@ -75,245 +69,36 @@ import ( thirdpartyresourcedataetcd "k8s.io/kubernetes/pkg/registry/thirdpartyresourcedata/etcd" "k8s.io/kubernetes/pkg/storage" etcdutil "k8s.io/kubernetes/pkg/storage/etcd/util" - "k8s.io/kubernetes/pkg/ui" "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/sets" daemonetcd "k8s.io/kubernetes/pkg/registry/daemonset/etcd" horizontalpodautoscaleretcd "k8s.io/kubernetes/pkg/registry/horizontalpodautoscaler/etcd" - "github.com/emicklei/go-restful" - "github.com/emicklei/go-restful/swagger" "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" - "golang.org/x/net/context" "k8s.io/kubernetes/pkg/registry/service/allocator" "k8s.io/kubernetes/pkg/registry/service/portallocator" ) -const ( - DefaultEtcdPathPrefix = "/registry" -) - -// StorageDestinations is a mapping from API group & resource to -// the underlying storage interfaces. -type StorageDestinations struct { - APIGroups map[string]*StorageDestinationsForAPIGroup -} - -type StorageDestinationsForAPIGroup struct { - Default storage.Interface - Overrides map[string]storage.Interface -} - -func NewStorageDestinations() StorageDestinations { - return StorageDestinations{ - APIGroups: map[string]*StorageDestinationsForAPIGroup{}, - } -} - -func (s *StorageDestinations) AddAPIGroup(group string, defaultStorage storage.Interface) { - s.APIGroups[group] = &StorageDestinationsForAPIGroup{ - Default: defaultStorage, - Overrides: map[string]storage.Interface{}, - } -} - -func (s *StorageDestinations) AddStorageOverride(group, resource string, override storage.Interface) { - if _, ok := s.APIGroups[group]; !ok { - s.AddAPIGroup(group, nil) - } - if s.APIGroups[group].Overrides == nil { - s.APIGroups[group].Overrides = map[string]storage.Interface{} - } - s.APIGroups[group].Overrides[resource] = override -} - -func (s *StorageDestinations) get(group, resource string) storage.Interface { - apigroup, ok := s.APIGroups[group] - if !ok { - glog.Errorf("No storage defined for API group: '%s'", apigroup) - return nil - } - if apigroup.Overrides != nil { - if client, exists := apigroup.Overrides[resource]; exists { - return client - } - } - return apigroup.Default -} - -// Get all backends for all registered storage destinations. -// Used for getting all instances for health validations. -func (s *StorageDestinations) backends() []string { - backends := sets.String{} - for _, group := range s.APIGroups { - if group.Default != nil { - for _, backend := range group.Default.Backends(context.TODO()) { - backends.Insert(backend) - } - } - if group.Overrides != nil { - for _, storage := range group.Overrides { - for _, backend := range storage.Backends(context.TODO()) { - backends.Insert(backend) - } - } - } - } - return backends.List() -} - -// Specifies the overrides for various API group versions. -// This can be used to enable/disable entire group versions or specific resources. -type APIGroupVersionOverride struct { - // Whether to enable or disable this group version. - Disable bool - // List of overrides for individual resources in this group version. - ResourceOverrides map[string]bool -} - -// Config is a structure used to configure a Master. type Config struct { - StorageDestinations StorageDestinations - // StorageVersions is a map between groups and their storage versions - StorageVersions map[string]string - EventTTL time.Duration - KubeletClient kubeletclient.KubeletClient - // allow downstream consumers to disable the core controller loops + *genericapiserver.Config + EnableCoreControllers bool - EnableLogsSupport bool - EnableUISupport bool - // allow downstream consumers to disable swagger - EnableSwaggerSupport bool - // Allows api group versions or specific resources to be conditionally enabled/disabled. - APIGroupVersionOverrides map[string]APIGroupVersionOverride - // allow downstream consumers to disable the index route - EnableIndex bool - EnableProfiling bool - EnableWatchCache bool - APIPrefix string - APIGroupPrefix string - CorsAllowedOriginList []string - Authenticator authenticator.Request - // TODO(roberthbailey): Remove once the server no longer supports http basic auth. - SupportsBasicAuth bool - Authorizer authorizer.Authorizer - AdmissionControl admission.Interface - MasterServiceNamespace string - - // Map requests to contexts. Exported so downstream consumers can provider their own mappers - RequestContextMapper api.RequestContextMapper - - // If specified, all web services will be registered into this container - RestfulContainer *restful.Container - - // If specified, requests will be allocated a random timeout between this value, and twice this value. - // Note that it is up to the request handlers to ignore or honor this timeout. In seconds. - MinRequestTimeout int - - // Number of masters running; all masters must be started with the - // same value for this field. (Numbers > 1 currently untested.) - MasterCount int - - // The port on PublicAddress where a read-write server will be installed. - // Defaults to 6443 if not set. - ReadWritePort int - - // ExternalHost is the host name to use for external (public internet) facing URLs (e.g. Swagger) - ExternalHost string - - // PublicAddress is the IP address where members of the cluster (kubelet, - // kube-proxy, services, etc.) can reach the master. - // If nil or 0.0.0.0, the host's default interface will be used. - PublicAddress net.IP - - // Control the interval that pod, node IP, and node heath status caches - // expire. - CacheTimeout time.Duration - - // The range of IPs to be assigned to services with type=ClusterIP or greater - ServiceClusterIPRange *net.IPNet - - // The IP address for the master service (must be inside ServiceClusterIPRange - ServiceReadWriteIP net.IP - - // The range of ports to be assigned to services with type=NodePort or greater - ServiceNodePortRange util.PortRange - - // Used to customize default proxy dial/tls options - ProxyDialer apiserver.ProxyDialerFunc - ProxyTLSClientConfig *tls.Config - + EventTTL time.Duration + KubeletClient kubeletclient.KubeletClient // Used to start and monitor tunneling Tunneler Tunneler - - // Additional ports to be exposed on the master service - // extraServicePorts is injectable in the event that more ports - // (other than the default 443/tcp) are exposed on the master - // and those ports need to be load balanced by the master - // service because this pkg is linked by out-of-tree projects - // like openshift which want to use the master but also do - // more stuff. - ExtraServicePorts []api.ServicePort - // Additional ports to be exposed on the master endpoints - // Port names should align with ports defined in ExtraServicePorts - ExtraEndpointPorts []api.EndpointPort - - KubernetesServiceNodePort int } -func (c *Config) storageDecorator() generic.StorageDecorator { - if c.EnableWatchCache { - return genericetcd.StorageWithCacher - } - return generic.UndecoratedStorage -} - -type InstallSSHKey func(user string, data []byte) error - // Master contains state for a Kubernetes cluster master/api server. type Master struct { - // "Inputs", Copied from Config - serviceClusterIPRange *net.IPNet - serviceNodePortRange util.PortRange - cacheTimeout time.Duration - minRequestTimeout time.Duration + *genericapiserver.GenericAPIServer - mux apiserver.Mux - muxHelper *apiserver.MuxHelper - handlerContainer *restful.Container - rootWebService *restful.WebService - enableCoreControllers bool - enableLogsSupport bool - enableUISupport bool - enableSwaggerSupport bool - enableProfiling bool - enableWatchCache bool - apiPrefix string - apiGroupPrefix string - corsAllowedOriginList []string - authenticator authenticator.Request - authorizer authorizer.Authorizer - admissionControl admission.Interface - masterCount int - apiGroupVersionOverrides map[string]APIGroupVersionOverride - requestContextMapper api.RequestContextMapper - - // External host is the name that should be used in external (public internet) URLs for this master - externalHost string - // clusterIP is the IP address of the master within the cluster. - clusterIP net.IP - publicReadWritePort int - serviceReadWriteIP net.IP - serviceReadWritePort int - masterServices *util.Runner - extraServicePorts []api.ServicePort - extraEndpointPorts []api.EndpointPort - - // storage contains the RESTful endpoints exposed by this master - storage map[string]rest.Storage + // Map of v1 resources to their REST storages. + v1ResourcesStorage map[string]rest.Storage + enableCoreControllers bool // registries are internal client APIs for accessing the storage layer // TODO: define the internal typed interface in a way that clients can // also be replaced @@ -324,288 +109,59 @@ type Master struct { serviceClusterIPAllocator service.RangeRegistry serviceNodePortAllocator service.RangeRegistry - // "Outputs" - Handler http.Handler - InsecureHandler http.Handler - - // Used for custom proxy dialing, and proxy TLS options - proxyTransport http.RoundTripper - - // Used to start and monitor tunneling - tunneler Tunneler - // storage for third party objects thirdPartyStorage storage.Interface // map from api path to storage for those objects thirdPartyResources map[string]*thirdpartyresourcedataetcd.REST // protects the map - thirdPartyResourcesLock sync.RWMutex - KubernetesServiceNodePort int -} + thirdPartyResourcesLock sync.RWMutex -// setDefaults fills in any fields not set that are required to have valid data. -func setDefaults(c *Config) { - if c.ServiceClusterIPRange == nil { - defaultNet := "10.0.0.0/24" - glog.Warningf("Network range for service cluster IPs is unspecified. Defaulting to %v.", defaultNet) - _, serviceClusterIPRange, err := net.ParseCIDR(defaultNet) - if err != nil { - glog.Fatalf("Unable to parse CIDR: %v", err) - } - if size := ipallocator.RangeSize(serviceClusterIPRange); size < 8 { - glog.Fatalf("The service cluster IP range must be at least %d IP addresses", 8) - } - c.ServiceClusterIPRange = serviceClusterIPRange - } - if c.ServiceReadWriteIP == nil { - // Select the first valid IP from ServiceClusterIPRange to use as the master service IP. - serviceReadWriteIP, err := ipallocator.GetIndexedIP(c.ServiceClusterIPRange, 1) - if err != nil { - glog.Fatalf("Failed to generate service read-write IP for master service: %v", err) - } - glog.V(4).Infof("Setting master service IP to %q (read-write).", serviceReadWriteIP) - c.ServiceReadWriteIP = serviceReadWriteIP - } - if c.ServiceNodePortRange.Size == 0 { - // TODO: Currently no way to specify an empty range (do we need to allow this?) - // We should probably allow this for clouds that don't require NodePort to do load-balancing (GCE) - // but then that breaks the strict nestedness of ServiceType. - // Review post-v1 - defaultServiceNodePortRange := util.PortRange{Base: 30000, Size: 2768} - c.ServiceNodePortRange = defaultServiceNodePortRange - glog.Infof("Node port range unspecified. Defaulting to %v.", c.ServiceNodePortRange) - } - if c.MasterCount == 0 { - // Clearly, there will be at least one master. - c.MasterCount = 1 - } - if c.ReadWritePort == 0 { - c.ReadWritePort = 6443 - } - if c.CacheTimeout == 0 { - c.CacheTimeout = 5 * time.Second - } - if c.RequestContextMapper == nil { - c.RequestContextMapper = api.NewRequestContextMapper() - } + // Used to start and monitor tunneling + tunneler Tunneler } // New returns a new instance of Master from the given config. -// Certain config fields will be set to a default value if unset, -// including: -// ServiceClusterIPRange -// ServiceNodePortRange -// MasterCount -// ReadWritePort -// PublicAddress +// Certain config fields will be set to a default value if unset. // Certain config fields must be specified, including: // KubeletClient -// Public fields: -// Handler -- The returned master has a field TopHandler which is an -// http.Handler which handles all the endpoints provided by the master, -// including the API, the UI, and miscellaneous debugging endpoints. All -// these are subject to authorization and authentication. -// InsecureHandler -- an http.Handler which handles all the same -// endpoints as Handler, but no authorization and authentication is done. -// Public methods: -// HandleWithAuth -- Allows caller to add an http.Handler for an endpoint -// that uses the same authentication and authorization (if any is configured) -// as the master's built-in endpoints. -// If the caller wants to add additional endpoints not using the master's -// auth, then the caller should create a handler for those endpoints, which delegates the -// any unhandled paths to "Handler". func New(c *Config) *Master { - setDefaults(c) if c.KubeletClient == nil { - glog.Fatalf("master.New() called with config.KubeletClient == nil") + glog.Fatalf("Master.New() called with config.KubeletClient == nil") } + s := genericapiserver.New(c.Config) + m := &Master{ - serviceClusterIPRange: c.ServiceClusterIPRange, - serviceNodePortRange: c.ServiceNodePortRange, - rootWebService: new(restful.WebService), - enableCoreControllers: c.EnableCoreControllers, - enableLogsSupport: c.EnableLogsSupport, - enableUISupport: c.EnableUISupport, - enableSwaggerSupport: c.EnableSwaggerSupport, - enableProfiling: c.EnableProfiling, - enableWatchCache: c.EnableWatchCache, - apiPrefix: c.APIPrefix, - apiGroupPrefix: c.APIGroupPrefix, - corsAllowedOriginList: c.CorsAllowedOriginList, - authenticator: c.Authenticator, - authorizer: c.Authorizer, - admissionControl: c.AdmissionControl, - apiGroupVersionOverrides: c.APIGroupVersionOverrides, - requestContextMapper: c.RequestContextMapper, + GenericAPIServer: s, + enableCoreControllers: c.EnableCoreControllers, + tunneler: c.Tunneler, + } + m.InstallAPIs(c) - cacheTimeout: c.CacheTimeout, - minRequestTimeout: time.Duration(c.MinRequestTimeout) * time.Second, - - masterCount: c.MasterCount, - externalHost: c.ExternalHost, - clusterIP: c.PublicAddress, - publicReadWritePort: c.ReadWritePort, - serviceReadWriteIP: c.ServiceReadWriteIP, - // TODO: serviceReadWritePort should be passed in as an argument, it may not always be 443 - serviceReadWritePort: 443, - extraServicePorts: c.ExtraServicePorts, - extraEndpointPorts: c.ExtraEndpointPorts, - - tunneler: c.Tunneler, - - KubernetesServiceNodePort: c.KubernetesServiceNodePort, + // TODO: Move this to generic api server. + if c.EnableSwaggerSupport { + m.InstallSwaggerAPI() } - var handlerContainer *restful.Container - if c.RestfulContainer != nil { - m.mux = c.RestfulContainer.ServeMux - handlerContainer = c.RestfulContainer - } else { - mux := http.NewServeMux() - m.mux = mux - handlerContainer = NewHandlerContainer(mux) + // TODO: Attempt clean shutdown? + if m.enableCoreControllers { + m.NewBootstrapController().Start() } - m.handlerContainer = handlerContainer - // Use CurlyRouter to be able to use regular expressions in paths. Regular expressions are required in paths for example for proxy (where the path is proxy/{kind}/{name}/{*}) - m.handlerContainer.Router(restful.CurlyRouter{}) - m.muxHelper = &apiserver.MuxHelper{Mux: m.mux, RegisteredPaths: []string{}} - - m.init(c) return m } -// HandleWithAuth adds an http.Handler for pattern to an http.ServeMux -// Applies the same authentication and authorization (if any is configured) -// to the request is used for the master's built-in endpoints. -func (m *Master) HandleWithAuth(pattern string, handler http.Handler) { - // TODO: Add a way for plugged-in endpoints to translate their - // URLs into attributes that an Authorizer can understand, and have - // sensible policy defaults for plugged-in endpoints. This will be different - // for generic endpoints versus REST object endpoints. - // TODO: convert to go-restful - m.muxHelper.Handle(pattern, handler) -} - -// HandleFuncWithAuth adds an http.Handler for pattern to an http.ServeMux -// Applies the same authentication and authorization (if any is configured) -// to the request is used for the master's built-in endpoints. -func (m *Master) HandleFuncWithAuth(pattern string, handler func(http.ResponseWriter, *http.Request)) { - // TODO: convert to go-restful - m.muxHelper.HandleFunc(pattern, handler) -} - -func NewHandlerContainer(mux *http.ServeMux) *restful.Container { - container := restful.NewContainer() - container.ServeMux = mux - apiserver.InstallRecoverHandler(container) - return container -} - -// init initializes master. -func (m *Master) init(c *Config) { - - if c.ProxyDialer != nil || c.ProxyTLSClientConfig != nil { - m.proxyTransport = util.SetTransportDefaults(&http.Transport{ - Dial: c.ProxyDialer, - TLSClientConfig: c.ProxyTLSClientConfig, - }) +func (m *Master) InstallAPIs(c *Config) { + apiVersions := []string{} + // Install v1 unless disabled. + if !m.ApiGroupVersionOverrides["api/v1"].Disable { + if err := m.api_v1(c).InstallREST(m.HandlerContainer); err != nil { + glog.Fatalf("Unable to setup API v1: %v", err) + } + apiVersions = append(apiVersions, "v1") } - + // Run the tunnel. healthzChecks := []healthz.HealthzChecker{} - - storageDecorator := c.storageDecorator() - dbClient := func(resource string) storage.Interface { return c.StorageDestinations.get("", resource) } - - podTemplateStorage := podtemplateetcd.NewREST(dbClient("podTemplates"), storageDecorator) - - eventStorage := eventetcd.NewREST(dbClient("events"), storageDecorator, uint64(c.EventTTL.Seconds())) - limitRangeStorage := limitrangeetcd.NewREST(dbClient("limitRanges"), storageDecorator) - - resourceQuotaStorage, resourceQuotaStatusStorage := resourcequotaetcd.NewREST(dbClient("resourceQuotas"), storageDecorator) - secretStorage := secretetcd.NewREST(dbClient("secrets"), storageDecorator) - serviceAccountStorage := serviceaccountetcd.NewREST(dbClient("serviceAccounts"), storageDecorator) - persistentVolumeStorage, persistentVolumeStatusStorage := pvetcd.NewREST(dbClient("persistentVolumes"), storageDecorator) - persistentVolumeClaimStorage, persistentVolumeClaimStatusStorage := pvcetcd.NewREST(dbClient("persistentVolumeClaims"), storageDecorator) - - namespaceStorage, namespaceStatusStorage, namespaceFinalizeStorage := namespaceetcd.NewREST(dbClient("namespaces"), storageDecorator) - m.namespaceRegistry = namespace.NewRegistry(namespaceStorage) - - endpointsStorage := endpointsetcd.NewREST(dbClient("endpoints"), storageDecorator) - m.endpointRegistry = endpoint.NewRegistry(endpointsStorage) - - nodeStorage, nodeStatusStorage := nodeetcd.NewREST(dbClient("nodes"), storageDecorator, c.KubeletClient, m.proxyTransport) - m.nodeRegistry = node.NewRegistry(nodeStorage) - - podStorage := podetcd.NewStorage( - dbClient("pods"), - storageDecorator, - kubeletclient.ConnectionInfoGetter(nodeStorage), - m.proxyTransport, - ) - - serviceStorage := serviceetcd.NewREST(dbClient("services"), storageDecorator) - m.serviceRegistry = service.NewRegistry(serviceStorage) - - var serviceClusterIPRegistry service.RangeRegistry - serviceClusterIPAllocator := ipallocator.NewAllocatorCIDRRange(m.serviceClusterIPRange, func(max int, rangeSpec string) allocator.Interface { - mem := allocator.NewAllocationMap(max, rangeSpec) - etcd := etcdallocator.NewEtcd(mem, "/ranges/serviceips", "serviceipallocation", dbClient("services")) - serviceClusterIPRegistry = etcd - return etcd - }) - m.serviceClusterIPAllocator = serviceClusterIPRegistry - - var serviceNodePortRegistry service.RangeRegistry - serviceNodePortAllocator := portallocator.NewPortAllocatorCustom(m.serviceNodePortRange, func(max int, rangeSpec string) allocator.Interface { - mem := allocator.NewAllocationMap(max, rangeSpec) - etcd := etcdallocator.NewEtcd(mem, "/ranges/servicenodeports", "servicenodeportallocation", dbClient("services")) - serviceNodePortRegistry = etcd - return etcd - }) - m.serviceNodePortAllocator = serviceNodePortRegistry - - controllerStorage, controllerStatusStorage := controlleretcd.NewREST(dbClient("replicationControllers"), storageDecorator) - - // TODO: Factor out the core API registration - m.storage = map[string]rest.Storage{ - "pods": podStorage.Pod, - "pods/attach": podStorage.Attach, - "pods/status": podStorage.Status, - "pods/log": podStorage.Log, - "pods/exec": podStorage.Exec, - "pods/portforward": podStorage.PortForward, - "pods/proxy": podStorage.Proxy, - "pods/binding": podStorage.Binding, - "bindings": podStorage.Binding, - - "podTemplates": podTemplateStorage, - - "replicationControllers": controllerStorage, - "replicationControllers/status": controllerStatusStorage, - "services": service.NewStorage(m.serviceRegistry, m.endpointRegistry, serviceClusterIPAllocator, serviceNodePortAllocator, m.proxyTransport), - "endpoints": endpointsStorage, - "nodes": nodeStorage, - "nodes/status": nodeStatusStorage, - "events": eventStorage, - - "limitRanges": limitRangeStorage, - "resourceQuotas": resourceQuotaStorage, - "resourceQuotas/status": resourceQuotaStatusStorage, - "namespaces": namespaceStorage, - "namespaces/status": namespaceStatusStorage, - "namespaces/finalize": namespaceFinalizeStorage, - "secrets": secretStorage, - "serviceAccounts": serviceAccountStorage, - "persistentVolumes": persistentVolumeStorage, - "persistentVolumes/status": persistentVolumeStatusStorage, - "persistentVolumeClaims": persistentVolumeClaimStorage, - "persistentVolumeClaims/status": persistentVolumeClaimStatusStorage, - - "componentStatuses": componentstatus.NewStorage(func() map[string]apiserver.Server { return m.getServersToValidate(c) }), - } - if m.tunneler != nil { m.tunneler.Run(m.getNodeAddresses) healthzChecks = append(healthzChecks, healthz.NamedCheck("SSH Tunnel Check", m.IsTunnelSyncHealthy)) @@ -615,29 +171,24 @@ func (m *Master) init(c *Config) { }, func() float64 { return float64(m.tunneler.SecondsSinceSync()) }) } - apiVersions := []string{} - // Install v1 unless disabled. - if !m.apiGroupVersionOverrides["api/v1"].Disable { - if err := m.api_v1().InstallREST(m.handlerContainer); err != nil { - glog.Fatalf("Unable to setup API v1: %v", err) - } - apiVersions = append(apiVersions, "v1") - } + // TODO(nikhiljindal): Refactor generic parts of support services (like /versions) to genericapiserver. + apiserver.InstallSupport(m.MuxHelper, m.RootWebService, c.EnableProfiling, healthzChecks...) + // Install root web services + m.HandlerContainer.Add(m.RootWebService) - apiserver.InstallSupport(m.muxHelper, m.rootWebService, c.EnableProfiling, healthzChecks...) - apiserver.AddApiWebService(m.handlerContainer, c.APIPrefix, apiVersions) - apiserver.InstallServiceErrorHandler(m.handlerContainer, m.newRequestInfoResolver(), apiVersions) + apiserver.AddApiWebService(m.HandlerContainer, c.APIPrefix, apiVersions) + apiserver.InstallServiceErrorHandler(m.HandlerContainer, m.NewRequestInfoResolver(), apiVersions) // allGroups records all supported groups at /apis allGroups := []unversioned.APIGroup{} // Install extensions unless disabled. - if !m.apiGroupVersionOverrides["extensions/v1beta1"].Disable { + if !m.ApiGroupVersionOverrides["extensions/v1beta1"].Disable { m.thirdPartyStorage = c.StorageDestinations.APIGroups[extensions.GroupName].Default m.thirdPartyResources = map[string]*thirdpartyresourcedataetcd.REST{} expVersion := m.experimental(c) - if err := expVersion.InstallREST(m.handlerContainer); err != nil { + if err := expVersion.InstallREST(m.HandlerContainer); err != nil { glog.Fatalf("Unable to setup experimental api: %v", err) } g, err := latest.Group(extensions.GroupName) @@ -659,93 +210,110 @@ func (m *Master) init(c *Config) { Versions: expAPIVersions, PreferredVersion: unversioned.GroupVersionForDiscovery{GroupVersion: storageVersion, Version: apiutil.GetVersion(storageVersion)}, } - apiserver.AddGroupWebService(m.handlerContainer, c.APIGroupPrefix+"/"+latest.GroupOrDie(extensions.GroupName).GroupVersion.Group, group) + apiserver.AddGroupWebService(m.HandlerContainer, c.APIGroupPrefix+"/"+latest.GroupOrDie(extensions.GroupName).GroupVersion.Group, group) allGroups = append(allGroups, group) - apiserver.InstallServiceErrorHandler(m.handlerContainer, m.newRequestInfoResolver(), []string{expVersion.GroupVersion.String()}) + apiserver.InstallServiceErrorHandler(m.HandlerContainer, m.NewRequestInfoResolver(), []string{expVersion.GroupVersion.String()}) } // This should be done after all groups are registered // TODO: replace the hardcoded "apis". - apiserver.AddApisWebService(m.handlerContainer, "/apis", allGroups) + apiserver.AddApisWebService(m.HandlerContainer, "/apis", allGroups) +} - // Register root handler. - // We do not register this using restful Webservice since we do not want to surface this in api docs. - // Allow master to be embedded in contexts which already have something registered at the root - if c.EnableIndex { - m.mux.HandleFunc("/", apiserver.IndexHandler(m.handlerContainer, m.muxHelper)) +func (m *Master) initV1ResourcesStorage(c *Config) { + storageDecorator := m.StorageDecorator() + dbClient := func(resource string) storage.Interface { return c.StorageDestinations.Get("", resource) } + + podTemplateStorage := podtemplateetcd.NewREST(dbClient("podTemplates"), storageDecorator) + + eventStorage := eventetcd.NewREST(dbClient("events"), storageDecorator, uint64(c.EventTTL.Seconds())) + limitRangeStorage := limitrangeetcd.NewREST(dbClient("limitRanges"), storageDecorator) + + resourceQuotaStorage, resourceQuotaStatusStorage := resourcequotaetcd.NewREST(dbClient("resourceQuotas"), storageDecorator) + secretStorage := secretetcd.NewREST(dbClient("secrets"), storageDecorator) + serviceAccountStorage := serviceaccountetcd.NewREST(dbClient("serviceAccounts"), storageDecorator) + persistentVolumeStorage, persistentVolumeStatusStorage := pvetcd.NewREST(dbClient("persistentVolumes"), storageDecorator) + persistentVolumeClaimStorage, persistentVolumeClaimStatusStorage := pvcetcd.NewREST(dbClient("persistentVolumeClaims"), storageDecorator) + + namespaceStorage, namespaceStatusStorage, namespaceFinalizeStorage := namespaceetcd.NewREST(dbClient("namespaces"), storageDecorator) + m.namespaceRegistry = namespace.NewRegistry(namespaceStorage) + + endpointsStorage := endpointsetcd.NewREST(dbClient("endpoints"), storageDecorator) + m.endpointRegistry = endpoint.NewRegistry(endpointsStorage) + + nodeStorage, nodeStatusStorage := nodeetcd.NewREST(dbClient("nodes"), storageDecorator, c.KubeletClient, m.ProxyTransport) + m.nodeRegistry = node.NewRegistry(nodeStorage) + + podStorage := podetcd.NewStorage( + dbClient("pods"), + storageDecorator, + kubeletclient.ConnectionInfoGetter(nodeStorage), + m.ProxyTransport, + ) + + serviceStorage := serviceetcd.NewREST(dbClient("services"), storageDecorator) + m.serviceRegistry = service.NewRegistry(serviceStorage) + + var serviceClusterIPRegistry service.RangeRegistry + serviceClusterIPRange := m.ServiceClusterIPRange + if serviceClusterIPRange == nil { + glog.Fatalf("service clusterIPRange is nil") + return } + serviceClusterIPAllocator := ipallocator.NewAllocatorCIDRRange(serviceClusterIPRange, func(max int, rangeSpec string) allocator.Interface { + mem := allocator.NewAllocationMap(max, rangeSpec) + etcd := etcdallocator.NewEtcd(mem, "/ranges/serviceips", "serviceipallocation", dbClient("services")) + serviceClusterIPRegistry = etcd + return etcd + }) + m.serviceClusterIPAllocator = serviceClusterIPRegistry - if c.EnableLogsSupport { - apiserver.InstallLogsSupport(m.muxHelper) - } - if c.EnableUISupport { - ui.InstallSupport(m.muxHelper, m.enableSwaggerSupport) - } + var serviceNodePortRegistry service.RangeRegistry + serviceNodePortAllocator := portallocator.NewPortAllocatorCustom(m.ServiceNodePortRange, func(max int, rangeSpec string) allocator.Interface { + mem := allocator.NewAllocationMap(max, rangeSpec) + etcd := etcdallocator.NewEtcd(mem, "/ranges/servicenodeports", "servicenodeportallocation", dbClient("services")) + serviceNodePortRegistry = etcd + return etcd + }) + m.serviceNodePortAllocator = serviceNodePortRegistry - if c.EnableProfiling { - m.mux.HandleFunc("/debug/pprof/", pprof.Index) - m.mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - m.mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - } + controllerStorage, controllerStatusStorage := controlleretcd.NewREST(dbClient("replicationControllers"), storageDecorator) - handler := http.Handler(m.mux.(*http.ServeMux)) - insecureHandler := handler + m.v1ResourcesStorage = map[string]rest.Storage{ + "pods": podStorage.Pod, + "pods/attach": podStorage.Attach, + "pods/status": podStorage.Status, + "pods/log": podStorage.Log, + "pods/exec": podStorage.Exec, + "pods/portforward": podStorage.PortForward, + "pods/proxy": podStorage.Proxy, + "pods/binding": podStorage.Binding, + "bindings": podStorage.Binding, - // TODO: handle CORS and auth using go-restful - // See github.com/emicklei/go-restful/blob/master/examples/restful-CORS-filter.go, and - // github.com/emicklei/go-restful/blob/master/examples/restful-basic-authentication.go + "podTemplates": podTemplateStorage, - attributeGetter := apiserver.NewRequestAttributeGetter(m.requestContextMapper, m.newRequestInfoResolver()) - handler = apiserver.WithAuthorizationCheck(handler, attributeGetter, m.authorizer) + "replicationControllers": controllerStorage, + "replicationControllers/status": controllerStatusStorage, + "services": service.NewStorage(m.serviceRegistry, m.endpointRegistry, serviceClusterIPAllocator, serviceNodePortAllocator, m.ProxyTransport), + "endpoints": endpointsStorage, + "nodes": nodeStorage, + "nodes/status": nodeStatusStorage, + "events": eventStorage, - // Install Authenticator - if c.Authenticator != nil { - authenticatedHandler, err := handlers.NewRequestAuthenticator(m.requestContextMapper, c.Authenticator, handlers.Unauthorized(c.SupportsBasicAuth), handler) - if err != nil { - glog.Fatalf("Could not initialize authenticator: %v", err) - } - handler = authenticatedHandler - } + "limitRanges": limitRangeStorage, + "resourceQuotas": resourceQuotaStorage, + "resourceQuotas/status": resourceQuotaStatusStorage, + "namespaces": namespaceStorage, + "namespaces/status": namespaceStatusStorage, + "namespaces/finalize": namespaceFinalizeStorage, + "secrets": secretStorage, + "serviceAccounts": serviceAccountStorage, + "persistentVolumes": persistentVolumeStorage, + "persistentVolumes/status": persistentVolumeStatusStorage, + "persistentVolumeClaims": persistentVolumeClaimStorage, + "persistentVolumeClaims/status": persistentVolumeClaimStatusStorage, - // Since OPTIONS request cannot carry authn headers (by w3c standards), we are doing CORS check - // before auth check. Otherwise all the CORS request will be rejected. - if len(c.CorsAllowedOriginList) > 0 { - allowedOriginRegexps, err := util.CompileRegexps(c.CorsAllowedOriginList) - if err != nil { - glog.Fatalf("Invalid CORS allowed origin, --cors-allowed-origins flag was set to %v - %v", strings.Join(c.CorsAllowedOriginList, ","), err) - } - handler = apiserver.CORS(handler, allowedOriginRegexps, nil, nil, "true") - insecureHandler = apiserver.CORS(insecureHandler, allowedOriginRegexps, nil, nil, "true") - } - - m.InsecureHandler = insecureHandler - - // Install root web services - m.handlerContainer.Add(m.rootWebService) - - // TODO: Make this optional? Consumers of master depend on this currently. - m.Handler = handler - - if m.enableSwaggerSupport { - m.InstallSwaggerAPI() - } - - // After all wrapping is done, put a context filter around both handlers - if handler, err := api.NewRequestContextFilter(m.requestContextMapper, m.Handler); err != nil { - glog.Fatalf("Could not initialize request context filter: %v", err) - } else { - m.Handler = handler - } - - if handler, err := api.NewRequestContextFilter(m.requestContextMapper, m.InsecureHandler); err != nil { - glog.Fatalf("Could not initialize request context filter: %v", err) - } else { - m.InsecureHandler = handler - } - - // TODO: Attempt clean shutdown? - if m.enableCoreControllers { - m.NewBootstrapController().Start() + "componentStatuses": componentstatus.NewStorage(func() map[string]apiserver.Server { return m.getServersToValidate(c) }), } } @@ -754,66 +322,37 @@ func (m *Master) NewBootstrapController() *Controller { return &Controller{ NamespaceRegistry: m.namespaceRegistry, ServiceRegistry: m.serviceRegistry, - MasterCount: m.masterCount, + MasterCount: m.MasterCount, EndpointRegistry: m.endpointRegistry, EndpointInterval: 10 * time.Second, ServiceClusterIPRegistry: m.serviceClusterIPAllocator, - ServiceClusterIPRange: m.serviceClusterIPRange, + ServiceClusterIPRange: m.ServiceClusterIPRange, ServiceClusterIPInterval: 3 * time.Minute, ServiceNodePortRegistry: m.serviceNodePortAllocator, - ServiceNodePortRange: m.serviceNodePortRange, + ServiceNodePortRange: m.ServiceNodePortRange, ServiceNodePortInterval: 3 * time.Minute, - PublicIP: m.clusterIP, + PublicIP: m.ClusterIP, - ServiceIP: m.serviceReadWriteIP, - ServicePort: m.serviceReadWritePort, - ExtraServicePorts: m.extraServicePorts, - ExtraEndpointPorts: m.extraEndpointPorts, - PublicServicePort: m.publicReadWritePort, + ServiceIP: m.ServiceReadWriteIP, + ServicePort: m.ServiceReadWritePort, + ExtraServicePorts: m.ExtraServicePorts, + ExtraEndpointPorts: m.ExtraEndpointPorts, + PublicServicePort: m.PublicReadWritePort, KubernetesServiceNodePort: m.KubernetesServiceNodePort, } } -// InstallSwaggerAPI installs the /swaggerapi/ endpoint to allow schema discovery -// and traversal. It is optional to allow consumers of the Kubernetes master to -// register their own web services into the Kubernetes mux prior to initialization -// of swagger, so that other resource types show up in the documentation. -func (m *Master) InstallSwaggerAPI() { - hostAndPort := m.externalHost - protocol := "https://" - - // TODO: this is kind of messed up, we should just pipe in the full URL from the outside, rather - // than guessing at it. - if len(m.externalHost) == 0 && m.clusterIP != nil { - host := m.clusterIP.String() - if m.publicReadWritePort != 0 { - hostAndPort = net.JoinHostPort(host, strconv.Itoa(m.publicReadWritePort)) - } - } - webServicesUrl := protocol + hostAndPort - - // Enable swagger UI and discovery API - swaggerConfig := swagger.Config{ - WebServicesUrl: webServicesUrl, - WebServices: m.handlerContainer.RegisteredWebServices(), - ApiPath: "/swaggerapi/", - SwaggerPath: "/swaggerui/", - SwaggerFilePath: "/swagger-ui/", - } - swagger.RegisterSwaggerService(swaggerConfig, m.handlerContainer) -} - func (m *Master) getServersToValidate(c *Config) map[string]apiserver.Server { serversToValidate := map[string]apiserver.Server{ "controller-manager": {Addr: "127.0.0.1", Port: ports.ControllerManagerPort, Path: "/healthz"}, "scheduler": {Addr: "127.0.0.1", Port: ports.SchedulerPort, Path: "/healthz"}, } - for ix, machine := range c.StorageDestinations.backends() { + for ix, machine := range c.StorageDestinations.Backends() { etcdUrl, err := url.Parse(machine) if err != nil { glog.Errorf("Failed to parse etcd url for validation: %v", err) @@ -839,17 +378,10 @@ func (m *Master) getServersToValidate(c *Config) map[string]apiserver.Server { return serversToValidate } -func (m *Master) newRequestInfoResolver() *apiserver.RequestInfoResolver { - return &apiserver.RequestInfoResolver{ - sets.NewString(strings.Trim(m.apiPrefix, "/"), strings.Trim(thirdpartyprefix, "/")), // all possible API prefixes - sets.NewString(strings.Trim(m.apiPrefix, "/")), // APIPrefixes that won't have groups (legacy) - } -} - func (m *Master) defaultAPIGroupVersion() *apiserver.APIGroupVersion { return &apiserver.APIGroupVersion{ - Root: m.apiPrefix, - RequestInfoResolver: m.newRequestInfoResolver(), + Root: m.ApiPrefix, + RequestInfoResolver: m.NewRequestInfoResolver(), Mapper: latest.GroupOrDie(api.GroupName).RESTMapper, @@ -858,17 +390,18 @@ func (m *Master) defaultAPIGroupVersion() *apiserver.APIGroupVersion { Typer: api.Scheme, Linker: latest.GroupOrDie(api.GroupName).SelfLinker, - Admit: m.admissionControl, - Context: m.requestContextMapper, + Admit: m.AdmissionControl, + Context: m.RequestContextMapper, - MinRequestTimeout: m.minRequestTimeout, + MinRequestTimeout: m.MinRequestTimeout, } } // api_v1 returns the resources and codec for API version v1. -func (m *Master) api_v1() *apiserver.APIGroupVersion { +func (m *Master) api_v1(c *Config) *apiserver.APIGroupVersion { + m.initV1ResourcesStorage(c) storage := make(map[string]rest.Storage) - for k, v := range m.storage { + for k, v := range m.v1ResourcesStorage { storage[strings.ToLower(k)] = v } version := m.defaultAPIGroupVersion() @@ -885,7 +418,7 @@ func (m *Master) HasThirdPartyResource(rsrc *extensions.ThirdPartyResource) (boo return false, err } path := makeThirdPartyPath(group) - services := m.handlerContainer.RegisteredWebServices() + services := m.HandlerContainer.RegisteredWebServices() for ix := range services { if services[ix].RootPath() == path { return true, nil @@ -913,11 +446,11 @@ func (m *Master) RemoveThirdPartyResource(path string) error { return err } - services := m.handlerContainer.RegisteredWebServices() + services := m.HandlerContainer.RegisteredWebServices() for ix := range services { root := services[ix].RootPath() if root == path || strings.HasPrefix(root, path+"/") { - m.handlerContainer.Remove(services[ix]) + m.HandlerContainer.Remove(services[ix]) } } return nil @@ -972,7 +505,7 @@ func (m *Master) InstallThirdPartyResource(rsrc *extensions.ThirdPartyResource) return err } thirdparty := m.thirdpartyapi(group, kind, rsrc.Versions[0].Name) - if err := thirdparty.InstallREST(m.handlerContainer); err != nil { + if err := thirdparty.InstallREST(m.HandlerContainer); err != nil { glog.Fatalf("Unable to setup thirdparty api: %v", err) } path := makeThirdPartyPath(group) @@ -984,9 +517,9 @@ func (m *Master) InstallThirdPartyResource(rsrc *extensions.ThirdPartyResource) Name: group, Versions: []unversioned.GroupVersionForDiscovery{groupVersion}, } - apiserver.AddGroupWebService(m.handlerContainer, path, apiGroup) + apiserver.AddGroupWebService(m.HandlerContainer, path, apiGroup) m.addThirdPartyResourceStorage(path, thirdparty.Storage[strings.ToLower(kind)+"s"].(*thirdpartyresourcedataetcd.REST)) - apiserver.InstallServiceErrorHandler(m.handlerContainer, m.newRequestInfoResolver(), []string{thirdparty.GroupVersion.String()}) + apiserver.InstallServiceErrorHandler(m.HandlerContainer, m.NewRequestInfoResolver(), []string{thirdparty.GroupVersion.String()}) return nil } @@ -1004,7 +537,7 @@ func (m *Master) thirdpartyapi(group, kind, version string) *apiserver.APIGroupV return &apiserver.APIGroupVersion{ Root: apiRoot, GroupVersion: unversioned.GroupVersion{Group: group, Version: version}, - RequestInfoResolver: m.newRequestInfoResolver(), + RequestInfoResolver: m.NewRequestInfoResolver(), Creater: thirdpartyresourcedata.NewObjectCreator(group, version, api.Scheme), Convertor: api.Scheme, @@ -1016,9 +549,9 @@ func (m *Master) thirdpartyapi(group, kind, version string) *apiserver.APIGroupV Storage: storage, OptionsExternalVersion: &optionsExternalVersion, - Context: m.requestContextMapper, + Context: m.RequestContextMapper, - MinRequestTimeout: m.minRequestTimeout, + MinRequestTimeout: m.MinRequestTimeout, } } @@ -1026,7 +559,7 @@ func (m *Master) thirdpartyapi(group, kind, version string) *apiserver.APIGroupV func (m *Master) experimental(c *Config) *apiserver.APIGroupVersion { // All resources except these are disabled by default. enabledResources := sets.NewString("jobs", "horizontalpodautoscalers", "ingresses") - resourceOverrides := m.apiGroupVersionOverrides["extensions/v1beta1"].ResourceOverrides + resourceOverrides := m.ApiGroupVersionOverrides["extensions/v1beta1"].ResourceOverrides isEnabled := func(resource string) bool { // Check if the resource has been overriden. enabled, ok := resourceOverrides[resource] @@ -1035,9 +568,9 @@ func (m *Master) experimental(c *Config) *apiserver.APIGroupVersion { } return enabled } - storageDecorator := c.storageDecorator() + storageDecorator := m.StorageDecorator() dbClient := func(resource string) storage.Interface { - return c.StorageDestinations.get(extensions.GroupName, resource) + return c.StorageDestinations.Get(extensions.GroupName, resource) } storage := map[string]rest.Storage{} @@ -1045,7 +578,7 @@ func (m *Master) experimental(c *Config) *apiserver.APIGroupVersion { autoscalerStorage, autoscalerStatusStorage := horizontalpodautoscaleretcd.NewREST(dbClient("horizontalpodautoscalers"), storageDecorator) storage["horizontalpodautoscalers"] = autoscalerStorage storage["horizontalpodautoscalers/status"] = autoscalerStatusStorage - controllerStorage := expcontrolleretcd.NewStorage(c.StorageDestinations.get("", "replicationControllers"), storageDecorator) + controllerStorage := expcontrolleretcd.NewStorage(c.StorageDestinations.Get("", "replicationControllers"), storageDecorator) storage["replicationcontrollers"] = controllerStorage.ReplicationController storage["replicationcontrollers/scale"] = controllerStorage.Scale } @@ -1092,8 +625,8 @@ func (m *Master) experimental(c *Config) *apiserver.APIGroupVersion { optionsExternalVersion := latest.GroupOrDie(api.GroupName).GroupVersion return &apiserver.APIGroupVersion{ - Root: m.apiGroupPrefix, - RequestInfoResolver: m.newRequestInfoResolver(), + Root: m.ApiGroupPrefix, + RequestInfoResolver: m.NewRequestInfoResolver(), Creater: api.Scheme, Convertor: api.Scheme, @@ -1106,10 +639,10 @@ func (m *Master) experimental(c *Config) *apiserver.APIGroupVersion { GroupVersion: extensionsGroup.GroupVersion, OptionsExternalVersion: &optionsExternalVersion, - Admit: m.admissionControl, - Context: m.requestContextMapper, + Admit: m.AdmissionControl, + Context: m.RequestContextMapper, - MinRequestTimeout: m.minRequestTimeout, + MinRequestTimeout: m.MinRequestTimeout, } } diff --git a/pkg/master/master_test.go b/pkg/master/master_test.go index 6816f5544e4..ae9a235e99f 100644 --- a/pkg/master/master_test.go +++ b/pkg/master/master_test.go @@ -36,7 +36,7 @@ import ( apiutil "k8s.io/kubernetes/pkg/api/util" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/apis/extensions" - "k8s.io/kubernetes/pkg/apiserver" + "k8s.io/kubernetes/pkg/genericapiserver" "k8s.io/kubernetes/pkg/kubelet/client" "k8s.io/kubernetes/pkg/registry/endpoint" "k8s.io/kubernetes/pkg/registry/namespace" @@ -59,14 +59,19 @@ import ( func setUp(t *testing.T) (Master, *etcdtesting.EtcdTestServer, Config, *assert.Assertions) { server := etcdtesting.NewEtcdTestClientServer(t) - master := Master{} - config := Config{} + master := Master{ + GenericAPIServer: &genericapiserver.GenericAPIServer{}, + } + config := Config{ + Config: &genericapiserver.Config{}, + } storageVersions := make(map[string]string) - storageDestinations := NewStorageDestinations() + storageDestinations := genericapiserver.NewStorageDestinations() storageDestinations.AddAPIGroup( api.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Default.Codec(), etcdtest.PathPrefix())) storageDestinations.AddAPIGroup( extensions.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Extensions.Codec(), etcdtest.PathPrefix())) + config.StorageDestinations = storageDestinations storageVersions[api.GroupName] = testapi.Default.GroupVersion().String() storageVersions[extensions.GroupName] = testapi.Extensions.GroupVersion().String() @@ -77,11 +82,8 @@ func setUp(t *testing.T) (Master, *etcdtesting.EtcdTestServer, Config, *assert.A return master, server, config, assert.New(t) } -// TestNew verifies that the New function returns a Master -// using the configuration properly. -func TestNew(t *testing.T) { +func newMaster(t *testing.T) (*Master, *etcdtesting.EtcdTestServer, Config, *assert.Assertions) { _, etcdserver, config, assert := setUp(t) - defer etcdserver.Terminate(t) config.KubeletClient = client.FakeKubeletClient{} @@ -89,37 +91,34 @@ func TestNew(t *testing.T) { config.ProxyTLSClientConfig = &tls.Config{} master := New(&config) + return master, etcdserver, config, assert +} + +// TestNew verifies that the New function returns a Master +// using the configuration properly. +func TestNew(t *testing.T) { + master, etcdserver, config, assert := newMaster(t) + defer etcdserver.Terminate(t) // Verify many of the variables match their config counterparts assert.Equal(master.enableCoreControllers, config.EnableCoreControllers) - assert.Equal(master.enableLogsSupport, config.EnableLogsSupport) - assert.Equal(master.enableUISupport, config.EnableUISupport) - assert.Equal(master.enableSwaggerSupport, config.EnableSwaggerSupport) - assert.Equal(master.enableSwaggerSupport, config.EnableSwaggerSupport) - assert.Equal(master.enableProfiling, config.EnableProfiling) - assert.Equal(master.apiPrefix, config.APIPrefix) - assert.Equal(master.apiGroupPrefix, config.APIGroupPrefix) - assert.Equal(master.corsAllowedOriginList, config.CorsAllowedOriginList) - assert.Equal(master.authenticator, config.Authenticator) - assert.Equal(master.authorizer, config.Authorizer) - assert.Equal(master.admissionControl, config.AdmissionControl) - assert.Equal(master.apiGroupVersionOverrides, config.APIGroupVersionOverrides) - assert.Equal(master.requestContextMapper, config.RequestContextMapper) - assert.Equal(master.cacheTimeout, config.CacheTimeout) - assert.Equal(master.masterCount, config.MasterCount) - assert.Equal(master.externalHost, config.ExternalHost) - assert.Equal(master.clusterIP, config.PublicAddress) - assert.Equal(master.publicReadWritePort, config.ReadWritePort) - assert.Equal(master.serviceReadWriteIP, config.ServiceReadWriteIP) assert.Equal(master.tunneler, config.Tunneler) + assert.Equal(master.ApiPrefix, config.APIPrefix) + assert.Equal(master.ApiGroupPrefix, config.APIGroupPrefix) + assert.Equal(master.ApiGroupVersionOverrides, config.APIGroupVersionOverrides) + assert.Equal(master.RequestContextMapper, config.RequestContextMapper) + assert.Equal(master.MasterCount, config.MasterCount) + assert.Equal(master.ClusterIP, config.PublicAddress) + assert.Equal(master.PublicReadWritePort, config.ReadWritePort) + assert.Equal(master.ServiceReadWriteIP, config.ServiceReadWriteIP) // These functions should point to the same memory location - masterDialer, _ := util.Dialer(master.proxyTransport) + masterDialer, _ := util.Dialer(master.ProxyTransport) masterDialerFunc := fmt.Sprintf("%p", masterDialer) configDialerFunc := fmt.Sprintf("%p", config.ProxyDialer) assert.Equal(masterDialerFunc, configDialerFunc) - assert.Equal(master.proxyTransport.(*http.Transport).TLSClientConfig, config.ProxyTLSClientConfig) + assert.Equal(master.ProxyTransport.(*http.Transport).TLSClientConfig, config.ProxyTLSClientConfig) } // TestGetServersToValidate verifies the unexported getServersToValidate function @@ -165,14 +164,29 @@ func TestFindExternalAddress(t *testing.T) { // TestApi_v1 verifies that the unexported api_v1 function does indeed // utilize the correct Version and Codec. func TestApi_v1(t *testing.T) { - master, etcdserver, _, assert := setUp(t) + _, etcdserver, config, assert := setUp(t) defer etcdserver.Terminate(t) - version := master.api_v1() + // config.KubeletClient = client.FakeKubeletClient{} + + config.ProxyDialer = func(network, addr string) (net.Conn, error) { return nil, nil } + config.ProxyTLSClientConfig = &tls.Config{} + + s := genericapiserver.New(config.Config) + master := &Master{ + GenericAPIServer: s, + tunneler: config.Tunneler, + } + + version := master.api_v1(&config) assert.Equal(unversioned.GroupVersion{Version: "v1"}, version.GroupVersion, "Version was not v1: %s", version.GroupVersion) assert.Equal(v1.Codec, version.Codec, "version.Codec was not for v1: %s", version.Codec) - for k, v := range master.storage { - assert.Contains(version.Storage, v, "Value %s not found (key: %s)", k, v) + // Verify that version storage has all the resources. + for k, v := range master.v1ResourcesStorage { + k = strings.ToLower(k) + val, ok := version.Storage[k] + assert.True(ok, "ok: %s", ok) + assert.Equal(val, v) } } @@ -188,10 +202,10 @@ func TestNewBootstrapController(t *testing.T) { master.serviceRegistry = registrytest.NewServiceRegistry() master.endpointRegistry = endpoint.NewRegistry(nil) - master.serviceNodePortRange = portRange - master.masterCount = 1 - master.serviceReadWritePort = 1000 - master.publicReadWritePort = 1010 + master.ServiceNodePortRange = portRange + master.MasterCount = 1 + master.ServiceReadWritePort = 1000 + master.PublicReadWritePort = 1010 controller := master.NewBootstrapController() @@ -199,9 +213,9 @@ func TestNewBootstrapController(t *testing.T) { assert.Equal(controller.EndpointRegistry, master.endpointRegistry) assert.Equal(controller.ServiceRegistry, master.serviceRegistry) assert.Equal(controller.ServiceNodePortRange, portRange) - assert.Equal(controller.MasterCount, master.masterCount) - assert.Equal(controller.ServicePort, master.serviceReadWritePort) - assert.Equal(controller.PublicServicePort, master.publicReadWritePort) + assert.Equal(controller.MasterCount, master.MasterCount) + assert.Equal(controller.ServicePort, master.ServiceReadWritePort) + assert.Equal(controller.PublicServicePort, master.PublicReadWritePort) } // TestControllerServicePorts verifies master extraServicePorts are @@ -214,7 +228,7 @@ func TestControllerServicePorts(t *testing.T) { master.serviceRegistry = registrytest.NewServiceRegistry() master.endpointRegistry = endpoint.NewRegistry(nil) - master.extraServicePorts = []api.ServicePort{ + master.ExtraServicePorts = []api.ServicePort{ { Name: "additional-port-1", Port: 1000, @@ -235,79 +249,6 @@ func TestControllerServicePorts(t *testing.T) { assert.Equal(1010, controller.ExtraServicePorts[1].Port) } -// TestNewHandlerContainer verifies that NewHandlerContainer uses the -// mux provided -func TestNewHandlerContainer(t *testing.T) { - assert := assert.New(t) - mux := http.NewServeMux() - container := NewHandlerContainer(mux) - assert.Equal(mux, container.ServeMux, "ServerMux's do not match") -} - -// TestHandleWithAuth verifies HandleWithAuth adds the path -// to the muxHelper.RegisteredPaths. -func TestHandleWithAuth(t *testing.T) { - master, etcdserver, _, assert := setUp(t) - defer etcdserver.Terminate(t) - - mh := apiserver.MuxHelper{Mux: http.NewServeMux()} - master.muxHelper = &mh - handler := func(r http.ResponseWriter, w *http.Request) { w.Write(nil) } - master.HandleWithAuth("/test", http.HandlerFunc(handler)) - - assert.Contains(master.muxHelper.RegisteredPaths, "/test", "Path not found in muxHelper") -} - -// TestHandleFuncWithAuth verifies HandleFuncWithAuth adds the path -// to the muxHelper.RegisteredPaths. -func TestHandleFuncWithAuth(t *testing.T) { - master, etcdserver, _, assert := setUp(t) - defer etcdserver.Terminate(t) - - mh := apiserver.MuxHelper{Mux: http.NewServeMux()} - master.muxHelper = &mh - handler := func(r http.ResponseWriter, w *http.Request) { w.Write(nil) } - master.HandleFuncWithAuth("/test", handler) - - assert.Contains(master.muxHelper.RegisteredPaths, "/test", "Path not found in muxHelper") -} - -// TestInstallSwaggerAPI verifies that the swagger api is added -// at the proper endpoint. -func TestInstallSwaggerAPI(t *testing.T) { - master, etcdserver, _, assert := setUp(t) - defer etcdserver.Terminate(t) - - mux := http.NewServeMux() - master.handlerContainer = NewHandlerContainer(mux) - - // Ensure swagger isn't installed without the call - ws := master.handlerContainer.RegisteredWebServices() - if !assert.Equal(len(ws), 0) { - for x := range ws { - assert.NotEqual("/swaggerapi", ws[x].RootPath(), "SwaggerAPI was installed without a call to InstallSwaggerAPI()") - } - } - - // Install swagger and test - master.InstallSwaggerAPI() - ws = master.handlerContainer.RegisteredWebServices() - if assert.NotEqual(0, len(ws), "SwaggerAPI not installed.") { - assert.Equal("/swaggerapi/", ws[0].RootPath(), "SwaggerAPI did not install to the proper path. %s != /swaggerapi", ws[0].RootPath()) - } - - // Empty externalHost verification - mux = http.NewServeMux() - master.handlerContainer = NewHandlerContainer(mux) - master.externalHost = "" - master.clusterIP = net.IPv4(10, 10, 10, 10) - master.publicReadWritePort = 1010 - master.InstallSwaggerAPI() - if assert.NotEqual(0, len(ws), "SwaggerAPI not installed.") { - assert.Equal("/swaggerapi/", ws[0].RootPath(), "SwaggerAPI did not install to the proper path. %s != /swaggerapi", ws[0].RootPath()) - } -} - // TestDefaultAPIGroupVersion verifies that the unexported defaultAPIGroupVersion // creates the expected APIGroupVersion based off of master. func TestDefaultAPIGroupVersion(t *testing.T) { @@ -316,10 +257,10 @@ func TestDefaultAPIGroupVersion(t *testing.T) { apiGroup := master.defaultAPIGroupVersion() - assert.Equal(apiGroup.Root, master.apiPrefix) - assert.Equal(apiGroup.Admit, master.admissionControl) - assert.Equal(apiGroup.Context, master.requestContextMapper) - assert.Equal(apiGroup.MinRequestTimeout, master.minRequestTimeout) + assert.Equal(apiGroup.Root, master.ApiPrefix) + assert.Equal(apiGroup.Admit, master.AdmissionControl) + assert.Equal(apiGroup.Context, master.RequestContextMapper) + assert.Equal(apiGroup.MinRequestTimeout, master.MinRequestTimeout) } // TestExpapi verifies that the unexported exapi creates @@ -331,7 +272,7 @@ func TestExpapi(t *testing.T) { extensionsGroupMeta := latest.GroupOrDie(extensions.GroupName) expAPIGroup := master.experimental(&config) - assert.Equal(expAPIGroup.Root, master.apiGroupPrefix) + assert.Equal(expAPIGroup.Root, master.ApiGroupPrefix) assert.Equal(expAPIGroup.Mapper, extensionsGroupMeta.RESTMapper) assert.Equal(expAPIGroup.Codec, extensionsGroupMeta.Codec) assert.Equal(expAPIGroup.Linker, extensionsGroupMeta.SelfLinker) @@ -371,31 +312,10 @@ func TestGetNodeAddresses(t *testing.T) { } func TestDiscoveryAtAPIS(t *testing.T) { - master, etcdserver, config, assert := setUp(t) + master, etcdserver, config, assert := newMaster(t) defer etcdserver.Terminate(t) - // ================= preparation for master.init() ====================== - portRange := util.PortRange{Base: 10, Size: 10} - master.serviceNodePortRange = portRange - - _, ipnet, err := net.ParseCIDR("192.168.1.1/24") - if !assert.NoError(err) { - t.Errorf("unexpected error: %v", err) - } - master.serviceClusterIPRange = ipnet - - mh := apiserver.MuxHelper{Mux: http.NewServeMux()} - master.muxHelper = &mh - master.rootWebService = new(restful.WebService) - - master.handlerContainer = restful.NewContainer() - - master.mux = http.NewServeMux() - master.requestContextMapper = api.NewRequestContextMapper() - // ======================= end of preparation =========================== - - master.init(&config) - server := httptest.NewServer(master.handlerContainer.ServeMux) + server := httptest.NewServer(master.HandlerContainer.ServeMux) resp, err := http.Get(server.URL + "/apis") if !assert.NoError(err) { t.Errorf("unexpected error: %v", err) @@ -457,14 +377,14 @@ func initThirdParty(t *testing.T, version string) (*Master, *etcdtesting.EtcdTes }, }, } - master.handlerContainer = restful.NewContainer() + master.HandlerContainer = restful.NewContainer() master.thirdPartyStorage = etcdstorage.NewEtcdStorage(etcdserver.Client, testapi.Extensions.Codec(), etcdtest.PathPrefix()) if !assert.NoError(master.InstallThirdPartyResource(api)) { t.FailNow() } - server := httptest.NewServer(master.handlerContainer.ServeMux) + server := httptest.NewServer(master.HandlerContainer.ServeMux) return &master, etcdserver, server, assert } @@ -883,7 +803,7 @@ func testInstallThirdPartyResourceRemove(t *testing.T, version string) { if len(installed) != 0 { t.Errorf("Resource(s) still installed: %v", installed) } - services := master.handlerContainer.RegisteredWebServices() + services := master.HandlerContainer.RegisteredWebServices() for ix := range services { if strings.HasPrefix(services[ix].RootPath(), "/apis/company.com") { t.Errorf("Web service still installed at %s: %#v", services[ix].RootPath(), services[ix]) diff --git a/pkg/master/tunneler.go b/pkg/master/tunneler.go index f07ddde0785..2cef89d2093 100644 --- a/pkg/master/tunneler.go +++ b/pkg/master/tunneler.go @@ -31,6 +31,8 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +type InstallSSHKey func(user string, data []byte) error + type AddressFunc func() (addresses []string, err error) type Tunneler interface { diff --git a/test/integration/framework/master_utils.go b/test/integration/framework/master_utils.go index 256c259f006..9afd8e6bc03 100644 --- a/test/integration/framework/master_utils.go +++ b/test/integration/framework/master_utils.go @@ -36,6 +36,7 @@ import ( "k8s.io/kubernetes/pkg/controller" replicationcontroller "k8s.io/kubernetes/pkg/controller/replication" "k8s.io/kubernetes/pkg/fields" + "k8s.io/kubernetes/pkg/genericapiserver" "k8s.io/kubernetes/pkg/kubectl" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" "k8s.io/kubernetes/pkg/master" @@ -138,22 +139,26 @@ func startMasterOrDie(masterConfig *master.Config) (*master.Master, *httptest.Se func NewMasterConfig() *master.Config { etcdClient := NewEtcdClient() storageVersions := make(map[string]string) + etcdStorage := etcdstorage.NewEtcdStorage(etcdClient, testapi.Default.Codec(), etcdtest.PathPrefix()) storageVersions[api.GroupName] = testapi.Default.GroupVersion().String() expEtcdStorage := NewExtensionsEtcdStorage(etcdClient) storageVersions[extensions.GroupName] = testapi.Extensions.GroupVersion().String() - storageDestinations := master.NewStorageDestinations() + + storageDestinations := genericapiserver.NewStorageDestinations() storageDestinations.AddAPIGroup(api.GroupName, etcdStorage) storageDestinations.AddAPIGroup(extensions.GroupName, expEtcdStorage) return &master.Config{ - StorageDestinations: storageDestinations, - StorageVersions: storageVersions, - KubeletClient: kubeletclient.FakeKubeletClient{}, - APIPrefix: "/api", - APIGroupPrefix: "/apis", - Authorizer: apiserver.NewAlwaysAllowAuthorizer(), - AdmissionControl: admit.NewAlwaysAdmit(), + Config: &genericapiserver.Config{ + StorageDestinations: storageDestinations, + StorageVersions: storageVersions, + APIPrefix: "/api", + APIGroupPrefix: "/apis", + Authorizer: apiserver.NewAlwaysAllowAuthorizer(), + AdmissionControl: admit.NewAlwaysAdmit(), + }, + KubeletClient: kubeletclient.FakeKubeletClient{}, } } diff --git a/test/integration/openshift_test.go b/test/integration/openshift_test.go index c8dae483a40..e3456027061 100644 --- a/test/integration/openshift_test.go +++ b/test/integration/openshift_test.go @@ -19,6 +19,7 @@ package integration import ( "testing" + "k8s.io/kubernetes/pkg/genericapiserver" "k8s.io/kubernetes/pkg/master" ) @@ -26,10 +27,15 @@ import ( // are not referenced directly by a master. func TestMasterExportsSymbols(t *testing.T) { _ = &master.Config{ + Config: &genericapiserver.Config{ + EnableUISupport: false, + EnableSwaggerSupport: false, + RestfulContainer: nil, + }, EnableCoreControllers: false, - EnableUISupport: false, - EnableSwaggerSupport: false, - RestfulContainer: nil, } - _ = (&master.Master{}).NewBootstrapController() + m := &master.Master{ + GenericAPIServer: &genericapiserver.GenericAPIServer{}, + } + _ = (m).NewBootstrapController() }