Merge pull request #13978 from jayunit100/api_server_append_13152

NodePort option: Allowing for apiservers behind load-balanced endpoint.

cmd/kube-apiserver/app/server.go

@@ -114,6 +114,7 @@ type APIServer struct {
 	SSHUser                    string
 	SSHKeyfile                 string
 	MaxConnectionBytesPerSec   int64
+	KubernetesServiceNodePort  int
 }
 
 // NewAPIServer creates a new APIServer object with default parameters
@@ -242,6 +243,8 @@ func (s *APIServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.KubeletConfig.CertFile, "kubelet-client-certificate", s.KubeletConfig.CertFile, "Path to a client cert file for TLS.")
 	fs.StringVar(&s.KubeletConfig.KeyFile, "kubelet-client-key", s.KubeletConfig.KeyFile, "Path to a client key file for TLS.")
 	fs.StringVar(&s.KubeletConfig.CAFile, "kubelet-certificate-authority", s.KubeletConfig.CAFile, "Path to a cert. file for the certificate authority.")
+	//See #14282 for details on how to test/try this option out.  TODO remove this comment once this option is tested in CI.
+	fs.IntVar(&s.KubernetesServiceNodePort, "kubernetes-service-node-port", 0, "If non-zero, the Kubernetes master service (which apiserver creates/maintains) will be of type NodePort, using this as the value of the port. If zero, the Kubernetes master service will be of type ClusterIP.")
 }
 
 // TODO: Longer term we should read this from some config store, rather than a flag.
@@ -353,6 +356,10 @@ func (s *APIServer) Run(_ []string) error {
 		glog.Fatalf("specify either --etcd-servers or --etcd-config")
 	}
 
+	if s.KubernetesServiceNodePort > 0 && !s.ServiceNodePortRange.Contains(s.KubernetesServiceNodePort) {
+		glog.Fatalf("Kubernetes service port range %v doesn't contain %v", s.ServiceNodePortRange, (s.KubernetesServiceNodePort))
+	}
+
 	capabilities.Initialize(capabilities.Capabilities{
 		AllowPrivileged: s.AllowPrivileged,
 		// TODO(vmarmol): Implement support for HostNetworkSources.
@@ -508,37 +515,38 @@ func (s *APIServer) Run(_ []string) error {
 		}
 	}
 	config := &master.Config{
-		StorageDestinations:    storageDestinations,
+		StorageDestinations:       storageDestinations,
-		StorageVersions:        storageVersions,
+		StorageVersions:           storageVersions,
-		EventTTL:               s.EventTTL,
+		EventTTL:                  s.EventTTL,
-		KubeletClient:          kubeletClient,
+		KubeletClient:             kubeletClient,
-		ServiceClusterIPRange:  &n,
+		ServiceClusterIPRange:     &n,
-		EnableCoreControllers:  true,
+		EnableCoreControllers:     true,
-		EnableLogsSupport:      s.EnableLogsSupport,
+		EnableLogsSupport:         s.EnableLogsSupport,
-		EnableUISupport:        true,
+		EnableUISupport:           true,
-		EnableSwaggerSupport:   true,
+		EnableSwaggerSupport:      true,
-		EnableProfiling:        s.EnableProfiling,
+		EnableProfiling:           s.EnableProfiling,
-		EnableWatchCache:       s.EnableWatchCache,
+		EnableWatchCache:          s.EnableWatchCache,
-		EnableIndex:            true,
+		EnableIndex:               true,
-		APIPrefix:              s.APIPrefix,
+		APIPrefix:                 s.APIPrefix,
-		APIGroupPrefix:         s.APIGroupPrefix,
+		APIGroupPrefix:            s.APIGroupPrefix,
-		CorsAllowedOriginList:  s.CorsAllowedOriginList,
+		CorsAllowedOriginList:     s.CorsAllowedOriginList,
-		ReadWritePort:          s.SecurePort,
+		ReadWritePort:             s.SecurePort,
-		PublicAddress:          s.AdvertiseAddress,
+		PublicAddress:             s.AdvertiseAddress,
-		Authenticator:          authenticator,
+		Authenticator:             authenticator,
-		SupportsBasicAuth:      len(s.BasicAuthFile) > 0,
+		SupportsBasicAuth:         len(s.BasicAuthFile) > 0,
-		Authorizer:             authorizer,
+		Authorizer:                authorizer,
-		AdmissionControl:       admissionController,
+		AdmissionControl:          admissionController,
-		DisableV1:              disableV1,
+		DisableV1:                 disableV1,
-		EnableExp:              enableExp,
+		EnableExp:                 enableExp,
-		MasterServiceNamespace: s.MasterServiceNamespace,
+		MasterServiceNamespace:    s.MasterServiceNamespace,
-		ClusterName:            s.ClusterName,
+		ClusterName:               s.ClusterName,
-		ExternalHost:           s.ExternalHost,
+		ExternalHost:              s.ExternalHost,
-		MinRequestTimeout:      s.MinRequestTimeout,
+		MinRequestTimeout:         s.MinRequestTimeout,
-		SSHUser:                s.SSHUser,
+		SSHUser:                   s.SSHUser,
-		SSHKeyfile:             s.SSHKeyfile,
+		SSHKeyfile:                s.SSHKeyfile,
-		InstallSSHKey:          installSSH,
+		InstallSSHKey:             installSSH,
-		ServiceNodePortRange:   s.ServiceNodePortRange,
+		ServiceNodePortRange:      s.ServiceNodePortRange,
+		KubernetesServiceNodePort: s.KubernetesServiceNodePort,
 	}
 	m := master.New(config)
 

hack/verify-flags/known-flags.txt

@@ -17,6 +17,7 @@ api-servers
 api-server-port
 api-token
 api-version
+kubernetes-service-node-port
 authorization-mode
 authorization-policy-file
 auth-path

pkg/master/controller.go

@@ -57,9 +57,10 @@ type Controller struct {
 
 	PublicIP net.IP
 
-	ServiceIP         net.IP
+	ServiceIP                 net.IP
-	ServicePort       int
+	ServicePort               int
-	PublicServicePort int
+	PublicServicePort         int
+	KubernetesServiceNodePort int
 
 	runner *util.Runner
 }
@@ -110,7 +111,7 @@ func (c *Controller) UpdateKubernetesService() error {
 		return err
 	}
 	if c.ServiceIP != nil {
-		if err := c.CreateMasterServiceIfNeeded("kubernetes", c.ServiceIP, c.ServicePort); err != nil {
+		if err := c.CreateMasterServiceIfNeeded("kubernetes", c.ServiceIP, c.ServicePort, c.KubernetesServiceNodePort); err != nil {
 			return err
 		}
 		if err := c.SetEndpoints("kubernetes", c.PublicIP, c.PublicServicePort); err != nil {
@@ -140,14 +141,33 @@ func (c *Controller) CreateNamespaceIfNeeded(ns string) error {
 	return err
 }
 
+// createPortAndServiceSpec creates an array of service ports.
+// If the NodePort value is 0, just the servicePort is used, otherwise, a node port is exposed.
+func createPortAndServiceSpec(servicePort int, nodePort int) ([]api.ServicePort, api.ServiceType) {
+	//Use the Cluster IP type for the service port if NodePort isn't provided.
+	//Otherwise, we will be binding the master service to a NodePort.
+	if nodePort <= 0 {
+		return []api.ServicePort{{Protocol: api.ProtocolTCP,
+			Port:       servicePort,
+			TargetPort: util.NewIntOrStringFromInt(servicePort)}}, api.ServiceTypeClusterIP
+	}
+	return []api.ServicePort{{Protocol: api.ProtocolTCP,
+		Port:       servicePort,
+		TargetPort: util.NewIntOrStringFromInt(servicePort),
+		NodePort:   nodePort,
+	}}, api.ServiceTypeNodePort
+}
+
 // CreateMasterServiceIfNeeded will create the specified service if it
 // doesn't already exist.
-func (c *Controller) CreateMasterServiceIfNeeded(serviceName string, serviceIP net.IP, servicePort int) error {
+func (c *Controller) CreateMasterServiceIfNeeded(serviceName string, serviceIP net.IP, servicePort, nodePort int) error {
 	ctx := api.NewDefaultContext()
 	if _, err := c.ServiceRegistry.GetService(ctx, serviceName); err == nil {
 		// The service already exists.
 		return nil
 	}
+
+	ports, serviceType := createPortAndServiceSpec(servicePort, nodePort)
 	svc := &api.Service{
 		ObjectMeta: api.ObjectMeta{
 			Name:      serviceName,
@@ -155,15 +175,14 @@ func (c *Controller) CreateMasterServiceIfNeeded(serviceName string, serviceIP n
 			Labels:    map[string]string{"provider": "kubernetes", "component": "apiserver"},
 		},
 		Spec: api.ServiceSpec{
-			Ports: []api.ServicePort{{Port: servicePort, Protocol: api.ProtocolTCP, TargetPort: util.NewIntOrStringFromInt(servicePort)}},
+			Ports: ports,
 			// maintained by this code, not by the pod selector
 			Selector:        nil,
 			ClusterIP:       serviceIP.String(),
 			SessionAffinity: api.ServiceAffinityNone,
-			Type:            api.ServiceTypeClusterIP,
+			Type:            serviceType,
 		},
 	}
-
 	if err := rest.BeforeCreate(rest.Services, ctx, svc); err != nil {
 		return err
 	}

pkg/master/controller_test.go

@@ -259,7 +259,7 @@ func TestSetEndpoints(t *testing.T) {
 		}
 		if test.expectUpdate != nil {
 			if len(registry.Updates) != 1 {
-				t.Errorf("case %q: unexpected updates: %v", test.testName, registry.Updates)
+				t.Errorf("case %q: unexpected updates: (%v).  Expected exactly 1 change. ", test.testName, registry.Updates)
 			} else if e, a := test.expectUpdate, &registry.Updates[0]; !reflect.DeepEqual(e, a) {
 				t.Errorf("case %q: expected update:\n%#v\ngot:\n%#v\n", test.testName, e, a)
 			}

pkg/master/master.go

@@ -244,6 +244,8 @@ type Config struct {
 	SSHUser       string
 	SSHKeyfile    string
 	InstallSSHKey InstallSSHKey
+
+	KubernetesServiceNodePort int
 }
 
 type InstallSSHKey func(user string, data []byte) error
@@ -317,7 +319,8 @@ type Master struct {
 	// map from api path to storage for those objects
 	thirdPartyResources map[string]*thirdpartyresourcedataetcd.REST
 	// protects the map
-	thirdPartyResourcesLock sync.RWMutex
+	thirdPartyResourcesLock   sync.RWMutex
+	KubernetesServiceNodePort int
 }
 
 // NewEtcdStorage returns a storage.Interface for the provided arguments or an error if the version
@@ -450,7 +453,8 @@ func New(c *Config) *Master {
 		// TODO: serviceReadWritePort should be passed in as an argument, it may not always be 443
 		serviceReadWritePort: 443,
 
-		installSSHKey: c.InstallSSHKey,
+		installSSHKey:             c.InstallSSHKey,
+		KubernetesServiceNodePort: c.KubernetesServiceNodePort,
 	}
 
 	var handlerContainer *restful.Container
@@ -781,9 +785,10 @@ func (m *Master) NewBootstrapController() *Controller {
 
 		PublicIP: m.clusterIP,
 
-		ServiceIP:         m.serviceReadWriteIP,
+		ServiceIP:                 m.serviceReadWriteIP,
-		ServicePort:       m.serviceReadWritePort,
+		ServicePort:               m.serviceReadWritePort,
-		PublicServicePort: m.publicReadWritePort,
+		PublicServicePort:         m.publicReadWritePort,
+		KubernetesServiceNodePort: m.KubernetesServiceNodePort,
 	}
 }