Merge pull request #33957 from bprashanth/esipp-beta

Automatic merge from submit-queue Loadbalanced client src ip preservation enters beta Sounds like we're going to try out the proposal (https://github.com/kubernetes/kubernetes/issues/30819#issuecomment-249877334) for annotations -> fields on just one feature in 1.5 (scheduler). Or do we want to just convert to fields right now?
2016-10-20 06:53:07 -07:00
parent 10ebd65913 5029bb06e9
commit e6b5b076b8
9 changed files with 610 additions and 131 deletions
--- a/pkg/api/service/annotations.go
+++ b/pkg/api/service/annotations.go
@@ -33,29 +33,44 @@ const (
 	// Not all cloud providers support this annotation, though AWS & GCE do.
 	AnnotationLoadBalancerSourceRangesKey = "service.beta.kubernetes.io/load-balancer-source-ranges"

-	// AnnotationExternalTraffic An annotation that denotes if this Service desires to route external traffic to local
-	// endpoints only. This preserves Source IP and avoids a second hop.
-	AnnotationExternalTraffic = "service.alpha.kubernetes.io/external-traffic"
 	// AnnotationValueExternalTrafficLocal Value of annotation to specify local endpoints behaviour
 	AnnotationValueExternalTrafficLocal = "OnlyLocal"
 	// AnnotationValueExternalTrafficGlobal Value of annotation to specify global (legacy) behaviour
 	AnnotationValueExternalTrafficGlobal = "Global"
-	// AnnotationHealthCheckNodePort Annotation specifying the healthcheck nodePort for the service
+
+	// TODO: The alpha annotations have been deprecated, remove them when we move this feature to GA.
+
+	// AlphaAnnotationHealthCheckNodePort Annotation specifying the healthcheck nodePort for the service
 	// If not specified, annotation is created by the service api backend with the allocated nodePort
 	// Will use user-specified nodePort value if specified by the client
-	AnnotationHealthCheckNodePort = "service.alpha.kubernetes.io/healthcheck-nodeport"
+	AlphaAnnotationHealthCheckNodePort = "service.alpha.kubernetes.io/healthcheck-nodeport"
+
+	// AlphaAnnotationExternalTraffic An annotation that denotes if this Service desires to route external traffic to local
+	// endpoints only. This preserves Source IP and avoids a second hop.
+	AlphaAnnotationExternalTraffic = "service.alpha.kubernetes.io/external-traffic"
+
+	// BetaAnnotationHealthCheckNodePort is the beta version of AlphaAnnotationHealthCheckNodePort.
+	BetaAnnotationHealthCheckNodePort = "service.beta.kubernetes.io/healthcheck-nodeport"
+
+	// BetaAnnotationExternalTraffic is the beta version of AlphaAnnotationExternalTraffic.
+	BetaAnnotationExternalTraffic = "service.beta.kubernetes.io/external-traffic"
 )

 // NeedsHealthCheck Check service for health check annotations
 func NeedsHealthCheck(service *api.Service) bool {
-	if l, ok := service.Annotations[AnnotationExternalTraffic]; ok {
-		if l == AnnotationValueExternalTrafficLocal {
-			return true
-		} else if l == AnnotationValueExternalTrafficGlobal {
-			return false
-		} else {
-			glog.Errorf("Invalid value for annotation %v", AnnotationExternalTraffic)
-			return false
+	// First check the alpha annotation and then the beta. This is so existing
+	// Services continue to work till the user decides to transition to beta.
+	// If they transition to beta, there's no way to go back to alpha without
+	// rolling back the cluster.
+	for _, annotation := range []string{AlphaAnnotationExternalTraffic, BetaAnnotationExternalTraffic} {
+		if l, ok := service.Annotations[annotation]; ok {
+			if l == AnnotationValueExternalTrafficLocal {
+				return true
+			} else if l == AnnotationValueExternalTrafficGlobal {
+				return false
+			} else {
+				glog.Errorf("Invalid value for annotation %v: %v", annotation, l)
+			}
 		}
 	}
 	return false
@@ -63,12 +78,19 @@ func NeedsHealthCheck(service *api.Service) bool {

 // GetServiceHealthCheckNodePort Return health check node port annotation for service, if one exists
 func GetServiceHealthCheckNodePort(service *api.Service) int32 {
-	if NeedsHealthCheck(service) {
-		if l, ok := service.Annotations[AnnotationHealthCheckNodePort]; ok {
+	if !NeedsHealthCheck(service) {
+		return 0
+	}
+	// First check the alpha annotation and then the beta. This is so existing
+	// Services continue to work till the user decides to transition to beta.
+	// If they transition to beta, there's no way to go back to alpha without
+	// rolling back the cluster.
+	for _, annotation := range []string{AlphaAnnotationHealthCheckNodePort, BetaAnnotationHealthCheckNodePort} {
+		if l, ok := service.Annotations[annotation]; ok {
 			p, err := strconv.Atoi(l)
 			if err != nil {
-				glog.Errorf("Failed to parse annotation %v: %v", AnnotationHealthCheckNodePort, err)
-				return 0
+				glog.Errorf("Failed to parse annotation %v: %v", annotation, err)
+				continue
 			}
 			return int32(p)
 		}
--- a/pkg/api/validation/validation.go
+++ b/pkg/api/validation/validation.go
@@ -2385,8 +2385,15 @@ var supportedSessionAffinityType = sets.NewString(string(api.ServiceAffinityClie
 var supportedServiceType = sets.NewString(string(api.ServiceTypeClusterIP), string(api.ServiceTypeNodePort),
 	string(api.ServiceTypeLoadBalancer), string(api.ServiceTypeExternalName))

-// ValidateService tests if required fields in the service are set.
+// ValidateService tests if required fields/annotations of a Service are valid.
 func ValidateService(service *api.Service) field.ErrorList {
+	allErrs := validateServiceFields(service)
+	allErrs = append(allErrs, validateServiceAnnotations(service, nil)...)
+	return allErrs
+}
+
+// validateServiceFields tests if required fields in the service are set.
+func validateServiceFields(service *api.Service) field.ErrorList {
 	allErrs := ValidateObjectMeta(&service.ObjectMeta, true, ValidateServiceName, field.NewPath("metadata"))

 	specPath := field.NewPath("spec")
@@ -2567,6 +2574,63 @@ func validateServicePort(sp *api.ServicePort, requireName, isHeadlessService boo
 	return allErrs
 }

+func validateServiceAnnotations(service *api.Service, oldService *api.Service) (allErrs field.ErrorList) {
+	// 2 annotations went from alpha to beta in 1.5: healthcheck-nodeport and
+	// external-traffic. The user cannot mix these. All updates to the alpha
+	// annotation are disallowed. The user must change both alpha annotations
+	// to beta before making any modifications, even though the system continues
+	// to respect the alpha version.
+	hcAlpha, healthCheckAlphaOk := service.Annotations[apiservice.AlphaAnnotationHealthCheckNodePort]
+	onlyLocalAlpha, onlyLocalAlphaOk := service.Annotations[apiservice.AlphaAnnotationExternalTraffic]
+
+	_, healthCheckBetaOk := service.Annotations[apiservice.BetaAnnotationHealthCheckNodePort]
+	_, onlyLocalBetaOk := service.Annotations[apiservice.BetaAnnotationExternalTraffic]
+
+	var oldHealthCheckAlpha, oldOnlyLocalAlpha string
+	var oldHealthCheckAlphaOk, oldOnlyLocalAlphaOk bool
+	if oldService != nil {
+		oldHealthCheckAlpha, oldHealthCheckAlphaOk = oldService.Annotations[apiservice.AlphaAnnotationHealthCheckNodePort]
+		oldOnlyLocalAlpha, oldOnlyLocalAlphaOk = oldService.Annotations[apiservice.AlphaAnnotationExternalTraffic]
+	}
+	hcValueChanged := oldHealthCheckAlphaOk && healthCheckAlphaOk && oldHealthCheckAlpha != hcAlpha
+	hcValueNew := !oldHealthCheckAlphaOk && healthCheckAlphaOk
+	hcValueGone := !healthCheckAlphaOk && !healthCheckBetaOk && oldHealthCheckAlphaOk
+	onlyLocalHCMismatch := onlyLocalBetaOk && healthCheckAlphaOk
+
+	// On upgrading to a 1.5 cluster, the user is locked in at the current
+	// alpha setting, till they modify the Service such that the pair of
+	// annotations are both beta. Basically this means we need to:
+	// Disallow updates to the alpha annotation.
+	// Disallow creating a Service with the alpha annotation.
+	// Disallow removing both alpha annotations. Removing the health-check
+	// annotation is rejected at a later stage anyway, so if we allow removing
+	// just onlyLocal we might leak the port.
+	// Disallow a single field from transitioning to beta. Mismatched annotations
+	// cause confusion.
+	// Ignore changes to the fields if they're both transitioning to beta.
+	// Allow modifications to Services in fields other than the alpha annotation.
+
+	if hcValueNew || hcValueChanged || hcValueGone || onlyLocalHCMismatch {
+		fieldPath := field.NewPath("metadata", "annotations").Key(apiservice.AlphaAnnotationHealthCheckNodePort)
+		msg := fmt.Sprintf("please replace the alpha annotation with the beta version %v",
+			apiservice.BetaAnnotationHealthCheckNodePort)
+		allErrs = append(allErrs, field.Invalid(fieldPath, apiservice.AlphaAnnotationHealthCheckNodePort, msg))
+	}
+
+	onlyLocalValueChanged := oldOnlyLocalAlphaOk && onlyLocalAlphaOk && oldOnlyLocalAlpha != onlyLocalAlpha
+	onlyLocalValueNew := !oldOnlyLocalAlphaOk && onlyLocalAlphaOk
+	onlyLocalValueGone := !onlyLocalAlphaOk && !onlyLocalBetaOk && oldOnlyLocalAlphaOk
+	hcOnlyLocalMismatch := onlyLocalAlphaOk && healthCheckBetaOk
+
+	if onlyLocalValueNew || onlyLocalValueChanged || onlyLocalValueGone || hcOnlyLocalMismatch {
+		fieldPath := field.NewPath("metadata", "annotations").Key(apiservice.AlphaAnnotationExternalTraffic)
+		msg := fmt.Sprintf("please replace the alpha annotation with the beta version %v",
+			apiservice.BetaAnnotationExternalTraffic)
+		allErrs = append(allErrs, field.Invalid(fieldPath, apiservice.AlphaAnnotationExternalTraffic, msg))
+	}
+	return
+}
+
 // ValidateServiceUpdate tests if required fields in the service are set during an update
 func ValidateServiceUpdate(service, oldService *api.Service) field.ErrorList {
 	allErrs := ValidateObjectMetaUpdate(&service.ObjectMeta, &oldService.ObjectMeta, field.NewPath("metadata"))
@@ -2578,7 +2642,8 @@ func ValidateServiceUpdate(service, oldService *api.Service) field.ErrorList {
 	// TODO(freehan): allow user to update loadbalancerSourceRanges
 	allErrs = append(allErrs, ValidateImmutableField(service.Spec.LoadBalancerSourceRanges, oldService.Spec.LoadBalancerSourceRanges, field.NewPath("spec", "loadBalancerSourceRanges"))...)

-	allErrs = append(allErrs, ValidateService(service)...)
+	allErrs = append(allErrs, validateServiceFields(service)...)
+	allErrs = append(allErrs, validateServiceAnnotations(service, oldService)...)
 	return allErrs
 }

--- a/pkg/api/validation/validation_test.go
+++ b/pkg/api/validation/validation_test.go
@@ -5207,6 +5207,13 @@ func TestValidateService(t *testing.T) {
 			},
 			numErrs: 1,
 		},
+		{
+			name: "LoadBalancer disallows onlyLocal alpha annotations",
+			tweakSvc: func(s *api.Service) {
+				s.Annotations[service.AlphaAnnotationExternalTraffic] = service.AnnotationValueExternalTrafficLocal
+			},
+			numErrs: 1,
+		},
 	}

 	for _, tc := range testCases {
@@ -6474,6 +6481,44 @@ func TestValidateServiceUpdate(t *testing.T) {
 			},
 			numErrs: 1,
 		},
+		{
+			name: "Service disallows removing one onlyLocal alpha annotation",
+			tweakSvc: func(oldSvc, newSvc *api.Service) {
+				oldSvc.Annotations[service.AlphaAnnotationExternalTraffic] = service.AnnotationValueExternalTrafficLocal
+				oldSvc.Annotations[service.AlphaAnnotationHealthCheckNodePort] = "3001"
+			},
+			numErrs: 2,
+		},
+		{
+			name: "Service disallows modifying onlyLocal alpha annotations",
+			tweakSvc: func(oldSvc, newSvc *api.Service) {
+				oldSvc.Annotations[service.AlphaAnnotationExternalTraffic] = service.AnnotationValueExternalTrafficLocal
+				oldSvc.Annotations[service.AlphaAnnotationHealthCheckNodePort] = "3001"
+				newSvc.Annotations[service.AlphaAnnotationExternalTraffic] = service.AnnotationValueExternalTrafficGlobal
+				newSvc.Annotations[service.AlphaAnnotationHealthCheckNodePort] = oldSvc.Annotations[service.AlphaAnnotationHealthCheckNodePort]
+			},
+			numErrs: 1,
+		},
+		{
+			name: "Service disallows promoting one of the onlyLocal pair to beta",
+			tweakSvc: func(oldSvc, newSvc *api.Service) {
+				oldSvc.Annotations[service.AlphaAnnotationExternalTraffic] = service.AnnotationValueExternalTrafficLocal
+				oldSvc.Annotations[service.AlphaAnnotationHealthCheckNodePort] = "3001"
+				newSvc.Annotations[service.BetaAnnotationExternalTraffic] = service.AnnotationValueExternalTrafficGlobal
+				newSvc.Annotations[service.AlphaAnnotationHealthCheckNodePort] = oldSvc.Annotations[service.AlphaAnnotationHealthCheckNodePort]
+			},
+			numErrs: 1,
+		},
+		{
+			name: "Service allows changing both onlyLocal annotations from alpha to beta",
+			tweakSvc: func(oldSvc, newSvc *api.Service) {
+				oldSvc.Annotations[service.AlphaAnnotationExternalTraffic] = service.AnnotationValueExternalTrafficLocal
+				oldSvc.Annotations[service.AlphaAnnotationHealthCheckNodePort] = "3001"
+				newSvc.Annotations[service.BetaAnnotationExternalTraffic] = service.AnnotationValueExternalTrafficLocal
+				newSvc.Annotations[service.BetaAnnotationHealthCheckNodePort] = oldSvc.Annotations[service.AlphaAnnotationHealthCheckNodePort]
+			},
+			numErrs: 0,
+		},
 	}

 	for _, tc := range testCases {
--- a/pkg/cloudprovider/providers/gce/gce.go
+++ b/pkg/cloudprovider/providers/gce/gce.go
@@ -736,9 +736,7 @@ func (gce *GCECloud) EnsureLoadBalancer(clusterName string, apiService *api.Serv
 			// This logic exists to detect a transition for a pre-existing service and turn on
 			// the tpNeedsUpdate flag to delete/recreate fwdrule/tpool adding the health check
 			// to the target pool.
-			glog.V(2).Infof("Annotation %s=%s added to new or pre-existing service",
-				apiservice.AnnotationExternalTraffic,
-				apiservice.AnnotationValueExternalTrafficLocal)
+			glog.V(2).Infof("Annotation external-traffic=OnlyLocal added to new or pre-existing service")
 			tpNeedsUpdate = true
 		}
 		hcToCreate, err = gce.ensureHttpHealthCheck(loadBalancerName, path, healthCheckNodePort)
--- a/pkg/proxy/iptables/proxier.go
+++ b/pkg/proxy/iptables/proxier.go
@@ -455,18 +455,20 @@ func (proxier *Proxier) OnServiceUpdate(allServices []api.Service) {
 			info.loadBalancerStatus = *api.LoadBalancerStatusDeepCopy(&service.Status.LoadBalancer)
 			info.sessionAffinityType = service.Spec.SessionAffinity
 			info.loadBalancerSourceRanges = service.Spec.LoadBalancerSourceRanges
-			info.onlyNodeLocalEndpoints = apiservice.NeedsHealthCheck(service) && featuregate.DefaultFeatureGate.ExternalTrafficLocalOnly()
+			info.onlyNodeLocalEndpoints = apiservice.NeedsHealthCheck(service) && featuregate.DefaultFeatureGate.ExternalTrafficLocalOnly() && (service.Spec.Type == api.ServiceTypeLoadBalancer || service.Spec.Type == api.ServiceTypeNodePort)
 			if info.onlyNodeLocalEndpoints {
 				p := apiservice.GetServiceHealthCheckNodePort(service)
 				if p == 0 {
 					glog.Errorf("Service does not contain necessary annotation %v",
-						apiservice.AnnotationHealthCheckNodePort)
+						apiservice.BetaAnnotationHealthCheckNodePort)
 				} else {
+					glog.V(4).Infof("Adding health check for %+v, port %v", serviceName.NamespacedName, p)
 					info.healthCheckNodePort = int(p)
 					// Turn on healthcheck responder to listen on the health check nodePort
 					healthcheck.AddServiceListener(serviceName.NamespacedName, info.healthCheckNodePort)
 				}
 			} else {
+				glog.V(4).Infof("Deleting health check for %+v", serviceName.NamespacedName)
 				// Delete healthcheck responders, if any, previously listening for this service
 				healthcheck.DeleteServiceListener(serviceName.NamespacedName, 0)
 			}
@@ -488,6 +490,7 @@ func (proxier *Proxier) OnServiceUpdate(allServices []api.Service) {
 			if info.onlyNodeLocalEndpoints && info.healthCheckNodePort > 0 {
 				// Remove ServiceListener health check nodePorts from the health checker
 				// TODO - Stats
+				glog.V(4).Infof("Deleting health check for %+v, port %v", name.NamespacedName, info.healthCheckNodePort)
 				healthcheck.DeleteServiceListener(name.NamespacedName, info.healthCheckNodePort)
 			}
 		}
--- a/pkg/registry/core/service/rest.go
+++ b/pkg/registry/core/service/rest.go
@@ -161,10 +161,10 @@ func (rs *REST) Create(ctx api.Context, obj runtime.Object) (runtime.Object, err
 	if shouldCheckOrAssignHealthCheckNodePort(service) {
 		var healthCheckNodePort int
 		var err error
-		if l, ok := service.Annotations[apiservice.AnnotationHealthCheckNodePort]; ok {
+		if l, ok := service.Annotations[apiservice.BetaAnnotationHealthCheckNodePort]; ok {
 			healthCheckNodePort, err = strconv.Atoi(l)
 			if err != nil || healthCheckNodePort <= 0 {
-				return nil, errors.NewInternalError(fmt.Errorf("Failed to parse annotation %v: %v", apiservice.AnnotationHealthCheckNodePort, err))
+				return nil, errors.NewInternalError(fmt.Errorf("Failed to parse annotation %v: %v", apiservice.BetaAnnotationHealthCheckNodePort, err))
 			}
 		}
 		if healthCheckNodePort > 0 {
@@ -183,7 +183,7 @@ func (rs *REST) Create(ctx api.Context, obj runtime.Object) (runtime.Object, err
 				return nil, errors.NewInternalError(fmt.Errorf("failed to allocate a nodePort: %v", err))
 			}
 			// Insert the newly allocated health check port as an annotation (plan of record for Alpha)
-			service.Annotations[apiservice.AnnotationHealthCheckNodePort] = fmt.Sprintf("%d", healthCheckNodePort)
+			service.Annotations[apiservice.BetaAnnotationHealthCheckNodePort] = fmt.Sprintf("%d", healthCheckNodePort)
 		}
 	}

@@ -313,7 +313,7 @@ func (rs *REST) healthCheckNodePortUpdate(oldService, service *api.Service) (boo
 				errmsg := fmt.Sprintf("Failed to allocate requested HealthCheck nodePort %v:%v",
 					requestedHealthCheckNodePort, err)
 				el := field.ErrorList{field.Invalid(field.NewPath("metadata", "annotations"),
-					apiservice.AnnotationHealthCheckNodePort, errmsg)}
+					apiservice.BetaAnnotationHealthCheckNodePort, errmsg)}
 				return false, errors.NewInvalid(api.Kind("Service"), service.Name, el)
 			}
 			glog.Infof("Reserved user requested nodePort: %d", requestedHealthCheckNodePort)
@@ -327,7 +327,7 @@ func (rs *REST) healthCheckNodePortUpdate(oldService, service *api.Service) (boo
 				return false, errors.NewInternalError(fmt.Errorf("failed to allocate a nodePort: %v", err))
 			}
 			// Insert the newly allocated health check port as an annotation (plan of record for Alpha)
-			service.Annotations[apiservice.AnnotationHealthCheckNodePort] = fmt.Sprintf("%d", healthCheckNodePort)
+			service.Annotations[apiservice.BetaAnnotationHealthCheckNodePort] = fmt.Sprintf("%d", healthCheckNodePort)
 			glog.Infof("Reserved health check nodePort: %d", healthCheckNodePort)
 		}

@@ -338,29 +338,30 @@ func (rs *REST) healthCheckNodePortUpdate(oldService, service *api.Service) (boo
 			glog.Warningf("Error releasing service health check %s node port %d: %v", service.Name, oldHealthCheckNodePort, err)
 			return false, errors.NewInternalError(fmt.Errorf("failed to free health check nodePort: %v", err))
 		} else {
-			delete(service.Annotations, apiservice.AnnotationHealthCheckNodePort)
+			delete(service.Annotations, apiservice.BetaAnnotationHealthCheckNodePort)
+			delete(service.Annotations, apiservice.AlphaAnnotationHealthCheckNodePort)
 			glog.Infof("Freed health check nodePort: %d", oldHealthCheckNodePort)
 		}

 	case !oldServiceHasHealthCheckNodePort && !assignHealthCheckNodePort:
-		if _, ok := service.Annotations[apiservice.AnnotationHealthCheckNodePort]; ok {
+		if _, ok := service.Annotations[apiservice.BetaAnnotationHealthCheckNodePort]; ok {
 			glog.Warningf("Attempt to insert health check node port annotation DENIED")
 			el := field.ErrorList{field.Invalid(field.NewPath("metadata", "annotations"),
-				apiservice.AnnotationHealthCheckNodePort, "Cannot insert healthcheck nodePort annotation")}
+				apiservice.BetaAnnotationHealthCheckNodePort, "Cannot insert healthcheck nodePort annotation")}
 			return false, errors.NewInvalid(api.Kind("Service"), service.Name, el)
 		}

 	case oldServiceHasHealthCheckNodePort && assignHealthCheckNodePort:
-		if _, ok := service.Annotations[apiservice.AnnotationHealthCheckNodePort]; !ok {
+		if _, ok := service.Annotations[apiservice.BetaAnnotationHealthCheckNodePort]; !ok {
 			glog.Warningf("Attempt to delete health check node port annotation DENIED")
 			el := field.ErrorList{field.Invalid(field.NewPath("metadata", "annotations"),
-				apiservice.AnnotationHealthCheckNodePort, "Cannot delete healthcheck nodePort annotation")}
+				apiservice.BetaAnnotationHealthCheckNodePort, "Cannot delete healthcheck nodePort annotation")}
 			return false, errors.NewInvalid(api.Kind("Service"), service.Name, el)
 		}
 		if oldHealthCheckNodePort != requestedHealthCheckNodePort {
 			glog.Warningf("Attempt to change value of health check node port annotation DENIED")
 			el := field.ErrorList{field.Invalid(field.NewPath("metadata", "annotations"),
-				apiservice.AnnotationHealthCheckNodePort, "Cannot change healthcheck nodePort during update")}
+				apiservice.BetaAnnotationHealthCheckNodePort, "Cannot change healthcheck nodePort during update")}
 			return false, errors.NewInvalid(api.Kind("Service"), service.Name, el)
 		}
 	}
--- a/pkg/registry/core/service/rest_test.go
+++ b/pkg/registry/core/service/rest_test.go
@@ -955,7 +955,7 @@ func TestServiceRegistryExternalTrafficAnnotationHealthCheckNodePortAllocation(t
 	svc := &api.Service{
 		ObjectMeta: api.ObjectMeta{Name: "external-lb-esipp",
 			Annotations: map[string]string{
-				service.AnnotationExternalTraffic: service.AnnotationValueExternalTrafficLocal,
+				service.BetaAnnotationExternalTraffic: service.AnnotationValueExternalTrafficLocal,
 			},
 		},
 		Spec: api.ServiceSpec{
@@ -975,25 +975,25 @@ func TestServiceRegistryExternalTrafficAnnotationHealthCheckNodePortAllocation(t
 	}
 	created_service := created_svc.(*api.Service)
 	if !service.NeedsHealthCheck(created_service) {
-		t.Errorf("Unexpected missing annotation %s", service.AnnotationExternalTraffic)
+		t.Errorf("Unexpected missing annotation %s", service.BetaAnnotationExternalTraffic)
 	}
 	port := service.GetServiceHealthCheckNodePort(created_service)
 	if port == 0 {
-		t.Errorf("Failed to allocate and create the health check node port annotation %s", service.AnnotationHealthCheckNodePort)
+		t.Errorf("Failed to allocate and create the health check node port annotation %s", service.BetaAnnotationHealthCheckNodePort)
 	}

 }

 // Validate using the user specified nodePort when the externalTraffic=OnlyLocal annotation is set
 // and type is LoadBalancer
-func TestServiceRegistryExternalTrafficAnnotationHealthCheckNodePortUserAllocation(t *testing.T) {
+func TestServiceRegistryExternalTrafficBetaAnnotationHealthCheckNodePortUserAllocation(t *testing.T) {
 	ctx := api.NewDefaultContext()
 	storage, _ := NewTestREST(t, nil)
 	svc := &api.Service{
 		ObjectMeta: api.ObjectMeta{Name: "external-lb-esipp",
 			Annotations: map[string]string{
-				service.AnnotationExternalTraffic:     service.AnnotationValueExternalTrafficLocal,
-				service.AnnotationHealthCheckNodePort: "30200",
+				service.BetaAnnotationExternalTraffic:     service.AnnotationValueExternalTrafficLocal,
+				service.BetaAnnotationHealthCheckNodePort: "30200",
 			},
 		},
 		Spec: api.ServiceSpec{
@@ -1013,11 +1013,11 @@ func TestServiceRegistryExternalTrafficAnnotationHealthCheckNodePortUserAllocati
 	}
 	created_service := created_svc.(*api.Service)
 	if !service.NeedsHealthCheck(created_service) {
-		t.Errorf("Unexpected missing annotation %s", service.AnnotationExternalTraffic)
+		t.Errorf("Unexpected missing annotation %s", service.BetaAnnotationExternalTraffic)
 	}
 	port := service.GetServiceHealthCheckNodePort(created_service)
 	if port == 0 {
-		t.Errorf("Failed to allocate and create the health check node port annotation %s", service.AnnotationHealthCheckNodePort)
+		t.Errorf("Failed to allocate and create the health check node port annotation %s", service.BetaAnnotationHealthCheckNodePort)
 	}
 	if port != 30200 {
 		t.Errorf("Failed to allocate requested nodePort expected 30200, got %d", port)
@@ -1031,8 +1031,8 @@ func TestServiceRegistryExternalTrafficAnnotationNegative(t *testing.T) {
 	svc := &api.Service{
 		ObjectMeta: api.ObjectMeta{Name: "external-lb-esipp",
 			Annotations: map[string]string{
-				service.AnnotationExternalTraffic:     service.AnnotationValueExternalTrafficLocal,
-				service.AnnotationHealthCheckNodePort: "-1",
+				service.BetaAnnotationExternalTraffic:     service.AnnotationValueExternalTrafficLocal,
+				service.BetaAnnotationHealthCheckNodePort: "-1",
 			},
 		},
 		Spec: api.ServiceSpec{
@@ -1060,7 +1060,7 @@ func TestServiceRegistryExternalTrafficAnnotationGlobal(t *testing.T) {
 	svc := &api.Service{
 		ObjectMeta: api.ObjectMeta{Name: "external-lb-esipp",
 			Annotations: map[string]string{
-				service.AnnotationExternalTraffic: service.AnnotationValueExternalTrafficGlobal,
+				service.BetaAnnotationExternalTraffic: service.AnnotationValueExternalTrafficGlobal,
 			},
 		},
 		Spec: api.ServiceSpec{
@@ -1081,12 +1081,12 @@ func TestServiceRegistryExternalTrafficAnnotationGlobal(t *testing.T) {
 	created_service := created_svc.(*api.Service)
 	// Make sure the service does not have the annotation
 	if service.NeedsHealthCheck(created_service) {
-		t.Errorf("Unexpected value for annotation %s", service.AnnotationExternalTraffic)
+		t.Errorf("Unexpected value for annotation %s", service.BetaAnnotationExternalTraffic)
 	}
 	// Make sure the service does not have the health check node port allocated
 	port := service.GetServiceHealthCheckNodePort(created_service)
 	if port != 0 {
-		t.Errorf("Unexpected allocation of health check node port annotation %s", service.AnnotationHealthCheckNodePort)
+		t.Errorf("Unexpected allocation of health check node port annotation %s", service.BetaAnnotationHealthCheckNodePort)
 	}
 }

@@ -1097,7 +1097,7 @@ func TestServiceRegistryExternalTrafficAnnotationClusterIP(t *testing.T) {
 	svc := &api.Service{
 		ObjectMeta: api.ObjectMeta{Name: "external-lb-esipp",
 			Annotations: map[string]string{
-				service.AnnotationExternalTraffic: service.AnnotationValueExternalTrafficGlobal,
+				service.BetaAnnotationExternalTraffic: service.AnnotationValueExternalTrafficGlobal,
 			},
 		},
 		Spec: api.ServiceSpec{
@@ -1119,6 +1119,6 @@ func TestServiceRegistryExternalTrafficAnnotationClusterIP(t *testing.T) {
 	// Make sure that ClusterIP services do not have the health check node port allocated
 	port := service.GetServiceHealthCheckNodePort(created_service)
 	if port != 0 {
-		t.Errorf("Unexpected allocation of health check node port annotation %s", service.AnnotationHealthCheckNodePort)
+		t.Errorf("Unexpected allocation of health check node port annotation %s", service.BetaAnnotationHealthCheckNodePort)
 	}
 }
--- a/pkg/util/config/feature_gate.go
+++ b/pkg/util/config/feature_gate.go
@@ -49,7 +49,7 @@ var (
 	// represented here.
 	knownFeatures = map[string]featureSpec{
 		allAlphaGate:              {false, alpha},
-		externalTrafficLocalOnly:  {false, alpha},
+		externalTrafficLocalOnly:  {true, beta},
 		appArmor:                  {true, beta},
 		dynamicKubeletConfig:      {false, alpha},
 		dynamicVolumeProvisioning: {true, alpha},