Merge pull request #106683 from panslava/rbs-annotation

Disable GCE external load balancer when services handled by Ingress-GCE

staging/src/k8s.io/legacy-cloud-providers/gce/gce_annotations.go

@@ -74,6 +74,13 @@ const (
 
 	// NetworkTierAnnotationPremium is an annotation to indicate the Service is on the Premium network tier
 	NetworkTierAnnotationPremium = cloud.NetworkTierPremium
+
+	// RBSAnnotationKey is annotated on a Service object to indicate
+	// opt-in mode for RBS NetLB
+	RBSAnnotationKey = "cloud.google.com/l4-rbs"
+
+	// RBSEnabled is an annotation to indicate the Service is opt-in for RBS
+	RBSEnabled = "enabled"
 )
 
 // GetLoadBalancerAnnotationType returns the type of GCP load balancer which should be assembled.

staging/src/k8s.io/legacy-cloud-providers/gce/gce_loadbalancer_external.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
+	cloudprovider "k8s.io/cloud-provider"
 	servicehelpers "k8s.io/cloud-provider/service/helpers"
 	utilnet "k8s.io/utils/net"
 
@@ -40,6 +41,8 @@ import (
 
 const (
 	errStrLbNoHosts = "cannot EnsureLoadBalancer() with no hosts"
+
+	ELBRbsFinalizer = "gke.networking.io/l4-netlb-v2"
 )
 
 // ensureExternalLoadBalancer is the external implementation of LoadBalancer.EnsureLoadBalancer.
@@ -51,6 +54,19 @@ const (
 // new load balancers and updating existing load balancers, recognizing when
 // each is needed.
 func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string, apiService *v1.Service, existingFwdRule *compute.ForwardingRule, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) {
+	// Skip service handling if managed by ingress-gce using Regional Backend Services
+	if val, ok := apiService.Annotations[RBSAnnotationKey]; ok && val == RBSEnabled {
+		return nil, cloudprovider.ImplementedElsewhere
+	}
+	// Skip service handling if service has Regional Backend Services finalizer
+	if hasFinalizer(apiService, ELBRbsFinalizer) {
+		return nil, cloudprovider.ImplementedElsewhere
+	}
+	// Skip service handling if it has Regional Backend Service created by Ingress-GCE
+	if existingFwdRule != nil && existingFwdRule.BackendService != "" {
+		return nil, cloudprovider.ImplementedElsewhere
+	}
+
 	if len(nodes) == 0 {
 		return nil, fmt.Errorf(errStrLbNoHosts)
 	}

staging/src/k8s.io/legacy-cloud-providers/gce/gce_loadbalancer_external_test.go

@@ -22,7 +22,6 @@ package gce
 import (
 	"context"
 	"fmt"
-
 	"reflect"
 	"strings"
 	"testing"
@@ -30,6 +29,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	compute "google.golang.org/api/compute/v1"
+	cloudprovider "k8s.io/cloud-provider"
 
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
@@ -580,6 +580,128 @@ func TestEnsureExternalLoadBalancerFailsWithNoNodes(t *testing.T) {
 	assert.EqualError(t, err, errStrLbNoHosts)
 }
 
+func TestEnsureExternalLoadBalancerRBSAnnotation(t *testing.T) {
+	t.Parallel()
+
+	for desc, tc := range map[string]struct {
+		annotations map[string]string
+		expectError *error
+	}{
+		"When RBS enabled": {
+			annotations: map[string]string{RBSAnnotationKey: RBSEnabled},
+			expectError: &cloudprovider.ImplementedElsewhere,
+		},
+		"When RBS not enabled": {
+			annotations: map[string]string{},
+			expectError: nil,
+		},
+		"When RBS annotation has wrong value": {
+			annotations: map[string]string{RBSAnnotationKey: "WrongValue"},
+			expectError: nil,
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			vals := DefaultTestClusterValues()
+			gce, err := fakeGCECloud(DefaultTestClusterValues())
+			require.NoError(t, err)
+			nodeNames := []string{"test-node-1"}
+
+			nodes, err := createAndInsertNodes(gce, nodeNames, vals.ZoneName)
+			require.NoError(t, err)
+
+			svc := fakeLoadbalancerService("")
+			svc.Annotations = tc.annotations
+			_, err = gce.ensureExternalLoadBalancer(vals.ClusterName, vals.ClusterID, svc, nil, nodes)
+			if tc.expectError != nil {
+				assert.EqualError(t, err, (*tc.expectError).Error())
+			} else {
+				assert.NoError(t, err, "Should not return an error "+desc)
+			}
+		})
+	}
+}
+
+func TestEnsureExternalLoadBalancerRBSFinalizer(t *testing.T) {
+	t.Parallel()
+
+	for desc, tc := range map[string]struct {
+		finalizers  []string
+		expectError *error
+	}{
+		"When has ELBRbsFinalizer": {
+			finalizers:  []string{ELBRbsFinalizer},
+			expectError: &cloudprovider.ImplementedElsewhere,
+		},
+		"When has no finalizer": {
+			finalizers:  []string{},
+			expectError: nil,
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			vals := DefaultTestClusterValues()
+			gce, err := fakeGCECloud(DefaultTestClusterValues())
+			require.NoError(t, err)
+			nodeNames := []string{"test-node-1"}
+
+			nodes, err := createAndInsertNodes(gce, nodeNames, vals.ZoneName)
+			require.NoError(t, err)
+
+			svc := fakeLoadbalancerService("")
+			svc.Finalizers = tc.finalizers
+			_, err = gce.ensureExternalLoadBalancer(vals.ClusterName, vals.ClusterID, svc, nil, nodes)
+			if tc.expectError != nil {
+				assert.EqualError(t, err, (*tc.expectError).Error())
+			} else {
+				assert.NoError(t, err, "Should not return an error "+desc)
+			}
+		})
+	}
+}
+
+func TestEnsureExternalLoadBalancerExistingFwdRule(t *testing.T) {
+	t.Parallel()
+
+	for desc, tc := range map[string]struct {
+		existingForwardingRule *compute.ForwardingRule
+		expectError            *error
+	}{
+		"When has existingForwardingRule with backend service": {
+			existingForwardingRule: &compute.ForwardingRule{
+				BackendService: "exists",
+			},
+			expectError: &cloudprovider.ImplementedElsewhere,
+		},
+		"When has existingForwardingRule with empty backend service": {
+			existingForwardingRule: &compute.ForwardingRule{
+				BackendService: "",
+			},
+			expectError: nil,
+		},
+		"When has no existingForwardingRule": {
+			existingForwardingRule: nil,
+			expectError:            nil,
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			vals := DefaultTestClusterValues()
+			gce, err := fakeGCECloud(DefaultTestClusterValues())
+			require.NoError(t, err)
+			nodeNames := []string{"test-node-1"}
+
+			nodes, err := createAndInsertNodes(gce, nodeNames, vals.ZoneName)
+			require.NoError(t, err)
+
+			svc := fakeLoadbalancerService("")
+			_, err = gce.ensureExternalLoadBalancer(vals.ClusterName, vals.ClusterID, svc, tc.existingForwardingRule, nodes)
+			if tc.expectError != nil {
+				assert.EqualError(t, err, (*tc.expectError).Error())
+			} else {
+				assert.NoError(t, err, "Should not return an error "+desc)
+			}
+		})
+	}
+}
+
 func TestForwardingRuleNeedsUpdate(t *testing.T) {
 	t.Parallel()