Mixed protocol support for Services with type=LoadBalancer (#94028)

* Mixed protocol support for Services with type=LoadBalancer

KEP: https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/20200103-mixed-protocol-lb.md
Add new feature gate to control the support of mixed protocols in Services with type=LoadBalancer
Add new fields to the ServiceStatus
  Add Ports to the LoadBalancerIngress, so cloud provider implementations can report the status of the requested load balanc
er ports
  Add ServiceCondition to the ServiceStatus so Service controllers can indicate the conditions of the Service

* regenerate conflicting stuff

pkg/apis/core/types.go

@@ -3475,12 +3475,23 @@ const (
 	ServiceExternalTrafficPolicyTypeCluster ServiceExternalTrafficPolicyType = "Cluster"
 )
 
+// These are the valid conditions of a service.
+const (
+	// LoadBalancerPortsError represents the condition of the requested ports
+	// on the cloud load balancer instance.
+	LoadBalancerPortsError = "LoadBalancerPortsError"
+)
+
 // ServiceStatus represents the current status of a service
 type ServiceStatus struct {
 	// LoadBalancer contains the current status of the load-balancer,
 	// if one is present.
 	// +optional
 	LoadBalancer LoadBalancerStatus
+
+	// Current service condition
+	// +optional
+	Conditions []metav1.Condition
 }
 
 // LoadBalancerStatus represents the status of a load-balancer
@@ -3503,6 +3514,11 @@ type LoadBalancerIngress struct {
 	// (typically AWS load-balancers)
 	// +optional
 	Hostname string
+
+	// Ports is a list of records of service ports
+	// If used, every port defined in the service should have an entry in it
+	// +optional
+	Ports []PortStatus
 }
 
 const (
@@ -5395,3 +5411,32 @@ type TopologySpreadConstraint struct {
 	// +optional
 	LabelSelector *metav1.LabelSelector
 }
+
+// These are the built-in errors for PortStatus.
+const (
+	// MixedProtocolNotSupported error in PortStatus means that the cloud provider
+	// can't ensure the port on the load balancer because mixed values of protocols
+	// on the same LoadBalancer type of Service are not supported by the cloud provider.
+	MixedProtocolNotSupported = "MixedProtocolNotSupported"
+)
+
+// PortStatus represents the error condition of a service port
+type PortStatus struct {
+	// Port is the port number of the service port of which status is recorded here
+	Port int32
+	// Protocol is the protocol of the service port of which status is recorded here
+	Protocol Protocol
+	// Error is to record the problem with the service port
+	// The format of the error shall comply with the following rules:
+	// - built-in error values shall be specified in this file and those shall use
+	//   CamelCase names
+	// - cloud provider specific error values must have names that comply with the
+	//   format foo.example.com/CamelCase.
+	// ---
+	// The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
+	// +optional
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:Pattern=`^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$`
+	// +kubebuilder:validation:MaxLength=316
+	Error *string
+}

pkg/apis/core/v1/zz_generated.conversion.go

@@ -1391,6 +1391,16 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*v1.PortStatus)(nil), (*core.PortStatus)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1_PortStatus_To_core_PortStatus(a.(*v1.PortStatus), b.(*core.PortStatus), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*core.PortStatus)(nil), (*v1.PortStatus)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_core_PortStatus_To_v1_PortStatus(a.(*core.PortStatus), b.(*v1.PortStatus), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddGeneratedConversionFunc((*v1.PortworxVolumeSource)(nil), (*core.PortworxVolumeSource)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_v1_PortworxVolumeSource_To_core_PortworxVolumeSource(a.(*v1.PortworxVolumeSource), b.(*core.PortworxVolumeSource), scope)
 	}); err != nil {
@@ -4291,6 +4301,7 @@ func Convert_core_List_To_v1_List(in *core.List, out *v1.List, s conversion.Scop
 func autoConvert_v1_LoadBalancerIngress_To_core_LoadBalancerIngress(in *v1.LoadBalancerIngress, out *core.LoadBalancerIngress, s conversion.Scope) error {
 	out.IP = in.IP
 	out.Hostname = in.Hostname
+	out.Ports = *(*[]core.PortStatus)(unsafe.Pointer(&in.Ports))
 	return nil
 }
 
@@ -4302,6 +4313,7 @@ func Convert_v1_LoadBalancerIngress_To_core_LoadBalancerIngress(in *v1.LoadBalan
 func autoConvert_core_LoadBalancerIngress_To_v1_LoadBalancerIngress(in *core.LoadBalancerIngress, out *v1.LoadBalancerIngress, s conversion.Scope) error {
 	out.IP = in.IP
 	out.Hostname = in.Hostname
+	out.Ports = *(*[]v1.PortStatus)(unsafe.Pointer(&in.Ports))
 	return nil
 }
 
@@ -6322,6 +6334,30 @@ func autoConvert_core_PodTemplateSpec_To_v1_PodTemplateSpec(in *core.PodTemplate
 	return nil
 }
 
+func autoConvert_v1_PortStatus_To_core_PortStatus(in *v1.PortStatus, out *core.PortStatus, s conversion.Scope) error {
+	out.Port = in.Port
+	out.Protocol = core.Protocol(in.Protocol)
+	out.Error = (*string)(unsafe.Pointer(in.Error))
+	return nil
+}
+
+// Convert_v1_PortStatus_To_core_PortStatus is an autogenerated conversion function.
+func Convert_v1_PortStatus_To_core_PortStatus(in *v1.PortStatus, out *core.PortStatus, s conversion.Scope) error {
+	return autoConvert_v1_PortStatus_To_core_PortStatus(in, out, s)
+}
+
+func autoConvert_core_PortStatus_To_v1_PortStatus(in *core.PortStatus, out *v1.PortStatus, s conversion.Scope) error {
+	out.Port = in.Port
+	out.Protocol = v1.Protocol(in.Protocol)
+	out.Error = (*string)(unsafe.Pointer(in.Error))
+	return nil
+}
+
+// Convert_core_PortStatus_To_v1_PortStatus is an autogenerated conversion function.
+func Convert_core_PortStatus_To_v1_PortStatus(in *core.PortStatus, out *v1.PortStatus, s conversion.Scope) error {
+	return autoConvert_core_PortStatus_To_v1_PortStatus(in, out, s)
+}
+
 func autoConvert_v1_PortworxVolumeSource_To_core_PortworxVolumeSource(in *v1.PortworxVolumeSource, out *core.PortworxVolumeSource, s conversion.Scope) error {
 	out.VolumeID = in.VolumeID
 	out.FSType = in.FSType
@@ -7636,6 +7672,7 @@ func autoConvert_v1_ServiceStatus_To_core_ServiceStatus(in *v1.ServiceStatus, ou
 	if err := Convert_v1_LoadBalancerStatus_To_core_LoadBalancerStatus(&in.LoadBalancer, &out.LoadBalancer, s); err != nil {
 		return err
 	}
+	out.Conditions = *(*[]metav1.Condition)(unsafe.Pointer(&in.Conditions))
 	return nil
 }
 
@@ -7648,6 +7685,7 @@ func autoConvert_core_ServiceStatus_To_v1_ServiceStatus(in *core.ServiceStatus,
 	if err := Convert_core_LoadBalancerStatus_To_v1_LoadBalancerStatus(&in.LoadBalancer, &out.LoadBalancer, s); err != nil {
 		return err
 	}
+	out.Conditions = *(*[]metav1.Condition)(unsafe.Pointer(&in.Conditions))
 	return nil
 }
 

pkg/apis/core/validation/BUILD

@@ -7,6 +7,7 @@ load(
 go_library(
     name = "go_default_library",
     srcs = [
+        "conditional_validation.go",
         "doc.go",
         "events.go",
         "validation.go",
@@ -48,6 +49,7 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
+        "conditional_validation_test.go",
         "events_test.go",
         "validation_test.go",
     ],

pkg/apis/core/validation/conditional_validation.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validation
+
+import (
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	api "k8s.io/kubernetes/pkg/apis/core"
+	"k8s.io/kubernetes/pkg/features"
+)
+
+// ValidateConditionalService validates conditionally valid fields.
+func ValidateConditionalService(service, oldService *api.Service) field.ErrorList {
+	var errs field.ErrorList
+
+	errs = append(errs, validateMixedProtocolLBService(service, oldService)...)
+
+	return errs
+}
+
+// validateMixedProtocolLBService checks if the old Service has type=LoadBalancer and whether the Service has different Protocols
+// on its ports. If the MixedProtocolLBService feature flag is disabled the usage of different Protocols in the new Service is
+// valid only if the old Service has different Protocols, too.
+func validateMixedProtocolLBService(service, oldService *api.Service) (errs field.ErrorList) {
+	if service.Spec.Type != api.ServiceTypeLoadBalancer {
+		return
+	}
+	if utilfeature.DefaultFeatureGate.Enabled(features.MixedProtocolLBService) {
+		return
+	}
+
+	if serviceHasMixedProtocols(service) && !serviceHasMixedProtocols(oldService) {
+		errs = append(errs, field.Invalid(field.NewPath("spec", "ports"), service.Spec.Ports, "may not contain more than 1 protocol when type is 'LoadBalancer'"))
+	}
+	return
+}
+
+func serviceHasMixedProtocols(service *api.Service) bool {
+	if service == nil {
+		return false
+	}
+	protos := map[string]bool{}
+	for _, port := range service.Spec.Ports {
+		protos[string(port.Protocol)] = true
+	}
+	return len(protos) > 1
+}

pkg/apis/core/validation/conditional_validation_test.go

@@ -0,0 +1,277 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validation
+
+import (
+	"strings"
+	"testing"
+
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
+	api "k8s.io/kubernetes/pkg/apis/core"
+	"k8s.io/kubernetes/pkg/features"
+)
+
+func TestValidateMixedProtocolLBService(t *testing.T) {
+	newLBServiceDifferentProtocols := &api.Service{
+		Spec: api.ServiceSpec{
+			Ports: []api.ServicePort{
+				{
+					Protocol: api.ProtocolTCP,
+				},
+				{
+					Protocol: api.ProtocolUDP,
+				},
+			},
+			Type: api.ServiceTypeLoadBalancer,
+		},
+	}
+	newLBServiceSameProtocols := &api.Service{
+		Spec: api.ServiceSpec{
+			Ports: []api.ServicePort{
+				{
+					Protocol: api.ProtocolTCP,
+				},
+				{
+					Protocol: api.ProtocolTCP,
+				},
+			},
+			Type: api.ServiceTypeLoadBalancer,
+		},
+	}
+	newNonLBServiceDifferentProtocols := &api.Service{
+		Spec: api.ServiceSpec{
+			Ports: []api.ServicePort{
+				{
+					Protocol: api.ProtocolTCP,
+				},
+				{
+					Protocol: api.ProtocolUDP,
+				},
+			},
+			Type: api.ServiceTypeNodePort,
+		},
+	}
+	newNonLBServiceSameProtocols := &api.Service{
+		Spec: api.ServiceSpec{
+			Ports: []api.ServicePort{
+				{
+					Protocol: api.ProtocolUDP,
+				},
+				{
+					Protocol: api.ProtocolUDP,
+				},
+			},
+			Type: api.ServiceTypeNodePort,
+		},
+	}
+	oldLBServiceDifferentProtocols := &api.Service{
+		Spec: api.ServiceSpec{
+			Ports: []api.ServicePort{
+				{
+					Protocol: api.ProtocolTCP,
+				},
+				{
+					Protocol: api.ProtocolUDP,
+				},
+			},
+			Type: api.ServiceTypeLoadBalancer,
+		},
+	}
+	oldLBServiceSameProtocols := &api.Service{
+		Spec: api.ServiceSpec{
+			Ports: []api.ServicePort{
+				{
+					Protocol: api.ProtocolTCP,
+				},
+				{
+					Protocol: api.ProtocolTCP,
+				},
+			},
+			Type: api.ServiceTypeLoadBalancer,
+		},
+	}
+	oldNonLBServiceDifferentProtocols := &api.Service{
+		Spec: api.ServiceSpec{
+			Ports: []api.ServicePort{
+				{
+					Protocol: api.ProtocolTCP,
+				},
+				{
+					Protocol: api.ProtocolUDP,
+				},
+			},
+			Type: api.ServiceTypeNodePort,
+		},
+	}
+	oldNonLBServiceSameProtocols := &api.Service{
+		Spec: api.ServiceSpec{
+			Ports: []api.ServicePort{
+				{
+					Protocol: api.ProtocolUDP,
+				},
+				{
+					Protocol: api.ProtocolUDP,
+				},
+			},
+			Type: api.ServiceTypeNodePort,
+		},
+	}
+	cases := map[string]struct {
+		oldService    *api.Service
+		newService    *api.Service
+		fgEnabled     bool
+		expectedError []string
+	}{
+		"Old service is nil, new service has different protocols, feature gate false": {
+			oldService:    nil,
+			newService:    newLBServiceDifferentProtocols,
+			fgEnabled:     false,
+			expectedError: []string{`spec.ports: Invalid value: []core.ServicePort{core.ServicePort{Name:"", Protocol:"TCP", AppProtocol:(*string)(nil), Port:0, TargetPort:intstr.IntOrString{Type:0, IntVal:0, StrVal:""}, NodePort:0}, core.ServicePort{Name:"", Protocol:"UDP", AppProtocol:(*string)(nil), Port:0, TargetPort:intstr.IntOrString{Type:0, IntVal:0, StrVal:""}, NodePort:0}}: may not contain more than 1 protocol when type is 'LoadBalancer'`},
+		},
+		"Old service is nil, new service has different protocols, feature gate true": {
+			oldService: nil,
+			newService: newLBServiceDifferentProtocols,
+			fgEnabled:  true,
+		},
+		"Old service is nil, new service does not have different protocols, feature gate false": {
+			oldService: nil,
+			newService: newLBServiceSameProtocols,
+			fgEnabled:  false,
+		},
+		"Old service is nil, new service does not have different protocols, feature gate true": {
+			oldService: nil,
+			newService: newLBServiceSameProtocols,
+			fgEnabled:  true,
+		},
+		"Old service is nil, new non-LB service has different protocols, feature gate false": {
+			oldService: nil,
+			newService: newNonLBServiceDifferentProtocols,
+			fgEnabled:  false,
+		},
+		"Old service is nil, new non-LB service has different protocols, feature gate true": {
+			oldService: nil,
+			newService: newNonLBServiceDifferentProtocols,
+			fgEnabled:  true,
+		},
+		"Old service is nil, new non-LB service does not have different protocols, feature gate false": {
+			oldService: nil,
+			newService: newNonLBServiceSameProtocols,
+			fgEnabled:  false,
+		},
+		"Old service is nil, new non-LB service does not have different protocols, feature gate true": {
+			oldService: nil,
+			newService: newNonLBServiceSameProtocols,
+			fgEnabled:  true,
+		},
+		"Non-LB services, both services have different protocols, feature gate false": {
+			oldService: oldNonLBServiceDifferentProtocols,
+			newService: newNonLBServiceDifferentProtocols,
+			fgEnabled:  false,
+		},
+		"Non-LB services, old service has same protocols, new service has different protocols, feature gate false": {
+			oldService: oldNonLBServiceSameProtocols,
+			newService: newNonLBServiceDifferentProtocols,
+			fgEnabled:  false,
+		},
+		"Non-LB services, old service has different protocols, new service has identical protocols, feature gate false": {
+			oldService: oldNonLBServiceDifferentProtocols,
+			newService: newNonLBServiceSameProtocols,
+			fgEnabled:  false,
+		},
+		"Non-LB services, both services have same protocols, feature gate false": {
+			oldService: oldNonLBServiceSameProtocols,
+			newService: newNonLBServiceSameProtocols,
+			fgEnabled:  false,
+		},
+		"Non-LB services, both services have different protocols, feature gate true": {
+			oldService: oldNonLBServiceDifferentProtocols,
+			newService: newNonLBServiceDifferentProtocols,
+			fgEnabled:  true,
+		},
+		"Non-LB services, old service has same protocols, new service has different protocols, feature gate true": {
+			oldService: oldNonLBServiceSameProtocols,
+			newService: newNonLBServiceDifferentProtocols,
+			fgEnabled:  true,
+		},
+		"Non-LB services, old service has different protocols, new service has identical protocols, feature gate true": {
+			oldService: oldNonLBServiceDifferentProtocols,
+			newService: newNonLBServiceSameProtocols,
+			fgEnabled:  true,
+		},
+		"Non-LB services, both services have same protocols, feature gate true": {
+			oldService: oldNonLBServiceSameProtocols,
+			newService: newNonLBServiceSameProtocols,
+			fgEnabled:  true,
+		},
+		"LB service, neither service has different protocols, feature gate false": {
+			oldService: oldLBServiceSameProtocols,
+			newService: newLBServiceSameProtocols,
+			fgEnabled:  false,
+		},
+		"LB service, old service does not have different protocols, new service has different protocols, feature gate false": {
+			oldService:    oldLBServiceSameProtocols,
+			newService:    newLBServiceDifferentProtocols,
+			fgEnabled:     false,
+			expectedError: []string{`spec.ports: Invalid value: []core.ServicePort{core.ServicePort{Name:"", Protocol:"TCP", AppProtocol:(*string)(nil), Port:0, TargetPort:intstr.IntOrString{Type:0, IntVal:0, StrVal:""}, NodePort:0}, core.ServicePort{Name:"", Protocol:"UDP", AppProtocol:(*string)(nil), Port:0, TargetPort:intstr.IntOrString{Type:0, IntVal:0, StrVal:""}, NodePort:0}}: may not contain more than 1 protocol when type is 'LoadBalancer'`},
+		},
+		"LB service, old service has different protocols, new service does not have different protocols, feature gate false": {
+			oldService: oldLBServiceDifferentProtocols,
+			newService: newLBServiceSameProtocols,
+			fgEnabled:  false,
+		},
+		"LB service, both services have different protocols, feature gate false": {
+			oldService: oldLBServiceDifferentProtocols,
+			newService: newLBServiceDifferentProtocols,
+			fgEnabled:  false,
+		},
+		"LB service, neither service has different protocols, feature gate true": {
+			oldService: oldLBServiceSameProtocols,
+			newService: newLBServiceSameProtocols,
+			fgEnabled:  true,
+		},
+		"LB service, old service has different protocols, new service does not have different protocols, feature gate true": {
+			oldService: oldLBServiceDifferentProtocols,
+			newService: newLBServiceSameProtocols,
+			fgEnabled:  true,
+		},
+		"LB service, old service does not have different protocols, new service has different protocols, feature gate true": {
+			oldService: oldLBServiceSameProtocols,
+			newService: newLBServiceDifferentProtocols,
+			fgEnabled:  true,
+		},
+		"LB service, both services have different protocols, feature gate true": {
+			oldService: oldLBServiceDifferentProtocols,
+			newService: newLBServiceDifferentProtocols,
+			fgEnabled:  true,
+		},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MixedProtocolLBService, tc.fgEnabled)()
+			errs := validateMixedProtocolLBService(tc.newService, tc.oldService)
+			if len(errs) != len(tc.expectedError) {
+				t.Fatalf("unexpected number of errors: %v", errs)
+			}
+			for i := range errs {
+				if !strings.Contains(errs[i].Error(), tc.expectedError[i]) {
+					t.Errorf("unexpected error %d: %v", i, errs[i])
+				}
+			}
+		})
+	}
+}

pkg/apis/core/validation/validation.go

@@ -4249,22 +4249,6 @@ func ValidateService(service *core.Service) field.ErrorList {
 		allErrs = append(allErrs, field.NotSupported(specPath.Child("type"), service.Spec.Type, supportedServiceType.List()))
 	}
 
-	if service.Spec.Type == core.ServiceTypeLoadBalancer {
-		portsPath := specPath.Child("ports")
-		includeProtocols := sets.NewString()
-		for i := range service.Spec.Ports {
-			portPath := portsPath.Index(i)
-			if !supportedPortProtocols.Has(string(service.Spec.Ports[i].Protocol)) {
-				allErrs = append(allErrs, field.Invalid(portPath.Child("protocol"), service.Spec.Ports[i].Protocol, "cannot create an external load balancer with non-TCP/UDP/SCTP ports"))
-			} else {
-				includeProtocols.Insert(string(service.Spec.Ports[i].Protocol))
-			}
-		}
-		if includeProtocols.Len() > 1 {
-			allErrs = append(allErrs, field.Invalid(portsPath, service.Spec.Ports, "cannot create an external load balancer with mix protocols"))
-		}
-	}
-
 	if service.Spec.Type == core.ServiceTypeClusterIP {
 		portsPath := specPath.Child("ports")
 		for i := range service.Spec.Ports {

pkg/apis/core/validation/validation_test.go

@@ -10288,12 +10288,12 @@ func TestValidateServiceCreate(t *testing.T) {
 			numErrs: 0,
 		},
 		{
-			name: "invalid load balancer with mix protocol",
+			name: "load balancer with mix protocol",
 			tweakSvc: func(s *core.Service) {
 				s.Spec.Type = core.ServiceTypeLoadBalancer
 				s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 12345, Protocol: "UDP", TargetPort: intstr.FromInt(12345)})
 			},
-			numErrs: 1,
+			numErrs: 0,
 		},
 		{
 			name: "valid 1",

pkg/apis/core/zz_generated.deepcopy.go

@@ -2146,6 +2146,13 @@ func (in *List) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *LoadBalancerIngress) DeepCopyInto(out *LoadBalancerIngress) {
 	*out = *in
+	if in.Ports != nil {
+		in, out := &in.Ports, &out.Ports
+		*out = make([]PortStatus, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	return
 }
 
@@ -2165,7 +2172,9 @@ func (in *LoadBalancerStatus) DeepCopyInto(out *LoadBalancerStatus) {
 	if in.Ingress != nil {
 		in, out := &in.Ingress, &out.Ingress
 		*out = make([]LoadBalancerIngress, len(*in))
-		copy(*out, *in)
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
 	}
 	return
 }
@@ -4081,6 +4090,27 @@ func (in *PodTemplateSpec) DeepCopy() *PodTemplateSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PortStatus) DeepCopyInto(out *PortStatus) {
+	*out = *in
+	if in.Error != nil {
+		in, out := &in.Error, &out.Error
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortStatus.
+func (in *PortStatus) DeepCopy() *PortStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(PortStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *PortworxVolumeSource) DeepCopyInto(out *PortworxVolumeSource) {
 	*out = *in
@@ -5312,6 +5342,13 @@ func (in *ServiceSpec) DeepCopy() *ServiceSpec {
 func (in *ServiceStatus) DeepCopyInto(out *ServiceStatus) {
 	*out = *in
 	in.LoadBalancer.DeepCopyInto(&out.LoadBalancer)
+	if in.Conditions != nil {
+		in, out := &in.Conditions, &out.Conditions
+		*out = make([]v1.Condition, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	return
 }