Merge pull request #40176 from timothysc/affinity_cleanup

Automatic merge from submit-queue (batch tested with PRs 40428, 40176)

Cleaup Affinity post conversion from annotations to fields

**What this PR does / why we need it**:
Cleans up leftover work from the conversion of affinity from annotations to fields.

fixes #40016 
related #25319 

**Special notes for your reviewer**:
There are some TODO items left for @luxas or @errordeveloper b/c they were trying to use affinity in a way that is not possible.  

**Release note**:

```release-note
NONE
```

/cc @kubernetes/sig-scheduling-misc @rrati

cmd/kubeadm/app/master/addons.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net"
 	"path"
+	"runtime"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
@@ -81,6 +82,23 @@ func createKubeProxyPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec {
 				},
 			},
 		},
+		Affinity: &v1.Affinity{
+			NodeAffinity: &v1.NodeAffinity{
+				RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
+					NodeSelectorTerms: []v1.NodeSelectorTerm{
+						{
+							MatchExpressions: []v1.NodeSelectorRequirement{
+								{
+									Key:      "beta.kubernetes.io/arch",
+									Operator: v1.NodeSelectorOpIn,
+									Values:   []string{runtime.GOARCH},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 	}
 }
 
@@ -240,6 +258,23 @@ func createKubeDNSPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec {
 			},
 		},
 		DNSPolicy: v1.DNSDefault,
+		Affinity: &v1.Affinity{
+			NodeAffinity: &v1.NodeAffinity{
+				RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
+					NodeSelectorTerms: []v1.NodeSelectorTerm{
+						{
+							MatchExpressions: []v1.NodeSelectorRequirement{
+								{
+									Key:      "beta.kubernetes.io/arch",
+									Operator: v1.NodeSelectorOpIn,
+									Values:   []string{runtime.GOARCH},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 	}
 }
 
@@ -266,7 +301,6 @@ func createKubeDNSServiceSpec(cfg *kubeadmapi.MasterConfiguration) (*v1.ServiceS
 func CreateEssentialAddons(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset) error {
 	kubeProxyDaemonSet := NewDaemonSet(kubeProxy, createKubeProxyPodSpec(cfg))
 	SetMasterTaintTolerations(&kubeProxyDaemonSet.Spec.Template.ObjectMeta)
-	SetNodeAffinity(&kubeProxyDaemonSet.Spec.Template.ObjectMeta, NativeArchitectureNodeAffinity())
 
 	if _, err := client.Extensions().DaemonSets(metav1.NamespaceSystem).Create(kubeProxyDaemonSet); err != nil {
 		return fmt.Errorf("failed creating essential kube-proxy addon [%v]", err)
@@ -276,7 +310,6 @@ func CreateEssentialAddons(cfg *kubeadmapi.MasterConfiguration, client *clientse
 
 	kubeDNSDeployment := NewDeployment(KubeDNS, 1, createKubeDNSPodSpec(cfg))
 	SetMasterTaintTolerations(&kubeDNSDeployment.Spec.Template.ObjectMeta)
-	SetNodeAffinity(&kubeDNSDeployment.Spec.Template.ObjectMeta, NativeArchitectureNodeAffinity())
 	kubeDNSServiceAccount := &v1.ServiceAccount{}
 	kubeDNSServiceAccount.ObjectMeta.Name = KubeDNS
 	if _, err := client.ServiceAccounts(metav1.NamespaceSystem).Create(kubeDNSServiceAccount); err != nil {

cmd/kubeadm/app/master/apiclient.go

@@ -19,7 +19,6 @@ package master
 import (
 	"encoding/json"
 	"fmt"
-	"runtime"
 	"time"
 
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
@@ -229,37 +228,6 @@ func SetMasterTaintTolerations(meta *metav1.ObjectMeta) {
 	meta.Annotations[v1.TolerationsAnnotationKey] = string(tolerationsAnnotation)
 }
 
-// SetNodeAffinity is a basic helper to set meta.Annotations[v1.AffinityAnnotationKey] for one or more v1.NodeSelectorRequirement(s)
-func SetNodeAffinity(meta *metav1.ObjectMeta, expr ...v1.NodeSelectorRequirement) {
-	nodeAffinity := &v1.NodeAffinity{
-		RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
-			NodeSelectorTerms: []v1.NodeSelectorTerm{{MatchExpressions: expr}},
-		},
-	}
-	affinityAnnotation, _ := json.Marshal(v1.Affinity{NodeAffinity: nodeAffinity})
-	if meta.Annotations == nil {
-		meta.Annotations = map[string]string{}
-	}
-	meta.Annotations[v1.AffinityAnnotationKey] = string(affinityAnnotation)
-}
-
-// MasterNodeAffinity returns v1.NodeSelectorRequirement to be used with SetNodeAffinity to set affinity to master node
-func MasterNodeAffinity() v1.NodeSelectorRequirement {
-	return v1.NodeSelectorRequirement{
-		Key:      metav1.NodeLabelKubeadmAlphaRole,
-		Operator: v1.NodeSelectorOpIn,
-		Values:   []string{metav1.NodeLabelRoleMaster},
-	}
-}
-
-// NativeArchitectureNodeAffinity returns v1.NodeSelectorRequirement to be used with SetNodeAffinity to nodes with CPU architecture
-// the same as master node
-func NativeArchitectureNodeAffinity() v1.NodeSelectorRequirement {
-	return v1.NodeSelectorRequirement{
-		Key: "beta.kubernetes.io/arch", Operator: v1.NodeSelectorOpIn, Values: []string{runtime.GOARCH},
-	}
-}
-
 func createDummyDeployment(client *clientset.Clientset) {
 	fmt.Println("[apiclient] Creating a test deployment")
 	dummyDeployment := NewDeployment("dummy", 1, v1.PodSpec{

cmd/kubeadm/app/master/discovery.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"path"
+	"runtime"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -103,6 +104,23 @@ func newKubeDiscoveryPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec {
 				Secret: &v1.SecretVolumeSource{SecretName: kubeDiscoverySecretName},
 			}},
 		},
+		Affinity: &v1.Affinity{
+			NodeAffinity: &v1.NodeAffinity{
+				RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
+					NodeSelectorTerms: []v1.NodeSelectorTerm{
+						{
+							MatchExpressions: []v1.NodeSelectorRequirement{
+								{
+									Key:      "beta.kubernetes.io/arch",
+									Operator: v1.NodeSelectorOpIn,
+									Values:   []string{runtime.GOARCH},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 	}
 }
 
@@ -117,7 +135,6 @@ func newKubeDiscovery(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certific
 	}
 
 	SetMasterTaintTolerations(&kd.Deployment.Spec.Template.ObjectMeta)
-	SetNodeAffinity(&kd.Deployment.Spec.Template.ObjectMeta, MasterNodeAffinity(), NativeArchitectureNodeAffinity())
 
 	return kd
 }

pkg/api/helpers.go

@@ -429,10 +429,6 @@ func NodeSelectorRequirementsAsSelector(nsm []NodeSelectorRequirement) (labels.S
 }
 
 const (
-	// AffinityAnnotationKey represents the key of affinity data (json serialized)
-	// in the Annotations of a Pod.
-	AffinityAnnotationKey string = "scheduler.alpha.kubernetes.io/affinity"
-
 	// TolerationsAnnotationKey represents the key of tolerations data (json serialized)
 	// in the Annotations of a Pod.
 	TolerationsAnnotationKey string = "scheduler.alpha.kubernetes.io/tolerations"

pkg/api/v1/helpers.go

@@ -233,10 +233,6 @@ func NodeSelectorRequirementsAsSelector(nsm []NodeSelectorRequirement) (labels.S
 }
 
 const (
-	// AffinityAnnotationKey represents the key of affinity data (json serialized)
-	// in the Annotations of a Pod.
-	AffinityAnnotationKey string = "scheduler.alpha.kubernetes.io/affinity"
-
 	// TolerationsAnnotationKey represents the key of tolerations data (json serialized)
 	// in the Annotations of a Pod.
 	TolerationsAnnotationKey string = "scheduler.alpha.kubernetes.io/tolerations"
@@ -276,20 +272,6 @@ const (
 	UnsafeSysctlsPodAnnotationKey string = "security.alpha.kubernetes.io/unsafe-sysctls"
 )
 
-// GetAffinityFromPod gets the json serialized affinity data from Pod.Annotations
-// and converts it to the Affinity type in api.
-func GetAffinityFromPodAnnotations(annotations map[string]string) (*Affinity, error) {
-	if len(annotations) > 0 && annotations[AffinityAnnotationKey] != "" {
-		var affinity Affinity
-		err := json.Unmarshal([]byte(annotations[AffinityAnnotationKey]), &affinity)
-		if err != nil {
-			return nil, err
-		}
-		return &affinity, nil
-	}
-	return nil, nil
-}
-
 // GetTolerationsFromPodAnnotations gets the json serialized tolerations data from Pod.Annotations
 // and converts it to the []Toleration type in api.
 func GetTolerationsFromPodAnnotations(annotations map[string]string) ([]Toleration, error) {

pkg/api/v1/helpers_test.go

@@ -179,62 +179,6 @@ func TestNodeSelectorRequirementsAsSelector(t *testing.T) {
 	}
 }
 
-func TestGetAffinityFromPod(t *testing.T) {
-	testCases := []struct {
-		pod       *Pod
-		expectErr bool
-	}{
-		{
-			pod:       &Pod{},
-			expectErr: false,
-		},
-		{
-			pod: &Pod{
-				ObjectMeta: metav1.ObjectMeta{
-					Annotations: map[string]string{
-						AffinityAnnotationKey: `
-						{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
-							"nodeSelectorTerms": [{
-								"matchExpressions": [{
-									"key": "foo",
-									"operator": "In",
-									"values": ["value1", "value2"]
-								}]
-							}]
-						}}}`,
-					},
-				},
-			},
-			expectErr: false,
-		},
-		{
-			pod: &Pod{
-				ObjectMeta: metav1.ObjectMeta{
-					Annotations: map[string]string{
-						AffinityAnnotationKey: `
-						{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
-							"nodeSelectorTerms": [{
-								"matchExpressions": [{
-									"key": "foo",
-						`,
-					},
-				},
-			},
-			expectErr: true,
-		},
-	}
-
-	for i, tc := range testCases {
-		_, err := GetAffinityFromPodAnnotations(tc.pod.Annotations)
-		if err == nil && tc.expectErr {
-			t.Errorf("[%v]expected error but got none.", i)
-		}
-		if err != nil && !tc.expectErr {
-			t.Errorf("[%v]did not expect error but got: %v", i, err)
-		}
-	}
-}
-
 func TestTaintToString(t *testing.T) {
 	testCases := []struct {
 		taint          *Taint

staging/src/k8s.io/client-go/pkg/api/helpers.go

@@ -429,10 +429,6 @@ func NodeSelectorRequirementsAsSelector(nsm []NodeSelectorRequirement) (labels.S
 }
 
 const (
-	// AffinityAnnotationKey represents the key of affinity data (json serialized)
-	// in the Annotations of a Pod.
-	AffinityAnnotationKey string = "scheduler.alpha.kubernetes.io/affinity"
-
 	// TolerationsAnnotationKey represents the key of tolerations data (json serialized)
 	// in the Annotations of a Pod.
 	TolerationsAnnotationKey string = "scheduler.alpha.kubernetes.io/tolerations"

staging/src/k8s.io/client-go/pkg/api/v1/helpers.go

@@ -233,10 +233,6 @@ func NodeSelectorRequirementsAsSelector(nsm []NodeSelectorRequirement) (labels.S
 }
 
 const (
-	// AffinityAnnotationKey represents the key of affinity data (json serialized)
-	// in the Annotations of a Pod.
-	AffinityAnnotationKey string = "scheduler.alpha.kubernetes.io/affinity"
-
 	// TolerationsAnnotationKey represents the key of tolerations data (json serialized)
 	// in the Annotations of a Pod.
 	TolerationsAnnotationKey string = "scheduler.alpha.kubernetes.io/tolerations"
@@ -276,20 +272,6 @@ const (
 	UnsafeSysctlsPodAnnotationKey string = "security.alpha.kubernetes.io/unsafe-sysctls"
 )
 
-// GetAffinityFromPod gets the json serialized affinity data from Pod.Annotations
-// and converts it to the Affinity type in api.
-func GetAffinityFromPodAnnotations(annotations map[string]string) (*Affinity, error) {
-	if len(annotations) > 0 && annotations[AffinityAnnotationKey] != "" {
-		var affinity Affinity
-		err := json.Unmarshal([]byte(annotations[AffinityAnnotationKey]), &affinity)
-		if err != nil {
-			return nil, err
-		}
-		return &affinity, nil
-	}
-	return nil, nil
-}
-
 // GetTolerationsFromPodAnnotations gets the json serialized tolerations data from Pod.Annotations
 // and converts it to the []Toleration type in api.
 func GetTolerationsFromPodAnnotations(annotations map[string]string) ([]Toleration, error) {

test/integration/scheduler_perf/BUILD

@@ -41,7 +41,6 @@ go_test(
         "//test/integration/framework:go_default_library",
         "//test/utils:go_default_library",
         "//vendor:github.com/golang/glog",
-        "//vendor:github.com/renstrom/dedent",
         "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
     ],
 )

test/integration/scheduler_perf/scheduler_test.go

@@ -28,7 +28,6 @@ import (
 	testutils "k8s.io/kubernetes/test/utils"
 
 	"github.com/golang/glog"
-	"github.com/renstrom/dedent"
 	"k8s.io/kubernetes/plugin/pkg/scheduler"
 )
 
@@ -79,31 +78,34 @@ func TestSchedule100Node3KNodeAffinityPods(t *testing.T) {
 		"scheduler-perf-",
 	)
 
-	affinityTemplate := dedent.Dedent(`
-		{
-			"nodeAffinity": {
-				"requiredDuringSchedulingIgnoredDuringExecution": {
-					"nodeSelectorTerms": [{
-						"matchExpressions": [{
-							"key": "` + nodeAffinityKey + `",
-							"operator": "In",
-							"values": ["%v"]
-						}]
-					}]
-				}
-			}
-		}`)
-
 	podCreatorConfig := testutils.NewTestPodCreatorConfig()
 	for i := 0; i < numGroups; i++ {
-		podCreatorConfig.AddStrategy("sched-perf-node-affinity", config.numPods/numGroups,
+		pod := &v1.Pod{
-			testutils.NewCustomCreatePodStrategy(&v1.Pod{
 			ObjectMeta: metav1.ObjectMeta{
 				GenerateName: "sched-perf-node-affinity-pod-",
-					Annotations:  map[string]string{v1.AffinityAnnotationKey: fmt.Sprintf(affinityTemplate, i)},
 			},
 			Spec: testutils.MakePodSpec(),
-			}),
+		}
+		pod.Spec.Affinity = &v1.Affinity{
+			NodeAffinity: &v1.NodeAffinity{
+				RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
+					NodeSelectorTerms: []v1.NodeSelectorTerm{
+						{
+							MatchExpressions: []v1.NodeSelectorRequirement{
+								{
+									Key:      nodeAffinityKey,
+									Operator: v1.NodeSelectorOpIn,
+									Values:   []string{string(i)},
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+
+		podCreatorConfig.AddStrategy("sched-perf-node-affinity", config.numPods/numGroups,
+			testutils.NewCustomCreatePodStrategy(pod),
 		)
 	}
 	config.podCreator = testutils.NewTestPodCreator(config.schedulerSupportFunctions.GetClient(), podCreatorConfig)