Add no admit on node side

Update generated code

Refactored predicates & restore helper

plugin/pkg/scheduler/algorithm/predicates/error.go

@@ -29,7 +29,6 @@ var (
 	ErrVolumeZoneConflict        = newPredicateFailureError("NoVolumeZoneConflict")
 	ErrNodeSelectorNotMatch      = newPredicateFailureError("MatchNodeSelector")
 	ErrPodAffinityNotMatch       = newPredicateFailureError("MatchInterPodAffinity")
-	ErrTaintsTolerationsNotMatch = newPredicateFailureError("PodToleratesNodeTaints")
 	ErrPodNotMatchHostName       = newPredicateFailureError("HostName")
 	ErrPodNotFitsHostPorts       = newPredicateFailureError("PodFitsHostPorts")
 	ErrNodeLabelPresenceViolated = newPredicateFailureError("CheckNodeLabelPresence")
@@ -42,6 +41,25 @@ var (
 	ErrFakePredicate = newPredicateFailureError("FakePredicateError")
 )
 
+// ErrTaintsTolerationsNotMatch is an error type that indicates with if it should be aware by kubelet.
+type ErrTaintsTolerationsNotMatch struct {
+	SomeUntoleratedTaintIsNoAdmit bool
+}
+
+func newErrTaintsTolerationsNotMatch(someUntoleratedTaintIsNoAdmit bool) *ErrTaintsTolerationsNotMatch {
+	return &ErrTaintsTolerationsNotMatch{
+		SomeUntoleratedTaintIsNoAdmit: someUntoleratedTaintIsNoAdmit,
+	}
+}
+
+func (e *ErrTaintsTolerationsNotMatch) Error() string {
+	return fmt.Sprintf("Taint Toleration unmatched with SomeUntoleratedTaintIsNoAdmit is: %v", e.SomeUntoleratedTaintIsNoAdmit)
+}
+
+func (e *ErrTaintsTolerationsNotMatch) GetReason() string {
+	return fmt.Sprintf("ErrTaintsTolerationsNotMatch, and SomeUntoleratedTaintIsNoAdmit is: %v", e.SomeUntoleratedTaintIsNoAdmit)
+}
+
 // InsufficientResourceError is an error type that indicates what kind of resource limit is
 // hit and caused the unfitting failure.
 type InsufficientResourceError struct {

plugin/pkg/scheduler/algorithm/predicates/predicates.go

@@ -1083,23 +1083,37 @@ func PodToleratesNodeTaints(pod *api.Pod, meta interface{}, nodeInfo *schedulerc
 		return false, nil, err
 	}
 
-	if tolerationsToleratesTaints(tolerations, taints) {
+	if tolerated, someUntoleratedTaintIsNoAdmit := tolerationsToleratesTaints(tolerations, taints); tolerated {
 		return true, nil, nil
+	} else {
+		return false, []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(someUntoleratedTaintIsNoAdmit)}, nil
 	}
-	return false, []algorithm.PredicateFailureReason{ErrTaintsTolerationsNotMatch}, nil
 }
 
-func tolerationsToleratesTaints(tolerations []api.Toleration, taints []api.Taint) bool {
+// tolerationsToleratesTaints checks if given tolerations can live with given taints.
+// It returns:
+// 1. whether tolerated or not;
+// 2. whether kubelet should be aware of (1).
+func tolerationsToleratesTaints(tolerations []api.Toleration, taints []api.Taint) (bool, bool) {
 	// If the taint list is nil/empty, it is tolerated by all tolerations by default.
 	if len(taints) == 0 {
-		return true
+		return true, false
 	}
 
 	// The taint list isn't nil/empty, a nil/empty toleration list can't tolerate them.
 	if len(tolerations) == 0 {
-		return false
+		// if there's taint has TaintEffectNoScheduleNoAdmit, kubelet should also be aware of this.
+		for _, taint := range taints {
+			if taint.Effect == api.TaintEffectNoScheduleNoAdmit {
+				return false, true
+			}
+		}
+		return false, false
 	}
 
+	someUntoleratedTaintIsNoAdmit := false
+	fits := true
+
 	for i := range taints {
 		taint := &taints[i]
 		// skip taints that have effect PreferNoSchedule, since it is for priorities
@@ -1107,12 +1121,16 @@ func tolerationsToleratesTaints(tolerations []api.Toleration, taints []api.Taint
 			continue
 		}
 
-		if !api.TaintToleratedByTolerations(taint, tolerations) {
-			return false
+		if tolerated := api.TaintToleratedByTolerations(taint, tolerations); !tolerated {
+			fits = false
+			if taint.Effect == api.TaintEffectNoScheduleNoAdmit {
+				someUntoleratedTaintIsNoAdmit = true
+				return fits, someUntoleratedTaintIsNoAdmit
+			}
 		}
 	}
 
-	return true
+	return fits, someUntoleratedTaintIsNoAdmit
 }
 
 // Determine if a pod is scheduled with best-effort QoS

plugin/pkg/scheduler/algorithm/predicates/predicates_test.go

@@ -2563,10 +2563,11 @@ func TestInterPodAffinityWithMultipleNodes(t *testing.T) {
 
 func TestPodToleratesTaints(t *testing.T) {
 	podTolerateTaintsTests := []struct {
-		pod  *api.Pod
-		node api.Node
-		fits bool
-		test string
+		pod                    *api.Pod
+		node                   api.Node
+		fits                   bool
+		expectedFailureReasons []algorithm.PredicateFailureReason
+		test                   string
 	}{
 		{
 			pod: &api.Pod{
@@ -2587,6 +2588,7 @@ func TestPodToleratesTaints(t *testing.T) {
 				},
 			},
 			fits: false,
+			expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(false)},
 			test: "a pod having no tolerations can't be scheduled onto a node with nonempty taints",
 		},
 		{
@@ -2652,6 +2654,7 @@ func TestPodToleratesTaints(t *testing.T) {
 				},
 			},
 			fits: false,
+			expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(false)},
 			test: "a pod which can't be scheduled on a dedicated node assigned to user2 with effect NoSchedule",
 		},
 		{
@@ -2758,6 +2761,7 @@ func TestPodToleratesTaints(t *testing.T) {
 				},
 			},
 			fits: false,
+			expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(false)},
 			test: "a pod has a toleration that keys and values match the taint on the node, but (non-empty) effect doesn't match, " +
 				"can't be scheduled onto the node",
 		},
@@ -2791,7 +2795,7 @@ func TestPodToleratesTaints(t *testing.T) {
 				},
 			},
 			fits: true,
-			test: "The pod has a toleration that keys and values match the taint on the node, the effect of toleration is empty, " +
+			test: "the pod has a toleration that keys and values match the taint on the node, the effect of toleration is empty, " +
 				"and the effect of taint is NoSchedule. Pod can be scheduled onto the node",
 		},
 		{
@@ -2828,8 +2832,63 @@ func TestPodToleratesTaints(t *testing.T) {
 			test: "The pod has a toleration that key and value don't match the taint on the node, " +
 				"but the effect of taint on node is PreferNochedule. Pod can be scheduled onto the node",
 		},
+		{
+			pod: &api.Pod{
+				ObjectMeta: api.ObjectMeta{
+					Name: "podadmit1",
+				},
+			},
+			node: api.Node{
+				ObjectMeta: api.ObjectMeta{
+					Annotations: map[string]string{
+						api.TaintsAnnotationKey: `
+						[{
+							"key": "dedicated",
+							"value": "user1",
+							"effect": "NoScheduleNoAdmit"
+						}]`,
+					},
+				},
+			},
+			fits: false,
+			expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(true)},
+			test: "node should aware that that a pod having no tolerations can't be scheduled or started on a node with nonempty taints",
+		},
+		{
+			pod: &api.Pod{
+				ObjectMeta: api.ObjectMeta{
+					Name: "podadmit2",
+					Annotations: map[string]string{
+						api.TolerationsAnnotationKey: `
+						[{
+							"key": "dedicated",
+							"operator": "Equal",
+							"value": "user2",
+							"effect": "NoScheduleNoAdmit"
+						}]`,
+					},
+				},
+				Spec: api.PodSpec{
+					Containers: []api.Container{{Image: "pod2:V1"}},
+				},
+			},
+			node: api.Node{
+				ObjectMeta: api.ObjectMeta{
+					Annotations: map[string]string{
+						api.TaintsAnnotationKey: `
+						[{
+							"key": "dedicated",
+							"value": "user1",
+							"effect": "NoScheduleNoAdmit"
+						}]`,
+					},
+				},
+			},
+			fits: false,
+			expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(true)},
+			test: "node should aware that a pod which can't be scheduled or start on a dedicated node assgined to user2 with effect NoScheduleNoAdmit",
+		},
 	}
-	expectedFailureReasons := []algorithm.PredicateFailureReason{ErrTaintsTolerationsNotMatch}
 
 	for _, test := range podTolerateTaintsTests {
 		nodeInfo := schedulercache.NewNodeInfo()
@@ -2838,11 +2897,11 @@ func TestPodToleratesTaints(t *testing.T) {
 		if err != nil {
 			t.Errorf("%s, unexpected error: %v", test.test, err)
 		}
-		if !fits && !reflect.DeepEqual(reasons, expectedFailureReasons) {
-			t.Errorf("%s, unexpected failure reason: %v, want: %v", test.test, reasons, expectedFailureReasons)
+		if !fits && !reflect.DeepEqual(reasons, test.expectedFailureReasons) {
+			t.Errorf("%s, unexpected failure reason: %v, want: %v", test.test, reasons, test.expectedFailureReasons)
 		}
 		if fits != test.fits {
-			t.Errorf("%s, expected: %v got %v", test.test, test.fits, fits)
+			t.Errorf("%s,\n expected: %v got %v", test.test, test.fits, fits)
 		}
 	}
 }

plugin/pkg/scheduler/algorithm/priorities/taint_toleration.go

@@ -32,7 +32,7 @@ func countIntolerableTaintsPreferNoSchedule(taints []api.Taint, tolerations []ap
 			continue
 		}
 
-		if !api.TaintToleratedByTolerations(taint, tolerations) {
+		if tolerable := api.TaintToleratedByTolerations(taint, tolerations); !tolerable {
 			intolerableTaints++
 		}
 	}