Merge pull request #118389 from amewayne/support_nodeannotationschanged

support nodeAnnotationsChanged event to trigger rescheduling

pkg/scheduler/eventhandlers.go

@@ -19,12 +19,12 @@ package scheduler
 import (
 	"context"
 	"fmt"
-	"reflect"
 	"strings"
 	"time"
 
 	v1 "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -540,20 +540,23 @@ func nodeSchedulingPropertiesChange(newNode *v1.Node, oldNode *v1.Node) []framew
 	if nodeConditionsChanged(newNode, oldNode) {
 		events = append(events, queue.NodeConditionChange)
 	}
+	if nodeAnnotationsChanged(newNode, oldNode) {
+		events = append(events, queue.NodeAnnotationChange)
+	}
 
 	return events
 }
 
 func nodeAllocatableChanged(newNode *v1.Node, oldNode *v1.Node) bool {
-	return !reflect.DeepEqual(oldNode.Status.Allocatable, newNode.Status.Allocatable)
+	return !equality.Semantic.DeepEqual(oldNode.Status.Allocatable, newNode.Status.Allocatable)
 }
 
 func nodeLabelsChanged(newNode *v1.Node, oldNode *v1.Node) bool {
-	return !reflect.DeepEqual(oldNode.GetLabels(), newNode.GetLabels())
+	return !equality.Semantic.DeepEqual(oldNode.GetLabels(), newNode.GetLabels())
 }
 
 func nodeTaintsChanged(newNode *v1.Node, oldNode *v1.Node) bool {
-	return !reflect.DeepEqual(newNode.Spec.Taints, oldNode.Spec.Taints)
+	return !equality.Semantic.DeepEqual(newNode.Spec.Taints, oldNode.Spec.Taints)
 }
 
 func nodeConditionsChanged(newNode *v1.Node, oldNode *v1.Node) bool {
@@ -564,13 +567,17 @@ func nodeConditionsChanged(newNode *v1.Node, oldNode *v1.Node) bool {
 		}
 		return conditionStatuses
 	}
-	return !reflect.DeepEqual(strip(oldNode.Status.Conditions), strip(newNode.Status.Conditions))
+	return !equality.Semantic.DeepEqual(strip(oldNode.Status.Conditions), strip(newNode.Status.Conditions))
 }
 
 func nodeSpecUnschedulableChanged(newNode *v1.Node, oldNode *v1.Node) bool {
 	return newNode.Spec.Unschedulable != oldNode.Spec.Unschedulable && !newNode.Spec.Unschedulable
 }
 
+func nodeAnnotationsChanged(newNode *v1.Node, oldNode *v1.Node) bool {
+	return !equality.Semantic.DeepEqual(oldNode.GetAnnotations(), newNode.GetAnnotations())
+}
+
 func preCheckForNode(nodeInfo *framework.NodeInfo) queue.PreEnqueueCheck {
 	// Note: the following checks doesn't take preemption into considerations, in very rare
 	// cases (e.g., node resizing), "pod" may still fail a check but preemption helps. We deliberately

pkg/scheduler/eventhandlers_test.go

@@ -574,6 +574,12 @@ func TestNodeSchedulingPropertiesChange(t *testing.T) {
 			}).Obj(),
 			wantEvents: []framework.ClusterEvent{queue.NodeTaintChange},
 		},
+		{
+			name:       "only node annotation changed",
+			newNode:    st.MakeNode().Annotation("foo", "bar").Obj(),
+			oldNode:    st.MakeNode().Annotation("foo", "fuz").Obj(),
+			wantEvents: []framework.ClusterEvent{queue.NodeAnnotationChange},
+		},
 		{
 			name:    "only node condition changed",
 			newNode: st.MakeNode().Obj(),

pkg/scheduler/framework/types.go

@@ -53,11 +53,12 @@ const (
 	UpdateNodeLabel       // 1000
 	UpdateNodeTaint       // 10000
 	UpdateNodeCondition   // 100000
+	UpdateNodeAnnotation  // 1000000
 
-	All ActionType = 1<<iota - 1 // 111111
+	All ActionType = 1<<iota - 1 // 1111111
 
 	// Use the general Update type if you don't either know or care the specific sub-Update type to use.
-	Update = UpdateNodeAllocatable | UpdateNodeLabel | UpdateNodeTaint | UpdateNodeCondition
+	Update = UpdateNodeAllocatable | UpdateNodeLabel | UpdateNodeTaint | UpdateNodeCondition | UpdateNodeAnnotation
 )
 
 // GVK is short for group/version/kind, which can uniquely represent a particular API resource.

pkg/scheduler/internal/queue/events.go

@@ -52,6 +52,8 @@ var (
 	NodeAllocatableChange = framework.ClusterEvent{Resource: framework.Node, ActionType: framework.UpdateNodeAllocatable, Label: "NodeAllocatableChange"}
 	// NodeLabelChange is the event when node label is changed.
 	NodeLabelChange = framework.ClusterEvent{Resource: framework.Node, ActionType: framework.UpdateNodeLabel, Label: "NodeLabelChange"}
+	// NodeAnnotationChange is the event when node annotation is changed.
+	NodeAnnotationChange = framework.ClusterEvent{Resource: framework.Node, ActionType: framework.UpdateNodeAnnotation, Label: "NodeAnnotationChange"}
 	// NodeTaintChange is the event when node taint is changed.
 	NodeTaintChange = framework.ClusterEvent{Resource: framework.Node, ActionType: framework.UpdateNodeTaint, Label: "NodeTaintChange"}
 	// NodeConditionChange is the event when node condition is changed.

pkg/scheduler/internal/queue/scheduling_queue_test.go

@@ -1217,6 +1217,7 @@ func BenchmarkMoveAllToActiveOrBackoffQueue(b *testing.B) {
 		NodeAllocatableChange,
 		NodeConditionChange,
 		NodeLabelChange,
+		NodeAnnotationChange,
 		PvcAdd,
 		PvcUpdate,
 		PvAdd,