github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #66480 from Huang-Wei/stateless-MatchNodeSelectorTerms

Browse Source 
Automatic merge from submit-queue (batch tested with PRs 67042, 66480, 67053). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

ensure MatchNodeSelectorTerms() runs statelessly

**What this PR does**:

Fix sorting behavior in selector.go:

- move sorting from NewRequirement() out to String()
- add related unit tests
- add unit tests in one of outer callers (pkg/apis/core/v1/helper)

**Why we need it**:
- Without this fix, scheduling and daemonset controller doesn't work well in some (corner) cases

**Which issue(s) this PR fixes**:
Fixes #66298

**Special notes for your reviewer**:
Parameter `nodeSelectorTerms` in method MatchNodeSelectorTerms() is a slice, which is fundamentally a {*elements, len, cap} tuple - i.e. it's passing in a pointer. In that method, NodeSelectorRequirementsAsSelector() -> NewRequirement() is invoked, and the `matchExpressions[*].values` is passed into and **modified** via `sort.Strings(vals)`.

This will cause following daemonset pod fall into an infinite create/delete loop:

```yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: problem
spec:
  selector:
    matchLabels:
      app: sleeper
  template:
    metadata:
      labels:
        app: sleeper
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/hostname
                operator: In
                values:
                - 127.0.0.2
                - 127.0.0.1
      containers:
      - name: busybox
        image: busybox
        command: ["/bin/sleep", "7200"]
```

(the problem can be stably reproduced on a local cluster started by `hack/local-up-cluster.sh`)

The first time daemonset yaml is handled by apiserver and persisted in etcd with original format (original order of values was kept - 127.0.0.2, 127.0.0.1). After that, daemonset controller tries to schedule pod, and it reuses the predicates logic in scheduler component - where the values are **sorted** deeply. This not only causes the pod to be created in sorted order (127.0.0.1, 127.0.0.2), but also introduced a bug when updating daemonset - internally ds controller use a "rawMessage" (bytes of an object) to calculate hash acting as a "controller-revision-hash" to control revision rollingUpdate/rollBack, so it keeps killing "old" pod and spawning "new" pod back and forth, and fall into an infinite loop.

The issue exists in `master`, `release-1.11` and `release-1.10`.

**Release note**:
```release-note
NONE
```
This commit is contained in:
Kubernetes Submit Queue
2018-08-07 14:27:59 -07:00
committed by GitHub
parent d863adedfe e5e0de1b9d
commit 00bf292cdc
4 changed files with 364 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

297
pkg/apis/core/v1/helper/helpers_test.go
View File

@@ -811,6 +811,303 @@ func TestMatchNodeSelectorTerms(t *testing.T) {
}
}
// TestMatchNodeSelectorTermsStateless ensures MatchNodeSelectorTerms()
// is invoked in a "stateless" manner, i.e. nodeSelectorTerms should NOT
// be deeply modifed after invoking
func TestMatchNodeSelectorTermsStateless(t *testing.T) {
type args struct {
nodeSelectorTerms []v1.NodeSelectorTerm
nodeLabels labels.Set
nodeFields fields.Set
}
tests := []struct {
name string
args args
want []v1.NodeSelectorTerm
}{
{
name: "nil terms",
args: args{
nodeSelectorTerms: nil,
nodeLabels: nil,
nodeFields: nil,
},
want: nil,
},
{
name: "nodeLabels: preordered matchExpressions and nil matchFields",
args: args{
nodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{{
Key: "label_1",
Operator: v1.NodeSelectorOpIn,
Values: []string{"label_1_val", "label_2_val"},
}},
},
},
nodeLabels: map[string]string{"label_1": "label_1_val"},
nodeFields: nil,
},
want: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{{
Key: "label_1",
Operator: v1.NodeSelectorOpIn,
Values: []string{"label_1_val", "label_2_val"},
}},
},
},
},
{
name: "nodeLabels: unordered matchExpressions and nil matchFields",
args: args{
nodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{{
Key: "label_1",
Operator: v1.NodeSelectorOpIn,
Values: []string{"label_2_val", "label_1_val"},
}},
},
},
nodeLabels: map[string]string{"label_1": "label_1_val"},
nodeFields: nil,
},
want: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{{
Key: "label_1",
Operator: v1.NodeSelectorOpIn,
Values: []string{"label_2_val", "label_1_val"},
}},
},
},
},
{
name: "nodeFields: nil matchExpressions and preordered matchFields",
args: args{
nodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchFields: []v1.NodeSelectorRequirement{{
Key: "metadata.name",
Operator: v1.NodeSelectorOpIn,
Values: []string{"host_1", "host_2"},
}},
},
},
nodeLabels: nil,
nodeFields: map[string]string{
"metadata.name": "host_1",
},
},
want: []v1.NodeSelectorTerm{
{
MatchFields: []v1.NodeSelectorRequirement{{
Key: "metadata.name",
Operator: v1.NodeSelectorOpIn,
Values: []string{"host_1", "host_2"},
}},
},
},
},
{
name: "nodeFields: nil matchExpressions and unordered matchFields",
args: args{
nodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchFields: []v1.NodeSelectorRequirement{{
Key: "metadata.name",
Operator: v1.NodeSelectorOpIn,
Values: []string{"host_2", "host_1"},
}},
},
},
nodeLabels: nil,
nodeFields: map[string]string{
"metadata.name": "host_1",
},
},
want: []v1.NodeSelectorTerm{
{
MatchFields: []v1.NodeSelectorRequirement{{
Key: "metadata.name",
Operator: v1.NodeSelectorOpIn,
Values: []string{"host_2", "host_1"},
}},
},
},
},
{
name: "nodeLabels and nodeFields: ordered matchExpressions and ordered matchFields",
args: args{
nodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{{
Key: "label_1",
Operator: v1.NodeSelectorOpIn,
Values: []string{"label_1_val", "label_2_val"},
}},
MatchFields: []v1.NodeSelectorRequirement{{
Key: "metadata.name",
Operator: v1.NodeSelectorOpIn,
Values: []string{"host_1", "host_2"},
}},
},
},
nodeLabels: map[string]string{
"label_1": "label_1_val",
},
nodeFields: map[string]string{
"metadata.name": "host_1",
},
},
want: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{{
Key: "label_1",
Operator: v1.NodeSelectorOpIn,
Values: []string{"label_1_val", "label_2_val"},
}},
MatchFields: []v1.NodeSelectorRequirement{{
Key: "metadata.name",
Operator: v1.NodeSelectorOpIn,
Values: []string{"host_1", "host_2"},
}},
},
},
},
{
name: "nodeLabels and nodeFields: ordered matchExpressions and unordered matchFields",
args: args{
nodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{{
Key: "label_1",
Operator: v1.NodeSelectorOpIn,
Values: []string{"label_1_val", "label_2_val"},
}},
MatchFields: []v1.NodeSelectorRequirement{{
Key: "metadata.name",
Operator: v1.NodeSelectorOpIn,
Values: []string{"host_2", "host_1"},
}},
},
},
nodeLabels: map[string]string{
"label_1": "label_1_val",
},
nodeFields: map[string]string{
"metadata.name": "host_1",
},
},
want: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{{
Key: "label_1",
Operator: v1.NodeSelectorOpIn,
Values: []string{"label_1_val", "label_2_val"},
}},
MatchFields: []v1.NodeSelectorRequirement{{
Key: "metadata.name",
Operator: v1.NodeSelectorOpIn,
Values: []string{"host_2", "host_1"},
}},
},
},
},
{
name: "nodeLabels and nodeFields: unordered matchExpressions and ordered matchFields",
args: args{
nodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{{
Key: "label_1",
Operator: v1.NodeSelectorOpIn,
Values: []string{"label_2_val", "label_1_val"},
}},
MatchFields: []v1.NodeSelectorRequirement{{
Key: "metadata.name",
Operator: v1.NodeSelectorOpIn,
Values: []string{"host_1", "host_2"},
}},
},
},
nodeLabels: map[string]string{
"label_1": "label_1_val",
},
nodeFields: map[string]string{
"metadata.name": "host_1",
},
},
want: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{{
Key: "label_1",
Operator: v1.NodeSelectorOpIn,
Values: []string{"label_2_val", "label_1_val"},
}},
MatchFields: []v1.NodeSelectorRequirement{{
Key: "metadata.name",
Operator: v1.NodeSelectorOpIn,
Values: []string{"host_1", "host_2"},
}},
},
},
},
{
name: "nodeLabels and nodeFields: unordered matchExpressions and unordered matchFields",
args: args{
nodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{{
Key: "label_1",
Operator: v1.NodeSelectorOpIn,
Values: []string{"label_2_val", "label_1_val"},
}},
MatchFields: []v1.NodeSelectorRequirement{{
Key: "metadata.name",
Operator: v1.NodeSelectorOpIn,
Values: []string{"host_2", "host_1"},
}},
},
},
nodeLabels: map[string]string{
"label_1": "label_1_val",
},
nodeFields: map[string]string{
"metadata.name": "host_1",
},
},
want: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{{
Key: "label_1",
Operator: v1.NodeSelectorOpIn,
Values: []string{"label_2_val", "label_1_val"},
}},
MatchFields: []v1.NodeSelectorRequirement{{
Key: "metadata.name",
Operator: v1.NodeSelectorOpIn,
Values: []string{"host_2", "host_1"},
}},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
MatchNodeSelectorTerms(tt.args.nodeSelectorTerms, tt.args.nodeLabels, tt.args.nodeFields)
if !apiequality.Semantic.DeepEqual(tt.args.nodeSelectorTerms, tt.want) {
// fail when tt.args.nodeSelectorTerms is deeply modified
t.Errorf("MatchNodeSelectorTerms() got = %v, want %v", tt.args.nodeSelectorTerms, tt.want)
}
})
}
}
func TestMatchTopologySelectorTerms(t *testing.T) {
type args struct {
topologySelectorTerms []v1.TopologySelectorTerm