github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[scheduling] Moved node affinity from annotations to api fields. #35518

Browse Source
This commit is contained in:
Robert Rati
2016-11-30 11:51:12 -05:00
parent 1eb9176455
commit 91931c138e
26 changed files with 648 additions and 501 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/api/serialization_test.go
View File

@@ -332,6 +332,7 @@ func TestEncode_Ptr(t *testing.T) {
TerminationGracePeriodSeconds: &grace,
SecurityContext: &api.PodSecurityContext{},
Affinity: &api.Affinity{},
},
}
obj := runtime.Object(pod)

3
pkg/api/testing/fuzzer.go
View File

@@ -130,6 +130,9 @@ func FuzzerFor(t *testing.T, version schema.GroupVersion, src rand.Source) *fuzz
if s.SecurityContext == nil {
s.SecurityContext = new(api.PodSecurityContext)
}
if s.Affinity == nil {
s.Affinity = new(api.Affinity)
}
},
func(j *api.PodPhase, c fuzz.Continue) {
statuses := []api.PodPhase{api.PodPending, api.PodRunning, api.PodFailed, api.PodUnknown}

1
pkg/api/testing/pod_specs.go
View File

@@ -29,6 +29,7 @@ func DeepEqualSafePodSpec() api.PodSpec {
DNSPolicy: api.DNSClusterFirst,
TerminationGracePeriodSeconds: &grace,
SecurityContext: &api.PodSecurityContext{},
Affinity: &api.Affinity{},
}
}

3
pkg/api/types.go
View File

@@ -1824,6 +1824,9 @@ type PodSpec struct {
// If not specified, the pod will not have a domainname at all.
// +optional
Subdomain string
// If specified, the pod's scheduling constraints
// +optional
Affinity *Affinity
}
// Sysctl defines a kernel parameter to be set

3
pkg/api/v1/defaults.go
View File

@@ -175,6 +175,9 @@ func SetDefaults_PodSpec(obj *PodSpec) {
period := int64(DefaultTerminationGracePeriodSeconds)
obj.TerminationGracePeriodSeconds = &period
}
if obj.Affinity == nil {
obj.Affinity = &Affinity{}
}
}
func SetDefaults_Probe(obj *Probe) {
if obj.TimeoutSeconds == 0 {

3
pkg/api/v1/types.go
View File

@@ -2107,6 +2107,9 @@ type PodSpec struct {
// If not specified, the pod will not have a domainname at all.
// +optional
Subdomain string `json:"subdomain,omitempty" protobuf:"bytes,17,opt,name=subdomain"`
// If specified, the pod's scheduling constraints
// +optional
Affinity *Affinity `json:"affinity,omitempty" protobuf:"bytes,18,opt,name=affinity"`
}
// PodSecurityContext holds pod-level security attributes and common container settings.

40
pkg/api/validation/validation.go
View File

@@ -1811,6 +1811,29 @@ func validateImagePullSecrets(imagePullSecrets []api.LocalObjectReference, fldPa
return allErrors
}
// validateAffinity checks if given affinities are valid
func validateAffinity(affinity *api.Affinity, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if affinity != nil {
if na := affinity.NodeAffinity; na != nil {
// TODO: Uncomment the next three lines once RequiredDuringSchedulingRequiredDuringExecution is implemented.
// if na.RequiredDuringSchedulingRequiredDuringExecution != nil {
// allErrs = append(allErrs, ValidateNodeSelector(na.RequiredDuringSchedulingRequiredDuringExecution, fldPath.Child("requiredDuringSchedulingRequiredDuringExecution"))...)
// }
if na.RequiredDuringSchedulingIgnoredDuringExecution != nil {
allErrs = append(allErrs, ValidateNodeSelector(na.RequiredDuringSchedulingIgnoredDuringExecution, fldPath.Child("requiredDuringSchedulingIgnoredDuringExecution"))...)
}
if len(na.PreferredDuringSchedulingIgnoredDuringExecution) > 0 {
allErrs = append(allErrs, ValidatePreferredSchedulingTerms(na.PreferredDuringSchedulingIgnoredDuringExecution, fldPath.Child("preferredDuringSchedulingIgnoredDuringExecution"))...)
}
}
}
return allErrs
}
func validateTaintEffect(effect *api.TaintEffect, allowEmpty bool, fldPath *field.Path) field.ErrorList {
if !allowEmpty && len(*effect) == 0 {
return field.ErrorList{field.Required(fldPath, "")}
@@ -1890,6 +1913,7 @@ func ValidatePodSpec(spec *api.PodSpec, fldPath *field.Path) field.ErrorList {
allErrs = append(allErrs, unversionedvalidation.ValidateLabels(spec.NodeSelector, fldPath.Child("nodeSelector"))...)
allErrs = append(allErrs, ValidatePodSecurityContext(spec.SecurityContext, spec, fldPath, fldPath.Child("securityContext"))...)
allErrs = append(allErrs, validateImagePullSecrets(spec.ImagePullSecrets, fldPath.Child("imagePullSecrets"))...)
allErrs = append(allErrs, validateAffinity(spec.Affinity, fldPath.Child("affinity"))...)
if len(spec.ServiceAccountName) > 0 {
for _, msg := range ValidateServiceAccountName(spec.ServiceAccountName, false) {
allErrs = append(allErrs, field.Invalid(fldPath.Child("serviceAccountName"), spec.ServiceAccountName, msg))
@@ -2116,22 +2140,6 @@ func ValidateAffinityInPodAnnotations(annotations map[string]string, fldPath *fi
}
affinityFldPath := fldPath.Child(api.AffinityAnnotationKey)
if affinity.NodeAffinity != nil {
na := affinity.NodeAffinity
naFldPath := affinityFldPath.Child("nodeAffinity")
// TODO: Uncomment the next three lines once RequiredDuringSchedulingRequiredDuringExecution is implemented.
// if na.RequiredDuringSchedulingRequiredDuringExecution != nil {
// allErrs = append(allErrs, ValidateNodeSelector(na.RequiredDuringSchedulingRequiredDuringExecution, naFldPath.Child("requiredDuringSchedulingRequiredDuringExecution"))...)
// }
if na.RequiredDuringSchedulingIgnoredDuringExecution != nil {
allErrs = append(allErrs, ValidateNodeSelector(na.RequiredDuringSchedulingIgnoredDuringExecution, naFldPath.Child("requiredDuringSchedulingIgnoredDuringExecution"))...)
}
if len(na.PreferredDuringSchedulingIgnoredDuringExecution) > 0 {
allErrs = append(allErrs, ValidatePreferredSchedulingTerms(na.PreferredDuringSchedulingIgnoredDuringExecution, naFldPath.Child("preferredDuringSchedulingIgnoredDuringExecution"))...)
}
}
if affinity.PodAffinity != nil {
allErrs = append(allErrs, validatePodAffinity(affinity.PodAffinity, affinityFldPath.Child("podAffinity"))...)
}

213
pkg/api/validation/validation_test.go
View File

@@ -3482,50 +3482,60 @@ func TestValidatePod(t *testing.T) {
NodeName: "foobar",
},
},
{ // Serialized affinity requirements in annotations.
{ // Serialized node affinity requirements.
ObjectMeta: api.ObjectMeta{
Name: "123",
Namespace: "ns",
// TODO: Uncomment and move this block into Annotations map once
// RequiredDuringSchedulingRequiredDuringExecution is implemented
// "requiredDuringSchedulingRequiredDuringExecution": {
// "nodeSelectorTerms": [{
// "matchExpressions": [{
// "key": "key1",
// "operator": "Exists"
// }]
// }]
// },
Annotations: map[string]string{
api.AffinityAnnotationKey: `
{"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "key2",
"operator": "In",
"values": ["value1", "value2"]
}]
}]
},
"preferredDuringSchedulingIgnoredDuringExecution": [
{
"weight": 10,
"preference": {"matchExpressions": [
{
"key": "foo",
"operator": "In", "values": ["bar"]
}
]}
}
]
}}`,
},
},
Spec: api.PodSpec{
Containers: []api.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent"}},
RestartPolicy: api.RestartPolicyAlways,
DNSPolicy: api.DNSClusterFirst,
// TODO: Uncomment and move this block and move inside NodeAffinity once
// RequiredDuringSchedulingRequiredDuringExecution is implemented
// RequiredDuringSchedulingRequiredDuringExecution: &api.NodeSelector{
// NodeSelectorTerms: []api.NodeSelectorTerm{
// {
// MatchExpressions: []api.NodeSelectorRequirement{
// {
// Key: "key1",
// Operator: api.NodeSelectorOpExists
// },
// },
// },
// },
// },
Affinity: &api.Affinity{
NodeAffinity: &api.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &api.NodeSelector{
NodeSelectorTerms: []api.NodeSelectorTerm{
{
MatchExpressions: []api.NodeSelectorRequirement{
{
Key: "key2",
Operator: api.NodeSelectorOpIn,
Values: []string{"value1", "value2"},
},
},
},
},
},
PreferredDuringSchedulingIgnoredDuringExecution: []api.PreferredSchedulingTerm{
{
Weight: 10,
Preference: api.NodeSelectorTerm{
MatchExpressions: []api.NodeSelectorRequirement{
{
Key: "foo",
Operator: api.NodeSelectorOpIn,
Values: []string{"bar"},
},
},
},
},
},
},
},
},
},
{ // Serialized pod affinity in affinity requirements in annotations.
@@ -3863,90 +3873,101 @@ func TestValidatePod(t *testing.T) {
Containers: []api.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent"}},
},
},
"invalid json of node affinity in pod annotations": {
"invalid node selector requirement in node affinity, operator can't be null": {
ObjectMeta: api.ObjectMeta{
Name: "123",
Namespace: "ns",
Annotations: map[string]string{
api.AffinityAnnotationKey: `
{"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
`,
},
},
Spec: validPodSpec,
},
"invalid node selector requirement in node affinity in pod annotations, operator can't be null": {
ObjectMeta: api.ObjectMeta{
Name: "123",
Namespace: "ns",
Annotations: map[string]string{
api.AffinityAnnotationKey: `
{"nodeAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "key1",
}]
}]
}}}`,
},
},
Spec: validPodSpec,
},
"invalid preferredSchedulingTerm in node affinity in pod annotations, weight should be in range 1-100": {
ObjectMeta: api.ObjectMeta{
Name: "123",
Namespace: "ns",
Annotations: map[string]string{
api.AffinityAnnotationKey: `
{"nodeAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": [
Spec: api.PodSpec{
Containers: []api.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent"}},
RestartPolicy: api.RestartPolicyAlways,
DNSPolicy: api.DNSClusterFirst,
Affinity: &api.Affinity{
NodeAffinity: &api.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &api.NodeSelector{
NodeSelectorTerms: []api.NodeSelectorTerm{
{
"weight": 199,
"preference": {"matchExpressions": [
MatchExpressions: []api.NodeSelectorRequirement{
{
"key": "foo",
"operator": "In",
"values": ["bar"]
}
]}
}
]}}`,
Key: "key1",
},
},
},
},
},
},
},
},
},
"invalid preferredSchedulingTerm in node affinity, weight should be in range 1-100": {
ObjectMeta: api.ObjectMeta{
Name: "123",
Namespace: "ns",
},
Spec: api.PodSpec{
Containers: []api.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent"}},
RestartPolicy: api.RestartPolicyAlways,
DNSPolicy: api.DNSClusterFirst,
Affinity: &api.Affinity{
NodeAffinity: &api.NodeAffinity{
PreferredDuringSchedulingIgnoredDuringExecution: []api.PreferredSchedulingTerm{
{
Weight: 199,
Preference: api.NodeSelectorTerm{
MatchExpressions: []api.NodeSelectorRequirement{
{
Key: "foo",
Operator: api.NodeSelectorOpIn,
Values: []string{"bar"},
},
},
},
},
},
},
},
},
Spec: validPodSpec,
},
"invalid requiredDuringSchedulingIgnoredDuringExecution node selector, nodeSelectorTerms must have at least one term": {
ObjectMeta: api.ObjectMeta{
Name: "123",
Namespace: "ns",
Annotations: map[string]string{
api.AffinityAnnotationKey: `
{"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": []
},
}}`,
Spec: api.PodSpec{
Containers: []api.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent"}},
RestartPolicy: api.RestartPolicyAlways,
DNSPolicy: api.DNSClusterFirst,
Affinity: &api.Affinity{
NodeAffinity: &api.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &api.NodeSelector{
NodeSelectorTerms: []api.NodeSelectorTerm{},
},
},
},
},
Spec: validPodSpec,
},
"invalid requiredDuringSchedulingIgnoredDuringExecution node selector term, matchExpressions must have at least one node selector requirement": {
ObjectMeta: api.ObjectMeta{
Name: "123",
Namespace: "ns",
Annotations: map[string]string{
api.AffinityAnnotationKey: `
{"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": []
}]
},
}}`,
Spec: api.PodSpec{
Containers: []api.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent"}},
RestartPolicy: api.RestartPolicyAlways,
DNSPolicy: api.DNSClusterFirst,
Affinity: &api.Affinity{
NodeAffinity: &api.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &api.NodeSelector{
NodeSelectorTerms: []api.NodeSelectorTerm{
{
MatchExpressions: []api.NodeSelectorRequirement{},
},
},
},
},
},
},
Spec: validPodSpec,
},
"invalid weight in preferredDuringSchedulingIgnoredDuringExecution in pod affinity annotations, weight should be in range 1-100": {
ObjectMeta: api.ObjectMeta{

3
pkg/apis/extensions/v1beta1/defaults_test.go
View File

@@ -40,6 +40,7 @@ func TestSetDefaultDaemonSet(t *testing.T) {
RestartPolicy: v1.RestartPolicyAlways,
SecurityContext: &v1.PodSecurityContext{},
TerminationGracePeriodSeconds: &period,
Affinity: &v1.Affinity{},
},
ObjectMeta: v1.ObjectMeta{
Labels: defaultLabels,
@@ -51,6 +52,7 @@ func TestSetDefaultDaemonSet(t *testing.T) {
RestartPolicy: v1.RestartPolicyAlways,
SecurityContext: &v1.PodSecurityContext{},
TerminationGracePeriodSeconds: &period,
Affinity: &v1.Affinity{},
},
}
tests := []struct {
@@ -155,6 +157,7 @@ func TestSetDefaultDeployment(t *testing.T) {
RestartPolicy: v1.RestartPolicyAlways,
SecurityContext: &v1.PodSecurityContext{},
TerminationGracePeriodSeconds: &period,
Affinity: &v1.Affinity{},
},
}
tests := []struct {

28
pkg/controller/daemon/daemoncontroller_test.go
View File

@@ -553,19 +553,23 @@ func TestNodeAffinityDaemonLaunchesPods(t *testing.T) {
addNodes(manager.nodeStore.Store, 0, 4, nil)
addNodes(manager.nodeStore.Store, 4, 3, simpleNodeLabel)
daemon := newDaemonSet("foo")
affinity := map[string]string{
v1.AffinityAnnotationKey: fmt.Sprintf(`
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "color",
"operator": "In",
"values": ["%s"]
}]
}]
}}}`, simpleNodeLabel["color"]),
daemon.Spec.Template.Spec.Affinity = &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "color",
Operator: v1.NodeSelectorOpIn,
Values: []string{simpleNodeLabel["color"]},
},
},
},
},
},
},
}
daemon.Spec.Template.ObjectMeta.Annotations = affinity
manager.dsStore.Add(daemon)
syncAndValidateDaemonSets(t, manager, daemon, podControl, 3, 0)
}

1
pkg/kubectl/cmd/util/helpers_test.go
View File

@@ -132,6 +132,7 @@ func TestMerge(t *testing.T) {
DNSPolicy: api.DNSClusterFirst,
TerminationGracePeriodSeconds: &grace,
SecurityContext: &api.PodSecurityContext{},
Affinity: &api.Affinity{},
},
},
},

2
pkg/kubelet/config/common_test.go
View File

@@ -54,6 +54,7 @@ func TestDecodeSinglePod(t *testing.T) {
SecurityContext: securitycontext.ValidSecurityContextWithContainerDefaults(),
}},
SecurityContext: &v1.PodSecurityContext{},
Affinity: &v1.Affinity{},
},
}
json, err := runtime.Encode(testapi.Default.Codec(), pod)
@@ -114,6 +115,7 @@ func TestDecodePodList(t *testing.T) {
SecurityContext: securitycontext.ValidSecurityContextWithContainerDefaults(),
}},
SecurityContext: &v1.PodSecurityContext{},
Affinity: &v1.Affinity{},
},
}
podList := &v1.PodList{

2
pkg/kubelet/config/file_linux_test.go
View File

@@ -188,6 +188,7 @@ func getTestCases(hostname types.NodeName) []*testCase {
Spec: v1.PodSpec{
Containers: []v1.Container{{Name: "image", Image: "test/image", SecurityContext: securitycontext.ValidSecurityContextWithContainerDefaults()}},
SecurityContext: &v1.PodSecurityContext{},
Affinity: &v1.Affinity{},
},
Status: v1.PodStatus{
Phase: v1.PodPending,
@@ -213,6 +214,7 @@ func getTestCases(hostname types.NodeName) []*testCase {
ImagePullPolicy: "Always",
SecurityContext: securitycontext.ValidSecurityContextWithContainerDefaults()}},
SecurityContext: &v1.PodSecurityContext{},
Affinity: &v1.Affinity{},
},
Status: v1.PodStatus{
Phase: v1.PodPending,

6
pkg/kubelet/config/http_test.go
View File

@@ -148,6 +148,7 @@ func TestExtractPodsFromHTTP(t *testing.T) {
NodeName: string(nodeName),
Containers: []v1.Container{{Name: "1", Image: "foo", ImagePullPolicy: v1.PullAlways}},
SecurityContext: &v1.PodSecurityContext{},
Affinity: &v1.Affinity{},
},
Status: v1.PodStatus{
Phase: v1.PodPending,
@@ -169,6 +170,7 @@ func TestExtractPodsFromHTTP(t *testing.T) {
DNSPolicy: v1.DNSClusterFirst,
SecurityContext: &v1.PodSecurityContext{},
TerminationGracePeriodSeconds: &grace,
Affinity: &v1.Affinity{},
Containers: []v1.Container{{
Name: "1",
@@ -199,6 +201,7 @@ func TestExtractPodsFromHTTP(t *testing.T) {
NodeName: nodeName,
Containers: []v1.Container{{Name: "1", Image: "foo", ImagePullPolicy: v1.PullAlways}},
SecurityContext: &v1.PodSecurityContext{},
Affinity: &v1.Affinity{},
},
Status: v1.PodStatus{
Phase: v1.PodPending,
@@ -213,6 +216,7 @@ func TestExtractPodsFromHTTP(t *testing.T) {
NodeName: nodeName,
Containers: []v1.Container{{Name: "2", Image: "bar:bartag", ImagePullPolicy: ""}},
SecurityContext: &v1.PodSecurityContext{},
Affinity: &v1.Affinity{},
},
Status: v1.PodStatus{
Phase: v1.PodPending,
@@ -236,6 +240,7 @@ func TestExtractPodsFromHTTP(t *testing.T) {
DNSPolicy: v1.DNSClusterFirst,
TerminationGracePeriodSeconds: &grace,
SecurityContext: &v1.PodSecurityContext{},
Affinity: &v1.Affinity{},
Containers: []v1.Container{{
Name: "1",
@@ -262,6 +267,7 @@ func TestExtractPodsFromHTTP(t *testing.T) {
DNSPolicy: v1.DNSClusterFirst,
TerminationGracePeriodSeconds: &grace,
SecurityContext: &v1.PodSecurityContext{},
Affinity: &v1.Affinity{},
Containers: []v1.Container{{
Name: "2",

5
pkg/registry/core/pod/etcd/etcd_test.go
View File

@@ -72,6 +72,7 @@ func validNewPod() *api.Pod {
},
},
SecurityContext: &api.PodSecurityContext{},
Affinity: &api.Affinity{},
},
}
}
@@ -658,6 +659,7 @@ func TestEtcdUpdateScheduled(t *testing.T) {
},
},
SecurityContext: &api.PodSecurityContext{},
Affinity: &api.Affinity{},
},
}, nil, 1)
if err != nil {
@@ -686,6 +688,7 @@ func TestEtcdUpdateScheduled(t *testing.T) {
TerminationGracePeriodSeconds: &grace,
SecurityContext: &api.PodSecurityContext{},
Affinity: &api.Affinity{},
},
}
_, _, err = storage.Update(ctx, podIn.Name, rest.DefaultUpdatedObjectInfo(&podIn, api.Scheme))
@@ -726,6 +729,7 @@ func TestEtcdUpdateStatus(t *testing.T) {
},
},
SecurityContext: &api.PodSecurityContext{},
Affinity: &api.Affinity{},
},
}
err := storage.Storage.Create(ctx, key, &podStart, nil, 0)
@@ -750,6 +754,7 @@ func TestEtcdUpdateStatus(t *testing.T) {
},
},
SecurityContext: &api.PodSecurityContext{},
Affinity: &api.Affinity{},
},
Status: api.PodStatus{
Phase: api.PodRunning,

9
plugin/pkg/scheduler/algorithm/predicates/predicates.go
View File

@@ -582,14 +582,6 @@ func podMatchesNodeLabels(pod *v1.Pod, node *v1.Node) bool {
}
}
// Parse required node affinity scheduling requirements
// and check if the current node match the requirements.
affinity, err := v1.GetAffinityFromPodAnnotations(pod.Annotations)
if err != nil {
glog.V(10).Infof("Failed to get Affinity from Pod %+v, err: %+v", podName(pod), err)
return false
}
// 1. nil NodeSelector matches all nodes (i.e. does not filter out any nodes)
// 2. nil []NodeSelectorTerm (equivalent to non-nil empty NodeSelector) matches no nodes
// 3. zero-length non-nil []NodeSelectorTerm matches no nodes also, just for simplicity
@@ -597,6 +589,7 @@ func podMatchesNodeLabels(pod *v1.Pod, node *v1.Node) bool {
// 5. zero-length non-nil []NodeSelectorRequirement matches no nodes also, just for simplicity
// 6. non-nil empty NodeSelectorRequirement is not allowed
nodeAffinityMatches := true
affinity := pod.Spec.Affinity
if affinity != nil && affinity.NodeAffinity != nil {
nodeAffinity := affinity.NodeAffinity
// if no required NodeAffinity requirements, will do no-op, means select all nodes.

422
plugin/pkg/scheduler/algorithm/predicates/predicates_test.go
View File

@@ -866,18 +866,23 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "foo",
"operator": "In",
"values": ["bar", "value2"]
}]
}]
}}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "foo",
Operator: v1.NodeSelectorOpIn,
Values: []string{"bar", "value2"},
},
},
},
},
},
},
},
},
},
@@ -889,18 +894,23 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "kernel-version",
"operator": "Gt",
"values": ["0204"]
}]
}]
}}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "kernel-version",
Operator: v1.NodeSelectorOpGt,
Values: []string{"0204"},
},
},
},
},
},
},
},
},
},
@@ -913,18 +923,23 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "mem-type",
"operator": "NotIn",
"values": ["DDR", "DDR2"]
}]
}]
}}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "mem-type",
Operator: v1.NodeSelectorOpNotIn,
Values: []string{"DDR", "DDR2"},
},
},
},
},
},
},
},
},
},
@@ -936,17 +951,22 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "GPU",
"operator": "Exists"
}]
}]
}}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "GPU",
Operator: v1.NodeSelectorOpExists,
},
},
},
},
},
},
},
},
},
@@ -958,18 +978,23 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "foo",
"operator": "In",
"values": ["value1", "value2"]
}]
}]
}}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "foo",
Operator: v1.NodeSelectorOpIn,
Values: []string{"value1", "value2"},
},
},
},
},
},
},
},
},
},
@@ -981,12 +1006,13 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": null
}}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: nil,
},
},
},
},
},
@@ -998,12 +1024,13 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": []
}}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{},
},
},
},
},
},
@@ -1013,6 +1040,7 @@ func TestPodFitsSelector(t *testing.T) {
fits: false,
test: "Pod with an empty []NodeSelectorTerm in affinity, can't match the node's labels and won't schedule onto the node",
},
/*
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
@@ -1023,6 +1051,22 @@ func TestPodFitsSelector(t *testing.T) {
}}}`,
},
},
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: {},
},
{
MatchExpressions: {},
},
},
},
},
},
},
},
labels: map[string]string{
"foo": "bar",
@@ -1030,14 +1074,20 @@ func TestPodFitsSelector(t *testing.T) {
fits: false,
test: "Pod with invalid NodeSelectTerms in affinity will match no objects and won't schedule onto the node",
},
*/
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{"matchExpressions": [{}]}]
}}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{},
},
},
},
},
},
},
},
@@ -1048,13 +1098,7 @@ func TestPodFitsSelector(t *testing.T) {
test: "Pod with empty MatchExpressions is not a valid value will match no objects and won't schedule onto the node",
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
"some-key": "some-value",
},
},
},
pod: &v1.Pod{},
labels: map[string]string{
"foo": "bar",
},
@@ -1063,11 +1107,11 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": null
}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: nil,
},
},
},
},
@@ -1079,21 +1123,26 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "GPU",
"operator": "Exists"
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "GPU",
Operator: v1.NodeSelectorOpExists,
}, {
"key": "GPU",
"operator": "NotIn",
"values": ["AMD", "INTER"]
}]
}]
}}}`,
Key: "GPU",
Operator: v1.NodeSelectorOpNotIn,
Values: []string{"AMD", "INTER"},
},
},
},
},
},
},
},
},
},
@@ -1105,21 +1154,26 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "GPU",
"operator": "Exists"
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "GPU",
Operator: v1.NodeSelectorOpExists,
}, {
"key": "GPU",
"operator": "In",
"values": ["AMD", "INTER"]
}]
}]
}}}`,
Key: "GPU",
Operator: v1.NodeSelectorOpIn,
Values: []string{"AMD", "INTER"},
},
},
},
},
},
},
},
},
},
@@ -1131,27 +1185,32 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
"matchExpressions": [{
"key": "foo",
"operator": "In",
"values": ["bar", "value2"]
}]
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "foo",
Operator: v1.NodeSelectorOpIn,
Values: []string{"bar", "value2"},
},
},
},
{
"matchExpressions": [{
"key": "diffkey",
"operator": "In",
"values": ["wrong", "value2"]
}]
}
]
}}}`,
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "diffkey",
Operator: v1.NodeSelectorOpIn,
Values: []string{"wrong", "value2"},
},
},
},
},
},
},
},
},
},
@@ -1199,23 +1258,26 @@ func TestPodFitsSelector(t *testing.T) {
// },
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "foo",
"operator": "Exists"
}]
}]
}}}`,
},
},
Spec: v1.PodSpec{
NodeSelector: map[string]string{
"foo": "bar",
},
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "foo",
Operator: v1.NodeSelectorOpExists,
},
},
},
},
},
},
},
},
},
labels: map[string]string{
@@ -1227,23 +1289,26 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "foo",
"operator": "Exists"
}]
}]
}}}`,
},
},
Spec: v1.PodSpec{
NodeSelector: map[string]string{
"foo": "bar",
},
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "foo",
Operator: v1.NodeSelectorOpExists,
},
},
},
},
},
},
},
},
},
labels: map[string]string{
@@ -2604,17 +2669,6 @@ func TestInterPodAffinityWithMultipleNodes(t *testing.T) {
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "hostname",
"operator": "NotIn",
"values": ["h1"]
}]
}]
}
},
"podAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": [{
"labelSelector": {
@@ -2630,6 +2684,25 @@ func TestInterPodAffinityWithMultipleNodes(t *testing.T) {
}`,
},
},
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "hostname",
Operator: v1.NodeSelectorOpNotIn,
Values: []string{"h1"},
},
},
},
},
},
},
},
},
},
pods: []*v1.Pod{
{Spec: v1.PodSpec{NodeName: "nodeA"}, ObjectMeta: v1.ObjectMeta{Labels: map[string]string{"foo": "abc"}}},
@@ -2781,10 +2854,7 @@ func TestInterPodAffinityWithMultipleNodes(t *testing.T) {
if !fits && !reflect.DeepEqual(reasons, affinityExpectedFailureReasons) {
t.Errorf("%s: unexpected failure reasons: %v", test.test, reasons)
}
affinity, err := v1.GetAffinityFromPodAnnotations(test.pod.ObjectMeta.Annotations)
if err != nil {
t.Errorf("%s: unexpected error: %v", test.test, err)
}
affinity := test.pod.Spec.Affinity
if affinity != nil && affinity.NodeAffinity != nil {
nodeInfo := schedulercache.NewNodeInfo()
nodeInfo.SetNode(&node)

7
plugin/pkg/scheduler/algorithm/priorities/metadata.go
View File

@@ -38,10 +38,15 @@ func PriorityMetadata(pod *v1.Pod, nodeNameToInfo map[string]*schedulercache.Nod
if err != nil {
return nil
}
affinity, err := v1.GetAffinityFromPodAnnotations(pod.Annotations)
affinity := pod.Spec.Affinity
annotationAffinity, err := v1.GetAffinityFromPodAnnotations(pod.Annotations)
if err != nil {
return nil
}
if annotationAffinity != nil {
affinity.PodAffinity = annotationAffinity.PodAffinity
affinity.PodAntiAffinity = annotationAffinity.PodAntiAffinity
}
return &priorityMetadata{
nonZeroRequest: getNonZeroRequests(pod),
podTolerations: tolerations,

8
plugin/pkg/scheduler/algorithm/priorities/node_affinity.go
View File

@@ -41,12 +41,8 @@ func CalculateNodeAffinityPriorityMap(pod *v1.Pod, meta interface{}, nodeInfo *s
if priorityMeta, ok := meta.(*priorityMetadata); ok {
affinity = priorityMeta.affinity
} else {
// We couldn't parse metadata - fallback to computing it.
var err error
affinity, err = v1.GetAffinityFromPodAnnotations(pod.Annotations)
if err != nil {
return schedulerapi.HostPriority{}, err
}
// We couldn't parse metadata - fallback to the podspec.
affinity = pod.Spec.Affinity
}
var count int32

106
plugin/pkg/scheduler/algorithm/priorities/node_affinity_test.go
View File

@@ -32,62 +32,72 @@ func TestNodeAffinityPriority(t *testing.T) {
label4 := map[string]string{"abc": "az11", "def": "az22"}
label5 := map[string]string{"foo": "bar", "key": "value", "az": "az1"}
affinity1 := map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": [
{
"weight": 2,
"preference": {
"matchExpressions": [
{
"key": "foo",
"operator": "In", "values": ["bar"]
}
]
}
}
]}}`,
affinity1 := &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{{
Weight: 2,
Preference: v1.NodeSelectorTerm{
MatchExpressions: []v1.NodeSelectorRequirement{{
Key: "foo",
Operator: v1.NodeSelectorOpIn,
Values: []string{"bar"},
}},
},
}},
},
}
affinity2 := map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": [
affinity2 := &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{
{
"weight": 2,
"preference": {"matchExpressions": [
Weight: 2,
Preference: v1.NodeSelectorTerm{
MatchExpressions: []v1.NodeSelectorRequirement{
{
"key": "foo",
"operator": "In", "values": ["bar"]
}
]}
Key: "foo",
Operator: v1.NodeSelectorOpIn,
Values: []string{"bar"},
},
},
},
},
{
"weight": 4,
"preference": {"matchExpressions": [
Weight: 4,
Preference: v1.NodeSelectorTerm{
MatchExpressions: []v1.NodeSelectorRequirement{
{
"key": "key",
"operator": "In", "values": ["value"]
}
]}
Key: "key",
Operator: v1.NodeSelectorOpIn,
Values: []string{"value"},
},
},
},
},
{
"weight": 5,
"preference": {"matchExpressions": [
Weight: 5,
Preference: v1.NodeSelectorTerm{
MatchExpressions: []v1.NodeSelectorRequirement{
{
"key": "foo",
"operator": "In", "values": ["bar"]
Key: "foo",
Operator: v1.NodeSelectorOpIn,
Values: []string{"bar"},
},
{
"key": "key",
"operator": "In", "values": ["value"]
Key: "key",
Operator: v1.NodeSelectorOpIn,
Values: []string{"value"},
},
{
"key": "az",
"operator": "In", "values": ["az1"]
}
]}
}
]}}`,
Key: "az",
Operator: v1.NodeSelectorOpIn,
Values: []string{"az1"},
},
},
},
},
},
},
}
tests := []struct {
@@ -112,8 +122,8 @@ func TestNodeAffinityPriority(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: affinity1,
Spec: v1.PodSpec{
Affinity: affinity1,
},
},
nodes: []*v1.Node{
@@ -126,8 +136,8 @@ func TestNodeAffinityPriority(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: affinity1,
Spec: v1.PodSpec{
Affinity: affinity1,
},
},
nodes: []*v1.Node{
@@ -140,8 +150,8 @@ func TestNodeAffinityPriority(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: affinity2,
Spec: v1.PodSpec{
Affinity: affinity2,
},
},
nodes: []*v1.Node{

2
staging/src/k8s.io/client-go/pkg/api/types.go
View File

@@ -1824,6 +1824,8 @@ type PodSpec struct {
// If not specified, the pod will not have a domainname at all.
// +optional
Subdomain string `json:"subdomain,omitempty"`
// If specified, the pod's scheduling constraints
Affinity *Affinity `json:"affinity,omitempty"`
}
// Sysctl defines a kernel parameter to be set

3
staging/src/k8s.io/client-go/pkg/api/v1/defaults.go
View File

@@ -175,6 +175,9 @@ func SetDefaults_PodSpec(obj *PodSpec) {
period := int64(DefaultTerminationGracePeriodSeconds)
obj.TerminationGracePeriodSeconds = &period
}
if obj.Affinity == nil {
obj.Affinity = &Affinity{}
}
}
func SetDefaults_Probe(obj *Probe) {
if obj.TimeoutSeconds == 0 {

2
staging/src/k8s.io/client-go/pkg/api/v1/types.go
View File

@@ -2107,6 +2107,8 @@ type PodSpec struct {
// If not specified, the pod will not have a domainname at all.
// +optional
Subdomain string `json:"subdomain,omitempty" protobuf:"bytes,17,opt,name=subdomain"`
// If specified, the pod's scheduling constraints
Affinity *Affinity `json:"affinity,omitempty" protobuf:"bytes,18,opt,name=affinity"`
}
// PodSecurityContext holds pod-level security attributes and common container settings.

29
test/e2e/daemon_set.go
View File

@@ -209,17 +209,22 @@ var _ = framework.KubeDescribe("Daemon set [Serial]", func() {
complexLabel := map[string]string{daemonsetNameLabel: dsName}
nodeSelector := map[string]string{daemonsetColorLabel: "blue"}
framework.Logf("Creating daemon with a node affinity %s", dsName)
affinity := map[string]string{
v1.AffinityAnnotationKey: fmt.Sprintf(`
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "%s",
"operator": "In",
"values": ["%s"]
}]
}]
}}}`, daemonsetColorLabel, nodeSelector[daemonsetColorLabel]),
affinity := &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: daemonsetColorLabel,
Operator: v1.NodeSelectorOpIn,
Values: []string{nodeSelector[daemonsetColorLabel]},
},
},
},
},
},
},
}
_, err := c.Extensions().DaemonSets(ns).Create(&extensions.DaemonSet{
ObjectMeta: v1.ObjectMeta{
@@ -230,9 +235,9 @@ var _ = framework.KubeDescribe("Daemon set [Serial]", func() {
Template: v1.PodTemplateSpec{
ObjectMeta: v1.ObjectMeta{
Labels: complexLabel,
Annotations: affinity,
},
Spec: v1.PodSpec{
Affinity: affinity,
Containers: []v1.Container{
{
Name: dsName,

29
test/e2e/empty_dir_wrapper.go
View File

@@ -314,17 +314,22 @@ func testNoWrappedVolumeRace(f *framework.Framework, volumes []v1.Volume, volume
targetNode := nodeList.Items[0]
By("Creating RC which spawns configmap-volume pods")
affinity := map[string]string{
v1.AffinityAnnotationKey: fmt.Sprintf(`
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "kubernetes.io/hostname",
"operator": "In",
"values": ["%s"]
}]
}]
}}}`, targetNode.Name),
affinity := &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "kubernetes.io/hostname",
Operator: v1.NodeSelectorOpIn,
Values: []string{targetNode.Name},
},
},
},
},
},
},
}
rc := &v1.ReplicationController{
@@ -338,7 +343,6 @@ func testNoWrappedVolumeRace(f *framework.Framework, volumes []v1.Volume, volume
},
Template: &v1.PodTemplateSpec{
ObjectMeta: v1.ObjectMeta{
Annotations: affinity,
Labels: map[string]string{"name": rcName},
},
Spec: v1.PodSpec{
@@ -355,6 +359,7 @@ func testNoWrappedVolumeRace(f *framework.Framework, volumes []v1.Volume, volume
VolumeMounts: volumeMounts,
},
},
Affinity: affinity,
DNSPolicy: v1.DNSDefault,
Volumes: volumes,
},

152
test/e2e/scheduler_predicates.go
View File

@@ -45,6 +45,7 @@ var masterNodes sets.String
type pausePodConfig struct {
Name string
Affinity string
NodeAffinity *v1.Affinity
Annotations, Labels, NodeSelector map[string]string
Resources *v1.ResourceRequirements
}
@@ -240,15 +241,17 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
podName := "without-label"
_, err := cs.Core().Pods(ns).Create(initPausePod(f, pausePodConfig{
Name: podName,
Affinity: `{
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": []
}]
NodeAffinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{},
},
},
},
},
},
}
}`,
}))
if err == nil || !errors.IsInvalid(err) {
@@ -300,28 +303,31 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
createPausePod(f, pausePodConfig{
Name: podName,
Affinity: `{
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [
NodeAffinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
"matchExpressions": [{
"key": "foo",
"operator": "In",
"values": ["bar", "value2"]
}]
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "foo",
Operator: v1.NodeSelectorOpIn,
Values: []string{"bar", "value2"},
},
},
}, {
MatchExpressions: []v1.NodeSelectorRequirement{
{
"matchExpressions": [{
"key": "diffkey",
"operator": "In",
"values": ["wrong", "value2"]
}]
}
]
}
}
}`,
Key: "diffkey",
Operator: v1.NodeSelectorOpIn,
Values: []string{"wrong", "value2"},
},
},
},
},
},
},
},
Labels: map[string]string{"name": "restricted"},
})
waitForScheduler()
@@ -344,23 +350,27 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
labelPodName := "with-labels"
pod := createPausePod(f, pausePodConfig{
Name: labelPodName,
Affinity: `{
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "kubernetes.io/hostname",
"operator": "In",
"values": ["` + nodeName + `"]
NodeAffinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "kubernetes.io/hostname",
Operator: v1.NodeSelectorOpIn,
Values: []string{nodeName},
}, {
"key": "` + k + `",
"operator": "In",
"values": ["` + v + `"]
}]
}]
}
}
}`,
Key: k,
Operator: v1.NodeSelectorOpIn,
Values: []string{v},
},
},
},
},
},
},
},
})
// check that pod got scheduled. We intentionally DO NOT check that the
@@ -374,31 +384,6 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
Expect(labelPod.Spec.NodeName).To(Equal(nodeName))
})
// Verify that an escaped JSON string of NodeAffinity in a YAML PodSpec works.
It("validates that embedding the JSON NodeAffinity setting as a string in the annotation value work", func() {
nodeName := getNodeThatCanRunPod(f)
By("Trying to apply a label with fake az info on the found node.")
k := "kubernetes.io/e2e-az-name"
v := "e2e-az1"
framework.AddOrUpdateLabelOnNode(cs, nodeName, k, v)
framework.ExpectNodeHasLabel(cs, nodeName, k, v)
defer framework.RemoveLabelOffNode(cs, nodeName, k)
By("Trying to launch a pod that with NodeAffinity setting as embedded JSON string in the annotation value.")
pod := createPodWithNodeAffinity(f)
// check that pod got scheduled. We intentionally DO NOT check that the
// pod is running because this will create a race condition with the
// kubelet and the scheduler: the scheduler might have scheduled a pod
// already when the kubelet does not know about its new label yet. The
// kubelet will then refuse to launch the pod.
framework.ExpectNoError(framework.WaitForPodNotPending(cs, ns, pod.Name, ""))
labelPod, err := cs.Core().Pods(ns).Get(pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err)
Expect(labelPod.Spec.NodeName).To(Equal(nodeName))
})
// labelSelector Operator is DoesNotExist but values are there in requiredDuringSchedulingIgnoredDuringExecution
// part of podAffinity,so validation fails.
It("validates that a pod with an invalid podAffinity is rejected because of the LabelSelectorRequirement is invalid", func() {
@@ -777,6 +762,7 @@ func initPausePod(f *framework.Framework, conf pausePodConfig) *v1.Pod {
},
Spec: v1.PodSpec{
NodeSelector: conf.NodeSelector,
Affinity: conf.NodeAffinity,
Containers: []v1.Container{
{
Name: podName,
@@ -822,19 +808,23 @@ func runPodAndGetNodeName(f *framework.Framework, conf pausePodConfig) string {
func createPodWithNodeAffinity(f *framework.Framework) *v1.Pod {
return createPausePod(f, pausePodConfig{
Name: "with-nodeaffinity-" + string(uuid.NewUUID()),
Affinity: `{
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "kubernetes.io/e2e-az-name",
"operator": "In",
"values": ["e2e-az1", "e2e-az2"]
}]
}]
}
}
}`,
NodeAffinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "kubernetes.io/e2e-az-name",
Operator: v1.NodeSelectorOpIn,
Values: []string{"e2e-az1", "e2e-az2"},
},
},
},
},
},
},
},
})
}