63d1237102
Given that we give a default CPU/memory requests for containers that don't provide any, the calculated usage can exceed the allocatable. Change-Id: I72e249652acacfbe8cea0dd6f895dabe43ff6376
423 lines
14 KiB
Go
423 lines
14 KiB
Go
/*
|
|
Copyright 2019 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package noderesources
|
|
|
|
import (
|
|
"context"
|
|
"reflect"
|
|
"testing"
|
|
|
|
v1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
|
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
|
"k8s.io/kubernetes/pkg/features"
|
|
"k8s.io/kubernetes/pkg/scheduler/framework"
|
|
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature"
|
|
"k8s.io/kubernetes/pkg/scheduler/framework/runtime"
|
|
"k8s.io/kubernetes/pkg/scheduler/internal/cache"
|
|
)
|
|
|
|
// getExistingVolumeCountForNode gets the current number of volumes on node.
|
|
func getExistingVolumeCountForNode(podInfos []*framework.PodInfo, maxVolumes int) int {
|
|
volumeCount := 0
|
|
for _, p := range podInfos {
|
|
volumeCount += len(p.Pod.Spec.Volumes)
|
|
}
|
|
if maxVolumes-volumeCount > 0 {
|
|
return maxVolumes - volumeCount
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func TestNodeResourcesBalancedAllocation(t *testing.T) {
|
|
// Enable volumesOnNodeForBalancing to do balanced node resource allocation
|
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.BalanceAttachedNodeVolumes, true)()
|
|
podwithVol1 := v1.PodSpec{
|
|
Containers: []v1.Container{
|
|
{
|
|
Resources: v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("1000m"),
|
|
v1.ResourceMemory: resource.MustParse("2000"),
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Resources: v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("2000m"),
|
|
v1.ResourceMemory: resource.MustParse("3000"),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Volumes: []v1.Volume{
|
|
{
|
|
VolumeSource: v1.VolumeSource{
|
|
AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "ovp"},
|
|
},
|
|
},
|
|
},
|
|
NodeName: "machine4",
|
|
}
|
|
podwithVol2 := v1.PodSpec{
|
|
Containers: []v1.Container{
|
|
{
|
|
Resources: v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("0m"),
|
|
v1.ResourceMemory: resource.MustParse("0"),
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Resources: v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("0m"),
|
|
v1.ResourceMemory: resource.MustParse("0"),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Volumes: []v1.Volume{
|
|
{
|
|
VolumeSource: v1.VolumeSource{
|
|
AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "ovp1"},
|
|
},
|
|
},
|
|
},
|
|
NodeName: "machine4",
|
|
}
|
|
podwithVol3 := v1.PodSpec{
|
|
Containers: []v1.Container{
|
|
{
|
|
Resources: v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("0m"),
|
|
v1.ResourceMemory: resource.MustParse("0"),
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Resources: v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("0m"),
|
|
v1.ResourceMemory: resource.MustParse("0"),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Volumes: []v1.Volume{
|
|
{
|
|
VolumeSource: v1.VolumeSource{
|
|
AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "ovp1"},
|
|
},
|
|
},
|
|
},
|
|
NodeName: "machine4",
|
|
}
|
|
labels1 := map[string]string{
|
|
"foo": "bar",
|
|
"baz": "blah",
|
|
}
|
|
labels2 := map[string]string{
|
|
"bar": "foo",
|
|
"baz": "blah",
|
|
}
|
|
machine1Spec := v1.PodSpec{
|
|
NodeName: "machine1",
|
|
}
|
|
machine2Spec := v1.PodSpec{
|
|
NodeName: "machine2",
|
|
}
|
|
noResources := v1.PodSpec{
|
|
Containers: []v1.Container{},
|
|
}
|
|
cpuOnly := v1.PodSpec{
|
|
NodeName: "machine1",
|
|
Containers: []v1.Container{
|
|
{
|
|
Resources: v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("1000m"),
|
|
v1.ResourceMemory: resource.MustParse("0"),
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Resources: v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("2000m"),
|
|
v1.ResourceMemory: resource.MustParse("0"),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
cpuOnly2 := cpuOnly
|
|
cpuOnly2.NodeName = "machine2"
|
|
cpuAndMemory := v1.PodSpec{
|
|
NodeName: "machine2",
|
|
Containers: []v1.Container{
|
|
{
|
|
Resources: v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("1000m"),
|
|
v1.ResourceMemory: resource.MustParse("2000"),
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Resources: v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("2000m"),
|
|
v1.ResourceMemory: resource.MustParse("3000"),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
cpuAndMemory3 := v1.PodSpec{
|
|
NodeName: "machine3",
|
|
Containers: []v1.Container{
|
|
{
|
|
Resources: v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("1000m"),
|
|
v1.ResourceMemory: resource.MustParse("2000"),
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Resources: v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("2000m"),
|
|
v1.ResourceMemory: resource.MustParse("3000"),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
nonZeroContainer := v1.PodSpec{
|
|
Containers: []v1.Container{{}},
|
|
}
|
|
nonZeroContainer1 := v1.PodSpec{
|
|
NodeName: "machine1",
|
|
Containers: []v1.Container{{}},
|
|
}
|
|
tests := []struct {
|
|
pod *v1.Pod
|
|
pods []*v1.Pod
|
|
nodes []*v1.Node
|
|
expectedList framework.NodeScoreList
|
|
name string
|
|
}{
|
|
{
|
|
// Node1 scores (remaining resources) on 0-MaxNodeScore scale
|
|
// CPU Fraction: 0 / 4000 = 0%
|
|
// Memory Fraction: 0 / 10000 = 0%
|
|
// Node1 Score: MaxNodeScore - (0-0)*MaxNodeScore = MaxNodeScore
|
|
// Node2 scores (remaining resources) on 0-MaxNodeScore scale
|
|
// CPU Fraction: 0 / 4000 = 0 %
|
|
// Memory Fraction: 0 / 10000 = 0%
|
|
// Node2 Score: MaxNodeScore - (0-0)*MaxNodeScore = MaxNodeScore
|
|
pod: &v1.Pod{Spec: noResources},
|
|
nodes: []*v1.Node{makeNode("machine1", 4000, 10000), makeNode("machine2", 4000, 10000)},
|
|
expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: framework.MaxNodeScore}},
|
|
name: "nothing scheduled, nothing requested",
|
|
},
|
|
{
|
|
// Node1 scores on 0-MaxNodeScore scale
|
|
// CPU Fraction: 3000 / 4000= 75%
|
|
// Memory Fraction: 5000 / 10000 = 50%
|
|
// Node1 Score: MaxNodeScore - (0.75-0.5)*MaxNodeScore = 75
|
|
// Node2 scores on 0-MaxNodeScore scale
|
|
// CPU Fraction: 3000 / 6000= 50%
|
|
// Memory Fraction: 5000/10000 = 50%
|
|
// Node2 Score: MaxNodeScore - (0.5-0.5)*MaxNodeScore = MaxNodeScore
|
|
pod: &v1.Pod{Spec: cpuAndMemory},
|
|
nodes: []*v1.Node{makeNode("machine1", 4000, 10000), makeNode("machine2", 6000, 10000)},
|
|
expectedList: []framework.NodeScore{{Name: "machine1", Score: 75}, {Name: "machine2", Score: framework.MaxNodeScore}},
|
|
name: "nothing scheduled, resources requested, differently sized machines",
|
|
},
|
|
{
|
|
// Node1 scores on 0-MaxNodeScore scale
|
|
// CPU Fraction: 0 / 4000= 0%
|
|
// Memory Fraction: 0 / 10000 = 0%
|
|
// Node1 Score: MaxNodeScore - (0-0)*MaxNodeScore = MaxNodeScore
|
|
// Node2 scores on 0-MaxNodeScore scale
|
|
// CPU Fraction: 0 / 4000= 0%
|
|
// Memory Fraction: 0 / 10000 = 0%
|
|
// Node2 Score: MaxNodeScore - (0-0)*MaxNodeScore= MaxNodeScore
|
|
pod: &v1.Pod{Spec: noResources},
|
|
nodes: []*v1.Node{makeNode("machine1", 4000, 10000), makeNode("machine2", 4000, 10000)},
|
|
expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: framework.MaxNodeScore}},
|
|
name: "no resources requested, pods scheduled",
|
|
pods: []*v1.Pod{
|
|
{Spec: machine1Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels2}},
|
|
{Spec: machine1Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}},
|
|
{Spec: machine2Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}},
|
|
{Spec: machine2Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}},
|
|
},
|
|
},
|
|
{
|
|
// Node1 scores on 0-MaxNodeScore scale
|
|
// CPU Fraction: 300 / 250 = 100%
|
|
// Memory Fraction: 600 / 10000 = 60%
|
|
// Node1 Score: MaxNodeScore - (100-60)*MaxNodeScore = 60
|
|
// Node2 scores on 0-MaxNodeScore scale
|
|
// CPU Fraction: 100 / 250 = 40%
|
|
// Memory Fraction: 200 / 10000 = 20%
|
|
// Node2 Score: MaxNodeScore - (40-20)*MaxNodeScore= 80
|
|
pod: &v1.Pod{Spec: nonZeroContainer},
|
|
nodes: []*v1.Node{makeNode("machine1", 250, 1000*1024*1024), makeNode("machine2", 250, 1000*1024*1024)},
|
|
expectedList: []framework.NodeScore{{Name: "machine1", Score: 60}, {Name: "machine2", Score: 80}},
|
|
name: "no resources requested, pods scheduled",
|
|
pods: []*v1.Pod{
|
|
{Spec: nonZeroContainer1},
|
|
{Spec: nonZeroContainer1},
|
|
},
|
|
},
|
|
{
|
|
// Node1 scores on 0-MaxNodeScore scale
|
|
// CPU Fraction: 6000 / 10000 = 60%
|
|
// Memory Fraction: 0 / 20000 = 0%
|
|
// Node1 Score: MaxNodeScore - (0.6-0)*MaxNodeScore = 40
|
|
// Node2 scores on 0-MaxNodeScore scale
|
|
// CPU Fraction: 6000 / 10000 = 60%
|
|
// Memory Fraction: 5000 / 20000 = 25%
|
|
// Node2 Score: MaxNodeScore - (0.6-0.25)*MaxNodeScore = 65
|
|
pod: &v1.Pod{Spec: noResources},
|
|
nodes: []*v1.Node{makeNode("machine1", 10000, 20000), makeNode("machine2", 10000, 20000)},
|
|
expectedList: []framework.NodeScore{{Name: "machine1", Score: 40}, {Name: "machine2", Score: 65}},
|
|
name: "no resources requested, pods scheduled with resources",
|
|
pods: []*v1.Pod{
|
|
{Spec: cpuOnly, ObjectMeta: metav1.ObjectMeta{Labels: labels2}},
|
|
{Spec: cpuOnly, ObjectMeta: metav1.ObjectMeta{Labels: labels1}},
|
|
{Spec: cpuOnly2, ObjectMeta: metav1.ObjectMeta{Labels: labels1}},
|
|
{Spec: cpuAndMemory, ObjectMeta: metav1.ObjectMeta{Labels: labels1}},
|
|
},
|
|
},
|
|
{
|
|
// Node1 scores on 0-MaxNodeScore scale
|
|
// CPU Fraction: 6000 / 10000 = 60%
|
|
// Memory Fraction: 5000 / 20000 = 25%
|
|
// Node1 Score: MaxNodeScore - (0.6-0.25)*MaxNodeScore = 65
|
|
// Node2 scores on 0-MaxNodeScore scale
|
|
// CPU Fraction: 6000 / 10000 = 60%
|
|
// Memory Fraction: 10000 / 20000 = 50%
|
|
// Node2 Score: MaxNodeScore - (0.6-0.5)*MaxNodeScore = 90
|
|
pod: &v1.Pod{Spec: cpuAndMemory},
|
|
nodes: []*v1.Node{makeNode("machine1", 10000, 20000), makeNode("machine2", 10000, 20000)},
|
|
expectedList: []framework.NodeScore{{Name: "machine1", Score: 65}, {Name: "machine2", Score: 90}},
|
|
name: "resources requested, pods scheduled with resources",
|
|
pods: []*v1.Pod{
|
|
{Spec: cpuOnly},
|
|
{Spec: cpuAndMemory},
|
|
},
|
|
},
|
|
{
|
|
// Node1 scores on 0-MaxNodeScore scale
|
|
// CPU Fraction: 6000 / 10000 = 60%
|
|
// Memory Fraction: 5000 / 20000 = 25%
|
|
// Node1 Score: MaxNodeScore - (0.6-0.25)*MaxNodeScore = 65
|
|
// Node2 scores on 0-MaxNodeScore scale
|
|
// CPU Fraction: 6000 / 10000 = 60%
|
|
// Memory Fraction: 10000 / 50000 = 20%
|
|
// Node2 Score: MaxNodeScore - (0.6-0.2)*MaxNodeScore = 60
|
|
pod: &v1.Pod{Spec: cpuAndMemory},
|
|
nodes: []*v1.Node{makeNode("machine1", 10000, 20000), makeNode("machine2", 10000, 50000)},
|
|
expectedList: []framework.NodeScore{{Name: "machine1", Score: 65}, {Name: "machine2", Score: 60}},
|
|
name: "resources requested, pods scheduled with resources, differently sized machines",
|
|
pods: []*v1.Pod{
|
|
{Spec: cpuOnly},
|
|
{Spec: cpuAndMemory},
|
|
},
|
|
},
|
|
{
|
|
// Node1 scores on 0-MaxNodeScore scale
|
|
// CPU Fraction: 6000 / 6000 = 1
|
|
// Memory Fraction: 0 / 10000 = 0
|
|
// Node1 Score: MaxNodeScore - (1 - 0) * MaxNodeScore = 0
|
|
// Node2 scores on 0-MaxNodeScore scale
|
|
// CPU Fraction: 6000 / 6000 = 1
|
|
// Memory Fraction 5000 / 10000 = 50%
|
|
// Node2 Score: MaxNodeScore - (1 - 0.5) * MaxNodeScore = 50
|
|
pod: &v1.Pod{Spec: cpuOnly},
|
|
nodes: []*v1.Node{makeNode("machine1", 6000, 10000), makeNode("machine2", 6000, 10000)},
|
|
expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: 50}},
|
|
name: "requested resources at node capacity",
|
|
pods: []*v1.Pod{
|
|
{Spec: cpuOnly},
|
|
{Spec: cpuAndMemory},
|
|
},
|
|
},
|
|
{
|
|
// Machine4 will be chosen here because it already has a existing volume making the variance
|
|
// of volume count, CPU usage, memory usage closer.
|
|
pod: &v1.Pod{
|
|
Spec: v1.PodSpec{
|
|
Volumes: []v1.Volume{
|
|
{
|
|
VolumeSource: v1.VolumeSource{
|
|
AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "ovp2"},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
nodes: []*v1.Node{makeNode("machine3", 3500, 40000), makeNode("machine4", 4000, 10000)},
|
|
expectedList: []framework.NodeScore{{Name: "machine3", Score: 89}, {Name: "machine4", Score: 98}},
|
|
name: "Include volume count on a node for balanced resource allocation",
|
|
pods: []*v1.Pod{
|
|
{Spec: cpuAndMemory3},
|
|
{Spec: podwithVol1},
|
|
{Spec: podwithVol2},
|
|
{Spec: podwithVol3},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
snapshot := cache.NewSnapshot(test.pods, test.nodes)
|
|
if len(test.pod.Spec.Volumes) > 0 {
|
|
maxVolumes := 5
|
|
nodeInfoList, _ := snapshot.NodeInfos().List()
|
|
for _, info := range nodeInfoList {
|
|
info.TransientInfo.TransNodeInfo.AllocatableVolumesCount = getExistingVolumeCountForNode(info.Pods, maxVolumes)
|
|
info.TransientInfo.TransNodeInfo.RequestedVolumes = len(test.pod.Spec.Volumes)
|
|
}
|
|
}
|
|
fh, _ := runtime.NewFramework(nil, nil, runtime.WithSnapshotSharedLister(snapshot))
|
|
p, _ := NewBalancedAllocation(nil, fh, feature.Features{EnablePodOverhead: true, EnableBalanceAttachedNodeVolumes: true})
|
|
|
|
for i := range test.nodes {
|
|
hostResult, err := p.(framework.ScorePlugin).Score(context.Background(), nil, test.pod, test.nodes[i].Name)
|
|
if err != nil {
|
|
t.Errorf("unexpected error: %v", err)
|
|
}
|
|
if !reflect.DeepEqual(test.expectedList[i].Score, hostResult) {
|
|
t.Errorf("got score %v for host %v, expected %v", hostResult, test.nodes[i].Name, test.expectedList[i].Score)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|