Merge pull request #54316 from dashpole/disk_request_eviction

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Take disk requests into account during evictions

fixes #54314

This PR is part of the local storage feature, and it makes the eviction manager take disk requests into account during disk evictions.
This uses the same eviction strategy as we do for memory.
Disk requests are only considered when the LocalStorageCapacityIsolation feature gate is enabled.  This is enforced by adding a check for the feature gate in getRequests().
I have added unit testing to ensure that previous behavior is preserved when the feature gate is disabled.
Most of the changes are testing.  Reviewers should focus on changes in **eviction/helpers.go**

/sig node
/assign @jingxu97  @vishh
This commit is contained in:
Kubernetes Submit Queue
2017-11-21 14:31:47 -08:00
committed by GitHub
4 changed files with 538 additions and 230 deletions

View File

@@ -27,6 +27,7 @@ import (
utilfeature "k8s.io/apiserver/pkg/util/feature" utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/tools/record" "k8s.io/client-go/tools/record"
kubeapi "k8s.io/kubernetes/pkg/apis/core" kubeapi "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/features"
statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api"
"k8s.io/kubernetes/pkg/kubelet/lifecycle" "k8s.io/kubernetes/pkg/kubelet/lifecycle"
@@ -184,15 +185,15 @@ type podToMake struct {
// TestMemoryPressure // TestMemoryPressure
func TestMemoryPressure(t *testing.T) { func TestMemoryPressure(t *testing.T) {
enablePodPriority(true) utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{string(features.PodPriority): true})
podMaker := makePodWithMemoryStats podMaker := makePodWithMemoryStats
summaryStatsMaker := makeMemoryStats summaryStatsMaker := makeMemoryStats
podsToMake := []podToMake{ podsToMake := []podToMake{
{name: "guaranteed-low-priority-high-usage", priority: lowPriority, requests: newResourceList("100m", "1Gi"), limits: newResourceList("100m", "1Gi"), memoryWorkingSet: "900Mi"}, {name: "guaranteed-low-priority-high-usage", priority: lowPriority, requests: newResourceList("100m", "1Gi", ""), limits: newResourceList("100m", "1Gi", ""), memoryWorkingSet: "900Mi"},
{name: "burstable-below-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi"), limits: newResourceList("200m", "1Gi"), memoryWorkingSet: "50Mi"}, {name: "burstable-below-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi", ""), limits: newResourceList("200m", "1Gi", ""), memoryWorkingSet: "50Mi"},
{name: "burstable-above-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi"), limits: newResourceList("200m", "1Gi"), memoryWorkingSet: "400Mi"}, {name: "burstable-above-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi", ""), limits: newResourceList("200m", "1Gi", ""), memoryWorkingSet: "400Mi"},
{name: "best-effort-high-priority-high-usage", priority: highPriority, requests: newResourceList("", ""), limits: newResourceList("", ""), memoryWorkingSet: "400Mi"}, {name: "best-effort-high-priority-high-usage", priority: highPriority, requests: newResourceList("", "", ""), limits: newResourceList("", "", ""), memoryWorkingSet: "400Mi"},
{name: "best-effort-low-priority-low-usage", priority: lowPriority, requests: newResourceList("", ""), limits: newResourceList("", ""), memoryWorkingSet: "100Mi"}, {name: "best-effort-low-priority-low-usage", priority: lowPriority, requests: newResourceList("", "", ""), limits: newResourceList("", "", ""), memoryWorkingSet: "100Mi"},
} }
pods := []*v1.Pod{} pods := []*v1.Pod{}
podStats := map[*v1.Pod]statsapi.PodStats{} podStats := map[*v1.Pod]statsapi.PodStats{}
@@ -248,8 +249,8 @@ func TestMemoryPressure(t *testing.T) {
} }
// create a best effort pod to test admission // create a best effort pod to test admission
bestEffortPodToAdmit, _ := podMaker("best-admit", defaultPriority, newResourceList("", ""), newResourceList("", ""), "0Gi") bestEffortPodToAdmit, _ := podMaker("best-admit", defaultPriority, newResourceList("", "", ""), newResourceList("", "", ""), "0Gi")
burstablePodToAdmit, _ := podMaker("burst-admit", defaultPriority, newResourceList("100m", "100Mi"), newResourceList("200m", "200Mi"), "0Gi") burstablePodToAdmit, _ := podMaker("burst-admit", defaultPriority, newResourceList("100m", "100Mi", ""), newResourceList("200m", "200Mi", ""), "0Gi")
// synchronize // synchronize
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider) manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
@@ -402,15 +403,18 @@ func parseQuantity(value string) resource.Quantity {
} }
func TestDiskPressureNodeFs(t *testing.T) { func TestDiskPressureNodeFs(t *testing.T) {
enablePodPriority(true) utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{
string(features.LocalStorageCapacityIsolation): true,
string(features.PodPriority): true,
})
podMaker := makePodWithDiskStats podMaker := makePodWithDiskStats
summaryStatsMaker := makeDiskStats summaryStatsMaker := makeDiskStats
podsToMake := []podToMake{ podsToMake := []podToMake{
{name: "low-priority-high-usage", priority: lowPriority, requests: newResourceList("100m", "1Gi"), limits: newResourceList("100m", "1Gi"), rootFsUsed: "900Mi"}, {name: "low-priority-high-usage", priority: lowPriority, requests: newResourceList("100m", "1Gi", ""), limits: newResourceList("100m", "1Gi", ""), rootFsUsed: "900Mi"},
{name: "below-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi"), limits: newResourceList("200m", "1Gi"), logsFsUsed: "50Mi"}, {name: "below-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi", ""), limits: newResourceList("200m", "1Gi", ""), logsFsUsed: "50Mi"},
{name: "above-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi"), limits: newResourceList("200m", "1Gi"), rootFsUsed: "400Mi"}, {name: "above-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi", ""), limits: newResourceList("200m", "1Gi", ""), rootFsUsed: "400Mi"},
{name: "high-priority-high-usage", priority: highPriority, requests: newResourceList("", ""), limits: newResourceList("", ""), perLocalVolumeUsed: "400Mi"}, {name: "high-priority-high-usage", priority: highPriority, requests: newResourceList("", "", ""), limits: newResourceList("", "", ""), perLocalVolumeUsed: "400Mi"},
{name: "low-priority-low-usage", priority: lowPriority, requests: newResourceList("", ""), limits: newResourceList("", ""), rootFsUsed: "100Mi"}, {name: "low-priority-low-usage", priority: lowPriority, requests: newResourceList("", "", ""), limits: newResourceList("", "", ""), rootFsUsed: "100Mi"},
} }
pods := []*v1.Pod{} pods := []*v1.Pod{}
podStats := map[*v1.Pod]statsapi.PodStats{} podStats := map[*v1.Pod]statsapi.PodStats{}
@@ -467,7 +471,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
} }
// create a best effort pod to test admission // create a best effort pod to test admission
podToAdmit, _ := podMaker("pod-to-admit", defaultPriority, newResourceList("", ""), newResourceList("", ""), "0Gi", "0Gi", "0Gi") podToAdmit, _ := podMaker("pod-to-admit", defaultPriority, newResourceList("", "", ""), newResourceList("", "", ""), "0Gi", "0Gi", "0Gi")
// synchronize // synchronize
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider) manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
@@ -601,15 +605,15 @@ func TestDiskPressureNodeFs(t *testing.T) {
// TestMinReclaim verifies that min-reclaim works as desired. // TestMinReclaim verifies that min-reclaim works as desired.
func TestMinReclaim(t *testing.T) { func TestMinReclaim(t *testing.T) {
enablePodPriority(true) utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{string(features.PodPriority): true})
podMaker := makePodWithMemoryStats podMaker := makePodWithMemoryStats
summaryStatsMaker := makeMemoryStats summaryStatsMaker := makeMemoryStats
podsToMake := []podToMake{ podsToMake := []podToMake{
{name: "guaranteed-low-priority-high-usage", priority: lowPriority, requests: newResourceList("100m", "1Gi"), limits: newResourceList("100m", "1Gi"), memoryWorkingSet: "900Mi"}, {name: "guaranteed-low-priority-high-usage", priority: lowPriority, requests: newResourceList("100m", "1Gi", ""), limits: newResourceList("100m", "1Gi", ""), memoryWorkingSet: "900Mi"},
{name: "burstable-below-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi"), limits: newResourceList("200m", "1Gi"), memoryWorkingSet: "50Mi"}, {name: "burstable-below-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi", ""), limits: newResourceList("200m", "1Gi", ""), memoryWorkingSet: "50Mi"},
{name: "burstable-above-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi"), limits: newResourceList("200m", "1Gi"), memoryWorkingSet: "400Mi"}, {name: "burstable-above-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi", ""), limits: newResourceList("200m", "1Gi", ""), memoryWorkingSet: "400Mi"},
{name: "best-effort-high-priority-high-usage", priority: highPriority, requests: newResourceList("", ""), limits: newResourceList("", ""), memoryWorkingSet: "400Mi"}, {name: "best-effort-high-priority-high-usage", priority: highPriority, requests: newResourceList("", "", ""), limits: newResourceList("", "", ""), memoryWorkingSet: "400Mi"},
{name: "best-effort-low-priority-low-usage", priority: lowPriority, requests: newResourceList("", ""), limits: newResourceList("", ""), memoryWorkingSet: "100Mi"}, {name: "best-effort-low-priority-low-usage", priority: lowPriority, requests: newResourceList("", "", ""), limits: newResourceList("", "", ""), memoryWorkingSet: "100Mi"},
} }
pods := []*v1.Pod{} pods := []*v1.Pod{}
podStats := map[*v1.Pod]statsapi.PodStats{} podStats := map[*v1.Pod]statsapi.PodStats{}
@@ -741,15 +745,18 @@ func TestMinReclaim(t *testing.T) {
} }
func TestNodeReclaimFuncs(t *testing.T) { func TestNodeReclaimFuncs(t *testing.T) {
enablePodPriority(true) utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{
string(features.PodPriority): true,
string(features.LocalStorageCapacityIsolation): true,
})
podMaker := makePodWithDiskStats podMaker := makePodWithDiskStats
summaryStatsMaker := makeDiskStats summaryStatsMaker := makeDiskStats
podsToMake := []podToMake{ podsToMake := []podToMake{
{name: "low-priority-high-usage", priority: lowPriority, requests: newResourceList("100m", "1Gi"), limits: newResourceList("100m", "1Gi"), rootFsUsed: "900Mi"}, {name: "low-priority-high-usage", priority: lowPriority, requests: newResourceList("100m", "1Gi", ""), limits: newResourceList("100m", "1Gi", ""), rootFsUsed: "900Mi"},
{name: "below-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi"), limits: newResourceList("200m", "1Gi"), logsFsUsed: "50Mi"}, {name: "below-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi", ""), limits: newResourceList("200m", "1Gi", ""), logsFsUsed: "50Mi"},
{name: "above-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi"), limits: newResourceList("200m", "1Gi"), rootFsUsed: "400Mi"}, {name: "above-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi", ""), limits: newResourceList("200m", "1Gi", ""), rootFsUsed: "400Mi"},
{name: "high-priority-high-usage", priority: highPriority, requests: newResourceList("", ""), limits: newResourceList("", ""), perLocalVolumeUsed: "400Mi"}, {name: "high-priority-high-usage", priority: highPriority, requests: newResourceList("", "", ""), limits: newResourceList("", "", ""), perLocalVolumeUsed: "400Mi"},
{name: "low-priority-low-usage", priority: lowPriority, requests: newResourceList("", ""), limits: newResourceList("", ""), rootFsUsed: "100Mi"}, {name: "low-priority-low-usage", priority: lowPriority, requests: newResourceList("", "", ""), limits: newResourceList("", "", ""), rootFsUsed: "100Mi"},
} }
pods := []*v1.Pod{} pods := []*v1.Pod{}
podStats := map[*v1.Pod]statsapi.PodStats{} podStats := map[*v1.Pod]statsapi.PodStats{}
@@ -915,7 +922,7 @@ func TestNodeReclaimFuncs(t *testing.T) {
} }
func TestInodePressureNodeFsInodes(t *testing.T) { func TestInodePressureNodeFsInodes(t *testing.T) {
enablePodPriority(true) utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{string(features.PodPriority): true})
podMaker := func(name string, priority int32, requests v1.ResourceList, limits v1.ResourceList, rootInodes, logInodes, volumeInodes string) (*v1.Pod, statsapi.PodStats) { podMaker := func(name string, priority int32, requests v1.ResourceList, limits v1.ResourceList, rootInodes, logInodes, volumeInodes string) (*v1.Pod, statsapi.PodStats) {
pod := newPod(name, priority, []v1.Container{ pod := newPod(name, priority, []v1.Container{
newContainer(name, requests, limits), newContainer(name, requests, limits),
@@ -943,11 +950,11 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
return result return result
} }
podsToMake := []podToMake{ podsToMake := []podToMake{
{name: "low-priority-high-usage", priority: lowPriority, requests: newResourceList("100m", "1Gi"), limits: newResourceList("100m", "1Gi"), rootFsInodesUsed: "900Mi"}, {name: "low-priority-high-usage", priority: lowPriority, requests: newResourceList("100m", "1Gi", ""), limits: newResourceList("100m", "1Gi", ""), rootFsInodesUsed: "900Mi"},
{name: "below-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi"), limits: newResourceList("200m", "1Gi"), rootFsInodesUsed: "50Mi"}, {name: "below-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi", ""), limits: newResourceList("200m", "1Gi", ""), rootFsInodesUsed: "50Mi"},
{name: "above-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi"), limits: newResourceList("200m", "1Gi"), rootFsInodesUsed: "400Mi"}, {name: "above-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi", ""), limits: newResourceList("200m", "1Gi", ""), rootFsInodesUsed: "400Mi"},
{name: "high-priority-high-usage", priority: highPriority, requests: newResourceList("", ""), limits: newResourceList("", ""), rootFsInodesUsed: "400Mi"}, {name: "high-priority-high-usage", priority: highPriority, requests: newResourceList("", "", ""), limits: newResourceList("", "", ""), rootFsInodesUsed: "400Mi"},
{name: "low-priority-low-usage", priority: lowPriority, requests: newResourceList("", ""), limits: newResourceList("", ""), rootFsInodesUsed: "100Mi"}, {name: "low-priority-low-usage", priority: lowPriority, requests: newResourceList("", "", ""), limits: newResourceList("", "", ""), rootFsInodesUsed: "100Mi"},
} }
pods := []*v1.Pod{} pods := []*v1.Pod{}
podStats := map[*v1.Pod]statsapi.PodStats{} podStats := map[*v1.Pod]statsapi.PodStats{}
@@ -1004,7 +1011,7 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
} }
// create a best effort pod to test admission // create a best effort pod to test admission
podToAdmit, _ := podMaker("pod-to-admit", defaultPriority, newResourceList("", ""), newResourceList("", ""), "0", "0", "0") podToAdmit, _ := podMaker("pod-to-admit", defaultPriority, newResourceList("", "", ""), newResourceList("", "", ""), "0", "0", "0")
// synchronize // synchronize
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider) manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
@@ -1138,11 +1145,11 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
// TestCriticalPodsAreNotEvicted // TestCriticalPodsAreNotEvicted
func TestCriticalPodsAreNotEvicted(t *testing.T) { func TestCriticalPodsAreNotEvicted(t *testing.T) {
enablePodPriority(true) utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{string(features.PodPriority): true})
podMaker := makePodWithMemoryStats podMaker := makePodWithMemoryStats
summaryStatsMaker := makeMemoryStats summaryStatsMaker := makeMemoryStats
podsToMake := []podToMake{ podsToMake := []podToMake{
{name: "critical", priority: defaultPriority, requests: newResourceList("100m", "1Gi"), limits: newResourceList("100m", "1Gi"), memoryWorkingSet: "800Mi"}, {name: "critical", priority: defaultPriority, requests: newResourceList("100m", "1Gi", ""), limits: newResourceList("100m", "1Gi", ""), memoryWorkingSet: "800Mi"},
} }
pods := []*v1.Pod{} pods := []*v1.Pod{}
podStats := map[*v1.Pod]statsapi.PodStats{} podStats := map[*v1.Pod]statsapi.PodStats{}
@@ -1209,7 +1216,7 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
} }
// Enable critical pod annotation feature gate // Enable critical pod annotation feature gate
utilfeature.DefaultFeatureGate.Set("ExperimentalCriticalPodAnnotation=True") utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{string(features.ExperimentalCriticalPodAnnotation): true})
// induce soft threshold // induce soft threshold
fakeClock.Step(1 * time.Minute) fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("1500Mi", podStats) summaryProvider.result = summaryStatsMaker("1500Mi", podStats)
@@ -1254,7 +1261,7 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
} }
// Disable critical pod annotation feature gate // Disable critical pod annotation feature gate
utilfeature.DefaultFeatureGate.Set("ExperimentalCriticalPodAnnotation=False") utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{string(features.ExperimentalCriticalPodAnnotation): false})
// induce memory pressure! // induce memory pressure!
fakeClock.Step(1 * time.Minute) fakeClock.Step(1 * time.Minute)
@@ -1274,16 +1281,16 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
// TestAllocatableMemoryPressure // TestAllocatableMemoryPressure
func TestAllocatableMemoryPressure(t *testing.T) { func TestAllocatableMemoryPressure(t *testing.T) {
enablePodPriority(true) utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{string(features.PodPriority): true})
podMaker := makePodWithMemoryStats podMaker := makePodWithMemoryStats
summaryStatsMaker := makeMemoryStats summaryStatsMaker := makeMemoryStats
constantCapacity := "4Gi" constantCapacity := "4Gi"
podsToMake := []podToMake{ podsToMake := []podToMake{
{name: "guaranteed-low-priority-high-usage", priority: lowPriority, requests: newResourceList("100m", "1Gi"), limits: newResourceList("100m", "1Gi"), memoryWorkingSet: "900Mi"}, {name: "guaranteed-low-priority-high-usage", priority: lowPriority, requests: newResourceList("100m", "1Gi", ""), limits: newResourceList("100m", "1Gi", ""), memoryWorkingSet: "900Mi"},
{name: "burstable-below-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi"), limits: newResourceList("200m", "1Gi"), memoryWorkingSet: "50Mi"}, {name: "burstable-below-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi", ""), limits: newResourceList("200m", "1Gi", ""), memoryWorkingSet: "50Mi"},
{name: "burstable-above-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi"), limits: newResourceList("200m", "1Gi"), memoryWorkingSet: "400Mi"}, {name: "burstable-above-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi", ""), limits: newResourceList("200m", "1Gi", ""), memoryWorkingSet: "400Mi"},
{name: "best-effort-high-priority-high-usage", priority: highPriority, requests: newResourceList("", ""), limits: newResourceList("", ""), memoryWorkingSet: "400Mi"}, {name: "best-effort-high-priority-high-usage", priority: highPriority, requests: newResourceList("", "", ""), limits: newResourceList("", "", ""), memoryWorkingSet: "400Mi"},
{name: "best-effort-low-priority-low-usage", priority: lowPriority, requests: newResourceList("", ""), limits: newResourceList("", ""), memoryWorkingSet: "100Mi"}, {name: "best-effort-low-priority-low-usage", priority: lowPriority, requests: newResourceList("", "", ""), limits: newResourceList("", "", ""), memoryWorkingSet: "100Mi"},
} }
pods := []*v1.Pod{} pods := []*v1.Pod{}
podStats := map[*v1.Pod]statsapi.PodStats{} podStats := map[*v1.Pod]statsapi.PodStats{}
@@ -1332,8 +1339,8 @@ func TestAllocatableMemoryPressure(t *testing.T) {
} }
// create a best effort pod to test admission // create a best effort pod to test admission
bestEffortPodToAdmit, _ := podMaker("best-admit", defaultPriority, newResourceList("", ""), newResourceList("", ""), "0Gi") bestEffortPodToAdmit, _ := podMaker("best-admit", defaultPriority, newResourceList("", "", ""), newResourceList("", "", ""), "0Gi")
burstablePodToAdmit, _ := podMaker("burst-admit", defaultPriority, newResourceList("100m", "100Mi"), newResourceList("200m", "200Mi"), "0Gi") burstablePodToAdmit, _ := podMaker("burst-admit", defaultPriority, newResourceList("100m", "100Mi", ""), newResourceList("200m", "200Mi", ""), "0Gi")
// synchronize // synchronize
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider) manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
@@ -1353,7 +1360,7 @@ func TestAllocatableMemoryPressure(t *testing.T) {
// induce memory pressure! // induce memory pressure!
fakeClock.Step(1 * time.Minute) fakeClock.Step(1 * time.Minute)
pod, podStat := podMaker("guaranteed-high-2", defaultPriority, newResourceList("100m", "1Gi"), newResourceList("100m", "1Gi"), "1Gi") pod, podStat := podMaker("guaranteed-high-2", defaultPriority, newResourceList("100m", "1Gi", ""), newResourceList("100m", "1Gi", ""), "1Gi")
podStats[pod] = podStat podStats[pod] = podStat
summaryProvider.result = summaryStatsMaker(constantCapacity, podStats) summaryProvider.result = summaryStatsMaker(constantCapacity, podStats)
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider) manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)

View File

@@ -390,20 +390,12 @@ func podDiskUsage(podStats statsapi.PodStats, pod *v1.Pod, statsToMeasure []fsSt
// podMemoryUsage aggregates pod memory usage. // podMemoryUsage aggregates pod memory usage.
func podMemoryUsage(podStats statsapi.PodStats) (v1.ResourceList, error) { func podMemoryUsage(podStats statsapi.PodStats) (v1.ResourceList, error) {
disk := resource.Quantity{Format: resource.BinarySI}
memory := resource.Quantity{Format: resource.BinarySI} memory := resource.Quantity{Format: resource.BinarySI}
for _, container := range podStats.Containers { for _, container := range podStats.Containers {
// disk usage (if known)
for _, fsStats := range []*statsapi.FsStats{container.Rootfs, container.Logs} {
disk.Add(*diskUsage(fsStats))
}
// memory usage (if known) // memory usage (if known)
memory.Add(*memoryUsage(container.Memory)) memory.Add(*memoryUsage(container.Memory))
} }
return v1.ResourceList{ return v1.ResourceList{v1.ResourceMemory: memory}, nil
v1.ResourceMemory: memory,
resourceDisk: disk,
}, nil
} }
// localEphemeralVolumeNames returns the set of ephemeral volumes for the pod that are local // localEphemeralVolumeNames returns the set of ephemeral volumes for the pod that are local
@@ -543,89 +535,84 @@ func priority(p1, p2 *v1.Pod) int {
// exceedMemoryRequests compares whether or not pods' memory usage exceeds their requests // exceedMemoryRequests compares whether or not pods' memory usage exceeds their requests
func exceedMemoryRequests(stats statsFunc) cmpFunc { func exceedMemoryRequests(stats statsFunc) cmpFunc {
return func(p1, p2 *v1.Pod) int { return func(p1, p2 *v1.Pod) int {
p1Stats, found := stats(p1) p1Stats, p1Found := stats(p1)
// if we have no usage stats for p1, we want p2 first p2Stats, p2Found := stats(p2)
if !found { if !p1Found || !p2Found {
return -1 // prioritize evicting the pod for which no stats were found
return cmpBool(!p1Found, !p2Found)
} }
// if we have no usage stats for p2, but p1 has usage, we want p1 first.
p2Stats, found := stats(p2) p1Usage, p1Err := podMemoryUsage(p1Stats)
if !found { p2Usage, p2Err := podMemoryUsage(p2Stats)
return 1 if p1Err != nil || p2Err != nil {
} // prioritize evicting the pod which had an error getting stats
// if we cant get usage for p1 measured, we want p2 first return cmpBool(p1Err != nil, p2Err != nil)
p1Usage, err := podMemoryUsage(p1Stats)
if err != nil {
return -1
}
// if we cant get usage for p2 measured, we want p1 first
p2Usage, err := podMemoryUsage(p2Stats)
if err != nil {
return 1
} }
p1Memory := p1Usage[v1.ResourceMemory] p1Memory := p1Usage[v1.ResourceMemory]
p2Memory := p2Usage[v1.ResourceMemory] p2Memory := p2Usage[v1.ResourceMemory]
p1ExceedsRequests := p1Memory.Cmp(podMemoryRequest(p1)) == 1 p1ExceedsRequests := p1Memory.Cmp(podRequest(p1, v1.ResourceMemory)) == 1
p2ExceedsRequests := p2Memory.Cmp(podMemoryRequest(p2)) == 1 p2ExceedsRequests := p2Memory.Cmp(podRequest(p2, v1.ResourceMemory)) == 1
if p1ExceedsRequests == p2ExceedsRequests { // prioritize evicting the pod which exceeds its requests
return 0 return cmpBool(p1ExceedsRequests, p2ExceedsRequests)
}
if p1ExceedsRequests && !p2ExceedsRequests {
// if p1 exceeds its requests, but p2 does not, then we want p2 first
return -1
}
return 1
} }
} }
// memory compares pods by largest consumer of memory relative to request. // memory compares pods by largest consumer of memory relative to request.
func memory(stats statsFunc) cmpFunc { func memory(stats statsFunc) cmpFunc {
return func(p1, p2 *v1.Pod) int { return func(p1, p2 *v1.Pod) int {
p1Stats, found := stats(p1) p1Stats, p1Found := stats(p1)
// if we have no usage stats for p1, we want p2 first p2Stats, p2Found := stats(p2)
if !found { if !p1Found || !p2Found {
return -1 // prioritize evicting the pod for which no stats were found
return cmpBool(!p1Found, !p2Found)
} }
// if we have no usage stats for p2, but p1 has usage, we want p1 first.
p2Stats, found := stats(p2) p1Usage, p1Err := podMemoryUsage(p1Stats)
if !found { p2Usage, p2Err := podMemoryUsage(p2Stats)
return 1 if p1Err != nil || p2Err != nil {
} // prioritize evicting the pod which had an error getting stats
// if we cant get usage for p1 measured, we want p2 first return cmpBool(p1Err != nil, p2Err != nil)
p1Usage, err := podMemoryUsage(p1Stats)
if err != nil {
return -1
}
// if we cant get usage for p2 measured, we want p1 first
p2Usage, err := podMemoryUsage(p2Stats)
if err != nil {
return 1
} }
// adjust p1, p2 usage relative to the request (if any) // adjust p1, p2 usage relative to the request (if any)
p1Memory := p1Usage[v1.ResourceMemory] p1Memory := p1Usage[v1.ResourceMemory]
p1Request := podMemoryRequest(p1) p1Request := podRequest(p1, v1.ResourceMemory)
p1Memory.Sub(p1Request) p1Memory.Sub(p1Request)
p2Memory := p2Usage[v1.ResourceMemory] p2Memory := p2Usage[v1.ResourceMemory]
p2Request := podMemoryRequest(p2) p2Request := podRequest(p2, v1.ResourceMemory)
p2Memory.Sub(p2Request) p2Memory.Sub(p2Request)
// if p2 is using more than p1, we want p2 first // prioritize evicting the pod which has the larger consumption of memory
return p2Memory.Cmp(p1Memory) return p2Memory.Cmp(p1Memory)
} }
} }
// podMemoryRequest returns the total memory request of a pod which is the // podRequest returns the total resource request of a pod which is the
// max(sum of init container requests, sum of container requests) // max(sum of init container requests, sum of container requests)
func podMemoryRequest(pod *v1.Pod) resource.Quantity { func podRequest(pod *v1.Pod, resourceName v1.ResourceName) resource.Quantity {
containerValue := resource.Quantity{Format: resource.BinarySI} containerValue := resource.Quantity{Format: resource.BinarySI}
if resourceName == resourceDisk && !utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolation) {
// if the local storage capacity isolation feature gate is disabled, pods request 0 disk
return containerValue
}
for i := range pod.Spec.Containers { for i := range pod.Spec.Containers {
switch resourceName {
case v1.ResourceMemory:
containerValue.Add(*pod.Spec.Containers[i].Resources.Requests.Memory()) containerValue.Add(*pod.Spec.Containers[i].Resources.Requests.Memory())
case resourceDisk:
containerValue.Add(*pod.Spec.Containers[i].Resources.Requests.StorageEphemeral())
}
} }
initValue := resource.Quantity{Format: resource.BinarySI} initValue := resource.Quantity{Format: resource.BinarySI}
for i := range pod.Spec.InitContainers { for i := range pod.Spec.InitContainers {
initValue.Add(*pod.Spec.InitContainers[i].Resources.Requests.Memory()) switch resourceName {
case v1.ResourceMemory:
containerValue.Add(*pod.Spec.Containers[i].Resources.Requests.Memory())
case resourceDisk:
containerValue.Add(*pod.Spec.Containers[i].Resources.Requests.StorageEphemeral())
}
} }
if containerValue.Cmp(initValue) > 0 { if containerValue.Cmp(initValue) > 0 {
return containerValue return containerValue
@@ -633,39 +620,71 @@ func podMemoryRequest(pod *v1.Pod) resource.Quantity {
return initValue return initValue
} }
// exceedDiskRequests compares whether or not pods' disk usage exceeds their requests
func exceedDiskRequests(stats statsFunc, fsStatsToMeasure []fsStatsType, diskResource v1.ResourceName) cmpFunc {
return func(p1, p2 *v1.Pod) int {
p1Stats, p1Found := stats(p1)
p2Stats, p2Found := stats(p2)
if !p1Found || !p2Found {
// prioritize evicting the pod for which no stats were found
return cmpBool(!p1Found, !p2Found)
}
p1Usage, p1Err := podDiskUsage(p1Stats, p1, fsStatsToMeasure)
p2Usage, p2Err := podDiskUsage(p2Stats, p2, fsStatsToMeasure)
if p1Err != nil || p2Err != nil {
// prioritize evicting the pod which had an error getting stats
return cmpBool(p1Err != nil, p2Err != nil)
}
p1Disk := p1Usage[diskResource]
p2Disk := p2Usage[diskResource]
p1ExceedsRequests := p1Disk.Cmp(podRequest(p1, diskResource)) == 1
p2ExceedsRequests := p2Disk.Cmp(podRequest(p2, diskResource)) == 1
// prioritize evicting the pod which exceeds its requests
return cmpBool(p1ExceedsRequests, p2ExceedsRequests)
}
}
// disk compares pods by largest consumer of disk relative to request for the specified disk resource. // disk compares pods by largest consumer of disk relative to request for the specified disk resource.
func disk(stats statsFunc, fsStatsToMeasure []fsStatsType, diskResource v1.ResourceName) cmpFunc { func disk(stats statsFunc, fsStatsToMeasure []fsStatsType, diskResource v1.ResourceName) cmpFunc {
return func(p1, p2 *v1.Pod) int { return func(p1, p2 *v1.Pod) int {
p1Stats, found := stats(p1) p1Stats, p1Found := stats(p1)
// if we have no usage stats for p1, we want p2 first p2Stats, p2Found := stats(p2)
if !found { if !p1Found || !p2Found {
return -1 // prioritize evicting the pod for which no stats were found
return cmpBool(!p1Found, !p2Found)
} }
// if we have no usage stats for p2, but p1 has usage, we want p1 first. p1Usage, p1Err := podDiskUsage(p1Stats, p1, fsStatsToMeasure)
p2Stats, found := stats(p2) p2Usage, p2Err := podDiskUsage(p2Stats, p2, fsStatsToMeasure)
if !found { if p1Err != nil || p2Err != nil {
return 1 // prioritize evicting the pod which had an error getting stats
} return cmpBool(p1Err != nil, p2Err != nil)
// if we cant get usage for p1 measured, we want p2 first
p1Usage, err := podDiskUsage(p1Stats, p1, fsStatsToMeasure)
if err != nil {
return -1
}
// if we cant get usage for p2 measured, we want p1 first
p2Usage, err := podDiskUsage(p2Stats, p2, fsStatsToMeasure)
if err != nil {
return 1
} }
// disk is best effort, so we don't measure relative to a request. // adjust p1, p2 usage relative to the request (if any)
// TODO: add disk as a guaranteed resource
p1Disk := p1Usage[diskResource] p1Disk := p1Usage[diskResource]
p2Disk := p2Usage[diskResource] p2Disk := p2Usage[diskResource]
// if p2 is using more than p1, we want p2 first p1Request := podRequest(p1, resourceDisk)
p1Disk.Sub(p1Request)
p2Request := podRequest(p2, resourceDisk)
p2Disk.Sub(p2Request)
// prioritize evicting the pod which has the larger consumption of disk
return p2Disk.Cmp(p1Disk) return p2Disk.Cmp(p1Disk)
} }
} }
// cmpBool compares booleans, placing true before false
func cmpBool(a, b bool) int {
if a == b {
return 0
}
if !b {
return -1
}
return 1
}
// rankMemoryPressure orders the input pods for eviction in response to memory pressure. // rankMemoryPressure orders the input pods for eviction in response to memory pressure.
// It ranks by whether or not the pod's usage exceeds its requests, then by priority, and // It ranks by whether or not the pod's usage exceeds its requests, then by priority, and
// finally by memory usage above requests. // finally by memory usage above requests.
@@ -676,7 +695,7 @@ func rankMemoryPressure(pods []*v1.Pod, stats statsFunc) {
// rankDiskPressureFunc returns a rankFunc that measures the specified fs stats. // rankDiskPressureFunc returns a rankFunc that measures the specified fs stats.
func rankDiskPressureFunc(fsStatsToMeasure []fsStatsType, diskResource v1.ResourceName) rankFunc { func rankDiskPressureFunc(fsStatsToMeasure []fsStatsType, diskResource v1.ResourceName) rankFunc {
return func(pods []*v1.Pod, stats statsFunc) { return func(pods []*v1.Pod, stats statsFunc) {
orderedBy(priority, disk(stats, fsStatsToMeasure, diskResource)).Sort(pods) orderedBy(exceedDiskRequests(stats, fsStatsToMeasure, diskResource), priority, disk(stats, fsStatsToMeasure, diskResource)).Sort(pods)
} }
} }

View File

@@ -409,17 +409,71 @@ func thresholdEqual(a evictionapi.Threshold, b evictionapi.Threshold) bool {
compareThresholdValue(a.Value, b.Value) compareThresholdValue(a.Value, b.Value)
} }
// TestOrderedByPriority ensures we order BestEffort < Burstable < Guaranteed func TestOrderedByExceedsRequestMemory(t *testing.T) {
utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.PodPriority))
below := newPod("below-requests", -1, []v1.Container{
newContainer("below-requests", newResourceList("", "200Mi", ""), newResourceList("", "", "")),
}, nil)
exceeds := newPod("exceeds-requests", 1, []v1.Container{
newContainer("exceeds-requests", newResourceList("", "100Mi", ""), newResourceList("", "", "")),
}, nil)
stats := map[*v1.Pod]statsapi.PodStats{
below: newPodMemoryStats(below, resource.MustParse("199Mi")), // -1 relative to request
exceeds: newPodMemoryStats(exceeds, resource.MustParse("101Mi")), // 1 relative to request
}
statsFn := func(pod *v1.Pod) (statsapi.PodStats, bool) {
result, found := stats[pod]
return result, found
}
pods := []*v1.Pod{below, exceeds}
orderedBy(exceedMemoryRequests(statsFn)).Sort(pods)
expected := []*v1.Pod{exceeds, below}
for i := range expected {
if pods[i] != expected[i] {
t.Errorf("Expected pod: %s, but got: %s", expected[i].Name, pods[i].Name)
}
}
}
func TestOrderedByExceedsRequestDisk(t *testing.T) {
utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.PodPriority))
utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.LocalStorageCapacityIsolation))
below := newPod("below-requests", -1, []v1.Container{
newContainer("below-requests", v1.ResourceList{v1.ResourceEphemeralStorage: resource.MustParse("200Mi")}, newResourceList("", "", "")),
}, nil)
exceeds := newPod("exceeds-requests", 1, []v1.Container{
newContainer("exceeds-requests", v1.ResourceList{v1.ResourceEphemeralStorage: resource.MustParse("100Mi")}, newResourceList("", "", "")),
}, nil)
stats := map[*v1.Pod]statsapi.PodStats{
below: newPodDiskStats(below, resource.MustParse("100Mi"), resource.MustParse("99Mi"), resource.MustParse("0Mi")), // -1 relative to request
exceeds: newPodDiskStats(exceeds, resource.MustParse("90Mi"), resource.MustParse("11Mi"), resource.MustParse("0Mi")), // 1 relative to request
}
statsFn := func(pod *v1.Pod) (statsapi.PodStats, bool) {
result, found := stats[pod]
return result, found
}
pods := []*v1.Pod{below, exceeds}
orderedBy(exceedDiskRequests(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceDisk)).Sort(pods)
expected := []*v1.Pod{exceeds, below}
for i := range expected {
if pods[i] != expected[i] {
t.Errorf("Expected pod: %s, but got: %s", expected[i].Name, pods[i].Name)
}
}
}
func TestOrderedByPriority(t *testing.T) { func TestOrderedByPriority(t *testing.T) {
enablePodPriority(true) utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.PodPriority))
low := newPod("low-priority", -134, []v1.Container{ low := newPod("low-priority", -134, []v1.Container{
newContainer("low-priority", newResourceList("", ""), newResourceList("", "")), newContainer("low-priority", newResourceList("", "", ""), newResourceList("", "", "")),
}, nil) }, nil)
medium := newPod("medium-priority", 1, []v1.Container{ medium := newPod("medium-priority", 1, []v1.Container{
newContainer("medium-priority", newResourceList("100m", "100Mi"), newResourceList("200m", "200Mi")), newContainer("medium-priority", newResourceList("100m", "100Mi", ""), newResourceList("200m", "200Mi", "")),
}, nil) }, nil)
high := newPod("high-priority", 12534, []v1.Container{ high := newPod("high-priority", 12534, []v1.Container{
newContainer("high-priority", newResourceList("200m", "200Mi"), newResourceList("200m", "200Mi")), newContainer("high-priority", newResourceList("200m", "200Mi", ""), newResourceList("200m", "200Mi", "")),
}, nil) }, nil)
pods := []*v1.Pod{high, medium, low} pods := []*v1.Pod{high, medium, low}
@@ -433,17 +487,16 @@ func TestOrderedByPriority(t *testing.T) {
} }
} }
// TestOrderedByPriority ensures we order BestEffort < Burstable < Guaranteed
func TestOrderedByPriorityDisabled(t *testing.T) { func TestOrderedByPriorityDisabled(t *testing.T) {
enablePodPriority(false) utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=false", features.PodPriority))
low := newPod("low-priority", lowPriority, []v1.Container{ low := newPod("low-priority", lowPriority, []v1.Container{
newContainer("low-priority", newResourceList("", ""), newResourceList("", "")), newContainer("low-priority", newResourceList("", "", ""), newResourceList("", "", "")),
}, nil) }, nil)
medium := newPod("medium-priority", defaultPriority, []v1.Container{ medium := newPod("medium-priority", defaultPriority, []v1.Container{
newContainer("medium-priority", newResourceList("100m", "100Mi"), newResourceList("200m", "200Mi")), newContainer("medium-priority", newResourceList("100m", "100Mi", ""), newResourceList("200m", "200Mi", "")),
}, nil) }, nil)
high := newPod("high-priority", highPriority, []v1.Container{ high := newPod("high-priority", highPriority, []v1.Container{
newContainer("high-priority", newResourceList("200m", "200Mi"), newResourceList("200m", "200Mi")), newContainer("high-priority", newResourceList("200m", "200Mi", ""), newResourceList("200m", "200Mi", "")),
}, nil) }, nil)
pods := []*v1.Pod{high, medium, low} pods := []*v1.Pod{high, medium, low}
@@ -459,74 +512,64 @@ func TestOrderedByPriorityDisabled(t *testing.T) {
} }
func TestOrderedbyDisk(t *testing.T) { func TestOrderedbyDisk(t *testing.T) {
testOrderedByResource(t, resourceDisk, newPodDiskStats) utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.LocalStorageCapacityIsolation))
}
func TestOrderedbyInodes(t *testing.T) {
testOrderedByResource(t, resourceInodes, newPodInodeStats)
}
// testOrderedByDisk ensures we order pods by greediest resource consumer
func testOrderedByResource(t *testing.T, orderedByResource v1.ResourceName,
newPodStatsFunc func(pod *v1.Pod, rootFsUsed, logsUsed, perLocalVolumeUsed resource.Quantity) statsapi.PodStats) {
enablePodPriority(true)
pod1 := newPod("best-effort-high", defaultPriority, []v1.Container{ pod1 := newPod("best-effort-high", defaultPriority, []v1.Container{
newContainer("best-effort-high", newResourceList("", ""), newResourceList("", "")), newContainer("best-effort-high", newResourceList("", "", ""), newResourceList("", "", "")),
}, []v1.Volume{ }, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{ newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{}, EmptyDir: &v1.EmptyDirVolumeSource{},
}), }),
}) })
pod2 := newPod("best-effort-low", defaultPriority, []v1.Container{ pod2 := newPod("best-effort-low", defaultPriority, []v1.Container{
newContainer("best-effort-low", newResourceList("", ""), newResourceList("", "")), newContainer("best-effort-low", newResourceList("", "", ""), newResourceList("", "", "")),
}, []v1.Volume{ }, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{ newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{}, EmptyDir: &v1.EmptyDirVolumeSource{},
}), }),
}) })
pod3 := newPod("burstable-high", defaultPriority, []v1.Container{ pod3 := newPod("burstable-high", defaultPriority, []v1.Container{
newContainer("burstable-high", newResourceList("100m", "100Mi"), newResourceList("200m", "1Gi")), newContainer("burstable-high", newResourceList("", "", "100Mi"), newResourceList("", "", "400Mi")),
}, []v1.Volume{ }, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{ newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{}, EmptyDir: &v1.EmptyDirVolumeSource{},
}), }),
}) })
pod4 := newPod("burstable-low", defaultPriority, []v1.Container{ pod4 := newPod("burstable-low", defaultPriority, []v1.Container{
newContainer("burstable-low", newResourceList("100m", "100Mi"), newResourceList("200m", "1Gi")), newContainer("burstable-low", newResourceList("", "", "100Mi"), newResourceList("", "", "400Mi")),
}, []v1.Volume{ }, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{ newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{}, EmptyDir: &v1.EmptyDirVolumeSource{},
}), }),
}) })
pod5 := newPod("guaranteed-high", defaultPriority, []v1.Container{ pod5 := newPod("guaranteed-high", defaultPriority, []v1.Container{
newContainer("guaranteed-high", newResourceList("100m", "1Gi"), newResourceList("100m", "1Gi")), newContainer("guaranteed-high", newResourceList("", "", "400Mi"), newResourceList("", "", "400Mi")),
}, []v1.Volume{ }, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{ newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{}, EmptyDir: &v1.EmptyDirVolumeSource{},
}), }),
}) })
pod6 := newPod("guaranteed-low", defaultPriority, []v1.Container{ pod6 := newPod("guaranteed-low", defaultPriority, []v1.Container{
newContainer("guaranteed-low", newResourceList("100m", "1Gi"), newResourceList("100m", "1Gi")), newContainer("guaranteed-low", newResourceList("", "", "400Mi"), newResourceList("", "", "400Mi")),
}, []v1.Volume{ }, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{ newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{}, EmptyDir: &v1.EmptyDirVolumeSource{},
}), }),
}) })
stats := map[*v1.Pod]statsapi.PodStats{ stats := map[*v1.Pod]statsapi.PodStats{
pod1: newPodStatsFunc(pod1, resource.MustParse("50Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 200Mi pod1: newPodDiskStats(pod1, resource.MustParse("50Mi"), resource.MustParse("100Mi"), resource.MustParse("150Mi")), // 300Mi - 0 = 300Mi
pod2: newPodStatsFunc(pod2, resource.MustParse("100Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 300Mi pod2: newPodDiskStats(pod2, resource.MustParse("25Mi"), resource.MustParse("25Mi"), resource.MustParse("50Mi")), // 100Mi - 0 = 100Mi
pod3: newPodStatsFunc(pod3, resource.MustParse("200Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 400Mi pod3: newPodDiskStats(pod3, resource.MustParse("150Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 350Mi - 100Mi = 250Mi
pod4: newPodStatsFunc(pod4, resource.MustParse("300Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 450Mi pod4: newPodDiskStats(pod4, resource.MustParse("25Mi"), resource.MustParse("35Mi"), resource.MustParse("50Mi")), // 110Mi - 100Mi = 10Mi
pod5: newPodStatsFunc(pod5, resource.MustParse("400Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 550Mi pod5: newPodDiskStats(pod5, resource.MustParse("225Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 375Mi - 400Mi = -25Mi
pod6: newPodStatsFunc(pod6, resource.MustParse("500Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 650Mi pod6: newPodDiskStats(pod6, resource.MustParse("25Mi"), resource.MustParse("45Mi"), resource.MustParse("50Mi")), // 120Mi - 400Mi = -280Mi
} }
statsFn := func(pod *v1.Pod) (statsapi.PodStats, bool) { statsFn := func(pod *v1.Pod) (statsapi.PodStats, bool) {
result, found := stats[pod] result, found := stats[pod]
return result, found return result, found
} }
pods := []*v1.Pod{pod1, pod2, pod3, pod4, pod5, pod6} pods := []*v1.Pod{pod1, pod2, pod3, pod4, pod5, pod6}
orderedBy(disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, orderedByResource)).Sort(pods) orderedBy(disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceDisk)).Sort(pods)
expected := []*v1.Pod{pod6, pod5, pod4, pod3, pod2, pod1} expected := []*v1.Pod{pod1, pod3, pod2, pod4, pod5, pod6}
for i := range expected { for i := range expected {
if pods[i] != expected[i] { if pods[i] != expected[i] {
t.Errorf("Expected pod[%d]: %s, but got: %s", i, expected[i].Name, pods[i].Name) t.Errorf("Expected pod[%d]: %s, but got: %s", i, expected[i].Name, pods[i].Name)
@@ -534,58 +577,244 @@ func testOrderedByResource(t *testing.T, orderedByResource v1.ResourceName,
} }
} }
func TestOrderedbyPriorityDisk(t *testing.T) { // Tests that we correctly ignore disk requests when the local storage feature gate is disabled.
testOrderedByPriorityResource(t, resourceDisk, newPodDiskStats) func TestOrderedbyDiskDisableLocalStorage(t *testing.T) {
} utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=false", features.LocalStorageCapacityIsolation))
pod1 := newPod("best-effort-high", defaultPriority, []v1.Container{
func TestOrderedbyPriorityInodes(t *testing.T) { newContainer("best-effort-high", newResourceList("", "", ""), newResourceList("", "", "")),
testOrderedByPriorityResource(t, resourceInodes, newPodInodeStats)
}
// testOrderedByPriorityDisk ensures we order pods by qos and then greediest resource consumer
func testOrderedByPriorityResource(t *testing.T, orderedByResource v1.ResourceName,
newPodStatsFunc func(pod *v1.Pod, rootFsUsed, logsUsed, perLocalVolumeUsed resource.Quantity) statsapi.PodStats) {
enablePodPriority(true)
pod1 := newPod("low-priority-high-usage", lowPriority, []v1.Container{
newContainer("low-priority-high-usage", newResourceList("", ""), newResourceList("", "")),
}, []v1.Volume{ }, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{ newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{}, EmptyDir: &v1.EmptyDirVolumeSource{},
}), }),
}) })
pod2 := newPod("low-priority-low-usage", lowPriority, []v1.Container{ pod2 := newPod("best-effort-low", defaultPriority, []v1.Container{
newContainer("low-priority-low-usage", newResourceList("", ""), newResourceList("", "")), newContainer("best-effort-low", newResourceList("", "", ""), newResourceList("", "", "")),
}, []v1.Volume{ }, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{ newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{}, EmptyDir: &v1.EmptyDirVolumeSource{},
}), }),
}) })
pod3 := newPod("high-priority-high-usage", highPriority, []v1.Container{ pod3 := newPod("burstable-high", defaultPriority, []v1.Container{
newContainer("high-priority-high-usage", newResourceList("100m", "100Mi"), newResourceList("200m", "1Gi")), newContainer("burstable-high", newResourceList("", "", "100Mi"), newResourceList("", "", "400Mi")),
}, []v1.Volume{ }, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{ newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{}, EmptyDir: &v1.EmptyDirVolumeSource{},
}), }),
}) })
pod4 := newPod("high-priority-low-usage", highPriority, []v1.Container{ pod4 := newPod("burstable-low", defaultPriority, []v1.Container{
newContainer("high-priority-low-usage", newResourceList("100m", "100Mi"), newResourceList("200m", "1Gi")), newContainer("burstable-low", newResourceList("", "", "100Mi"), newResourceList("", "", "400Mi")),
}, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
}),
})
pod5 := newPod("guaranteed-high", defaultPriority, []v1.Container{
newContainer("guaranteed-high", newResourceList("", "", "400Mi"), newResourceList("", "", "400Mi")),
}, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
}),
})
pod6 := newPod("guaranteed-low", defaultPriority, []v1.Container{
newContainer("guaranteed-low", newResourceList("", "", "400Mi"), newResourceList("", "", "400Mi")),
}, []v1.Volume{ }, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{ newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{}, EmptyDir: &v1.EmptyDirVolumeSource{},
}), }),
}) })
stats := map[*v1.Pod]statsapi.PodStats{ stats := map[*v1.Pod]statsapi.PodStats{
pod1: newPodStatsFunc(pod1, resource.MustParse("50Mi"), resource.MustParse("100Mi"), resource.MustParse("250Mi")), // 400Mi pod1: newPodDiskStats(pod1, resource.MustParse("50Mi"), resource.MustParse("100Mi"), resource.MustParse("150Mi")), // 300Mi
pod2: newPodStatsFunc(pod2, resource.MustParse("60Mi"), resource.MustParse("30Mi"), resource.MustParse("10Mi")), // 100Mi pod2: newPodDiskStats(pod2, resource.MustParse("25Mi"), resource.MustParse("25Mi"), resource.MustParse("50Mi")), // 100Mi
pod3: newPodStatsFunc(pod3, resource.MustParse("150Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 350Mi pod3: newPodDiskStats(pod3, resource.MustParse("150Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 350Mi
pod4: newPodStatsFunc(pod4, resource.MustParse("10Mi"), resource.MustParse("40Mi"), resource.MustParse("100Mi")), // 150Mi pod4: newPodDiskStats(pod4, resource.MustParse("25Mi"), resource.MustParse("35Mi"), resource.MustParse("50Mi")), // 110Mi
pod5: newPodDiskStats(pod5, resource.MustParse("225Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 375Mi
pod6: newPodDiskStats(pod6, resource.MustParse("25Mi"), resource.MustParse("45Mi"), resource.MustParse("50Mi")), // 120Mi
}
statsFn := func(pod *v1.Pod) (statsapi.PodStats, bool) {
result, found := stats[pod]
return result, found
}
pods := []*v1.Pod{pod1, pod3, pod2, pod4, pod5, pod6}
orderedBy(disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceDisk)).Sort(pods)
expected := []*v1.Pod{pod5, pod3, pod1, pod6, pod4, pod2}
for i := range expected {
if pods[i] != expected[i] {
t.Errorf("Expected pod[%d]: %s, but got: %s", i, expected[i].Name, pods[i].Name)
}
}
}
func TestOrderedbyInodes(t *testing.T) {
utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.PodPriority))
utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.LocalStorageCapacityIsolation))
low := newPod("low", defaultPriority, []v1.Container{
newContainer("low", newResourceList("", "", ""), newResourceList("", "", "")),
}, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
}),
})
medium := newPod("medium", defaultPriority, []v1.Container{
newContainer("medium", newResourceList("", "", ""), newResourceList("", "", "")),
}, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
}),
})
high := newPod("high", defaultPriority, []v1.Container{
newContainer("high", newResourceList("", "", ""), newResourceList("", "", "")),
}, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
}),
})
stats := map[*v1.Pod]statsapi.PodStats{
low: newPodInodeStats(low, resource.MustParse("50000"), resource.MustParse("100000"), resource.MustParse("50000")), // 200000
medium: newPodInodeStats(medium, resource.MustParse("100000"), resource.MustParse("150000"), resource.MustParse("50000")), // 300000
high: newPodInodeStats(high, resource.MustParse("200000"), resource.MustParse("150000"), resource.MustParse("50000")), // 400000
}
statsFn := func(pod *v1.Pod) (statsapi.PodStats, bool) {
result, found := stats[pod]
return result, found
}
pods := []*v1.Pod{low, medium, high}
orderedBy(disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceInodes)).Sort(pods)
expected := []*v1.Pod{high, medium, low}
for i := range expected {
if pods[i] != expected[i] {
t.Errorf("Expected pod[%d]: %s, but got: %s", i, expected[i].Name, pods[i].Name)
}
}
}
// TestOrderedByPriorityDisk ensures we order pods by priority and then greediest resource consumer
func TestOrderedByPriorityDisk(t *testing.T) {
utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.PodPriority))
utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.LocalStorageCapacityIsolation))
pod1 := newPod("above-requests-low-priority-high-usage", lowPriority, []v1.Container{
newContainer("above-requests-low-priority-high-usage", newResourceList("", "", ""), newResourceList("", "", "")),
}, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
}),
})
pod2 := newPod("above-requests-low-priority-low-usage", lowPriority, []v1.Container{
newContainer("above-requests-low-priority-low-usage", newResourceList("", "", ""), newResourceList("", "", "")),
}, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
}),
})
pod3 := newPod("above-requests-high-priority-high-usage", highPriority, []v1.Container{
newContainer("above-requests-high-priority-high-usage", newResourceList("", "", "100Mi"), newResourceList("", "", "")),
}, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
}),
})
pod4 := newPod("above-requests-high-priority-low-usage", highPriority, []v1.Container{
newContainer("above-requests-high-priority-low-usage", newResourceList("", "", "100Mi"), newResourceList("", "", "")),
}, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
}),
})
pod5 := newPod("below-requests-low-priority-high-usage", lowPriority, []v1.Container{
newContainer("below-requests-low-priority-high-usage", newResourceList("", "", "1Gi"), newResourceList("", "", "")),
}, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
}),
})
pod6 := newPod("below-requests-low-priority-low-usage", lowPriority, []v1.Container{
newContainer("below-requests-low-priority-low-usage", newResourceList("", "", "1Gi"), newResourceList("", "", "")),
}, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
}),
})
pod7 := newPod("below-requests-high-priority-high-usage", highPriority, []v1.Container{
newContainer("below-requests-high-priority-high-usage", newResourceList("", "", "1Gi"), newResourceList("", "", "")),
}, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
}),
})
pod8 := newPod("below-requests-high-priority-low-usage", highPriority, []v1.Container{
newContainer("below-requests-high-priority-low-usage", newResourceList("", "", "1Gi"), newResourceList("", "", "")),
}, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
}),
})
stats := map[*v1.Pod]statsapi.PodStats{
pod1: newPodDiskStats(pod1, resource.MustParse("200Mi"), resource.MustParse("100Mi"), resource.MustParse("200Mi")), // 500 relative to request
pod2: newPodDiskStats(pod2, resource.MustParse("10Mi"), resource.MustParse("10Mi"), resource.MustParse("30Mi")), // 50 relative to request
pod3: newPodDiskStats(pod3, resource.MustParse("200Mi"), resource.MustParse("150Mi"), resource.MustParse("250Mi")), // 500 relative to request
pod4: newPodDiskStats(pod4, resource.MustParse("90Mi"), resource.MustParse("50Mi"), resource.MustParse("10Mi")), // 50 relative to request
pod5: newPodDiskStats(pod5, resource.MustParse("500Mi"), resource.MustParse("200Mi"), resource.MustParse("100Mi")), // -200 relative to request
pod6: newPodDiskStats(pod6, resource.MustParse("50Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // -800 relative to request
pod7: newPodDiskStats(pod7, resource.MustParse("250Mi"), resource.MustParse("500Mi"), resource.MustParse("50Mi")), // -200 relative to request
pod8: newPodDiskStats(pod8, resource.MustParse("100Mi"), resource.MustParse("60Mi"), resource.MustParse("40Mi")), // -800 relative to request
}
statsFn := func(pod *v1.Pod) (statsapi.PodStats, bool) {
result, found := stats[pod]
return result, found
}
pods := []*v1.Pod{pod8, pod7, pod6, pod5, pod4, pod3, pod2, pod1}
expected := []*v1.Pod{pod1, pod2, pod3, pod4, pod5, pod6, pod7, pod8}
fsStatsToMeasure := []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}
orderedBy(exceedDiskRequests(statsFn, fsStatsToMeasure, resourceDisk), priority, disk(statsFn, fsStatsToMeasure, resourceDisk)).Sort(pods)
for i := range expected {
if pods[i] != expected[i] {
t.Errorf("Expected pod[%d]: %s, but got: %s", i, expected[i].Name, pods[i].Name)
}
}
}
// TestOrderedByPriorityInodes ensures we order pods by priority and then greediest resource consumer
func TestOrderedByPriorityInodes(t *testing.T) {
utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.PodPriority))
pod1 := newPod("low-priority-high-usage", lowPriority, []v1.Container{
newContainer("low-priority-high-usage", newResourceList("", "", ""), newResourceList("", "", "")),
}, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
}),
})
pod2 := newPod("low-priority-low-usage", lowPriority, []v1.Container{
newContainer("low-priority-low-usage", newResourceList("", "", ""), newResourceList("", "", "")),
}, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
}),
})
pod3 := newPod("high-priority-high-usage", highPriority, []v1.Container{
newContainer("high-priority-high-usage", newResourceList("", "", ""), newResourceList("", "", "")),
}, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
}),
})
pod4 := newPod("high-priority-low-usage", highPriority, []v1.Container{
newContainer("high-priority-low-usage", newResourceList("", "", ""), newResourceList("", "", "")),
}, []v1.Volume{
newVolume("local-volume", v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
}),
})
stats := map[*v1.Pod]statsapi.PodStats{
pod1: newPodInodeStats(pod1, resource.MustParse("50000"), resource.MustParse("100000"), resource.MustParse("250000")), // 400000
pod2: newPodInodeStats(pod2, resource.MustParse("60000"), resource.MustParse("30000"), resource.MustParse("10000")), // 100000
pod3: newPodInodeStats(pod3, resource.MustParse("150000"), resource.MustParse("150000"), resource.MustParse("50000")), // 350000
pod4: newPodInodeStats(pod4, resource.MustParse("10000"), resource.MustParse("40000"), resource.MustParse("100000")), // 150000
} }
statsFn := func(pod *v1.Pod) (statsapi.PodStats, bool) { statsFn := func(pod *v1.Pod) (statsapi.PodStats, bool) {
result, found := stats[pod] result, found := stats[pod]
return result, found return result, found
} }
pods := []*v1.Pod{pod4, pod3, pod2, pod1} pods := []*v1.Pod{pod4, pod3, pod2, pod1}
orderedBy(priority, disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, orderedByResource)).Sort(pods) orderedBy(priority, disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceInodes)).Sort(pods)
expected := []*v1.Pod{pod1, pod2, pod3, pod4} expected := []*v1.Pod{pod1, pod2, pod3, pod4}
for i := range expected { for i := range expected {
if pods[i] != expected[i] { if pods[i] != expected[i] {
@@ -597,22 +826,22 @@ func testOrderedByPriorityResource(t *testing.T, orderedByResource v1.ResourceNa
// TestOrderedByMemory ensures we order pods by greediest memory consumer relative to request. // TestOrderedByMemory ensures we order pods by greediest memory consumer relative to request.
func TestOrderedByMemory(t *testing.T) { func TestOrderedByMemory(t *testing.T) {
pod1 := newPod("best-effort-high", defaultPriority, []v1.Container{ pod1 := newPod("best-effort-high", defaultPriority, []v1.Container{
newContainer("best-effort-high", newResourceList("", ""), newResourceList("", "")), newContainer("best-effort-high", newResourceList("", "", ""), newResourceList("", "", "")),
}, nil) }, nil)
pod2 := newPod("best-effort-low", defaultPriority, []v1.Container{ pod2 := newPod("best-effort-low", defaultPriority, []v1.Container{
newContainer("best-effort-low", newResourceList("", ""), newResourceList("", "")), newContainer("best-effort-low", newResourceList("", "", ""), newResourceList("", "", "")),
}, nil) }, nil)
pod3 := newPod("burstable-high", defaultPriority, []v1.Container{ pod3 := newPod("burstable-high", defaultPriority, []v1.Container{
newContainer("burstable-high", newResourceList("100m", "100Mi"), newResourceList("200m", "1Gi")), newContainer("burstable-high", newResourceList("", "100Mi", ""), newResourceList("", "1Gi", "")),
}, nil) }, nil)
pod4 := newPod("burstable-low", defaultPriority, []v1.Container{ pod4 := newPod("burstable-low", defaultPriority, []v1.Container{
newContainer("burstable-low", newResourceList("100m", "100Mi"), newResourceList("200m", "1Gi")), newContainer("burstable-low", newResourceList("", "100Mi", ""), newResourceList("", "1Gi", "")),
}, nil) }, nil)
pod5 := newPod("guaranteed-high", defaultPriority, []v1.Container{ pod5 := newPod("guaranteed-high", defaultPriority, []v1.Container{
newContainer("guaranteed-high", newResourceList("100m", "1Gi"), newResourceList("100m", "1Gi")), newContainer("guaranteed-high", newResourceList("", "1Gi", ""), newResourceList("", "1Gi", "")),
}, nil) }, nil)
pod6 := newPod("guaranteed-low", defaultPriority, []v1.Container{ pod6 := newPod("guaranteed-low", defaultPriority, []v1.Container{
newContainer("guaranteed-low", newResourceList("100m", "1Gi"), newResourceList("100m", "1Gi")), newContainer("guaranteed-low", newResourceList("", "1Gi", ""), newResourceList("", "1Gi", "")),
}, nil) }, nil)
stats := map[*v1.Pod]statsapi.PodStats{ stats := map[*v1.Pod]statsapi.PodStats{
pod1: newPodMemoryStats(pod1, resource.MustParse("500Mi")), // 500 relative to request pod1: newPodMemoryStats(pod1, resource.MustParse("500Mi")), // 500 relative to request
@@ -638,30 +867,30 @@ func TestOrderedByMemory(t *testing.T) {
// TestOrderedByPriorityMemory ensures we order by priority and then memory consumption relative to request. // TestOrderedByPriorityMemory ensures we order by priority and then memory consumption relative to request.
func TestOrderedByPriorityMemory(t *testing.T) { func TestOrderedByPriorityMemory(t *testing.T) {
enablePodPriority(true) utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.PodPriority))
pod1 := newPod("above-requests-low-priority-high-usage", lowPriority, []v1.Container{ pod1 := newPod("above-requests-low-priority-high-usage", lowPriority, []v1.Container{
newContainer("above-requests-low-priority-high-usage", newResourceList("", ""), newResourceList("", "")), newContainer("above-requests-low-priority-high-usage", newResourceList("", "", ""), newResourceList("", "", "")),
}, nil) }, nil)
pod2 := newPod("above-requests-low-priority-low-usage", lowPriority, []v1.Container{ pod2 := newPod("above-requests-low-priority-low-usage", lowPriority, []v1.Container{
newContainer("above-requests-low-priority-low-usage", newResourceList("", ""), newResourceList("", "")), newContainer("above-requests-low-priority-low-usage", newResourceList("", "", ""), newResourceList("", "", "")),
}, nil) }, nil)
pod3 := newPod("above-requests-high-priority-high-usage", highPriority, []v1.Container{ pod3 := newPod("above-requests-high-priority-high-usage", highPriority, []v1.Container{
newContainer("above-requests-high-priority-high-usage", newResourceList("", "100Mi"), newResourceList("", "")), newContainer("above-requests-high-priority-high-usage", newResourceList("", "100Mi", ""), newResourceList("", "", "")),
}, nil) }, nil)
pod4 := newPod("above-requests-high-priority-low-usage", highPriority, []v1.Container{ pod4 := newPod("above-requests-high-priority-low-usage", highPriority, []v1.Container{
newContainer("above-requests-high-priority-low-usage", newResourceList("", "100Mi"), newResourceList("", "")), newContainer("above-requests-high-priority-low-usage", newResourceList("", "100Mi", ""), newResourceList("", "", "")),
}, nil) }, nil)
pod5 := newPod("below-requests-low-priority-high-usage", lowPriority, []v1.Container{ pod5 := newPod("below-requests-low-priority-high-usage", lowPriority, []v1.Container{
newContainer("below-requests-low-priority-high-usage", newResourceList("", "1Gi"), newResourceList("", "")), newContainer("below-requests-low-priority-high-usage", newResourceList("", "1Gi", ""), newResourceList("", "", "")),
}, nil) }, nil)
pod6 := newPod("below-requests-low-priority-low-usage", lowPriority, []v1.Container{ pod6 := newPod("below-requests-low-priority-low-usage", lowPriority, []v1.Container{
newContainer("below-requests-low-priority-low-usage", newResourceList("", "1Gi"), newResourceList("", "")), newContainer("below-requests-low-priority-low-usage", newResourceList("", "1Gi", ""), newResourceList("", "", "")),
}, nil) }, nil)
pod7 := newPod("below-requests-high-priority-high-usage", highPriority, []v1.Container{ pod7 := newPod("below-requests-high-priority-high-usage", highPriority, []v1.Container{
newContainer("below-requests-high-priority-high-usage", newResourceList("", "1Gi"), newResourceList("", "")), newContainer("below-requests-high-priority-high-usage", newResourceList("", "1Gi", ""), newResourceList("", "", "")),
}, nil) }, nil)
pod8 := newPod("below-requests-high-priority-low-usage", highPriority, []v1.Container{ pod8 := newPod("below-requests-high-priority-low-usage", highPriority, []v1.Container{
newContainer("below-requests-high-priority-low-usage", newResourceList("", "1Gi"), newResourceList("", "")), newContainer("below-requests-high-priority-low-usage", newResourceList("", "1Gi", ""), newResourceList("", "", "")),
}, nil) }, nil)
stats := map[*v1.Pod]statsapi.PodStats{ stats := map[*v1.Pod]statsapi.PodStats{
pod1: newPodMemoryStats(pod1, resource.MustParse("500Mi")), // 500 relative to request pod1: newPodMemoryStats(pod1, resource.MustParse("500Mi")), // 500 relative to request
@@ -678,7 +907,6 @@ func TestOrderedByPriorityMemory(t *testing.T) {
return result, found return result, found
} }
pods := []*v1.Pod{pod8, pod7, pod6, pod5, pod4, pod3, pod2, pod1} pods := []*v1.Pod{pod8, pod7, pod6, pod5, pod4, pod3, pod2, pod1}
// pods := []*v1.Pod{pod1, pod2, pod3, pod4, pod5, pod6, pod7, pod8}
expected := []*v1.Pod{pod1, pod2, pod3, pod4, pod5, pod6, pod7, pod8} expected := []*v1.Pod{pod1, pod2, pod3, pod4, pod5, pod6, pod7, pod8}
orderedBy(exceedMemoryRequests(statsFn), priority, memory(statsFn)).Sort(pods) orderedBy(exceedMemoryRequests(statsFn), priority, memory(statsFn)).Sort(pods)
for i := range expected { for i := range expected {
@@ -1589,7 +1817,7 @@ func newPodMemoryStats(pod *v1.Pod, workingSet resource.Quantity) statsapi.PodSt
return result return result
} }
func newResourceList(cpu, memory string) v1.ResourceList { func newResourceList(cpu, memory, disk string) v1.ResourceList {
res := v1.ResourceList{} res := v1.ResourceList{}
if cpu != "" { if cpu != "" {
res[v1.ResourceCPU] = resource.MustParse(cpu) res[v1.ResourceCPU] = resource.MustParse(cpu)
@@ -1597,6 +1825,9 @@ func newResourceList(cpu, memory string) v1.ResourceList {
if memory != "" { if memory != "" {
res[v1.ResourceMemory] = resource.MustParse(memory) res[v1.ResourceMemory] = resource.MustParse(memory)
} }
if disk != "" {
res[v1.ResourceEphemeralStorage] = resource.MustParse(disk)
}
return res return res
} }
@@ -1681,7 +1912,3 @@ func (s1 thresholdList) Equal(s2 thresholdList) bool {
} }
return true return true
} }
func enablePodPriority(enabled bool) {
utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=%t", features.PodPriority, enabled))
}

View File

@@ -50,6 +50,7 @@ const (
pressureDelay = 20 * time.Second pressureDelay = 20 * time.Second
testContextFmt = "when we run containers that should cause %s" testContextFmt = "when we run containers that should cause %s"
noPressure = v1.NodeConditionType("NoPressure") noPressure = v1.NodeConditionType("NoPressure")
lotsOfDisk = 10240 // 10 Gb in Mb
) )
// InodeEviction tests that the node responds to node disk pressure by evicting only responsible pods. // InodeEviction tests that the node responds to node disk pressure by evicting only responsible pods.
@@ -127,16 +128,16 @@ var _ = framework.KubeDescribe("LocalStorageEviction [Slow] [Serial] [Disruptive
expectedNodeCondition := v1.NodeDiskPressure expectedNodeCondition := v1.NodeDiskPressure
Context(fmt.Sprintf(testContextFmt, expectedNodeCondition), func() { Context(fmt.Sprintf(testContextFmt, expectedNodeCondition), func() {
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) { tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
diskConsumed := uint64(100000000) // At least 100 Mb for pods to consume diskConsumed := resource.MustParse("100Mi")
summary := eventuallyGetSummary() summary := eventuallyGetSummary()
availableBytes := *(summary.Node.Fs.AvailableBytes) availableBytes := *(summary.Node.Fs.AvailableBytes)
initialConfig.EvictionHard = map[string]string{"nodefs.available": fmt.Sprintf("%d", availableBytes-diskConsumed)} initialConfig.EvictionHard = map[string]string{"nodefs.available": fmt.Sprintf("%d", availableBytes-uint64(diskConsumed.Value()))}
initialConfig.EvictionMinimumReclaim = map[string]string{} initialConfig.EvictionMinimumReclaim = map[string]string{}
}) })
runEvictionTest(f, pressureTimeout, expectedNodeCondition, logDiskMetrics, []podEvictSpec{ runEvictionTest(f, pressureTimeout, expectedNodeCondition, logDiskMetrics, []podEvictSpec{
{ {
evictionPriority: 1, evictionPriority: 1,
pod: diskConsumingPod("container-disk-hog", 10000, nil, v1.ResourceRequirements{}), pod: diskConsumingPod("container-disk-hog", lotsOfDisk, nil, v1.ResourceRequirements{}),
}, },
{ {
evictionPriority: 0, evictionPriority: 0,
@@ -155,10 +156,13 @@ var _ = framework.KubeDescribe("LocalStorageSoftEviction [Slow] [Serial] [Disrup
expectedNodeCondition := v1.NodeDiskPressure expectedNodeCondition := v1.NodeDiskPressure
Context(fmt.Sprintf(testContextFmt, expectedNodeCondition), func() { Context(fmt.Sprintf(testContextFmt, expectedNodeCondition), func() {
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) { tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
diskConsumed := uint64(100000000) // At least 100 Mb for pods to consume diskConsumed := resource.MustParse("100Mi")
summary := eventuallyGetSummary() summary := eventuallyGetSummary()
availableBytes := *(summary.Node.Fs.AvailableBytes) availableBytes := *(summary.Node.Fs.AvailableBytes)
initialConfig.EvictionSoft = map[string]string{"nodefs.available": fmt.Sprintf("%d", availableBytes-diskConsumed)} if availableBytes <= uint64(diskConsumed.Value()) {
framework.Skipf("Too little disk free on the host for the LocalStorageSoftEviction test to run")
}
initialConfig.EvictionSoft = map[string]string{"nodefs.available": fmt.Sprintf("%d", availableBytes-uint64(diskConsumed.Value()))}
initialConfig.EvictionSoftGracePeriod = map[string]string{"nodefs.available": "1m"} initialConfig.EvictionSoftGracePeriod = map[string]string{"nodefs.available": "1m"}
// Defer to the pod default grace period // Defer to the pod default grace period
initialConfig.EvictionMaxPodGracePeriod = 30 initialConfig.EvictionMaxPodGracePeriod = 30
@@ -169,7 +173,7 @@ var _ = framework.KubeDescribe("LocalStorageSoftEviction [Slow] [Serial] [Disrup
runEvictionTest(f, pressureTimeout, expectedNodeCondition, logDiskMetrics, []podEvictSpec{ runEvictionTest(f, pressureTimeout, expectedNodeCondition, logDiskMetrics, []podEvictSpec{
{ {
evictionPriority: 1, evictionPriority: 1,
pod: diskConsumingPod("container-disk-hog", 10000, nil, v1.ResourceRequirements{}), pod: diskConsumingPod("container-disk-hog", lotsOfDisk, nil, v1.ResourceRequirements{}),
}, },
{ {
evictionPriority: 0, evictionPriority: 0,
@@ -189,34 +193,35 @@ var _ = framework.KubeDescribe("LocalStorageCapacityIsolationEviction [Slow] [Se
initialConfig.EvictionHard = map[string]string{} initialConfig.EvictionHard = map[string]string{}
}) })
sizeLimit := resource.MustParse("100Mi") sizeLimit := resource.MustParse("100Mi")
used := int64(200) // Consume 200 Mb useOverLimit := 101 /* Mb */
useUnderLimit := 99 /* Mb */
containerLimit := v1.ResourceList{v1.ResourceEphemeralStorage: sizeLimit} containerLimit := v1.ResourceList{v1.ResourceEphemeralStorage: sizeLimit}
runEvictionTest(f, evictionTestTimeout, noPressure, logDiskMetrics, []podEvictSpec{ runEvictionTest(f, evictionTestTimeout, noPressure, logDiskMetrics, []podEvictSpec{
{ {
evictionPriority: 1, // This pod should be evicted because emptyDir (default storage type) usage violation evictionPriority: 1, // This pod should be evicted because emptyDir (default storage type) usage violation
pod: diskConsumingPod("emptydir-disk-sizelimit", used, &v1.VolumeSource{ pod: diskConsumingPod("emptydir-disk-sizelimit", useOverLimit, &v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{SizeLimit: &sizeLimit}, EmptyDir: &v1.EmptyDirVolumeSource{SizeLimit: &sizeLimit},
}, v1.ResourceRequirements{}), }, v1.ResourceRequirements{}),
}, },
{ {
evictionPriority: 1, // This pod should be evicted because of memory emptyDir usage violation evictionPriority: 1, // This pod should be evicted because of memory emptyDir usage violation
pod: diskConsumingPod("emptydir-memory-sizelimit", used, &v1.VolumeSource{ pod: diskConsumingPod("emptydir-memory-sizelimit", useOverLimit, &v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{Medium: "Memory", SizeLimit: &sizeLimit}, EmptyDir: &v1.EmptyDirVolumeSource{Medium: "Memory", SizeLimit: &sizeLimit},
}, v1.ResourceRequirements{}), }, v1.ResourceRequirements{}),
}, },
{ {
evictionPriority: 1, // This pod should cross the container limit by writing to its writable layer. evictionPriority: 1, // This pod should cross the container limit by writing to its writable layer.
pod: diskConsumingPod("container-disk-limit", used, nil, v1.ResourceRequirements{Limits: containerLimit}), pod: diskConsumingPod("container-disk-limit", useOverLimit, nil, v1.ResourceRequirements{Limits: containerLimit}),
}, },
{ {
evictionPriority: 1, // This pod should hit the container limit by writing to an emptydir evictionPriority: 1, // This pod should hit the container limit by writing to an emptydir
pod: diskConsumingPod("container-emptydir-disk-limit", used, &v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}, pod: diskConsumingPod("container-emptydir-disk-limit", useOverLimit, &v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}},
v1.ResourceRequirements{Limits: containerLimit}), v1.ResourceRequirements{Limits: containerLimit}),
}, },
{ {
evictionPriority: 0, // This pod should not be evicted because it uses less than its limit evictionPriority: 0, // This pod should not be evicted because it uses less than its limit
pod: diskConsumingPod("emptydir-disk-below-sizelimit", int64(50), &v1.VolumeSource{ pod: diskConsumingPod("emptydir-disk-below-sizelimit", useUnderLimit, &v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{SizeLimit: &sizeLimit}, EmptyDir: &v1.EmptyDirVolumeSource{SizeLimit: &sizeLimit},
}, v1.ResourceRequirements{}), }, v1.ResourceRequirements{}),
}, },
@@ -224,11 +229,11 @@ var _ = framework.KubeDescribe("LocalStorageCapacityIsolationEviction [Slow] [Se
}) })
}) })
// PriorityEvictionOrdering tests that the node responds to node memory pressure by evicting pods. // PriorityMemoryEvictionOrdering tests that the node responds to node memory pressure by evicting pods.
// This test tests that the guaranteed pod is never evicted, and that the lower-priority pod is evicted before // This test tests that the guaranteed pod is never evicted, and that the lower-priority pod is evicted before
// the higher priority pod. // the higher priority pod.
var _ = framework.KubeDescribe("PriorityEvictionOrdering [Slow] [Serial] [Disruptive] [Flaky]", func() { var _ = framework.KubeDescribe("PriorityMemoryEvictionOrdering [Slow] [Serial] [Disruptive] [Flaky]", func() {
f := framework.NewDefaultFramework("priority-eviction-ordering-test") f := framework.NewDefaultFramework("priority-memory-eviction-ordering-test")
expectedNodeCondition := v1.NodeMemoryPressure expectedNodeCondition := v1.NodeMemoryPressure
pressureTimeout := 10 * time.Minute pressureTimeout := 10 * time.Minute
Context(fmt.Sprintf(testContextFmt, expectedNodeCondition), func() { Context(fmt.Sprintf(testContextFmt, expectedNodeCondition), func() {
@@ -237,6 +242,9 @@ var _ = framework.KubeDescribe("PriorityEvictionOrdering [Slow] [Serial] [Disrup
memoryConsumed := resource.MustParse("600Mi") memoryConsumed := resource.MustParse("600Mi")
summary := eventuallyGetSummary() summary := eventuallyGetSummary()
availableBytes := *(summary.Node.Memory.AvailableBytes) availableBytes := *(summary.Node.Memory.AvailableBytes)
if availableBytes <= uint64(memoryConsumed.Value()) {
framework.Skipf("Too little memory free on the host for the PriorityMemoryEvictionOrdering test to run")
}
initialConfig.EvictionHard = map[string]string{"memory.available": fmt.Sprintf("%d", availableBytes-uint64(memoryConsumed.Value()))} initialConfig.EvictionHard = map[string]string{"memory.available": fmt.Sprintf("%d", availableBytes-uint64(memoryConsumed.Value()))}
initialConfig.EvictionMinimumReclaim = map[string]string{} initialConfig.EvictionMinimumReclaim = map[string]string{}
}) })
@@ -267,6 +275,53 @@ var _ = framework.KubeDescribe("PriorityEvictionOrdering [Slow] [Serial] [Disrup
}) })
}) })
// PriorityLocalStorageEvictionOrdering tests that the node responds to node disk pressure by evicting pods.
// This test tests that the guaranteed pod is never evicted, and that the lower-priority pod is evicted before
// the higher priority pod.
var _ = framework.KubeDescribe("PriorityLocalStorageEvictionOrdering [Slow] [Serial] [Disruptive] [Flaky]", func() {
f := framework.NewDefaultFramework("priority-disk-eviction-ordering-test")
expectedNodeCondition := v1.NodeDiskPressure
pressureTimeout := 10 * time.Minute
Context(fmt.Sprintf(testContextFmt, expectedNodeCondition), func() {
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
initialConfig.FeatureGates[string(features.PodPriority)] = true
initialConfig.FeatureGates[string(features.LocalStorageCapacityIsolation)] = true
diskConsumed := resource.MustParse("350Mi")
summary := eventuallyGetSummary()
availableBytes := *(summary.Node.Fs.AvailableBytes)
if availableBytes <= uint64(diskConsumed.Value()) {
framework.Skipf("Too little disk free on the host for the PriorityLocalStorageEvictionOrdering test to run")
}
initialConfig.EvictionHard = map[string]string{"nodefs.available": fmt.Sprintf("%d", availableBytes-uint64(diskConsumed.Value()))}
initialConfig.EvictionMinimumReclaim = map[string]string{}
})
specs := []podEvictSpec{
{
evictionPriority: 2,
pod: diskConsumingPod("best-effort-disk", lotsOfDisk, nil, v1.ResourceRequirements{}),
},
{
evictionPriority: 1,
pod: diskConsumingPod("high-priority-disk", lotsOfDisk, nil, v1.ResourceRequirements{}),
},
{
evictionPriority: 0,
pod: diskConsumingPod("guaranteed-disk", 299 /* Mb */, nil, v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceEphemeralStorage: resource.MustParse("300Mi"),
},
Limits: v1.ResourceList{
v1.ResourceEphemeralStorage: resource.MustParse("300Mi"),
},
}),
},
}
systemPriority := int32(2147483647)
specs[1].pod.Spec.Priority = &systemPriority
runEvictionTest(f, pressureTimeout, expectedNodeCondition, logDiskMetrics, specs)
})
})
// Struct used by runEvictionTest that specifies the pod, and when that pod should be evicted, relative to other pods // Struct used by runEvictionTest that specifies the pod, and when that pod should be evicted, relative to other pods
type podEvictSpec struct { type podEvictSpec struct {
// P0 should never be evicted, P1 shouldn't evict before P2, etc. // P0 should never be evicted, P1 shouldn't evict before P2, etc.
@@ -571,9 +626,9 @@ func inodeConsumingPod(name string, volumeSource *v1.VolumeSource) *v1.Pod {
return podWithCommand(volumeSource, v1.ResourceRequirements{}, name, "i=0; while true; do touch %s${i}.txt; sleep 0.001; i=$((i+=1)); done;") return podWithCommand(volumeSource, v1.ResourceRequirements{}, name, "i=0; while true; do touch %s${i}.txt; sleep 0.001; i=$((i+=1)); done;")
} }
func diskConsumingPod(name string, diskConsumedMB int64, volumeSource *v1.VolumeSource, resources v1.ResourceRequirements) *v1.Pod { func diskConsumingPod(name string, diskConsumedMB int, volumeSource *v1.VolumeSource, resources v1.ResourceRequirements) *v1.Pod {
// Each iteration writes 1Mb to the file // Each iteration writes 1 Mb, so do diskConsumedMB iterations.
return podWithCommand(volumeSource, resources, name, fmt.Sprintf("i=0; while [ $i -lt %d ];", diskConsumedMB/100)+" do dd if=/dev/urandom of=%s${i} bs=100 count=1000000; i=$(($i+1)); done; while true; do sleep 5; done") return podWithCommand(volumeSource, resources, name, fmt.Sprintf("i=0; while [ $i -lt %d ];", diskConsumedMB)+" do dd if=/dev/urandom of=%s${i} bs=1048576 count=1; i=$(($i+1)); done; while true; do sleep 5; done")
} }
// podWithCommand returns a pod with the provided volumeSource and resourceRequirements. // podWithCommand returns a pod with the provided volumeSource and resourceRequirements.