aa50c0f54c
Automatic merge from submit-queue (batch tested with PRs 51628, 51637, 51490, 51279, 51302) Fix pod local ephemeral storage usage calculation We use podDiskUsage to calculate pod local ephemeral storage which is not correct, because podDiskUsage also contains HostPath volume which is considered as persistent storage This pr fixes it **Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #51489 **Special notes for your reviewer**: **Release note**: ```release-note NONE ``` /assign @jingxu97 @vishh cc @ddysher
467 lines
14 KiB
Go
467 lines
14 KiB
Go
/*
|
|
Copyright 2016 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package e2e_node
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
|
|
. "github.com/onsi/ginkgo"
|
|
. "github.com/onsi/gomega"
|
|
"k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
|
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
|
|
"k8s.io/kubernetes/test/e2e/framework"
|
|
)
|
|
|
|
type podEvictSpec struct {
|
|
evicted bool
|
|
pod v1.Pod
|
|
}
|
|
|
|
const (
|
|
totalEvict = 7
|
|
)
|
|
|
|
// Eviction Policy is described here:
|
|
// https://github.com/kubernetes/kubernetes/blob/master/docs/proposals/kubelet-eviction.md
|
|
|
|
var _ = framework.KubeDescribe("LocalStorageCapacityIsolationEviction [Slow] [Serial] [Disruptive] [Flaky] [Feature:LocalStorageCapacityIsolation]", func() {
|
|
|
|
f := framework.NewDefaultFramework("localstorage-eviction-test")
|
|
|
|
emptyDirVolumeName := "volume-emptydir-pod"
|
|
gitRepoVolumeName := "volume-gitrepo-pod"
|
|
configMapVolumeName := "volume-configmap-pod"
|
|
downwardAPIVolumeName := "volume-downwardapi-pod"
|
|
podTestSpecs := []podEvictSpec{
|
|
{evicted: true, // This pod should be evicted because emptyDir (default storage type) usage violation
|
|
pod: v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "emptydir-hog-pod"},
|
|
Spec: v1.PodSpec{
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
Containers: []v1.Container{
|
|
{
|
|
Image: "gcr.io/google_containers/busybox:1.24",
|
|
Name: "container-emptydir-hog-pod",
|
|
Command: []string{
|
|
"sh",
|
|
"-c",
|
|
"sleep 5; dd if=/dev/urandom of=target-file of=/cache/target-file bs=50000 count=1; while true; do sleep 5; done",
|
|
},
|
|
VolumeMounts: []v1.VolumeMount{
|
|
{
|
|
Name: emptyDirVolumeName,
|
|
MountPath: "/cache",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Volumes: []v1.Volume{
|
|
{
|
|
Name: emptyDirVolumeName,
|
|
VolumeSource: v1.VolumeSource{
|
|
EmptyDir: &v1.EmptyDirVolumeSource{
|
|
SizeLimit: resource.NewQuantity(int64(1000), resource.BinarySI),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
{evicted: true, // This pod should be evicted because emptyDir (memory type) usage violation
|
|
pod: v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "emptydir-memory-pod"},
|
|
Spec: v1.PodSpec{
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
Containers: []v1.Container{
|
|
{
|
|
Image: "gcr.io/google_containers/busybox:1.24",
|
|
Name: "container-emptydir-memory-pod",
|
|
Command: []string{
|
|
"sh",
|
|
"-c",
|
|
"sleep 5; dd if=/dev/urandom of=target-file of=/cache/target-file bs=50000 count=1; while true; do sleep 5; done",
|
|
},
|
|
VolumeMounts: []v1.VolumeMount{
|
|
{
|
|
Name: emptyDirVolumeName,
|
|
MountPath: "/cache",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Volumes: []v1.Volume{
|
|
{
|
|
Name: emptyDirVolumeName,
|
|
VolumeSource: v1.VolumeSource{
|
|
EmptyDir: &v1.EmptyDirVolumeSource{
|
|
Medium: "Memory",
|
|
SizeLimit: resource.NewQuantity(int64(10000), resource.BinarySI),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
{evicted: false,
|
|
pod: v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "container-emptydir-pod-critical"},
|
|
Spec: v1.PodSpec{
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
Containers: []v1.Container{
|
|
{
|
|
Image: "gcr.io/google_containers/busybox:1.24",
|
|
Name: "container-emptydir-hog-pod",
|
|
Command: []string{
|
|
"sh",
|
|
"-c",
|
|
"sleep 5; dd if=/dev/urandom of=target-file of=/cache/target-file bs=50000 count=1; while true; do sleep 5; done",
|
|
},
|
|
VolumeMounts: []v1.VolumeMount{
|
|
{
|
|
Name: emptyDirVolumeName,
|
|
MountPath: "/cache",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Volumes: []v1.Volume{
|
|
{
|
|
Name: emptyDirVolumeName,
|
|
VolumeSource: v1.VolumeSource{
|
|
EmptyDir: &v1.EmptyDirVolumeSource{
|
|
SizeLimit: resource.NewQuantity(int64(100000), resource.BinarySI),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
{evicted: true, // This pod should be evicted because container ephemeral storage usage violation
|
|
pod: v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "container-hog-pod"},
|
|
Spec: v1.PodSpec{
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
Containers: []v1.Container{
|
|
{
|
|
Image: "gcr.io/google_containers/busybox:1.24",
|
|
Name: "container-hog-pod",
|
|
Command: []string{
|
|
"sh",
|
|
"-c",
|
|
"sleep 5; dd if=/dev/urandom of=target-file bs=50000 count=1; while true; do sleep 5; done",
|
|
},
|
|
Resources: v1.ResourceRequirements{
|
|
Limits: v1.ResourceList{
|
|
v1.ResourceEphemeralStorage: *resource.NewMilliQuantity(
|
|
int64(40000),
|
|
resource.BinarySI),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
{evicted: true, // This pod should be evicted because pod ephemeral storage usage violation
|
|
pod: v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "emptydir-container-hog-pod"},
|
|
Spec: v1.PodSpec{
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
Containers: []v1.Container{
|
|
{
|
|
Image: "gcr.io/google_containers/busybox:1.24",
|
|
Name: "emptydir-container-hog-pod",
|
|
Command: []string{
|
|
"sh",
|
|
"-c",
|
|
"sleep 5; dd if=/dev/urandom of=target-file of=/cache/target-file bs=50000 count=1; while true; do sleep 5; done",
|
|
},
|
|
Resources: v1.ResourceRequirements{
|
|
Limits: v1.ResourceList{
|
|
v1.ResourceEphemeralStorage: *resource.NewMilliQuantity(
|
|
int64(40000),
|
|
resource.BinarySI),
|
|
},
|
|
},
|
|
VolumeMounts: []v1.VolumeMount{
|
|
{
|
|
Name: emptyDirVolumeName,
|
|
MountPath: "/cache",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Volumes: []v1.Volume{
|
|
{
|
|
Name: emptyDirVolumeName,
|
|
VolumeSource: v1.VolumeSource{
|
|
EmptyDir: &v1.EmptyDirVolumeSource{
|
|
SizeLimit: resource.NewQuantity(int64(100000), resource.BinarySI),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
{evicted: true, // This pod should be evicted because pod ephemeral storage usage violation
|
|
pod: v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "downward-api-container-hog-pod"},
|
|
Spec: v1.PodSpec{
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
Containers: []v1.Container{
|
|
{
|
|
Image: "gcr.io/google_containers/busybox:1.24",
|
|
Name: "downward-api-container-hog-pod",
|
|
Command: []string{
|
|
"sh",
|
|
"-c",
|
|
"sleep 5; dd if=/dev/urandom of=target-file of=/cache/target-file bs=50000 count=1; while true; do sleep 5; done",
|
|
},
|
|
Resources: v1.ResourceRequirements{
|
|
Limits: v1.ResourceList{
|
|
v1.ResourceEphemeralStorage: *resource.NewMilliQuantity(
|
|
int64(40000),
|
|
resource.BinarySI),
|
|
},
|
|
},
|
|
VolumeMounts: []v1.VolumeMount{
|
|
{
|
|
Name: downwardAPIVolumeName,
|
|
MountPath: "/cache",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Volumes: []v1.Volume{
|
|
{
|
|
Name: downwardAPIVolumeName,
|
|
VolumeSource: v1.VolumeSource{
|
|
DownwardAPI: &v1.DownwardAPIVolumeSource{},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
{evicted: true, // This pod should be evicted because pod ephemeral storage usage violation
|
|
pod: v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "configmap-container-hog-pod"},
|
|
Spec: v1.PodSpec{
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
Containers: []v1.Container{
|
|
{
|
|
Image: "gcr.io/google_containers/busybox:1.24",
|
|
Name: "configmap-container-hog-pod",
|
|
Command: []string{
|
|
"sh",
|
|
"-c",
|
|
"sleep 5; dd if=/dev/urandom of=target-file of=/cache/target-file bs=50000 count=1; while true; do sleep 5; done",
|
|
},
|
|
Resources: v1.ResourceRequirements{
|
|
Limits: v1.ResourceList{
|
|
v1.ResourceEphemeralStorage: *resource.NewMilliQuantity(
|
|
int64(40000),
|
|
resource.BinarySI),
|
|
},
|
|
},
|
|
VolumeMounts: []v1.VolumeMount{
|
|
{
|
|
Name: configMapVolumeName,
|
|
MountPath: "/cache",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Volumes: []v1.Volume{
|
|
{
|
|
Name: configMapVolumeName,
|
|
VolumeSource: v1.VolumeSource{
|
|
ConfigMap: &v1.ConfigMapVolumeSource{
|
|
LocalObjectReference: v1.LocalObjectReference{
|
|
Name: "my-cfgmap",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
{evicted: true, // This pod should be evicted because pod ephemeral storage usage violation
|
|
pod: v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "gitrepo-container-hog-pod"},
|
|
Spec: v1.PodSpec{
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
Containers: []v1.Container{
|
|
{
|
|
Image: "gcr.io/google_containers/busybox:1.24",
|
|
Name: "gitrepo-container-hog-pod",
|
|
Command: []string{
|
|
"sh",
|
|
"-c",
|
|
"sleep 5; dd if=/dev/urandom of=target-file of=/cache/target-file bs=50000 count=1; while true; do sleep 5; done",
|
|
},
|
|
Resources: v1.ResourceRequirements{
|
|
Limits: v1.ResourceList{
|
|
v1.ResourceEphemeralStorage: *resource.NewMilliQuantity(
|
|
int64(40000),
|
|
resource.BinarySI),
|
|
},
|
|
},
|
|
VolumeMounts: []v1.VolumeMount{
|
|
{
|
|
Name: gitRepoVolumeName,
|
|
MountPath: "/cache",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Volumes: []v1.Volume{
|
|
{
|
|
Name: gitRepoVolumeName,
|
|
VolumeSource: v1.VolumeSource{
|
|
GitRepo: &v1.GitRepoVolumeSource{
|
|
Repository: "my-repo",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
evictionTestTimeout := 10 * time.Minute
|
|
testCondition := "PodLocalEphemeralStorage/ContainerLocalEphemeralStorage usage limit violation"
|
|
Context(fmt.Sprintf("EmptyDirEviction when we run containers that should cause %s", testCondition), func() {
|
|
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
|
|
initialConfig.FeatureGates += ", LocalStorageCapacityIsolation=true"
|
|
})
|
|
err := utilfeature.DefaultFeatureGate.Set("LocalStorageCapacityIsolation=true")
|
|
if err != nil {
|
|
framework.Failf("Failed to enable feature gate for LocalStorageCapacityIsolation: %v", err)
|
|
return
|
|
}
|
|
|
|
runLocalStorageIsolationEvictionTest(f, testCondition, podTestSpecs, evictionTestTimeout, hasInodePressure)
|
|
})
|
|
})
|
|
|
|
// runLocalStorageEvictionTest sets up a testing environment given the provided nodes, and checks a few things:
|
|
// pods that exceed their local storage limit are evicted
|
|
// pods that didn't exceed their local storage limit are not evicted
|
|
// runLocalStorageEvictionTest then cleans up the testing environment by deleting provided nodes,
|
|
func runLocalStorageIsolationEvictionTest(f *framework.Framework, testCondition string, podTestSpecs []podEvictSpec, evictionTestTimeout time.Duration,
|
|
hasPressureCondition func(*framework.Framework, string) (bool, error)) {
|
|
|
|
Context(fmt.Sprintf("EmptyDirEviction when we run containers that should cause %s", testCondition), func() {
|
|
|
|
BeforeEach(func() {
|
|
By("seting up pods to be used by tests")
|
|
|
|
for _, spec := range podTestSpecs {
|
|
By(fmt.Sprintf("creating pod with container: %s", spec.pod.Name))
|
|
f.PodClient().CreateSync(&spec.pod)
|
|
}
|
|
})
|
|
|
|
It(fmt.Sprintf("Test should eventually see %s, and then evict the correct pods", testCondition), func() {
|
|
evictNum := 0
|
|
evictMap := make(map[string]string)
|
|
Eventually(func() error {
|
|
// Gather current information
|
|
updatedPodList, err := f.ClientSet.Core().Pods(f.Namespace.Name).List(metav1.ListOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get the list of pod: %v", err)
|
|
}
|
|
updatedPods := updatedPodList.Items
|
|
|
|
for _, p := range updatedPods {
|
|
framework.Logf("fetching pod %s; phase= %v", p.Name, p.Status.Phase)
|
|
for _, testPod := range podTestSpecs {
|
|
if p.Name == testPod.pod.Name {
|
|
if !testPod.evicted {
|
|
Expect(p.Status.Phase).NotTo(Equal(v1.PodFailed),
|
|
fmt.Sprintf("%s pod failed (and shouldn't have failed)", p.Name))
|
|
} else {
|
|
if _, ok := evictMap[p.Name]; !ok && p.Status.Phase == v1.PodFailed {
|
|
evictNum++
|
|
evictMap[p.Name] = p.Name
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
if evictNum == totalEvict {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("pods that caused %s have not been evicted", testCondition)
|
|
}, evictionTestTimeout, evictionPollInterval).Should(BeNil())
|
|
|
|
By("making sure we can start a new pod after the test")
|
|
podName := "test-admit-pod"
|
|
f.PodClient().CreateSync(&v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: podName,
|
|
},
|
|
Spec: v1.PodSpec{
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
Containers: []v1.Container{
|
|
{
|
|
Image: framework.GetPauseImageNameForHostArch(),
|
|
Name: podName,
|
|
},
|
|
},
|
|
},
|
|
})
|
|
})
|
|
|
|
AfterEach(func() {
|
|
By("deleting pods")
|
|
for _, spec := range podTestSpecs {
|
|
By(fmt.Sprintf("deleting pod: %s", spec.pod.Name))
|
|
f.PodClient().DeleteSync(spec.pod.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
|
|
}
|
|
|
|
if CurrentGinkgoTestDescription().Failed {
|
|
if framework.TestContext.DumpLogsOnFailure {
|
|
logPodEvents(f)
|
|
logNodeEvents(f)
|
|
}
|
|
By("sleeping to allow for cleanup of test")
|
|
time.Sleep(postTestConditionMonitoringPeriod)
|
|
}
|
|
})
|
|
})
|
|
}
|