Merge pull request #122745 from kannon92/swap-no-swap-default
[KEP-2400] add no swap as the default option for swap
This commit is contained in:
		| @@ -105,6 +105,7 @@ import ( | |||||||
| 	kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics" | 	kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics" | ||||||
| 	"k8s.io/kubernetes/pkg/kubelet/server" | 	"k8s.io/kubernetes/pkg/kubelet/server" | ||||||
| 	"k8s.io/kubernetes/pkg/kubelet/stats/pidlimit" | 	"k8s.io/kubernetes/pkg/kubelet/stats/pidlimit" | ||||||
|  | 	kubelettypes "k8s.io/kubernetes/pkg/kubelet/types" | ||||||
| 	kubeletutil "k8s.io/kubernetes/pkg/kubelet/util" | 	kubeletutil "k8s.io/kubernetes/pkg/kubelet/util" | ||||||
| 	utilfs "k8s.io/kubernetes/pkg/util/filesystem" | 	utilfs "k8s.io/kubernetes/pkg/util/filesystem" | ||||||
| 	"k8s.io/kubernetes/pkg/util/flock" | 	"k8s.io/kubernetes/pkg/util/flock" | ||||||
| @@ -799,6 +800,14 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend | |||||||
| 			return fmt.Errorf("topology manager policy options %v require feature gates %q enabled", | 			return fmt.Errorf("topology manager policy options %v require feature gates %q enabled", | ||||||
| 				s.TopologyManagerPolicyOptions, features.TopologyManagerPolicyOptions) | 				s.TopologyManagerPolicyOptions, features.TopologyManagerPolicyOptions) | ||||||
| 		} | 		} | ||||||
|  | 		if utilfeature.DefaultFeatureGate.Enabled(features.NodeSwap) { | ||||||
|  | 			if !isCgroup2UnifiedMode() && s.MemorySwap.SwapBehavior == kubelettypes.LimitedSwap { | ||||||
|  | 				klog.InfoS("Swap feature is enabled and LimitedSwap but it is only supported with cgroupv2", "CGroupV2", isCgroup2UnifiedMode(), "SwapBehavior", s.MemorySwap.SwapBehavior) | ||||||
|  | 			} | ||||||
|  | 			if !s.FailSwapOn && s.MemorySwap.SwapBehavior == "" { | ||||||
|  | 				klog.InfoS("NoSwap is set due to memorySwapBehavior not specified", "memorySwapBehavior", s.MemorySwap.SwapBehavior, "FailSwapOn", s.FailSwapOn) | ||||||
|  | 			} | ||||||
|  | 		} | ||||||
|  |  | ||||||
| 		kubeDeps.ContainerManager, err = cm.NewContainerManager( | 		kubeDeps.ContainerManager, err = cm.NewContainerManager( | ||||||
| 			kubeDeps.Mounter, | 			kubeDeps.Mounter, | ||||||
|   | |||||||
| @@ -539,11 +539,13 @@ const ( | |||||||
| 	// Allow pods to failover to a different node in case of non graceful node shutdown | 	// Allow pods to failover to a different node in case of non graceful node shutdown | ||||||
| 	NodeOutOfServiceVolumeDetach featuregate.Feature = "NodeOutOfServiceVolumeDetach" | 	NodeOutOfServiceVolumeDetach featuregate.Feature = "NodeOutOfServiceVolumeDetach" | ||||||
|  |  | ||||||
| 	// owner: @iholder101 | 	// owner: @iholder101 @kannon92 | ||||||
|  | 	// kep: https://kep.k8s.io/2400 | ||||||
| 	// alpha: v1.22 | 	// alpha: v1.22 | ||||||
| 	// beta1: v1.28. For more info, please look at the KEP: https://kep.k8s.io/2400. | 	// beta1: v1.28 (default=false) | ||||||
| 	// | 	// beta2: v.1.30 (default=true) | ||||||
| 	// Permits kubelet to run with swap enabled |  | ||||||
|  | 	// Permits kubelet to run with swap enabled. | ||||||
| 	NodeSwap featuregate.Feature = "NodeSwap" | 	NodeSwap featuregate.Feature = "NodeSwap" | ||||||
|  |  | ||||||
| 	// owner: @mortent, @atiratree, @ravig | 	// owner: @mortent, @atiratree, @ravig | ||||||
| @@ -1105,7 +1107,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS | |||||||
|  |  | ||||||
| 	NodeOutOfServiceVolumeDetach: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31 | 	NodeOutOfServiceVolumeDetach: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31 | ||||||
|  |  | ||||||
| 	NodeSwap: {Default: false, PreRelease: featuregate.Beta}, | 	NodeSwap: {Default: true, PreRelease: featuregate.Beta}, | ||||||
|  |  | ||||||
| 	PDBUnhealthyPodEvictionPolicy: {Default: true, PreRelease: featuregate.Beta}, | 	PDBUnhealthyPodEvictionPolicy: {Default: true, PreRelease: featuregate.Beta}, | ||||||
|  |  | ||||||
|   | |||||||
							
								
								
									
										2
									
								
								pkg/generated/openapi/zz_generated.openapi.go
									
									
									
										generated
									
									
									
								
							
							
						
						
									
										2
									
								
								pkg/generated/openapi/zz_generated.openapi.go
									
									
									
										generated
									
									
									
								
							| @@ -59686,7 +59686,7 @@ func schema_k8sio_kubelet_config_v1beta1_MemorySwapConfiguration(ref common.Refe | |||||||
| 				Properties: map[string]spec.Schema{ | 				Properties: map[string]spec.Schema{ | ||||||
| 					"swapBehavior": { | 					"swapBehavior": { | ||||||
| 						SchemaProps: spec.SchemaProps{ | 						SchemaProps: spec.SchemaProps{ | ||||||
| 							Description: "swapBehavior configures swap memory available to container workloads. May be one of \"\", \"LimitedSwap\": workload combined memory and swap usage cannot exceed pod memory limit \"UnlimitedSwap\": workloads can use unlimited swap, up to the allocatable limit.", | 							Description: "swapBehavior configures swap memory available to container workloads. May be one of \"\", \"NoSwap\": workloads can not use swap, default option. \"LimitedSwap\": workload swap usage is limited. The swap limit is proportionate to the container's memory request.", | ||||||
| 							Type:        []string{"string"}, | 							Type:        []string{"string"}, | ||||||
| 							Format:      "", | 							Format:      "", | ||||||
| 						}, | 						}, | ||||||
|   | |||||||
| @@ -666,8 +666,8 @@ type ShutdownGracePeriodByPodPriority struct { | |||||||
|  |  | ||||||
| type MemorySwapConfiguration struct { | type MemorySwapConfiguration struct { | ||||||
| 	// swapBehavior configures swap memory available to container workloads. May be one of | 	// swapBehavior configures swap memory available to container workloads. May be one of | ||||||
| 	// "", "LimitedSwap": workload combined memory and swap usage cannot exceed pod memory limit | 	// "", "NoSwap": workloads can not use swap, default option. | ||||||
| 	// "UnlimitedSwap": workloads can use unlimited swap, up to the allocatable limit. | 	// "LimitedSwap": workload swap usage is limited. The swap limit is proportionate to the container's memory request. | ||||||
| 	// +featureGate=NodeSwap | 	// +featureGate=NodeSwap | ||||||
| 	// +optional | 	// +optional | ||||||
| 	SwapBehavior string | 	SwapBehavior string | ||||||
|   | |||||||
| @@ -471,7 +471,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { | |||||||
| 				IPTablesMasqueradeBit:                     utilpointer.Int32(1), | 				IPTablesMasqueradeBit:                     utilpointer.Int32(1), | ||||||
| 				IPTablesDropBit:                           utilpointer.Int32(1), | 				IPTablesDropBit:                           utilpointer.Int32(1), | ||||||
| 				FailSwapOn:                                utilpointer.Bool(true), | 				FailSwapOn:                                utilpointer.Bool(true), | ||||||
| 				MemorySwap:                                v1beta1.MemorySwapConfiguration{SwapBehavior: "UnlimitedSwap"}, | 				MemorySwap:                                v1beta1.MemorySwapConfiguration{SwapBehavior: "NoSwap"}, | ||||||
| 				ContainerLogMaxSize:                       "1Mi", | 				ContainerLogMaxSize:                       "1Mi", | ||||||
| 				ContainerLogMaxFiles:                      utilpointer.Int32(1), | 				ContainerLogMaxFiles:                      utilpointer.Int32(1), | ||||||
| 				ContainerLogMaxWorkers:                    utilpointer.Int32(1), | 				ContainerLogMaxWorkers:                    utilpointer.Int32(1), | ||||||
| @@ -620,7 +620,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { | |||||||
| 				IPTablesMasqueradeBit:                     utilpointer.Int32(1), | 				IPTablesMasqueradeBit:                     utilpointer.Int32(1), | ||||||
| 				IPTablesDropBit:                           utilpointer.Int32(1), | 				IPTablesDropBit:                           utilpointer.Int32(1), | ||||||
| 				FailSwapOn:                                utilpointer.Bool(true), | 				FailSwapOn:                                utilpointer.Bool(true), | ||||||
| 				MemorySwap:                                v1beta1.MemorySwapConfiguration{SwapBehavior: "UnlimitedSwap"}, | 				MemorySwap:                                v1beta1.MemorySwapConfiguration{SwapBehavior: "NoSwap"}, | ||||||
| 				ContainerLogMaxSize:                       "1Mi", | 				ContainerLogMaxSize:                       "1Mi", | ||||||
| 				ContainerLogMaxFiles:                      utilpointer.Int32(1), | 				ContainerLogMaxFiles:                      utilpointer.Int32(1), | ||||||
| 				ContainerLogMaxWorkers:                    utilpointer.Int32(1), | 				ContainerLogMaxWorkers:                    utilpointer.Int32(1), | ||||||
|   | |||||||
| @@ -195,10 +195,10 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration, featur | |||||||
| 	if localFeatureGate.Enabled(features.NodeSwap) { | 	if localFeatureGate.Enabled(features.NodeSwap) { | ||||||
| 		switch kc.MemorySwap.SwapBehavior { | 		switch kc.MemorySwap.SwapBehavior { | ||||||
| 		case "": | 		case "": | ||||||
|  | 		case kubetypes.NoSwap: | ||||||
| 		case kubetypes.LimitedSwap: | 		case kubetypes.LimitedSwap: | ||||||
| 		case kubetypes.UnlimitedSwap: |  | ||||||
| 		default: | 		default: | ||||||
| 			allErrors = append(allErrors, fmt.Errorf("invalid configuration: memorySwap.swapBehavior %q must be one of: \"\", %q, or %q", kc.MemorySwap.SwapBehavior, kubetypes.LimitedSwap, kubetypes.UnlimitedSwap)) | 			allErrors = append(allErrors, fmt.Errorf("invalid configuration: memorySwap.swapBehavior %q must be one of: \"\", %q or %q", kc.MemorySwap.SwapBehavior, kubetypes.LimitedSwap, kubetypes.NoSwap)) | ||||||
| 		} | 		} | ||||||
| 	} | 	} | ||||||
| 	if !localFeatureGate.Enabled(features.NodeSwap) && kc.MemorySwap != (kubeletconfig.MemorySwapConfiguration{}) { | 	if !localFeatureGate.Enabled(features.NodeSwap) && kc.MemorySwap != (kubeletconfig.MemorySwapConfiguration{}) { | ||||||
|   | |||||||
| @@ -367,7 +367,7 @@ func TestValidateKubeletConfiguration(t *testing.T) { | |||||||
| 			conf.MemorySwap.SwapBehavior = "invalid-behavior" | 			conf.MemorySwap.SwapBehavior = "invalid-behavior" | ||||||
| 			return conf | 			return conf | ||||||
| 		}, | 		}, | ||||||
| 		errMsg: "invalid configuration: memorySwap.swapBehavior \"invalid-behavior\" must be one of: \"\", \"LimitedSwap\", or \"UnlimitedSwap\"", | 		errMsg: "invalid configuration: memorySwap.swapBehavior \"invalid-behavior\" must be one of: \"\", \"LimitedSwap\" or \"NoSwap\"", | ||||||
| 	}, { | 	}, { | ||||||
| 		name: "specify MemorySwap.SwapBehavior without enabling NodeSwap", | 		name: "specify MemorySwap.SwapBehavior without enabling NodeSwap", | ||||||
| 		configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { | 		configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { | ||||||
|   | |||||||
| @@ -168,6 +168,12 @@ func (m *kubeGenericRuntimeManager) configureContainerSwapResources(lcr *runtime | |||||||
| 		return | 		return | ||||||
| 	} | 	} | ||||||
| 	swapConfigurationHelper := newSwapConfigurationHelper(*m.machineInfo) | 	swapConfigurationHelper := newSwapConfigurationHelper(*m.machineInfo) | ||||||
|  | 	if m.memorySwapBehavior == kubelettypes.LimitedSwap { | ||||||
|  | 		if !isCgroup2UnifiedMode() { | ||||||
|  | 			swapConfigurationHelper.ConfigureNoSwap(lcr) | ||||||
|  | 			return | ||||||
|  | 		} | ||||||
|  | 	} | ||||||
|  |  | ||||||
| 	if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.NodeSwap) { | 	if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.NodeSwap) { | ||||||
| 		swapConfigurationHelper.ConfigureNoSwap(lcr) | 		swapConfigurationHelper.ConfigureNoSwap(lcr) | ||||||
| @@ -177,10 +183,12 @@ func (m *kubeGenericRuntimeManager) configureContainerSwapResources(lcr *runtime | |||||||
| 	// NOTE(ehashman): Behavior is defined in the opencontainers runtime spec: | 	// NOTE(ehashman): Behavior is defined in the opencontainers runtime spec: | ||||||
| 	// https://github.com/opencontainers/runtime-spec/blob/1c3f411f041711bbeecf35ff7e93461ea6789220/config-linux.md#memory | 	// https://github.com/opencontainers/runtime-spec/blob/1c3f411f041711bbeecf35ff7e93461ea6789220/config-linux.md#memory | ||||||
| 	switch m.memorySwapBehavior { | 	switch m.memorySwapBehavior { | ||||||
|  | 	case kubelettypes.NoSwap: | ||||||
|  | 		swapConfigurationHelper.ConfigureNoSwap(lcr) | ||||||
| 	case kubelettypes.LimitedSwap: | 	case kubelettypes.LimitedSwap: | ||||||
| 		swapConfigurationHelper.ConfigureLimitedSwap(lcr, pod, container) | 		swapConfigurationHelper.ConfigureLimitedSwap(lcr, pod, container) | ||||||
| 	default: | 	default: | ||||||
| 		swapConfigurationHelper.ConfigureUnlimitedSwap(lcr) | 		swapConfigurationHelper.ConfigureNoSwap(lcr) | ||||||
| 	} | 	} | ||||||
| } | } | ||||||
|  |  | ||||||
| @@ -401,19 +409,6 @@ func (m swapConfigurationHelper) ConfigureNoSwap(lcr *runtimeapi.LinuxContainerR | |||||||
| 	m.configureSwap(lcr, 0) | 	m.configureSwap(lcr, 0) | ||||||
| } | } | ||||||
|  |  | ||||||
| func (m swapConfigurationHelper) ConfigureUnlimitedSwap(lcr *runtimeapi.LinuxContainerResources) { |  | ||||||
| 	if !isCgroup2UnifiedMode() { |  | ||||||
| 		m.ConfigureNoSwap(lcr) |  | ||||||
| 		return |  | ||||||
| 	} |  | ||||||
|  |  | ||||||
| 	if lcr.Unified == nil { |  | ||||||
| 		lcr.Unified = map[string]string{} |  | ||||||
| 	} |  | ||||||
|  |  | ||||||
| 	lcr.Unified[cm.Cgroup2MaxSwapFilename] = "max" |  | ||||||
| } |  | ||||||
|  |  | ||||||
| func (m swapConfigurationHelper) configureSwap(lcr *runtimeapi.LinuxContainerResources, swapMemory int64) { | func (m swapConfigurationHelper) configureSwap(lcr *runtimeapi.LinuxContainerResources, swapMemory int64) { | ||||||
| 	if !isCgroup2UnifiedMode() { | 	if !isCgroup2UnifiedMode() { | ||||||
| 		klog.ErrorS(fmt.Errorf("swap configuration is not supported with cgroup v1"), "swap configuration under cgroup v1 is unexpected") | 		klog.ErrorS(fmt.Errorf("swap configuration is not supported with cgroup v1"), "swap configuration under cgroup v1 is unexpected") | ||||||
|   | |||||||
| @@ -918,19 +918,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) { | |||||||
| 		} | 		} | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	expectUnlimitedSwap := func(cgroupVersion CgroupVersion, resources ...*runtimeapi.LinuxContainerResources) { |  | ||||||
| 		const msg = "container is expected to have unlimited swap access" |  | ||||||
|  |  | ||||||
| 		for _, r := range resources { |  | ||||||
| 			switch cgroupVersion { |  | ||||||
| 			case cgroupV1: |  | ||||||
| 				assert.Equal(t, int64(-1), r.MemorySwapLimitInBytes, msg) |  | ||||||
| 			case cgroupV2: |  | ||||||
| 				assert.Equal(t, "max", r.Unified[cm.Cgroup2MaxSwapFilename], msg) |  | ||||||
| 			} |  | ||||||
| 		} |  | ||||||
| 	} |  | ||||||
|  |  | ||||||
| 	expectSwap := func(cgroupVersion CgroupVersion, swapBytesExpected int64, resources *runtimeapi.LinuxContainerResources) { | 	expectSwap := func(cgroupVersion CgroupVersion, swapBytesExpected int64, resources *runtimeapi.LinuxContainerResources) { | ||||||
| 		msg := fmt.Sprintf("container swap is expected to be limited by %d bytes", swapBytesExpected) | 		msg := fmt.Sprintf("container swap is expected to be limited by %d bytes", swapBytesExpected) | ||||||
|  |  | ||||||
| @@ -967,13 +954,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) { | |||||||
| 			nodeSwapFeatureGateEnabled: true, | 			nodeSwapFeatureGateEnabled: true, | ||||||
| 			swapBehavior:               types.LimitedSwap, | 			swapBehavior:               types.LimitedSwap, | ||||||
| 		}, | 		}, | ||||||
| 		{ |  | ||||||
| 			name:                       "cgroups v1, UnlimitedSwap, Burstable QoS", |  | ||||||
| 			cgroupVersion:              cgroupV1, |  | ||||||
| 			qosClass:                   v1.PodQOSBurstable, |  | ||||||
| 			nodeSwapFeatureGateEnabled: true, |  | ||||||
| 			swapBehavior:               types.UnlimitedSwap, |  | ||||||
| 		}, |  | ||||||
| 		{ | 		{ | ||||||
| 			name:                       "cgroups v1, LimitedSwap, Best-effort QoS", | 			name:                       "cgroups v1, LimitedSwap, Best-effort QoS", | ||||||
| 			cgroupVersion:              cgroupV1, | 			cgroupVersion:              cgroupV1, | ||||||
| @@ -990,17 +970,10 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) { | |||||||
| 			nodeSwapFeatureGateEnabled: false, | 			nodeSwapFeatureGateEnabled: false, | ||||||
| 			swapBehavior:               types.LimitedSwap, | 			swapBehavior:               types.LimitedSwap, | ||||||
| 		}, | 		}, | ||||||
| 		{ |  | ||||||
| 			name:                       "NodeSwap feature gate turned off, cgroups v2, UnlimitedSwap", |  | ||||||
| 			cgroupVersion:              cgroupV2, |  | ||||||
| 			qosClass:                   v1.PodQOSBurstable, |  | ||||||
| 			nodeSwapFeatureGateEnabled: false, |  | ||||||
| 			swapBehavior:               types.UnlimitedSwap, |  | ||||||
| 		}, |  | ||||||
|  |  | ||||||
| 		// With no swapBehavior, UnlimitedSwap should be the default | 		// With no swapBehavior, NoSwap should be the default | ||||||
| 		{ | 		{ | ||||||
| 			name:                       "With no swapBehavior - UnlimitedSwap should be the default", | 			name:                       "With no swapBehavior - NoSwap should be the default", | ||||||
| 			cgroupVersion:              cgroupV2, | 			cgroupVersion:              cgroupV2, | ||||||
| 			qosClass:                   v1.PodQOSBestEffort, | 			qosClass:                   v1.PodQOSBestEffort, | ||||||
| 			nodeSwapFeatureGateEnabled: true, | 			nodeSwapFeatureGateEnabled: true, | ||||||
| @@ -1008,6 +981,13 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) { | |||||||
| 		}, | 		}, | ||||||
|  |  | ||||||
| 		// With Guaranteed and Best-effort QoS | 		// With Guaranteed and Best-effort QoS | ||||||
|  | 		{ | ||||||
|  | 			name:                       "Best-effort QoS, cgroups v2, NoSwap", | ||||||
|  | 			cgroupVersion:              cgroupV2, | ||||||
|  | 			qosClass:                   v1.PodQOSBestEffort, | ||||||
|  | 			nodeSwapFeatureGateEnabled: true, | ||||||
|  | 			swapBehavior:               "NoSwap", | ||||||
|  | 		}, | ||||||
| 		{ | 		{ | ||||||
| 			name:                       "Best-effort QoS, cgroups v2, LimitedSwap", | 			name:                       "Best-effort QoS, cgroups v2, LimitedSwap", | ||||||
| 			cgroupVersion:              cgroupV2, | 			cgroupVersion:              cgroupV2, | ||||||
| @@ -1015,13 +995,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) { | |||||||
| 			nodeSwapFeatureGateEnabled: true, | 			nodeSwapFeatureGateEnabled: true, | ||||||
| 			swapBehavior:               types.LimitedSwap, | 			swapBehavior:               types.LimitedSwap, | ||||||
| 		}, | 		}, | ||||||
| 		{ |  | ||||||
| 			name:                       "Best-effort QoS, cgroups v2, UnlimitedSwap", |  | ||||||
| 			cgroupVersion:              cgroupV2, |  | ||||||
| 			qosClass:                   v1.PodQOSBurstable, |  | ||||||
| 			nodeSwapFeatureGateEnabled: true, |  | ||||||
| 			swapBehavior:               types.UnlimitedSwap, |  | ||||||
| 		}, |  | ||||||
| 		{ | 		{ | ||||||
| 			name:                       "Guaranteed QoS, cgroups v2, LimitedSwap", | 			name:                       "Guaranteed QoS, cgroups v2, LimitedSwap", | ||||||
| 			cgroupVersion:              cgroupV2, | 			cgroupVersion:              cgroupV2, | ||||||
| @@ -1029,13 +1002,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) { | |||||||
| 			nodeSwapFeatureGateEnabled: true, | 			nodeSwapFeatureGateEnabled: true, | ||||||
| 			swapBehavior:               types.LimitedSwap, | 			swapBehavior:               types.LimitedSwap, | ||||||
| 		}, | 		}, | ||||||
| 		{ |  | ||||||
| 			name:                       "Guaranteed QoS, cgroups v2, UnlimitedSwap", |  | ||||||
| 			cgroupVersion:              cgroupV2, |  | ||||||
| 			qosClass:                   v1.PodQOSGuaranteed, |  | ||||||
| 			nodeSwapFeatureGateEnabled: true, |  | ||||||
| 			swapBehavior:               types.UnlimitedSwap, |  | ||||||
| 		}, |  | ||||||
|  |  | ||||||
| 		// With a "guaranteed" container (when memory requests equal to limits) | 		// With a "guaranteed" container (when memory requests equal to limits) | ||||||
| 		{ | 		{ | ||||||
| @@ -1047,15 +1013,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) { | |||||||
| 			addContainerWithoutRequests: false, | 			addContainerWithoutRequests: false, | ||||||
| 			addGuaranteedContainer:      true, | 			addGuaranteedContainer:      true, | ||||||
| 		}, | 		}, | ||||||
| 		{ |  | ||||||
| 			name:                        "Burstable QoS, cgroups v2, UnlimitedSwap, with a guaranteed container", |  | ||||||
| 			cgroupVersion:               cgroupV2, |  | ||||||
| 			qosClass:                    v1.PodQOSBurstable, |  | ||||||
| 			nodeSwapFeatureGateEnabled:  true, |  | ||||||
| 			swapBehavior:                types.UnlimitedSwap, |  | ||||||
| 			addContainerWithoutRequests: false, |  | ||||||
| 			addGuaranteedContainer:      true, |  | ||||||
| 		}, |  | ||||||
|  |  | ||||||
| 		// Swap is expected to be allocated | 		// Swap is expected to be allocated | ||||||
| 		{ | 		{ | ||||||
| @@ -1067,15 +1024,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) { | |||||||
| 			addContainerWithoutRequests: false, | 			addContainerWithoutRequests: false, | ||||||
| 			addGuaranteedContainer:      false, | 			addGuaranteedContainer:      false, | ||||||
| 		}, | 		}, | ||||||
| 		{ |  | ||||||
| 			name:                        "Burstable QoS, cgroups v2, UnlimitedSwap", |  | ||||||
| 			cgroupVersion:               cgroupV2, |  | ||||||
| 			qosClass:                    v1.PodQOSBurstable, |  | ||||||
| 			nodeSwapFeatureGateEnabled:  true, |  | ||||||
| 			swapBehavior:                types.UnlimitedSwap, |  | ||||||
| 			addContainerWithoutRequests: false, |  | ||||||
| 			addGuaranteedContainer:      false, |  | ||||||
| 		}, |  | ||||||
| 		{ | 		{ | ||||||
| 			name:                        "Burstable QoS, cgroups v2, LimitedSwap, with a container with no requests", | 			name:                        "Burstable QoS, cgroups v2, LimitedSwap, with a container with no requests", | ||||||
| 			cgroupVersion:               cgroupV2, | 			cgroupVersion:               cgroupV2, | ||||||
| @@ -1085,15 +1033,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) { | |||||||
| 			addContainerWithoutRequests: true, | 			addContainerWithoutRequests: true, | ||||||
| 			addGuaranteedContainer:      false, | 			addGuaranteedContainer:      false, | ||||||
| 		}, | 		}, | ||||||
| 		{ |  | ||||||
| 			name:                        "Burstable QoS, cgroups v2, UnlimitedSwap, with a container with no requests", |  | ||||||
| 			cgroupVersion:               cgroupV2, |  | ||||||
| 			qosClass:                    v1.PodQOSBurstable, |  | ||||||
| 			nodeSwapFeatureGateEnabled:  true, |  | ||||||
| 			swapBehavior:                types.UnlimitedSwap, |  | ||||||
| 			addContainerWithoutRequests: true, |  | ||||||
| 			addGuaranteedContainer:      false, |  | ||||||
| 		}, |  | ||||||
| 		// All the above examples with Swap disabled on node | 		// All the above examples with Swap disabled on node | ||||||
| 		{ | 		{ | ||||||
| 			name:                       "Swap disabled on node, cgroups v1, LimitedSwap, Burstable QoS", | 			name:                       "Swap disabled on node, cgroups v1, LimitedSwap, Burstable QoS", | ||||||
| @@ -1103,14 +1042,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) { | |||||||
| 			nodeSwapFeatureGateEnabled: true, | 			nodeSwapFeatureGateEnabled: true, | ||||||
| 			swapBehavior:               types.LimitedSwap, | 			swapBehavior:               types.LimitedSwap, | ||||||
| 		}, | 		}, | ||||||
| 		{ |  | ||||||
| 			name:                       "Swap disabled on node, cgroups v1, UnlimitedSwap, Burstable QoS", |  | ||||||
| 			swapDisabledOnNode:         true, |  | ||||||
| 			cgroupVersion:              cgroupV1, |  | ||||||
| 			qosClass:                   v1.PodQOSBurstable, |  | ||||||
| 			nodeSwapFeatureGateEnabled: true, |  | ||||||
| 			swapBehavior:               types.UnlimitedSwap, |  | ||||||
| 		}, |  | ||||||
| 		{ | 		{ | ||||||
| 			name:                       "Swap disabled on node, cgroups v1, LimitedSwap, Best-effort QoS", | 			name:                       "Swap disabled on node, cgroups v1, LimitedSwap, Best-effort QoS", | ||||||
| 			swapDisabledOnNode:         true, | 			swapDisabledOnNode:         true, | ||||||
| @@ -1129,18 +1060,10 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) { | |||||||
| 			nodeSwapFeatureGateEnabled: false, | 			nodeSwapFeatureGateEnabled: false, | ||||||
| 			swapBehavior:               types.LimitedSwap, | 			swapBehavior:               types.LimitedSwap, | ||||||
| 		}, | 		}, | ||||||
| 		{ |  | ||||||
| 			name:                       "Swap disabled on node, NodeSwap feature gate turned off, cgroups v2, UnlimitedSwap", |  | ||||||
| 			swapDisabledOnNode:         true, |  | ||||||
| 			cgroupVersion:              cgroupV2, |  | ||||||
| 			qosClass:                   v1.PodQOSBurstable, |  | ||||||
| 			nodeSwapFeatureGateEnabled: false, |  | ||||||
| 			swapBehavior:               types.UnlimitedSwap, |  | ||||||
| 		}, |  | ||||||
|  |  | ||||||
| 		// With no swapBehavior, UnlimitedSwap should be the default | 		// With no swapBehavior, NoSwap should be the default | ||||||
| 		{ | 		{ | ||||||
| 			name:                       "Swap disabled on node, With no swapBehavior - UnlimitedSwap should be the default", | 			name:                       "Swap disabled on node, With no swapBehavior - NoSwap should be the default", | ||||||
| 			swapDisabledOnNode:         true, | 			swapDisabledOnNode:         true, | ||||||
| 			cgroupVersion:              cgroupV2, | 			cgroupVersion:              cgroupV2, | ||||||
| 			qosClass:                   v1.PodQOSBestEffort, | 			qosClass:                   v1.PodQOSBestEffort, | ||||||
| @@ -1157,14 +1080,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) { | |||||||
| 			nodeSwapFeatureGateEnabled: true, | 			nodeSwapFeatureGateEnabled: true, | ||||||
| 			swapBehavior:               types.LimitedSwap, | 			swapBehavior:               types.LimitedSwap, | ||||||
| 		}, | 		}, | ||||||
| 		{ |  | ||||||
| 			name:                       "Swap disabled on node, Best-effort QoS, cgroups v2, UnlimitedSwap", |  | ||||||
| 			swapDisabledOnNode:         true, |  | ||||||
| 			cgroupVersion:              cgroupV2, |  | ||||||
| 			qosClass:                   v1.PodQOSBurstable, |  | ||||||
| 			nodeSwapFeatureGateEnabled: true, |  | ||||||
| 			swapBehavior:               types.UnlimitedSwap, |  | ||||||
| 		}, |  | ||||||
| 		{ | 		{ | ||||||
| 			name:                       "Swap disabled on node, Guaranteed QoS, cgroups v2, LimitedSwap", | 			name:                       "Swap disabled on node, Guaranteed QoS, cgroups v2, LimitedSwap", | ||||||
| 			swapDisabledOnNode:         true, | 			swapDisabledOnNode:         true, | ||||||
| @@ -1173,14 +1088,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) { | |||||||
| 			nodeSwapFeatureGateEnabled: true, | 			nodeSwapFeatureGateEnabled: true, | ||||||
| 			swapBehavior:               types.LimitedSwap, | 			swapBehavior:               types.LimitedSwap, | ||||||
| 		}, | 		}, | ||||||
| 		{ |  | ||||||
| 			name:                       "Swap disabled on node, Guaranteed QoS, cgroups v2, UnlimitedSwap", |  | ||||||
| 			swapDisabledOnNode:         true, |  | ||||||
| 			cgroupVersion:              cgroupV2, |  | ||||||
| 			qosClass:                   v1.PodQOSGuaranteed, |  | ||||||
| 			nodeSwapFeatureGateEnabled: true, |  | ||||||
| 			swapBehavior:               types.UnlimitedSwap, |  | ||||||
| 		}, |  | ||||||
|  |  | ||||||
| 		// With a "guaranteed" container (when memory requests equal to limits) | 		// With a "guaranteed" container (when memory requests equal to limits) | ||||||
| 		{ | 		{ | ||||||
| @@ -1193,16 +1100,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) { | |||||||
| 			addContainerWithoutRequests: false, | 			addContainerWithoutRequests: false, | ||||||
| 			addGuaranteedContainer:      true, | 			addGuaranteedContainer:      true, | ||||||
| 		}, | 		}, | ||||||
| 		{ |  | ||||||
| 			name:                        "Swap disabled on node, Burstable QoS, cgroups v2, UnlimitedSwap, with a guaranteed container", |  | ||||||
| 			swapDisabledOnNode:          true, |  | ||||||
| 			cgroupVersion:               cgroupV2, |  | ||||||
| 			qosClass:                    v1.PodQOSBurstable, |  | ||||||
| 			nodeSwapFeatureGateEnabled:  true, |  | ||||||
| 			swapBehavior:                types.UnlimitedSwap, |  | ||||||
| 			addContainerWithoutRequests: false, |  | ||||||
| 			addGuaranteedContainer:      true, |  | ||||||
| 		}, |  | ||||||
|  |  | ||||||
| 		// Swap is expected to be allocated | 		// Swap is expected to be allocated | ||||||
| 		{ | 		{ | ||||||
| @@ -1215,16 +1112,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) { | |||||||
| 			addContainerWithoutRequests: false, | 			addContainerWithoutRequests: false, | ||||||
| 			addGuaranteedContainer:      false, | 			addGuaranteedContainer:      false, | ||||||
| 		}, | 		}, | ||||||
| 		{ |  | ||||||
| 			name:                        "Swap disabled on node, Burstable QoS, cgroups v2, UnlimitedSwap", |  | ||||||
| 			swapDisabledOnNode:          true, |  | ||||||
| 			cgroupVersion:               cgroupV2, |  | ||||||
| 			qosClass:                    v1.PodQOSBurstable, |  | ||||||
| 			nodeSwapFeatureGateEnabled:  true, |  | ||||||
| 			swapBehavior:                types.UnlimitedSwap, |  | ||||||
| 			addContainerWithoutRequests: false, |  | ||||||
| 			addGuaranteedContainer:      false, |  | ||||||
| 		}, |  | ||||||
| 		{ | 		{ | ||||||
| 			name:                        "Swap disabled on node, Burstable QoS, cgroups v2, LimitedSwap, with a container with no requests", | 			name:                        "Swap disabled on node, Burstable QoS, cgroups v2, LimitedSwap, with a container with no requests", | ||||||
| 			swapDisabledOnNode:          true, | 			swapDisabledOnNode:          true, | ||||||
| @@ -1235,16 +1122,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) { | |||||||
| 			addContainerWithoutRequests: true, | 			addContainerWithoutRequests: true, | ||||||
| 			addGuaranteedContainer:      false, | 			addGuaranteedContainer:      false, | ||||||
| 		}, | 		}, | ||||||
| 		{ |  | ||||||
| 			name:                        "Swap disabled on node, Burstable QoS, cgroups v2, UnlimitedSwap, with a container with no requests", |  | ||||||
| 			swapDisabledOnNode:          true, |  | ||||||
| 			cgroupVersion:               cgroupV2, |  | ||||||
| 			qosClass:                    v1.PodQOSBurstable, |  | ||||||
| 			nodeSwapFeatureGateEnabled:  true, |  | ||||||
| 			swapBehavior:                types.UnlimitedSwap, |  | ||||||
| 			addContainerWithoutRequests: true, |  | ||||||
| 			addGuaranteedContainer:      false, |  | ||||||
| 		}, |  | ||||||
| 	} { | 	} { | ||||||
| 		t.Run(tc.name, func(t *testing.T) { | 		t.Run(tc.name, func(t *testing.T) { | ||||||
| 			setCgroupVersionDuringTest(tc.cgroupVersion) | 			setCgroupVersionDuringTest(tc.cgroupVersion) | ||||||
| @@ -1294,8 +1171,8 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) { | |||||||
| 				return | 				return | ||||||
| 			} | 			} | ||||||
|  |  | ||||||
| 			if tc.swapBehavior == types.UnlimitedSwap || tc.swapBehavior == "" { | 			if tc.swapBehavior == types.NoSwap || tc.swapBehavior == "" { | ||||||
| 				expectUnlimitedSwap(tc.cgroupVersion, resourcesC1, resourcesC2) | 				expectNoSwap(tc.cgroupVersion, resourcesC1, resourcesC2) | ||||||
| 				return | 				return | ||||||
| 			} | 			} | ||||||
|  |  | ||||||
|   | |||||||
| @@ -36,5 +36,5 @@ const ( | |||||||
| // SwapBehavior types | // SwapBehavior types | ||||||
| const ( | const ( | ||||||
| 	LimitedSwap = "LimitedSwap" | 	LimitedSwap = "LimitedSwap" | ||||||
| 	UnlimitedSwap = "UnlimitedSwap" | 	NoSwap      = "NoSwap" | ||||||
| ) | ) | ||||||
|   | |||||||
| @@ -954,8 +954,8 @@ type ShutdownGracePeriodByPodPriority struct { | |||||||
|  |  | ||||||
| type MemorySwapConfiguration struct { | type MemorySwapConfiguration struct { | ||||||
| 	// swapBehavior configures swap memory available to container workloads. May be one of | 	// swapBehavior configures swap memory available to container workloads. May be one of | ||||||
| 	// "", "LimitedSwap": workload combined memory and swap usage cannot exceed pod memory limit | 	// "", "NoSwap": workloads can not use swap, default option. | ||||||
| 	// "UnlimitedSwap": workloads can use unlimited swap, up to the allocatable limit. | 	// "LimitedSwap": workload swap usage is limited. The swap limit is proportionate to the container's memory request. | ||||||
| 	// +featureGate=NodeSwap | 	// +featureGate=NodeSwap | ||||||
| 	// +optional | 	// +optional | ||||||
| 	SwapBehavior string `json:"swapBehavior,omitempty"` | 	SwapBehavior string `json:"swapBehavior,omitempty"` | ||||||
|   | |||||||
| @@ -24,8 +24,6 @@ import ( | |||||||
| 	v1 "k8s.io/api/core/v1" | 	v1 "k8s.io/api/core/v1" | ||||||
| 	"k8s.io/apimachinery/pkg/api/resource" | 	"k8s.io/apimachinery/pkg/api/resource" | ||||||
| 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||||||
| 	utilfeature "k8s.io/apiserver/pkg/util/feature" |  | ||||||
| 	"k8s.io/kubernetes/pkg/features" |  | ||||||
| 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" | 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" | ||||||
| 	"k8s.io/kubernetes/test/e2e/framework" | 	"k8s.io/kubernetes/test/e2e/framework" | ||||||
| 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod" | 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod" | ||||||
| @@ -119,18 +117,6 @@ func runOomKillerTest(f *framework.Framework, testCase testCase, kubeReservedMem | |||||||
| 		}) | 		}) | ||||||
|  |  | ||||||
| 		ginkgo.It("The containers terminated by OOM killer should have the reason set to OOMKilled", func() { | 		ginkgo.It("The containers terminated by OOM killer should have the reason set to OOMKilled", func() { | ||||||
| 			cfg, configErr := getCurrentKubeletConfig(context.TODO()) |  | ||||||
| 			framework.ExpectNoError(configErr) |  | ||||||
| 			if utilfeature.DefaultFeatureGate.Enabled(features.NodeSwap) { |  | ||||||
| 				// If Swap is enabled, we should test OOM with LimitedSwap. |  | ||||||
| 				// UnlimitedSwap allows for workloads to use unbounded swap which |  | ||||||
| 				// makes testing OOM challenging. |  | ||||||
| 				// We are not able to change the default for these conformance tests, |  | ||||||
| 				// so we will skip these tests if swap is enabled. |  | ||||||
| 				if cfg.MemorySwap.SwapBehavior == "" || cfg.MemorySwap.SwapBehavior == "UnlimitedSwap" { |  | ||||||
| 					ginkgo.Skip("OOMKiller should not run with UnlimitedSwap") |  | ||||||
| 				} |  | ||||||
| 			} |  | ||||||
| 			ginkgo.By("Waiting for the pod to be failed") | 			ginkgo.By("Waiting for the pod to be failed") | ||||||
| 			err := e2epod.WaitForPodTerminatedInNamespace(context.TODO(), f.ClientSet, testCase.podSpec.Name, "", f.Namespace.Name) | 			err := e2epod.WaitForPodTerminatedInNamespace(context.TODO(), f.ClientSet, testCase.podSpec.Name, "", f.Namespace.Name) | ||||||
| 			framework.ExpectNoError(err, "Failed waiting for pod to terminate, %s/%s", f.Namespace.Name, testCase.podSpec.Name) | 			framework.ExpectNoError(err, "Failed waiting for pod to terminate, %s/%s", f.Namespace.Name, testCase.podSpec.Name) | ||||||
|   | |||||||
| @@ -55,16 +55,17 @@ var _ = SIGDescribe("Swap", framework.WithNodeConformance(), "[LinuxOnly]", func | |||||||
|  |  | ||||||
| 		isCgroupV2 := isPodCgroupV2(f, pod) | 		isCgroupV2 := isPodCgroupV2(f, pod) | ||||||
| 		isLimitedSwap := isLimitedSwap(f, pod) | 		isLimitedSwap := isLimitedSwap(f, pod) | ||||||
|  | 		isNoSwap := isNoSwap(f, pod) | ||||||
|  |  | ||||||
| 		if !isSwapFeatureGateEnabled() || !isCgroupV2 || (isLimitedSwap && (qosClass != v1.PodQOSBurstable || memoryRequestEqualLimit)) { | 		if !isSwapFeatureGateEnabled() || !isCgroupV2 || isNoSwap || (isLimitedSwap && (qosClass != v1.PodQOSBurstable || memoryRequestEqualLimit)) { | ||||||
| 			ginkgo.By(fmt.Sprintf("Expecting no swap. feature gate on? %t isCgroupV2? %t is QoS burstable? %t", isSwapFeatureGateEnabled(), isCgroupV2, qosClass == v1.PodQOSBurstable)) | 			ginkgo.By(fmt.Sprintf("Expecting no swap. isNoSwap? %t, feature gate on? %t isCgroupV2? %t is QoS burstable? %t", isNoSwap, isSwapFeatureGateEnabled(), isCgroupV2, qosClass == v1.PodQOSBurstable)) | ||||||
| 			expectNoSwap(f, pod, isCgroupV2) | 			expectNoSwap(f, pod, isCgroupV2) | ||||||
| 			return | 			return | ||||||
| 		} | 		} | ||||||
|  |  | ||||||
| 		if !isLimitedSwap { | 		if !isLimitedSwap { | ||||||
| 			ginkgo.By("expecting unlimited swap") | 			ginkgo.By("expecting no swap") | ||||||
| 			expectUnlimitedSwap(f, pod, isCgroupV2) | 			expectNoSwap(f, pod, isCgroupV2) | ||||||
| 			return | 			return | ||||||
| 		} | 		} | ||||||
|  |  | ||||||
| @@ -176,16 +177,6 @@ func expectNoSwap(f *framework.Framework, pod *v1.Pod, isCgroupV2 bool) { | |||||||
| 	} | 	} | ||||||
| } | } | ||||||
|  |  | ||||||
| func expectUnlimitedSwap(f *framework.Framework, pod *v1.Pod, isCgroupV2 bool) { |  | ||||||
| 	if isCgroupV2 { |  | ||||||
| 		swapLimit := readCgroupFile(f, pod, cgroupV2SwapLimitFile) |  | ||||||
| 		gomega.ExpectWithOffset(1, swapLimit).To(gomega.Equal("max"), "max swap allowed should be \"max\"") |  | ||||||
| 	} else { |  | ||||||
| 		swapPlusMemLimit := readCgroupFile(f, pod, cgroupV1SwapLimitFile) |  | ||||||
| 		gomega.ExpectWithOffset(1, swapPlusMemLimit).To(gomega.Equal("-1")) |  | ||||||
| 	} |  | ||||||
| } |  | ||||||
|  |  | ||||||
| // supports v2 only as v1 shouldn't support LimitedSwap | // supports v2 only as v1 shouldn't support LimitedSwap | ||||||
| func expectLimitedSwap(f *framework.Framework, pod *v1.Pod, expectedSwapLimit int64) { | func expectLimitedSwap(f *framework.Framework, pod *v1.Pod, expectedSwapLimit int64) { | ||||||
| 	swapLimitStr := readCgroupFile(f, pod, cgroupV2SwapLimitFile) | 	swapLimitStr := readCgroupFile(f, pod, cgroupV2SwapLimitFile) | ||||||
| @@ -253,3 +244,10 @@ func isLimitedSwap(f *framework.Framework, pod *v1.Pod) bool { | |||||||
|  |  | ||||||
| 	return kubeletCfg.MemorySwap.SwapBehavior == types.LimitedSwap | 	return kubeletCfg.MemorySwap.SwapBehavior == types.LimitedSwap | ||||||
| } | } | ||||||
|  |  | ||||||
|  | func isNoSwap(f *framework.Framework, pod *v1.Pod) bool { | ||||||
|  | 	kubeletCfg, err := getCurrentKubeletConfig(context.Background()) | ||||||
|  | 	framework.ExpectNoError(err, "cannot get kubelet config") | ||||||
|  |  | ||||||
|  | 	return kubeletCfg.MemorySwap.SwapBehavior == types.NoSwap || kubeletCfg.MemorySwap.SwapBehavior == "" | ||||||
|  | } | ||||||
|   | |||||||
		Reference in New Issue
	
	Block a user
	 Kubernetes Prow Robot
					Kubernetes Prow Robot