github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #102218 from kolyshkin/cgroup-cleanups

Browse Source 
pkg/kubelet/cm: cgroup-related cleanups
This commit is contained in:
Kubernetes Prow Robot
2021-06-01 13:45:51 -07:00
committed by GitHub
parent e5b54d0769 f1aee7e049
commit 7c7a0865cd
4 changed files with 26 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
pkg/kubelet/cm/cgroup_manager_linux.go
View File

@@ -17,6 +17,7 @@ limitations under the License.
package cm
import (
"errors"
"fmt"
"io/ioutil"
"os"
@@ -29,6 +30,7 @@ import (
libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups"
cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
cgroupfs2 "github.com/opencontainers/runc/libcontainer/cgroups/fs2"
"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
cgroupsystemd "github.com/opencontainers/runc/libcontainer/cgroups/systemd"
libcontainerconfigs "github.com/opencontainers/runc/libcontainer/configs"
libcontainerdevices "github.com/opencontainers/runc/libcontainer/devices"
@@ -330,27 +332,6 @@ func (m *cgroupManagerImpl) Destroy(cgroupConfig *CgroupConfig) error {
return nil
}
type subsystem interface {
// Name returns the name of the subsystem.
Name() string
// Set the cgroup represented by cgroup.
Set(path string, cgroup *libcontainerconfigs.Resources) error
// GetStats returns the statistics associated with the cgroup
GetStats(path string, stats *libcontainercgroups.Stats) error
}
// getSupportedSubsystems returns a map of subsystem and if it must be mounted for the kubelet to function.
func getSupportedSubsystems() map[subsystem]bool {
supportedSubsystems := map[subsystem]bool{
&cgroupfs.MemoryGroup{}: true,
&cgroupfs.CpuGroup{}: true,
&cgroupfs.PidsGroup{}: true,
}
// not all hosts support hugetlb cgroup, and in the absent of hugetlb, we will fail silently by reporting no capacity.
supportedSubsystems[&cgroupfs.HugetlbGroup{}] = false
return supportedSubsystems
}
// getCpuWeight converts from the range [2, 262144] to [1, 10000]
func getCpuWeight(cpuShares *uint64) uint64 {
if cpuShares == nil {
@@ -393,51 +374,6 @@ func getSupportedUnifiedControllers() sets.String {
return supportedControllers.Intersection(availableRootControllers)
}
// propagateControllers on an unified hierarchy enables all the supported controllers for the specified cgroup
func propagateControllers(path string) error {
if err := os.MkdirAll(filepath.Join(cmutil.CgroupRoot, path), 0755); err != nil {
return fmt.Errorf("failed to create cgroup %q : %v", path, err)
}
// Retrieve all the supported controllers from the cgroup root
controllersFileContent, err := ioutil.ReadFile(filepath.Join(cmutil.CgroupRoot, "cgroup.controllers"))
if err != nil {
return fmt.Errorf("failed to read controllers from %q : %v", cmutil.CgroupRoot, err)
}
supportedControllers := getSupportedUnifiedControllers()
// The retrieved content looks like: "cpuset cpu io memory hugetlb pids". Prepend each of the controllers
// with '+', so we have something like "+cpuset +cpu +io +memory +hugetlb +pids"
controllers := ""
for _, controller := range strings.Fields(string(controllersFileContent)) {
// ignore controllers we don't care about
if !supportedControllers.Has(controller) {
continue
}
sep := " +"
if controllers == "" {
sep = "+"
}
controllers = controllers + sep + controller
}
current := cmutil.CgroupRoot
// Write the controllers list to each "cgroup.subtree_control" file until it reaches the parent cgroup.
// For the /foo/bar/baz cgroup, controllers must be enabled sequentially in the files:
// - /sys/fs/cgroup/foo/cgroup.subtree_control
// - /sys/fs/cgroup/foo/bar/cgroup.subtree_control
for _, p := range strings.Split(filepath.Dir(path), "/") {
current = filepath.Join(current, p)
if err := ioutil.WriteFile(filepath.Join(current, "cgroup.subtree_control"), []byte(controllers), 0755); err != nil {
return fmt.Errorf("failed to enable controllers on %q: %v", cmutil.CgroupRoot, err)
}
}
return nil
}
func (m *cgroupManagerImpl) toResources(resourceConfig *ResourceConfig) *libcontainerconfigs.Resources {
resources := &libcontainerconfigs.Resources{
Devices: []*libcontainerdevices.Rule{
@@ -535,10 +471,6 @@ func (m *cgroupManagerImpl) Update(cgroupConfig *CgroupConfig) error {
}
if unified {
if err := propagateControllers(libcontainerCgroupConfig.Path); err != nil {
return err
}
supportedControllers := getSupportedUnifiedControllers()
if !supportedControllers.Has("hugetlb") {
resources.HugetlbLimit = nil
@@ -669,53 +601,21 @@ func (m *cgroupManagerImpl) ReduceCPULimits(cgroupName CgroupName) error {
return m.Update(containerConfig)
}
func getStatsSupportedSubsystems(cgroupPaths map[string]string) (*libcontainercgroups.Stats, error) {
stats := libcontainercgroups.NewStats()
for sys, required := range getSupportedSubsystems() {
if _, ok := cgroupPaths[sys.Name()]; !ok {
if required {
return nil, fmt.Errorf("failed to find subsystem mount for required subsystem: %v", sys.Name())
}
// the cgroup is not mounted, but its not required so continue...
klog.V(6).InfoS("Unable to find subsystem mount for optional subsystem", "subsystemName", sys.Name())
continue
}
if err := sys.GetStats(cgroupPaths[sys.Name()], stats); err != nil {
return nil, fmt.Errorf("failed to get stats for supported subsystems : %v", err)
}
}
return stats, nil
}
func toResourceStats(stats *libcontainercgroups.Stats) *ResourceStats {
return &ResourceStats{
MemoryStats: &MemoryStats{
Usage: int64(stats.MemoryStats.Usage.Usage),
},
}
}
// Get sets the ResourceParameters of the specified cgroup as read from the cgroup fs
func (m *cgroupManagerImpl) GetResourceStats(name CgroupName) (*ResourceStats, error) {
var err error
var stats *libcontainercgroups.Stats
// MemoryUsage returns the current memory usage of the specified cgroup,
// as read from cgroupfs.
func (m *cgroupManagerImpl) MemoryUsage(name CgroupName) (int64, error) {
var path, file string
if libcontainercgroups.IsCgroup2UnifiedMode() {
cgroupPath := m.buildCgroupUnifiedPath(name)
manager, err := cgroupfs2.NewManager(nil, cgroupPath, false)
if err != nil {
return nil, fmt.Errorf("failed to create cgroup v2 manager: %v", err)
}
stats, err = manager.GetStats()
if err != nil {
return nil, fmt.Errorf("failed to get stats for cgroup %v: %v", name, err)
}
path = m.buildCgroupUnifiedPath(name)
file = "memory.current"
} else {
cgroupPaths := m.buildCgroupPaths(name)
stats, err = getStatsSupportedSubsystems(cgroupPaths)
if err != nil {
return nil, fmt.Errorf("failed to get stats supported cgroup subsystems for cgroup %v: %v", name, err)
mp, ok := m.subsystems.MountPoints["memory"]
if !ok { // should not happen
return -1, errors.New("no cgroup v1 mountpoint for memory controller found")
}
path = mp + "/" + m.Name(name)
file = "memory.usage_in_bytes"
}
return toResourceStats(stats), nil
val, err := fscommon.GetCgroupParamUint(path, file)
return int64(val), err
}

10
pkg/kubelet/cm/cgroup_manager_unsupported.go
View File

@@ -18,10 +18,12 @@ limitations under the License.
package cm
import "fmt"
import "errors"
type unsupportedCgroupManager struct{}
var errNotSupported = errors.New("Cgroup Manager is not supported in this build")
// Make sure that unsupportedCgroupManager implements the CgroupManager interface
var _ CgroupManager = &unsupportedCgroupManager{}
@@ -51,11 +53,11 @@ func (m *unsupportedCgroupManager) Update(_ *CgroupConfig) error {
}
func (m *unsupportedCgroupManager) Create(_ *CgroupConfig) error {
return fmt.Errorf("Cgroup Manager is not supported in this build")
return errNotSupported
}
func (m *unsupportedCgroupManager) GetResourceStats(name CgroupName) (*ResourceStats, error) {
return nil, fmt.Errorf("Cgroup Manager is not supported in this build")
func (m *unsupportedCgroupManager) MemoryUsage(_ CgroupName) (int64, error) {
return -1, errNotSupported
}
func (m *unsupportedCgroupManager) Pids(_ CgroupName) []int {

5
pkg/kubelet/cm/qos_container_manager_linux.go
View File

@@ -28,7 +28,7 @@ import (
units "github.com/docker/go-units"
cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/api/v1/resource"
v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
@@ -247,12 +247,11 @@ func (m *qosContainerManagerImpl) retrySetMemoryReserve(configs map[v1.PodQOSCla
// Attempt to set the limit near the current usage to put pressure
// on the cgroup and prevent further growth.
for qos, config := range configs {
stats, err := m.cgroupManager.GetResourceStats(config.Name)
usage, err := m.cgroupManager.MemoryUsage(config.Name)
if err != nil {
klog.V(2).InfoS("Failed to get resource stats", "err", err)
return
}
usage := stats.MemoryStats.Usage
// Because there is no good way to determine of the original Update()
// on the memory resource was successful, we determine failure of the

18
pkg/kubelet/cm/types.go
View File

@@ -17,7 +17,7 @@ limitations under the License.
package cm
import (
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
)
@@ -53,18 +53,6 @@ type CgroupConfig struct {
ResourceParameters *ResourceConfig
}
// MemoryStats holds the on-demand statistics from the memory cgroup
type MemoryStats struct {
// Memory usage (in bytes).
Usage int64
}
// ResourceStats holds on-demand statistics from various cgroup subsystems
type ResourceStats struct {
// Memory statistics.
MemoryStats *MemoryStats
}
// CgroupManager allows for cgroup management.
// Supports Cgroup Creation ,Deletion and Updates.
type CgroupManager interface {
@@ -90,8 +78,8 @@ type CgroupManager interface {
Pids(name CgroupName) []int
// ReduceCPULimits reduces the CPU CFS values to the minimum amount of shares.
ReduceCPULimits(cgroupName CgroupName) error
// GetResourceStats returns statistics of the specified cgroup as read from the cgroup fs.
GetResourceStats(name CgroupName) (*ResourceStats, error)
// MemoryUsage returns current memory usage of the specified cgroup, as read from the cgroupfs.
MemoryUsage(name CgroupName) (int64, error)
}
// QOSContainersInfo stores the names of containers per qos