github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

subtract inactive_file from usage when setting memcg threshold

Browse Source
This commit is contained in:
David Ashpole 2018-03-06 09:09:44 -08:00
parent a83aec0a0c
commit 54cf14ffcc
1 changed files with 19 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
pkg/kubelet/eviction/eviction_manager.go
View File

@ -184,15 +184,11 @@ func (m *managerImpl) IsUnderPIDPressure() bool {
return hasNodeCondition(m.nodeConditions, v1.NodePIDPressure) return hasNodeCondition(m.nodeConditions, v1.NodePIDPressure)
} }
func startMemoryThresholdNotifier(thresholds []evictionapi.Threshold, observations signalObservations, hard bool, handler thresholdNotifierHandlerFunc) error { func startMemoryThresholdNotifier(thresholds []evictionapi.Threshold, summary *statsapi.Summary, hard bool, handler thresholdNotifierHandlerFunc) error {
for _, threshold := range thresholds { for _, threshold := range thresholds {
if threshold.Signal != evictionapi.SignalMemoryAvailable || hard != isHardEvictionThreshold(threshold) { if threshold.Signal != evictionapi.SignalMemoryAvailable || hard != isHardEvictionThreshold(threshold) {
continue continue
} }
observed, found := observations[evictionapi.SignalMemoryAvailable]
if !found {
continue
}
cgroups, err := cm.GetCgroupSubsystems() cgroups, err := cm.GetCgroupSubsystems()
if err != nil { if err != nil {
return err return err
@ -203,11 +199,19 @@ func startMemoryThresholdNotifier(thresholds []evictionapi.Threshold, observatio
return fmt.Errorf("memory cgroup mount point not found") return fmt.Errorf("memory cgroup mount point not found")
} }
attribute := "memory.usage_in_bytes" attribute := "memory.usage_in_bytes"
quantity := evictionapi.GetThresholdQuantity(threshold.Value, observed.capacity) if summary.Node.Memory == nil || summary.Node.Memory.UsageBytes == nil || summary.Node.Memory.WorkingSetBytes == nil {
usageThreshold := resource.NewQuantity(observed.capacity.Value(), resource.DecimalSI) return fmt.Errorf("summary was incomplete")
usageThreshold.Sub(*quantity) }
// Set threshold on usage to capacity - eviction_hard + inactive_file,
// since we want to be notified when working_set = capacity - eviction_hard
inactiveFile := resource.NewQuantity(int64(*summary.Node.Memory.UsageBytes-*summary.Node.Memory.WorkingSetBytes), resource.BinarySI)
capacity := resource.NewQuantity(int64(*summary.Node.Memory.AvailableBytes+*summary.Node.Memory.WorkingSetBytes), resource.BinarySI)
evictionThresholdQuantity := evictionapi.GetThresholdQuantity(threshold.Value, capacity)
memcgThreshold := capacity.DeepCopy()
memcgThreshold.Sub(*evictionThresholdQuantity)
memcgThreshold.Add(*inactiveFile)
description := fmt.Sprintf("<%s available", formatThresholdValue(threshold.Value)) description := fmt.Sprintf("<%s available", formatThresholdValue(threshold.Value))
memcgThresholdNotifier, err := NewMemCGThresholdNotifier(cgpath, attribute, usageThreshold.String(), description, handler) memcgThresholdNotifier, err := NewMemCGThresholdNotifier(cgpath, attribute, memcgThreshold.String(), description, handler)
if err != nil { if err != nil {
return err return err
} }
@ -247,16 +251,12 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
return nil return nil
} }
// make observations and get a function to derive pod usage stats relative to those observations.
observations, statsFunc := makeSignalObservations(summary)
debugLogObservations("observations", observations)
// attempt to create a threshold notifier to improve eviction response time // attempt to create a threshold notifier to improve eviction response time
if m.config.KernelMemcgNotification && !m.notifiersInitialized { if m.config.KernelMemcgNotification && !m.notifiersInitialized {
glog.Infof("eviction manager attempting to integrate with kernel memcg notification api") glog.Infof("eviction manager attempting to integrate with kernel memcg notification api")
m.notifiersInitialized = true m.notifiersInitialized = true
// start soft memory notification // start soft memory notification
err = startMemoryThresholdNotifier(m.config.Thresholds, observations, false, func(desc string) { err = startMemoryThresholdNotifier(m.config.Thresholds, summary, false, func(desc string) {
glog.Infof("soft memory eviction threshold crossed at %s", desc) glog.Infof("soft memory eviction threshold crossed at %s", desc)
// TODO wait grace period for soft memory limit // TODO wait grace period for soft memory limit
m.synchronize(diskInfoProvider, podFunc) m.synchronize(diskInfoProvider, podFunc)
@ -265,7 +265,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
glog.Warningf("eviction manager: failed to create soft memory threshold notifier: %v", err) glog.Warningf("eviction manager: failed to create soft memory threshold notifier: %v", err)
} }
// start hard memory notification // start hard memory notification
err = startMemoryThresholdNotifier(m.config.Thresholds, observations, true, func(desc string) { err = startMemoryThresholdNotifier(m.config.Thresholds, summary, true, func(desc string) {
glog.Infof("hard memory eviction threshold crossed at %s", desc) glog.Infof("hard memory eviction threshold crossed at %s", desc)
m.synchronize(diskInfoProvider, podFunc) m.synchronize(diskInfoProvider, podFunc)
}) })
@ -274,6 +274,10 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
} }
} }
// make observations and get a function to derive pod usage stats relative to those observations.
observations, statsFunc := makeSignalObservations(summary)
debugLogObservations("observations", observations)
// determine the set of thresholds met independent of grace period // determine the set of thresholds met independent of grace period
thresholds = thresholdsMet(thresholds, observations, false) thresholds = thresholdsMet(thresholds, observations, false)
debugLogThresholdsWithObservation("thresholds - ignoring grace period", thresholds, observations) debugLogThresholdsWithObservation("thresholds - ignoring grace period", thresholds, observations)