1516 lines
47 KiB
Python
Executable File
1516 lines
47 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""A daemon that prevents OOM in Linux systems."""
|
||
import os
|
||
from operator import itemgetter
|
||
from time import sleep, time
|
||
from argparse import ArgumentParser
|
||
from sys import stdout
|
||
from signal import SIGKILL, SIGTERM
|
||
|
||
start_time = time()
|
||
|
||
sig_dict = {SIGKILL: 'SIGKILL',
|
||
SIGTERM: 'SIGTERM'}
|
||
|
||
##########################################################################
|
||
|
||
|
||
# function definition section
|
||
|
||
|
||
def root_notify_env():
|
||
"""
|
||
Get environment for notification sending from root.
|
||
|
||
Returns list [tuple username, tuple DISPLAY, tuple DBUS_SESSION_BUS_ADDRES]
|
||
"""
|
||
ps_output_list = Popen(['ps', 'ae'], stdout=PIPE
|
||
).communicate()[0].decode().split('\n')
|
||
|
||
lines_with_displays = []
|
||
for line in ps_output_list:
|
||
if ' DISPLAY=' in line and ' DBUS_SESSION_BUS_ADDRES' \
|
||
'S=' in line and ' USER=' in line:
|
||
lines_with_displays.append(line)
|
||
|
||
# list of tuples with needments
|
||
deus = []
|
||
for i in lines_with_displays:
|
||
for i in i.split(' '):
|
||
if i.startswith('USER='):
|
||
user = i.strip('\n').split('=')[1]
|
||
continue
|
||
if i.startswith('DISPLAY='):
|
||
disp_value = i.strip('\n').split('=')[1][0:2]
|
||
disp = 'DISPLAY=' + disp_value
|
||
continue
|
||
if i.startswith('DBUS_SESSION_BUS_ADDRESS='):
|
||
dbus = i.strip('\n')
|
||
deus.append(tuple([user, disp, dbus]))
|
||
|
||
# unique list of tuples
|
||
vult = []
|
||
for user_env_tuple in set(deus):
|
||
vult.append(user_env_tuple)
|
||
|
||
return vult
|
||
|
||
|
||
def string_to_float_convert_test(string):
|
||
"""Try to interprete string values as floats."""
|
||
try:
|
||
return float(string)
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
def string_to_int_convert_test(string):
|
||
"""Try to interpretst string values as integers."""
|
||
try:
|
||
return int(string)
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
# extracting the parameter from the config dictionary, str return
|
||
def conf_parse_string(param):
|
||
"""
|
||
Get string parameters from the config dict.
|
||
|
||
param: config_dict key
|
||
returns config_dict[param].strip()
|
||
"""
|
||
if param in config_dict:
|
||
return config_dict[param].strip()
|
||
else:
|
||
print('All the necessary parameters must be in the config')
|
||
print('There is no "{}" parameter in the config'.format(param))
|
||
exit()
|
||
|
||
|
||
# extracting the parameter from the config dictionary, bool return
|
||
def conf_parse_bool(param):
|
||
"""
|
||
Get bool parameters from the config_dict.
|
||
|
||
param: config_dicst key
|
||
returns bool
|
||
"""
|
||
if param in config_dict:
|
||
param_str = config_dict[param]
|
||
if param_str == 'True':
|
||
return True
|
||
elif param_str == 'False':
|
||
return False
|
||
else:
|
||
print('Invalid value of the "{}" parameter.'.format(param_str))
|
||
print('Valid values are True and False.')
|
||
print('Exit')
|
||
exit()
|
||
else:
|
||
print('All the necessary parameters must be in the config')
|
||
print('There is no "{}" parameter in the config'.format(param_str))
|
||
exit()
|
||
|
||
|
||
def func_decrease_oom_score_adj(oom_score_adj_max):
|
||
"""
|
||
Stupid function, must be remaked
|
||
"""
|
||
for i in os.listdir('/proc'):
|
||
if i.isdigit() is False:
|
||
continue
|
||
try:
|
||
oom_score_adj = int(rline1('/proc/' + i + '/oom_score_adj'))
|
||
if oom_score_adj > oom_score_adj_max:
|
||
write('/proc/' + i + '/oom_score_adj',
|
||
str(oom_score_adj_max) + '\n')
|
||
except FileNotFoundError:
|
||
pass
|
||
except ProcessLookupError:
|
||
pass
|
||
|
||
|
||
def rline1(path):
|
||
"""read 1st line from path."""
|
||
with open(path) as f:
|
||
for line in f:
|
||
return line[:-1]
|
||
|
||
|
||
def write(path, string):
|
||
"""Write string to path."""
|
||
with open(path, 'w') as f:
|
||
f.write(string)
|
||
|
||
|
||
def kib_to_mib(num):
|
||
"""Convert Kib values to Mib values."""
|
||
return round(num / 1024.0)
|
||
|
||
|
||
def percent(num):
|
||
"""Interprete mum as percentage."""
|
||
return round(num * 100, 1)
|
||
|
||
|
||
def just_percent_mem(num):
|
||
"""Pls, put description here..."""
|
||
return str(round(num * 100, 1)).rjust(4, ' ')
|
||
|
||
|
||
def just_percent_swap(num):
|
||
return str(round(num * 100, 1)).rjust(5, ' ')
|
||
|
||
|
||
def human(num, lenth):
|
||
"""Convert KiB values to MiB values with right alignment"""
|
||
return str(round(num / 1024)).rjust(lenth, ' ')
|
||
|
||
|
||
# return str with amount of bytes
|
||
def zram_stat(zram_id):
|
||
"""
|
||
Get zram state.
|
||
|
||
zram_id: str zram block-device id
|
||
returns bytes diskcize, str mem_used_total
|
||
"""
|
||
try:
|
||
disksize = rline1('/sys/block/' + zram_id + '/disksize')
|
||
except FileNotFoundError:
|
||
return '0', '0'
|
||
if disksize == ['0\n']:
|
||
return '0', '0'
|
||
try:
|
||
mm_stat = rline1('/sys/block/' + zram_id + '/mm_stat').split(' ')
|
||
mm_stat_list = []
|
||
for i in mm_stat:
|
||
if i != '':
|
||
mm_stat_list.append(i)
|
||
mem_used_total = mm_stat_list[2]
|
||
except FileNotFoundError:
|
||
mem_used_total = rline1('/sys/block/' + zram_id + '/mem_used_total')
|
||
return disksize, mem_used_total # BYTES, str
|
||
|
||
|
||
# return process name
|
||
def pid_to_name(pid):
|
||
"""
|
||
Get process name by pid.
|
||
|
||
pid: str pid of required process
|
||
returns string process_name
|
||
"""
|
||
try:
|
||
with open('/proc/' + pid + '/status') as f:
|
||
for line in f:
|
||
return line[:-1].split('\t')[1]
|
||
except FileNotFoundError:
|
||
return '<unknown>'
|
||
except ProcessLookupError:
|
||
return '<unknown>'
|
||
|
||
|
||
|
||
def pid_to_cmdline(pid):
|
||
"""
|
||
Get process cmdline by pid.
|
||
|
||
pid: str pid of required process
|
||
returns string cmdline
|
||
"""
|
||
with open('/proc/' + pid + '/cmdline') as file:
|
||
try:
|
||
return file.readlines()[0].replace('\x00', ' ').strip()
|
||
except IndexError:
|
||
return ''
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
def send_notify_warn():
|
||
"""
|
||
Look for process with maximum 'badness' and warn user with notification.
|
||
"""
|
||
# find process with max badness
|
||
fat_tuple = fattest()
|
||
pid = fat_tuple[1]
|
||
name = fat_tuple[0]
|
||
|
||
if mem_used_zram > 0:
|
||
low_mem_percent = '{}% {}% {}%'.format(
|
||
round(mem_available / mem_total * 100),
|
||
round(swap_free / (swap_total + 0.1) * 100),
|
||
round(mem_used_zram / mem_total * 100))
|
||
elif swap_free > 0:
|
||
low_mem_percent = '{}% {}%'.format(
|
||
round(mem_available / mem_total * 100),
|
||
round(swap_free / (swap_total + 0.1) * 100))
|
||
else:
|
||
low_mem_percent = '{}%'.format(
|
||
round(mem_available / mem_total * 100))
|
||
|
||
title = 'Low memory: {}'.format(low_mem_percent)
|
||
body = 'Fattest process: <b>{}</b>, <b>{}</b>'.format(pid, name)
|
||
|
||
if root: # If nohang was started by root
|
||
# send notification to all active users with special script
|
||
Popen([
|
||
'/usr/bin/nohang_notify_low_mem',
|
||
'--mem', low_mem_percent,
|
||
'--pid', pid,
|
||
'--name', name
|
||
])
|
||
else: # Or by regular user
|
||
# send notification to user that runs this nohang
|
||
Popen(['notify-send', '--icon=dialog-warning', '{}'.format(title), '{}'.format(body)])
|
||
|
||
|
||
def send_notify(signal, name, pid):
|
||
"""
|
||
Notificate about OOM Preventing.
|
||
|
||
signal: key for notify_sig_dict
|
||
name: str process name
|
||
pid: str process pid
|
||
"""
|
||
title = 'Preventing OOM'
|
||
body = '<b>{}</b> process <b>{}</b>, <b>{}</b>'.format(
|
||
notify_sig_dict[signal], pid, name.replace(
|
||
# сивол & может ломать уведомления в некоторых темах оформления, поэтому заменяется на *
|
||
'&', '*'))
|
||
if root:
|
||
# send notification to all active users with notify-send
|
||
b = root_notify_env()
|
||
if len(b) > 0:
|
||
for i in b:
|
||
username, display_env, dbus_env = i[0], i[1], i[2]
|
||
Popen(['sudo', '-u', username, 'env', display_env,
|
||
dbus_env, 'notify-send', '--icon=dialog-warning',
|
||
'{}'.format(title), '{}'.format(body)])
|
||
else:
|
||
# send notification to user that runs this nohang
|
||
Popen(['notify-send', '--icon=dialog-warning',
|
||
'{}'.format(title), '{}'.format(body)])
|
||
|
||
|
||
def send_notify_etc(pid, name, command):
|
||
"""
|
||
Notificate about OOM Preventing.
|
||
|
||
command: str command that will be executed
|
||
name: str process name
|
||
pid: str process pid
|
||
"""
|
||
title = 'Preventing OOM'
|
||
body = 'Victim is process <b>{}</b>, <b>{}</b>\nExecute the command:\n<b>{}</b>'.format(
|
||
pid, name.replace('&', '*'), command.replace('&', '*'))
|
||
if root:
|
||
# send notification to all active users with notify-send
|
||
b = root_notify_env()
|
||
if len(b) > 0:
|
||
for i in b:
|
||
username, display_env, dbus_env = i[0], i[1], i[2]
|
||
Popen(['sudo', '-u', username, 'env', display_env,
|
||
dbus_env, 'notify-send', '--icon=dialog-warning',
|
||
'{}'.format(title), '{}'.format(body)])
|
||
else:
|
||
# send notification to user that runs this nohang
|
||
Popen(['notify-send', '--icon=dialog-warning', '{}'.format(title), '{}'
|
||
.format(body)])
|
||
|
||
|
||
def sleep_after_send_signal(signal):
|
||
"""
|
||
Sleeping after signal was sent.
|
||
|
||
signal: sent signal
|
||
"""
|
||
if signal is SIGKILL:
|
||
if print_sleep_periods:
|
||
print(' sleep', min_delay_after_sigkill)
|
||
sleep(min_delay_after_sigterm)
|
||
else:
|
||
if print_sleep_periods:
|
||
print(' sleep', min_delay_after_sigterm)
|
||
sleep(min_delay_after_sigterm)
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
def find_victim_and_send_signal(signal):
|
||
"""
|
||
Find victim with highest badness and send SIGTERM/SIGKILL
|
||
"""
|
||
if decrease_oom_score_adj and root:
|
||
# это не оптимальное решение
|
||
func_decrease_oom_score_adj(oom_score_adj_max)
|
||
|
||
pid_badness_list = []
|
||
|
||
if regex_matching:
|
||
|
||
for pid in os.listdir('/proc'):
|
||
# только директории, имена которых состоят только из цифр, за исключением /proc/1/
|
||
if pid[0].isdecimal() is False or pid == '1':
|
||
continue
|
||
|
||
try:
|
||
badness = int(rline1('/proc/' + pid + '/oom_score'))
|
||
|
||
|
||
|
||
name = pid_to_name(pid)
|
||
cmdline = pid_to_cmdline(pid)
|
||
#uid = pid_to_uid(pid)
|
||
|
||
# skip kthreads
|
||
if cmdline == '':
|
||
continue
|
||
|
||
#print([pid], [name], [cmdline])
|
||
|
||
if search(avoid_regex, name) is not None:
|
||
badness = int(badness / avoid_factor)
|
||
print(' Name matches with avoid_regex \033[33m{}\033[0m: \033[33m{}\033[0m, CmdLine: {}'.format(
|
||
avoid_regex, name, cmdline))
|
||
|
||
if search(prefer_regex, name) is not None:
|
||
badness = int((badness + 1) * prefer_factor)
|
||
print(' Name matches with prefer_regex \033[33m{}\033[0m: \033[33m{}\033[0m, CmdLine: {}'.format(
|
||
prefer_regex, name, cmdline))
|
||
|
||
|
||
if search(avoid_re_cmdline, cmdline) is not None:
|
||
badness = int(badness / avoid_cmd_factor)
|
||
print(' Cmdline matches with avoid_re_cmdline \033[33m{}\033[0m: \033[33m{}\033[0m, Name: {}'.format(
|
||
avoid_re_cmdline, cmdline, name))
|
||
|
||
|
||
if search(prefer_re_cmdline, cmdline) is not None:
|
||
badness = int((badness + 1) * prefer_cmd_factor)
|
||
print(' Cmdline matches with prefer_re_cmdline \033[33m{}\033[0m: \033[33m{}\033[0m, Name: {}'.format(
|
||
prefer_re_cmdline, cmdline, name))
|
||
|
||
|
||
|
||
|
||
except FileNotFoundError:
|
||
badness = 0
|
||
except ProcessLookupError:
|
||
badness = 0
|
||
pid_badness_list.append((pid, badness))
|
||
|
||
|
||
|
||
else:
|
||
|
||
for pid in os.listdir('/proc'):
|
||
if pid[0].isdecimal() is False:
|
||
continue
|
||
try:
|
||
badness = int(rline1('/proc/' + pid + '/oom_score'))
|
||
except FileNotFoundError:
|
||
badness = 0
|
||
except ProcessLookupError:
|
||
badness = 0
|
||
pid_badness_list.append((pid, badness))
|
||
|
||
# Make list of (pid, badness) tuples, sorted by 'badness' values
|
||
pid_tuple_list = sorted(
|
||
pid_badness_list, key=itemgetter(1), reverse=True)[0]
|
||
|
||
# Get maximum 'badness' value
|
||
victim_badness = pid_tuple_list[1]
|
||
|
||
if victim_badness >= min_badness: # Try to send signal to found victim
|
||
|
||
pid = pid_tuple_list[0]
|
||
|
||
name = pid_to_name(pid)
|
||
|
||
# Get VmRSS and VmSwap and cmdline of victim process and try to send signal
|
||
try:
|
||
with open('/proc/' + pid + '/status') as f:
|
||
for n, line in enumerate(f):
|
||
if n is uid_index:
|
||
uid = line.split('\t')[1]
|
||
continue
|
||
if n is vm_rss_index:
|
||
vm_rss = kib_to_mib(int(line.split('\t')[1][:-4]))
|
||
continue
|
||
if n is vm_swap_index:
|
||
vm_swap = kib_to_mib(int(line.split('\t')[1][:-4]))
|
||
break
|
||
|
||
with open('/proc/' + pid + '/cmdline') as file:
|
||
try:
|
||
cmdline = file.readlines()[0].replace('\x00', ' ')
|
||
except IndexError:
|
||
cmdline = ''
|
||
except FileNotFoundError:
|
||
pass
|
||
except ProcessLookupError:
|
||
pass
|
||
except IndexError:
|
||
pass
|
||
except ValueError:
|
||
pass
|
||
|
||
if execute_the_command and signal is SIGTERM and name in etc_dict:
|
||
command = etc_dict[name]
|
||
exit_status = os.system(etc_dict[name])
|
||
response_time = time() - time0
|
||
|
||
etc_info = ''' Found the victim with highest badness:\n Name: \033[33m{}\033[0m\n PID: \033[33m{}\033[0m\n UID: \033[33m{}\033[0m\n Badness: \033[33m{}\033[0m\n VmRSS: \033[33m{}\033[0m MiB\n VmSwap: \033[33m{}\033[0m MiB\n CmdLine: \033[33m{}\033[0m\n Execute the command: \033[4m{}\033[0m\n Exit status: {}; response time: {} ms'''.format(name, pid, uid, victim_badness, vm_rss, vm_swap, cmdline, command, exit_status, round(response_time * 1000))
|
||
|
||
print(mem_info)
|
||
print(etc_info)
|
||
if gui_notifications: send_notify_etc(pid, name, command)
|
||
|
||
else:
|
||
|
||
try:
|
||
os.kill(int(pid), signal)
|
||
response_time = time() - time0
|
||
send_result = '\033[32mOK\033[0m; response time: {} ms'.format(
|
||
round(response_time * 1000))
|
||
|
||
if gui_notifications:
|
||
send_notify(signal, name, pid)
|
||
|
||
except FileNotFoundError:
|
||
response_time = time() - time0
|
||
send_result = 'no such process; response time: {} ms'.format(
|
||
round(response_time * 1000))
|
||
except ProcessLookupError:
|
||
response_time = time() - time0
|
||
send_result = 'no such process; response time: {} ms'.format(
|
||
round(response_time * 1000))
|
||
|
||
preventing_oom_message = ' Found the process with highest badness:\n Name: \033[33m{}\033[0m\n PID: \033[33m{}\033[0m\n UID: \033[33m{}\033[0m\n Badness: \033[33m{}\033[0m\n VmRSS: \033[33m{}\033[0m MiB\n VmSwap: \033[33m{}\033[0m MiB\n CmdLine: \033[33m{}\033[0m\n Sending \033[4m{}\033[0m to the victim; {}'.format(
|
||
name, pid, uid, victim_badness, vm_rss, vm_swap, cmdline, sig_dict[signal], send_result)
|
||
print(mem_info)
|
||
print(preventing_oom_message)
|
||
|
||
else:
|
||
|
||
response_time = time() - time0
|
||
victim_badness_is_too_small = ' victim badness {} < min_badness {}; nothing to do; response time: {} ms'.format(
|
||
victim_badness,
|
||
min_badness,
|
||
round(response_time * 1000))
|
||
|
||
print(victim_badness_is_too_small)
|
||
|
||
|
||
sleep_after_send_signal(signal)
|
||
|
||
|
||
def sleep_after_check_mem():
|
||
"""Specify sleep times depends on rates and avialable memory."""
|
||
t_mem = mem_available / rate_mem
|
||
t_swap = swap_free / rate_swap
|
||
t_zram = (mem_total - mem_used_zram) / rate_zram
|
||
|
||
t_mem_swap = t_mem + t_swap
|
||
t_mem_zram = t_mem + t_zram
|
||
|
||
if t_mem_swap <= t_mem_zram:
|
||
t = t_mem_swap
|
||
else:
|
||
t = t_mem_zram
|
||
|
||
try:
|
||
if print_sleep_periods:
|
||
print('sleep', round(t, 2),
|
||
' (t_mem={}, t_swap={}, t_zram={})'.format(
|
||
round(t_mem, 2),
|
||
round(t_swap, 2),
|
||
round(t_zram, 2)))
|
||
stdout.flush()
|
||
sleep(t)
|
||
except KeyboardInterrupt:
|
||
exit()
|
||
|
||
|
||
def fattest():
|
||
"""Find the 'fattest' process"""
|
||
pid_badness_list = []
|
||
|
||
if regex_matching:
|
||
|
||
for pid in os.listdir('/proc'):
|
||
if pid[0].isdecimal() is False:
|
||
continue
|
||
|
||
try:
|
||
badness = int(rline1('/proc/' + pid + '/oom_score'))
|
||
name = pid_to_name(pid)
|
||
|
||
if search(avoid_regex, name) is not None:
|
||
badness = int(badness / avoid_factor)
|
||
|
||
if search(prefer_regex, name) is not None:
|
||
badness = int((badness + 1) * prefer_factor)
|
||
|
||
except FileNotFoundError:
|
||
badness = 0
|
||
except ProcessLookupError:
|
||
badness = 0
|
||
pid_badness_list.append((pid, badness))
|
||
|
||
else:
|
||
|
||
for pid in os.listdir('/proc'):
|
||
if pid[0].isdecimal() is False:
|
||
continue
|
||
try:
|
||
badness = int(rline1('/proc/' + pid + '/oom_score'))
|
||
except FileNotFoundError:
|
||
badness = 0
|
||
except ProcessLookupError:
|
||
badness = 0
|
||
pid_badness_list.append((pid, badness))
|
||
|
||
# Make list of (pid, badness) tuples, sorted by 'badness' values
|
||
pid_tuple_list = sorted(
|
||
pid_badness_list, key=itemgetter(1), reverse=True)[0]
|
||
|
||
pid = pid_tuple_list[0]
|
||
|
||
name = pid_to_name(pid)
|
||
|
||
return (name, pid)
|
||
|
||
|
||
def calculate_percent(arg_key):
|
||
"""
|
||
parse conf dict
|
||
Calculate mem_min_KEY_percent.
|
||
|
||
Try use this one)
|
||
arg_key: str key for config_dict
|
||
returns int mem_min_percent or NoneType if got some error
|
||
"""
|
||
|
||
if arg_key in config_dict:
|
||
mem_min = config_dict[arg_key]
|
||
|
||
if mem_min.endswith('%'):
|
||
# truncate percents, so we have a number
|
||
mem_min_percent = mem_min[:-1].strip()
|
||
# then 'float test'
|
||
mem_min_percent = string_to_float_convert_test(mem_min_percent)
|
||
if mem_min_percent is None:
|
||
print('Invalid {} value, not float\nExit'.format(arg_key))
|
||
exit()
|
||
# Final validations...
|
||
if mem_min_percent < 0 or mem_min_percent > 100:
|
||
print(
|
||
'{}, as percents value, out of range [0; 100]\nExit'.format(arg_key))
|
||
exit()
|
||
|
||
# mem_min_sigterm_percent is clean and valid float percentage. Can
|
||
# translate into Kb
|
||
mem_min_kb = mem_min_percent / 100 * mem_total
|
||
mem_min_mb = round(mem_min_kb / 1024)
|
||
|
||
elif mem_min.endswith('M'):
|
||
mem_min_mb = string_to_float_convert_test(mem_min[:-1].strip())
|
||
if mem_min_mb is None:
|
||
print('Invalid {} value, not float\nExit'.format(arg_key))
|
||
exit()
|
||
mem_min_kb = mem_min_mb * 1024
|
||
if mem_min_kb > mem_total:
|
||
print(
|
||
'{} value can not be greater then MemTotal ({} MiB)\nExit'.format(
|
||
arg_key, round(
|
||
mem_total / 1024)))
|
||
exit()
|
||
mem_min_percent = mem_min_kb / mem_total * 100
|
||
|
||
else:
|
||
print('Invalid {} units in config.\n Exit'.format(arg_key))
|
||
mem_min_percent = None
|
||
|
||
else:
|
||
print('{} not in config\nExit'.format(arg_key))
|
||
mem_min_percent = None
|
||
|
||
return mem_min_kb, mem_min_mb, mem_min_percent
|
||
|
||
|
||
##########################################################################
|
||
|
||
|
||
# find mem_total
|
||
# find positions of SwapFree and SwapTotal in /proc/meminfo
|
||
|
||
with open('/proc/meminfo') as file:
|
||
mem_list = file.readlines()
|
||
|
||
mem_list_names = []
|
||
for s in mem_list:
|
||
mem_list_names.append(s.split(':')[0])
|
||
|
||
if mem_list_names[2] != 'MemAvailable':
|
||
print('Your Linux kernel is too old, Linux 3.14+ requied\nExit')
|
||
exit()
|
||
|
||
swap_total_index = mem_list_names.index('SwapTotal')
|
||
swap_free_index = swap_total_index + 1
|
||
|
||
mem_total = int(mem_list[0].split(':')[1].strip(' kB\n'))
|
||
|
||
# Get names from /proc/*/status to be able to get VmRSS and VmSwap values
|
||
|
||
with open('/proc/self/status') as file:
|
||
status_list = file.readlines()
|
||
|
||
status_names = []
|
||
for s in status_list:
|
||
status_names.append(s.split(':')[0])
|
||
|
||
vm_rss_index = status_names.index('VmRSS')
|
||
vm_swap_index = status_names.index('VmSwap')
|
||
uid_index = status_names.index('Uid')
|
||
|
||
##########################################################################
|
||
|
||
|
||
# Configurations
|
||
|
||
|
||
# directory where the script is running
|
||
cd = os.getcwd()
|
||
|
||
print('CD:', cd)
|
||
|
||
# where to look for a config if not specified via the -c/--config option
|
||
default_configs = (cd + '/nohang.conf', '/etc/nohang/nohang.conf')
|
||
|
||
# universal message if config is invalid
|
||
conf_err_mess = '\nSet up the path to the valid conf' \
|
||
'ig file with -c/--config option!\nExit'
|
||
|
||
|
||
# Cmd argparse
|
||
parser = ArgumentParser()
|
||
parser.add_argument(
|
||
'-c',
|
||
'--config',
|
||
help="""path to the config file, default values:
|
||
./nohang.conf, /etc/nohang/nohang.conf""",
|
||
default=None,
|
||
type=str
|
||
)
|
||
|
||
args = parser.parse_args()
|
||
|
||
arg_config = args.config
|
||
|
||
|
||
if arg_config is None:
|
||
|
||
config = None
|
||
for i in default_configs:
|
||
if os.path.exists(i):
|
||
config = i
|
||
break
|
||
if config is None:
|
||
print('Default configuration was not found\n', conf_err_mess)
|
||
exit()
|
||
|
||
else:
|
||
|
||
if os.path.exists(arg_config):
|
||
config = arg_config
|
||
else:
|
||
print("File {} doesn't exists{}".format(arg_config, conf_err_mess))
|
||
exit()
|
||
|
||
|
||
print('The path to the config:', config)
|
||
|
||
|
||
##########################################################################
|
||
|
||
|
||
# parsing the config with obtaining the parameters dictionary
|
||
|
||
# conf_parameters_dict
|
||
# conf_restart_dict
|
||
|
||
try:
|
||
with open(config) as f:
|
||
|
||
# dictionary with config options
|
||
config_dict = dict()
|
||
|
||
# dictionary with names and commands for the parameter
|
||
# execute_the_command
|
||
etc_dict = dict()
|
||
|
||
for line in f:
|
||
|
||
a = line.startswith('#')
|
||
b = line.startswith('\n')
|
||
c = line.startswith('\t')
|
||
d = line.startswith(' ')
|
||
|
||
etc = line.startswith('$ETC')
|
||
|
||
if not a and not b and not c and not d and not etc:
|
||
a = line.split('=')
|
||
config_dict[a[0].strip()] = a[1].strip()
|
||
|
||
if etc:
|
||
a = line[4:].split('///')
|
||
etc_name = a[0].strip()
|
||
etc_command = a[1].strip()
|
||
if len(etc_name) > 15:
|
||
print(
|
||
'Invalid config, the length of the process name must not exceed 15 characters\nExit')
|
||
exit()
|
||
etc_dict[etc_name] = etc_command
|
||
|
||
except PermissionError:
|
||
print('PermissionError', conf_err_mess)
|
||
exit()
|
||
except UnicodeDecodeError:
|
||
print('UnicodeDecodeError', conf_err_mess)
|
||
exit()
|
||
except IsADirectoryError:
|
||
print('IsADirectoryError', conf_err_mess)
|
||
exit()
|
||
except IndexError:
|
||
print('IndexError', conf_err_mess)
|
||
exit()
|
||
|
||
|
||
##########################################################################
|
||
|
||
|
||
# extracting parameters from the dictionary
|
||
# check for all necessary parameters
|
||
# validation of all parameters
|
||
|
||
print_config = conf_parse_bool('print_config')
|
||
print_mem_check_results = conf_parse_bool('print_mem_check_results')
|
||
print_sleep_periods = conf_parse_bool('print_sleep_periods')
|
||
realtime_ionice = conf_parse_bool('realtime_ionice')
|
||
mlockall = conf_parse_bool('mlockall')
|
||
gui_low_memory_warnings = conf_parse_bool('gui_low_memory_warnings')
|
||
gui_notifications = conf_parse_bool('gui_notifications')
|
||
decrease_oom_score_adj = conf_parse_bool('decrease_oom_score_adj')
|
||
regex_matching = conf_parse_bool('regex_matching')
|
||
execute_the_command = conf_parse_bool('execute_the_command')
|
||
|
||
prefer_regex = conf_parse_string('prefer_regex')
|
||
avoid_regex = conf_parse_string('avoid_regex')
|
||
|
||
prefer_re_cmdline = conf_parse_string('prefer_re_cmdline')
|
||
avoid_re_cmdline = conf_parse_string('avoid_re_cmdline')
|
||
|
||
|
||
|
||
|
||
|
||
|
||
mem_min_sigterm_kb, mem_min_sigterm_mb, mem_min_sigterm_percent = calculate_percent(
|
||
'mem_min_sigterm')
|
||
mem_min_sigkill_kb, mem_min_sigkill_mb, mem_min_sigkill_percent = calculate_percent(
|
||
'mem_min_sigkill')
|
||
|
||
zram_max_sigterm_kb, zram_max_sigterm_mb, zram_max_sigterm_percent = calculate_percent(
|
||
'zram_max_sigterm')
|
||
zram_max_sigkill_kb, zram_max_sigkill_mb, zram_max_sigkill_percent = calculate_percent(
|
||
'zram_max_sigkill')
|
||
|
||
mem_min_warnings_kb, mem_min_warnings_mb, mem_min_warnings_percent = calculate_percent(
|
||
'mem_min_warnings')
|
||
zram_max_warnings_kb, zram_max_warnings_mb, zram_max_warnings_percent = calculate_percent(
|
||
'zram_max_warnings')
|
||
|
||
|
||
if 'realtime_ionice_classdata' in config_dict:
|
||
realtime_ionice_classdata = string_to_int_convert_test(
|
||
config_dict['realtime_ionice_classdata'])
|
||
if realtime_ionice_classdata is None:
|
||
print('Invalid value of the "realtime_ionice_classdata" parameter.')
|
||
print('Valid values are integers from the range [0; 7].')
|
||
print('Exit')
|
||
exit()
|
||
if realtime_ionice_classdata < 0 or realtime_ionice_classdata > 7:
|
||
print('Invalid value of the "realtime_ionice_classdata" parameter.')
|
||
print('Valid values are integers from the range [0; 7].')
|
||
print('Exit')
|
||
exit()
|
||
else:
|
||
print('All the necessary parameters must be in the config')
|
||
print('There is no "realtime_ionice_classdata" parameter in the config')
|
||
exit()
|
||
|
||
|
||
if 'niceness' in config_dict:
|
||
niceness = string_to_int_convert_test(config_dict['niceness'])
|
||
if niceness is None:
|
||
print('Invalid niceness value, not integer\nExit')
|
||
exit()
|
||
if niceness < -20 or niceness > 19:
|
||
print('niceness out of range [-20; 19]\nExit')
|
||
exit()
|
||
else:
|
||
print('niceness not in config\nExit')
|
||
exit()
|
||
|
||
|
||
if 'oom_score_adj' in config_dict:
|
||
oom_score_adj = string_to_int_convert_test(
|
||
config_dict['oom_score_adj'])
|
||
if oom_score_adj is None:
|
||
print('Invalid oom_score_adj value, not integer\nExit')
|
||
exit()
|
||
if oom_score_adj < -1000 or oom_score_adj > 1000:
|
||
print('oom_score_adj out of range [-1000; 1000]\nExit')
|
||
exit()
|
||
else:
|
||
print('oom_score_adj not in config\nExit')
|
||
exit()
|
||
|
||
|
||
if 'rate_mem' in config_dict:
|
||
rate_mem = string_to_float_convert_test(config_dict['rate_mem'])
|
||
if rate_mem is None:
|
||
print('Invalid rate_mem value, not float\nExit')
|
||
exit()
|
||
if rate_mem <= 0:
|
||
print('rate_mem MUST be > 0\nExit')
|
||
exit()
|
||
else:
|
||
print('rate_mem not in config\nExit')
|
||
exit()
|
||
|
||
|
||
if 'rate_swap' in config_dict:
|
||
rate_swap = string_to_float_convert_test(config_dict['rate_swap'])
|
||
if rate_swap is None:
|
||
print('Invalid rate_swap value, not float\nExit')
|
||
exit()
|
||
if rate_swap <= 0:
|
||
print('rate_swap MUST be > 0\nExit')
|
||
exit()
|
||
else:
|
||
print('rate_swap not in config\nExit')
|
||
exit()
|
||
|
||
|
||
if 'rate_zram' in config_dict:
|
||
rate_zram = string_to_float_convert_test(config_dict['rate_zram'])
|
||
if rate_zram is None:
|
||
print('Invalid rate_zram value, not float\nExit')
|
||
exit()
|
||
if rate_zram <= 0:
|
||
print('rate_zram MUST be > 0\nExit')
|
||
exit()
|
||
else:
|
||
print('rate_zram not in config\nExit')
|
||
exit()
|
||
|
||
|
||
# НУЖНА ВАЛИДАЦИЯ НА МЕСТЕ!
|
||
if 'swap_min_sigterm' in config_dict:
|
||
swap_min_sigterm = config_dict['swap_min_sigterm']
|
||
else:
|
||
print('swap_min_sigterm not in config\nExit')
|
||
exit()
|
||
|
||
|
||
# НУЖНА ВАЛИДАЦИЯ НА МЕСТЕ!
|
||
if 'swap_min_sigkill' in config_dict:
|
||
swap_min_sigkill = config_dict['swap_min_sigkill']
|
||
else:
|
||
print('swap_min_sigkill not in config\nExit')
|
||
exit()
|
||
|
||
|
||
if 'min_delay_after_sigterm' in config_dict:
|
||
min_delay_after_sigterm = string_to_float_convert_test(
|
||
config_dict['min_delay_after_sigterm'])
|
||
if min_delay_after_sigterm is None:
|
||
print('Invalid min_delay_after_sigterm value, not float\nExit')
|
||
exit()
|
||
if min_delay_after_sigterm < 0:
|
||
print('min_delay_after_sigterm must be positiv\nExit')
|
||
exit()
|
||
else:
|
||
print('min_delay_after_sigterm not in config\nExit')
|
||
exit()
|
||
|
||
|
||
if 'min_delay_after_sigkill' in config_dict:
|
||
min_delay_after_sigkill = string_to_float_convert_test(
|
||
config_dict['min_delay_after_sigkill'])
|
||
if min_delay_after_sigkill is None:
|
||
print('Invalid min_delay_after_sigkill value, not float\nExit')
|
||
exit()
|
||
if min_delay_after_sigkill < 0:
|
||
print('min_delay_after_sigkill must be positiv\nExit')
|
||
exit()
|
||
else:
|
||
print('min_delay_after_sigkill not in config\nExit')
|
||
exit()
|
||
|
||
|
||
if 'min_badness' in config_dict:
|
||
min_badness = string_to_int_convert_test(
|
||
config_dict['min_badness'])
|
||
if min_badness is None:
|
||
print('Invalid min_badness value, not integer\nExit')
|
||
exit()
|
||
if min_badness < 0 or min_badness > 1000:
|
||
print('Invalud min_badness value\nExit')
|
||
exit()
|
||
else:
|
||
print('min_badness not in config\nExit')
|
||
exit()
|
||
|
||
|
||
if 'oom_score_adj_max' in config_dict:
|
||
oom_score_adj_max = string_to_int_convert_test(
|
||
config_dict['oom_score_adj_max'])
|
||
if oom_score_adj_max is None:
|
||
print('Invalid oom_score_adj_max value, not integer\nExit')
|
||
exit()
|
||
if oom_score_adj_max < 0 or oom_score_adj_max > 1000:
|
||
print('Invalid oom_score_adj_max value\nExit')
|
||
exit()
|
||
else:
|
||
print('oom_score_adj_max not in config\nExit')
|
||
exit()
|
||
|
||
|
||
if 'prefer_factor' in config_dict:
|
||
prefer_factor = string_to_float_convert_test(config_dict['prefer_factor'])
|
||
if prefer_factor is None:
|
||
print('Invalid prefer_factor value, not float\nExit')
|
||
exit()
|
||
if prefer_factor < 1 and prefer_factor > 1000:
|
||
print('prefer_factor value out of range [1; 1000]\nExit')
|
||
exit()
|
||
else:
|
||
print('prefer_factor not in config\nExit')
|
||
exit()
|
||
|
||
|
||
if 'avoid_factor' in config_dict:
|
||
avoid_factor = string_to_float_convert_test(config_dict['avoid_factor'])
|
||
if avoid_factor is None:
|
||
print('Invalid avoid_factor value, not float\nExit')
|
||
exit()
|
||
if avoid_factor < 1 and avoid_factor > 1000:
|
||
print('avoid_factor value out of range [1; 1000]\nExit')
|
||
exit()
|
||
else:
|
||
print('avoid_factor not in config\nExit')
|
||
exit()
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
if 'prefer_cmd_factor' in config_dict:
|
||
prefer_cmd_factor = string_to_float_convert_test(config_dict['prefer_cmd_factor'])
|
||
if prefer_cmd_factor is None:
|
||
print('Invalid prefer_cmd_factor value, not float\nExit')
|
||
exit()
|
||
if prefer_cmd_factor < 1 and prefer_cmd_factor > 1000:
|
||
print('prefer_cmd_factor value out of range [1; 1000]\nExit')
|
||
exit()
|
||
else:
|
||
print('prefer_cmd_factor not in config\nExit')
|
||
exit()
|
||
|
||
|
||
if 'avoid_cmd_factor' in config_dict:
|
||
avoid_cmd_factor = string_to_float_convert_test(config_dict['avoid_cmd_factor'])
|
||
if avoid_cmd_factor is None:
|
||
print('Invalid avoid_cmd_factor value, not float\nExit')
|
||
exit()
|
||
if avoid_cmd_factor < 1 and avoid_cmd_factor > 1000:
|
||
print('avoid_cmd_factor value out of range [1; 1000]\nExit')
|
||
exit()
|
||
else:
|
||
print('avoid_cmd_factor not in config\nExit')
|
||
exit()
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
if 'min_time_between_warnings' in config_dict:
|
||
min_time_between_warnings = string_to_float_convert_test(
|
||
config_dict['min_time_between_warnings'])
|
||
if min_time_between_warnings is None:
|
||
print('Invalid min_time_between_warnings value, not float\nExit')
|
||
exit()
|
||
if min_time_between_warnings < 1 or min_time_between_warnings > 300:
|
||
print('min_time_between_warnings value out of range [1; 300]\nExit')
|
||
exit()
|
||
else:
|
||
print('min_time_between_warnings not in config\nExit')
|
||
exit()
|
||
|
||
|
||
# НА МЕСТЕ!!!
|
||
if 'swap_min_warnings' in config_dict:
|
||
swap_min_warnings = config_dict['swap_min_warnings']
|
||
else:
|
||
print('swap_min_warnings not in config\nExit')
|
||
exit()
|
||
|
||
|
||
##########################################################################
|
||
|
||
|
||
# Get Kibibytes levels
|
||
|
||
|
||
# Returns Kibibytes value if absolute value was set in config,
|
||
# or tuple with percentage
|
||
def sig_level_to_kb_swap(string):
|
||
"""Returns Kibibytes value if abs val was set in config, or tuple with %"""
|
||
if string.endswith('%'):
|
||
return float(string[:-1].strip()), True
|
||
elif string.endswith('M'):
|
||
return float(string[:-1].strip()) * 1024
|
||
else:
|
||
print('Invalid config file. There are invalid units somewhere\nExit')
|
||
exit()
|
||
|
||
|
||
# So, get them
|
||
swap_min_sigterm_swap = sig_level_to_kb_swap(swap_min_sigterm)
|
||
swap_min_sigkill_swap = sig_level_to_kb_swap(swap_min_sigkill)
|
||
|
||
swap_min_warnings_swap = sig_level_to_kb_swap(swap_min_warnings)
|
||
|
||
|
||
if isinstance(swap_min_sigterm_swap, tuple):
|
||
swap_term_is_percent = True
|
||
swap_min_sigterm_percent = swap_min_sigterm_swap[0]
|
||
else:
|
||
swap_term_is_percent = False
|
||
swap_min_sigterm_kb = swap_min_sigterm_swap
|
||
|
||
if isinstance(swap_min_sigkill_swap, tuple):
|
||
swap_kill_is_percent = True
|
||
swap_min_sigkill_percent = swap_min_sigkill_swap[0]
|
||
else:
|
||
swap_kill_is_percent = False
|
||
swap_min_sigkill_kb = swap_min_sigkill_swap
|
||
|
||
|
||
if isinstance(swap_min_warnings_swap, tuple):
|
||
swap_warn_is_percent = True
|
||
swap_min_warnings_percent = swap_min_warnings_swap[0]
|
||
else:
|
||
swap_warn_is_percent = False
|
||
swap_min_warnings_kb = swap_min_warnings_swap
|
||
|
||
|
||
##########################################################################
|
||
|
||
# self-defense
|
||
|
||
|
||
# возожно стоит убрать поддержку mlockall и ionice
|
||
|
||
# Increase priority
|
||
try:
|
||
os.nice(niceness)
|
||
niceness_result = 'OK'
|
||
except PermissionError:
|
||
niceness_result = 'Fail'
|
||
pass
|
||
|
||
# Deny self-killing
|
||
try:
|
||
with open('/proc/self/oom_score_adj', 'w') as file:
|
||
file.write('{}\n'.format(oom_score_adj))
|
||
oom_score_adj_result = 'OK'
|
||
except PermissionError:
|
||
oom_score_adj_result = 'Fail'
|
||
except OSError:
|
||
oom_score_adj_result = 'Fail'
|
||
|
||
# Deny process swapping
|
||
if mlockall:
|
||
from ctypes import CDLL
|
||
result = CDLL('libc.so.6', use_errno=True).mlockall(3)
|
||
if result is 0:
|
||
mla_res = 'OK'
|
||
else:
|
||
mla_res = 'Fail'
|
||
else:
|
||
mla_res = ''
|
||
|
||
|
||
self_uid = os.geteuid()
|
||
self_pid = os.getpid()
|
||
|
||
|
||
if self_uid == 0:
|
||
root = True
|
||
decrease_res = 'OK'
|
||
else:
|
||
root = False
|
||
decrease_res = 'Impossible'
|
||
|
||
|
||
if root and realtime_ionice:
|
||
os.system('ionice -c 1 -n {} -p {}'.format(
|
||
realtime_ionice_classdata, self_pid))
|
||
|
||
|
||
##########################################################################
|
||
|
||
if print_config:
|
||
|
||
print(
|
||
'\n1. Memory levels to respond to as an OOM threat\n[displaying these options need fix]\n')
|
||
|
||
print('mem_min_sigterm: {} MiB, {} %'.format(
|
||
round(mem_min_sigterm_mb), round(mem_min_sigterm_percent, 1)))
|
||
print('mem_min_sigkill: {} MiB, {} %'.format(
|
||
round(mem_min_sigkill_mb), round(mem_min_sigkill_percent, 1)))
|
||
|
||
print('swap_min_sigterm: {}'.format(swap_min_sigterm))
|
||
print('swap_min_sigkill: {}'.format(swap_min_sigkill))
|
||
|
||
print('zram_max_sigterm: {} MiB, {} %'.format(
|
||
round(zram_max_sigterm_mb), round(zram_max_sigterm_percent, 1)))
|
||
print('zram_max_sigkill: {} MiB, {} %'.format(
|
||
round(zram_max_sigkill_mb), round(zram_max_sigkill_percent, 1)))
|
||
|
||
print('\n2. The frequency of checking the level of available memory (and CPU usage)\n')
|
||
print('rate_mem: {}'.format(rate_mem))
|
||
print('rate_swap: {}'.format(rate_swap))
|
||
print('rate_zram: {}'.format(rate_zram))
|
||
|
||
print('\n3. The prevention of killing innocent victims\n')
|
||
print('min_delay_after_sigterm: {}'.format(min_delay_after_sigterm))
|
||
print('min_delay_after_sigkill: {}'.format(min_delay_after_sigkill))
|
||
print('min_badness: {}'.format(min_badness))
|
||
|
||
# False (OK) - OK не нужен когда фолс
|
||
print('decrease_oom_score_adj: {} ({})'.format(
|
||
decrease_oom_score_adj, decrease_res
|
||
))
|
||
if decrease_oom_score_adj:
|
||
print('oom_score_adj_max: {}'.format(oom_score_adj_max))
|
||
|
||
print('\n4. Impact on the badness of processes via matching their names\nwith regular expressions\n')
|
||
print('regex_matching: {}'.format(regex_matching))
|
||
if regex_matching:
|
||
print('prefer_regex: {}'.format(prefer_regex))
|
||
print('prefer_factor: {}'.format(prefer_factor))
|
||
print('avoid_regex: {}'.format(avoid_regex))
|
||
print('avoid_factor: {}'.format(avoid_factor))
|
||
|
||
print('\n5. The execution of a specific command instead of sending the\nSIGTERM signal\n')
|
||
print('execute_the_command: {}'.format(execute_the_command))
|
||
if execute_the_command:
|
||
print('\nPROCESS NAME COMMAND TO EXECUTE')
|
||
for key in etc_dict:
|
||
print('{} {}'.format(key.ljust(15), etc_dict[key]))
|
||
|
||
print('\n6. GUI notifications:\n- OOM prevention results and\n- low memory warnings\n')
|
||
print('gui_notifications: {}'.format(gui_notifications))
|
||
|
||
print('gui_low_memory_warnings: {}'.format(gui_low_memory_warnings))
|
||
if gui_low_memory_warnings:
|
||
print('min_time_between_warnings: {}'.format(min_time_between_warnings))
|
||
|
||
print('mem_min_warnings: {} MiB, {} %'.format(
|
||
round(mem_min_warnings_mb), round(mem_min_warnings_percent, 1)))
|
||
|
||
print('swap_min_warnings: {}'.format(swap_min_warnings))
|
||
|
||
print('zram_max_warnings: {} MiB, {} %'.format(
|
||
round(zram_max_warnings_mb), round(zram_max_warnings_percent, 1)))
|
||
|
||
print(
|
||
'\n7. Preventing the slowing down of the program\n[displaying these options need fix]\n')
|
||
print('mlockall: {} ({})'.format(mlockall, mla_res))
|
||
print('niceness: {} ({})'.format(
|
||
niceness, niceness_result
|
||
))
|
||
print('oom_score_adj: {} ({})'.format(
|
||
oom_score_adj, oom_score_adj_result
|
||
))
|
||
|
||
print('realtime_ionice: {} ({})'.format(realtime_ionice, ''))
|
||
|
||
if realtime_ionice:
|
||
print('realtime_ionice_classdata: {}'.format(realtime_ionice_classdata))
|
||
|
||
print('\n8. Output verbosity\n')
|
||
print('print_config: {}'.format(print_config))
|
||
print('print_mem_check_results: {}'.format(print_mem_check_results))
|
||
print('print_sleep_periods: {}\n'.format(print_sleep_periods))
|
||
|
||
|
||
##########################################################################
|
||
|
||
|
||
# for calculating the column width when printing mem and zram
|
||
mem_len = len(str(round(mem_total / 1024.0)))
|
||
|
||
|
||
if gui_notifications or gui_low_memory_warnings:
|
||
from subprocess import Popen, PIPE
|
||
notify_sig_dict = {SIGKILL: 'Killing',
|
||
SIGTERM: 'Terminating'}
|
||
|
||
if regex_matching:
|
||
from re import search
|
||
|
||
|
||
rate_mem = rate_mem * 1048576
|
||
rate_swap = rate_swap * 1048576
|
||
rate_zram = rate_zram * 1048576
|
||
|
||
warn_time_now = 0
|
||
warn_time_delta = 1000
|
||
warn_timer = 0
|
||
|
||
x = time() - start_time
|
||
print(
|
||
'The duration of startup:', round(
|
||
x * 1000, 1), 'ms')
|
||
|
||
print('Monitoring started!')
|
||
|
||
stdout.flush()
|
||
|
||
##########################################################################
|
||
|
||
|
||
while True:
|
||
|
||
# find mem_available, swap_total, swap_free
|
||
with open('/proc/meminfo') as f:
|
||
for n, line in enumerate(f):
|
||
if n is 2:
|
||
mem_available = int(line.split(':')[1].strip(' kB\n'))
|
||
continue
|
||
if n is swap_total_index:
|
||
swap_total = int(line.split(':')[1].strip(' kB\n'))
|
||
continue
|
||
if n is swap_free_index:
|
||
swap_free = int(line.split(':')[1].strip(' kB\n'))
|
||
break
|
||
|
||
# if swap_min_sigkill is set in percent
|
||
if swap_kill_is_percent:
|
||
swap_min_sigkill_kb = swap_total * swap_min_sigkill_percent / 100.0
|
||
|
||
if swap_term_is_percent:
|
||
swap_min_sigterm_kb = swap_total * swap_min_sigterm_percent / 100.0
|
||
|
||
if swap_warn_is_percent:
|
||
swap_min_warnings_kb = swap_total * swap_min_warnings_percent / 100.0
|
||
|
||
|
||
# find MemUsedZram
|
||
disksize_sum = 0
|
||
mem_used_total_sum = 0
|
||
for dev in os.listdir('/sys/block'):
|
||
if dev.startswith('zram'):
|
||
stat = zram_stat(dev)
|
||
disksize_sum += int(stat[0])
|
||
mem_used_total_sum += int(stat[1])
|
||
|
||
# Означает, что при задани zram disksize = 10000M доступная память
|
||
# уменьшится на 42 MiB.
|
||
# Найден экспериментально, требует уточнения с разными ядрами и архитектурами.
|
||
# На небольших дисксайзах (до гигабайта) может быть больше, до 0.0045.
|
||
# Создатель модуля zram утверждает, что ZRAM_DISKSIZE_FACTOR доожен быть 0.001
|
||
# ("zram uses about 0.1% of the size of the disk"
|
||
# - https://www.kernel.org/doc/Documentation/blockdev/zram.txt),
|
||
# но это утверждение противоречит опытным данным.
|
||
# ZRAM_DISKSIZE_FACTOR = deltaMemAvailavle / disksize
|
||
# found experimentally
|
||
ZRAM_DISKSIZE_FACTOR = 0.0042
|
||
mem_used_zram = (
|
||
mem_used_total_sum + disksize_sum * ZRAM_DISKSIZE_FACTOR
|
||
) / 1024.0
|
||
|
||
|
||
if print_mem_check_results:
|
||
|
||
# Calculate 'swap-column' width
|
||
swap_len = len(str(round(swap_total / 1024.0)))
|
||
|
||
# Output avialable mem sizes
|
||
if swap_total == 0 and mem_used_zram == 0:
|
||
print('MemAvail: {} M, {} %'.format(
|
||
human(mem_available, mem_len),
|
||
just_percent_mem(mem_available / mem_total)))
|
||
|
||
elif swap_total > 0 and mem_used_zram == 0:
|
||
print('MemAvail: {} M, {} % | SwapFree: {} M, {} %'.format(
|
||
human(mem_available, mem_len),
|
||
just_percent_mem(mem_available / mem_total),
|
||
human(swap_free, swap_len),
|
||
just_percent_swap(swap_free / (swap_total + 0.1))))
|
||
|
||
else:
|
||
print('MemAvail: {} M, {} % | SwapFree: {} M, {} % | Mem'
|
||
'UsedZram: {} M, {} %'.format(
|
||
human(mem_available, mem_len),
|
||
just_percent_mem(mem_available / mem_total),
|
||
human(swap_free, swap_len),
|
||
just_percent_swap(swap_free / (swap_total + 0.1)),
|
||
human(mem_used_zram, mem_len),
|
||
just_percent_mem(mem_used_zram / mem_total)))
|
||
|
||
# если swap_min_sigkill задан в абсолютной величине и Swap_total = 0
|
||
if swap_total > swap_min_sigkill_kb: # If swap_min_sigkill is absolute
|
||
swap_sigkill_pc = percent(swap_min_sigkill_kb / (swap_total + 0.1))
|
||
else:
|
||
swap_sigkill_pc = '-'
|
||
|
||
if swap_total > swap_min_sigterm_kb:
|
||
swap_sigterm_pc = percent(swap_min_sigterm_kb / (swap_total + 0.1))
|
||
else:
|
||
|
||
# СТОИТ ПЕЧАТАТЬ СВОП ТОЛЬКО ПРИ SwapTotal > 0
|
||
# нет, печатать так: SwapTotal = 0 KiB, ignore swapspace
|
||
swap_sigterm_pc = '-'
|
||
|
||
# Limits overdrafting checks
|
||
# If overdrafted - try to prevent OOM
|
||
# else - just sleep
|
||
|
||
# MEM SWAP KILL
|
||
if mem_available <= mem_min_sigkill_kb and swap_free <= swap_min_sigkill_kb:
|
||
time0 = time()
|
||
|
||
mem_info = '\033[4mLow memory; corrective action required!\033[0m\n MemAvailable [{} MiB, {} %] <= mem_min_sigkill [{} MiB, {} %]\n Swa' \
|
||
'pFree [{} MiB, {} %] <= swap_min_sigkill [{} MiB, {} %]'.format(
|
||
kib_to_mib(mem_available),
|
||
percent(mem_available / mem_total),
|
||
|
||
kib_to_mib(mem_min_sigkill_kb),
|
||
percent(mem_min_sigkill_kb / mem_total),
|
||
|
||
kib_to_mib(swap_free),
|
||
percent(swap_free / (swap_total + 0.1)),
|
||
|
||
kib_to_mib(swap_min_sigkill_kb),
|
||
swap_sigkill_pc)
|
||
|
||
find_victim_and_send_signal(SIGKILL)
|
||
|
||
# ZRAM KILL
|
||
elif mem_used_zram >= zram_max_sigkill_kb:
|
||
time0 = time()
|
||
|
||
mem_info = '\033[4mLow memory; corrective action required!\033[0m\n MemUsedZram [{} MiB, {} %] >= zram_max_sigkill [{} MiB, {} %]'.format(
|
||
kib_to_mib(mem_used_zram),
|
||
percent(mem_used_zram / mem_total),
|
||
kib_to_mib(zram_max_sigkill_kb),
|
||
percent(zram_max_sigkill_kb / mem_total))
|
||
|
||
find_victim_and_send_signal(SIGKILL)
|
||
|
||
# MEM SWAP TERM
|
||
elif mem_available <= mem_min_sigterm_kb and swap_free <= swap_min_sigterm_kb:
|
||
|
||
time0 = time()
|
||
|
||
mem_info = '\033[4mLow memory; corrective action required!\033[0m\n MemAvailable [{} MiB, {} %] <= mem_min_sigterm [{} MiB, {} %]\n Sw' \
|
||
'apFree [{} MiB, {} %] <= swap_min_sigterm [{} MiB, {} %]'.format(
|
||
kib_to_mib(mem_available),
|
||
percent(mem_available / mem_total),
|
||
|
||
|
||
kib_to_mib(mem_min_sigterm_kb),
|
||
#percent(mem_min_sigterm_kb / mem_total),
|
||
|
||
# ОКРУГЛЯТЬ НА МЕСТЕ ВЫШЕ
|
||
round(mem_min_sigterm_percent, 1),
|
||
|
||
kib_to_mib(swap_free),
|
||
percent(swap_free / (swap_total + 0.1)),
|
||
|
||
|
||
kib_to_mib(swap_min_sigterm_kb),
|
||
swap_sigterm_pc)
|
||
|
||
find_victim_and_send_signal(SIGTERM)
|
||
|
||
# ZRAM TERM
|
||
elif mem_used_zram >= zram_max_sigterm_kb:
|
||
time0 = time()
|
||
|
||
mem_info = '\033[4mLow memory; corrective action required!\033[0m\n MemUsedZram [{} MiB, {} %] >= zram_max_sigter' \
|
||
'm [{} M, {} %]'.format(
|
||
kib_to_mib(mem_used_zram),
|
||
percent(mem_used_zram / mem_total),
|
||
kib_to_mib(zram_max_sigterm_kb),
|
||
percent(zram_max_sigterm_kb / mem_total))
|
||
|
||
find_victim_and_send_signal(SIGTERM)
|
||
|
||
# LOW MEMORY WARNINGS
|
||
elif gui_low_memory_warnings:
|
||
|
||
if mem_available <= mem_min_warnings_kb and swap_free <= swap_min_warnings_kb + \
|
||
0.1 or mem_used_zram >= zram_max_warnings_kb:
|
||
warn_time_delta = time() - warn_time_now
|
||
warn_time_now = time()
|
||
warn_timer += warn_time_delta
|
||
if warn_timer > min_time_between_warnings:
|
||
send_notify_warn()
|
||
warn_timer = 0
|
||
sleep_after_check_mem()
|
||
|
||
# SLEEP BETWEEN MEM CHECKS
|
||
else:
|
||
stdout.flush()
|
||
sleep_after_check_mem()
|