nohang/nohang
2018-08-07 15:03:01 +09:00

1479 lines
51 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
# A daemon that prevents out of memory
import os
import signal
from operator import itemgetter
from time import sleep, time
from argparse import ArgumentParser
from sys import stdout
sig_dict = {signal.SIGKILL: 'SIGKILL',
signal.SIGTERM: 'SIGTERM'}
notify_sig_dict = {signal.SIGKILL: 'Killing',
signal.SIGTERM: 'Terminating'}
# directory where the script is running
cd = os.getcwd()
# where to look for a config if not specified via the -c/--config option
default_configs = (cd + '/nohang.conf', '/etc/nohang/nohang.conf')
# universal message if config is invalid
conf_err_mess = '\nSet up the path to the valid conf' \
'ig file with -c/--config option!\nExit'
# означает, что при задани zram disksize = 10000M доступная память
# уменьшится на 42M
# найден экспериментально, требует уточнения с разными ядрами и архитектурами
# на небольших дисксайзах (до гигабайта) может быть больше, до 0.0045
# создатель модуля zram утверждает, что zram_disksize_factor доожен быть 0.001
# ("zram uses about 0.1% of the size of the disk"
# - https://www.kernel.org/doc/Documentation/blockdev/zram.txt),
# но это утверждение противоречит опытным данным
# zram_disksize_factor = deltaMemAvailavle / disksize
# found experimentally
zram_disksize_factor = 0.0042
##########################################################################
# function definition section
# return list of tuples with
# username, DISPLAY and DBUS_SESSION_BUS_ADDRESS
def root_notify_env():
ps_output_list = Popen(['ps', 'ae'], stdout=PIPE
).communicate()[0].decode().split('\n')
lines_with_displays = []
for line in ps_output_list:
if ' DISPLAY=' in line and ' DBUS_SESSION_BUS_ADDRES' \
'S=' in line and ' USER=' in line:
lines_with_displays.append(line)
# list of tuples with needments
deus = []
for i in lines_with_displays:
for i in i.split(' '):
if i.startswith('USER='):
user = i.strip('\n').split('=')[1]
continue
if i.startswith('DISPLAY='):
disp_value = i.strip('\n').split('=')[1][0:2]
disp = 'DISPLAY=' + disp_value
continue
if i.startswith('DBUS_SESSION_BUS_ADDRESS='):
dbus = i.strip('\n')
deus.append(tuple([user, disp, dbus]))
# unique list of tuples
vult = []
for user_env_tuple in set(deus):
vult.append(user_env_tuple)
return vult
def string_to_float_convert_test(string):
try:
return float(string)
except ValueError:
return None
def string_to_int_convert_test(string):
try:
return int(string)
except ValueError:
return None
# extracting the parameter from the config dictionary, str return
def conf_parse_string(param):
if param in config_dict:
return config_dict[param].strip()
else:
print('All the necessary parameters must be in the config')
print('There is no "{}" parameter in the config'.format(param))
exit()
# extracting the parameter from the config dictionary, bool return
def conf_parse_bool(param):
if param in config_dict:
param_str = config_dict[param]
if param_str == 'True':
return True
elif param_str == 'False':
return False
else:
print('Invalid value of the "{}" parameter.'.format(param_str))
print('Valid values are True and False.')
print('Exit')
exit()
else:
print('All the necessary parameters must be in the config')
print('There is no "{}" parameter in the config'.format(param_str))
exit()
def func_decrease_oom_score_adj(oom_score_adj_max):
for i in os.listdir('/proc'):
if i.isdigit() is not True:
continue
try:
oom_score_adj = int(rline1('/proc/' + i + '/oom_score_adj'))
if oom_score_adj > oom_score_adj_max:
write('/proc/' + i + '/oom_score_adj',
str(oom_score_adj_max) + '\n')
except FileNotFoundError:
pass
except ProcessLookupError:
pass
# read 1st line
def rline1(path):
with open(path) as f:
for line in f:
return line[:-1]
# write in file
def write(path, string):
with open(path, 'w') as f:
f.write(string)
def kib_to_mib(num):
return round(num / 1024.0)
def percent(num):
return round(num * 100, 1)
def just_percent_mem(num):
return str(round(num * 100, 1)).rjust(4, ' ')
def just_percent_swap(num):
return str(round(num * 100, 1)).rjust(5, ' ')
# KiB to MiB, right alignment
def human(num, lenth):
return str(round(num / 1024)).rjust(lenth, ' ')
# return str with amount of bytes
def zram_stat(zram_id):
try:
disksize = rline1('/sys/block/' + zram_id + '/disksize')
except FileNotFoundError:
return '0', '0'
if disksize == ['0\n']:
return '0', '0'
try:
mm_stat = rline1('/sys/block/' + zram_id + '/mm_stat').split(' ')
mm_stat_list = []
for i in mm_stat:
if i != '':
mm_stat_list.append(i)
mem_used_total = mm_stat_list[2]
except FileNotFoundError:
mem_used_total = rline1('/sys/block/' + zram_id + '/mem_used_total')
return disksize, mem_used_total # BYTES, str
# return process name
def pid_to_name(pid):
try:
with open('/proc/' + pid + '/status') as f:
for line in f:
return line[:-1].split('\t')[1]
except FileNotFoundError:
return '<unknown>'
except ProcessLookupError:
return '<unknown>'
def send_notify_warn():
title = 'LOW MEMORY'
if mem_used_zram > 0:
body = '<b>Mem Available: {} %\nSwap Free: {} %\nMem Used Zram: {} %</b>'.format(
round(mem_available / mem_total * 100),
round(swap_free / (swap_total + 0.1) * 100),
round(mem_used_zram / mem_total * 100))
elif swap_free > 0:
body = '<b>Mem Available: {} %\nSwap Free: {} %</b>'.format(
round(mem_available / mem_total * 100),
round(swap_free / (swap_total + 0.1) * 100))
else:
body = '<b>Mem Available: {} %</b>'.format(
round(mem_available / mem_total * 100))
if root:
# отправляем уведомление всем залогиненным пользователям
b = root_notify_env()
if len(b) > 0:
for i in b:
username, display_env, dbus_env = i[0], i[1], i[2]
Popen(['sudo', '-u', username, 'env', display_env,
dbus_env, 'notify-send', '--icon=dialog-warning',
'{}'.format(title), '{}'.format(body)])
else:
# отправляем уведомление пользователю, который запустил nohang
Popen(['notify-send', '--icon=dialog-warning', '{}'.format(title), '{}'.format(body)])
def send_notify(signal, name, pid):
title = 'NOHANG TRIGGERED'
body = '<b>{}</b> process <b>{}</b>, <b>{}</b>'.format(
notify_sig_dict[signal], pid, name.replace('&', '*'))
if root:
# отправляем уведомление всем залогиненным пользователям
b = root_notify_env()
if len(b) > 0:
for i in b:
username, display_env, dbus_env = i[0], i[1], i[2]
Popen(['sudo', '-u', username, 'env', display_env,
dbus_env, 'notify-send', '--icon=dialog-warning',
'{}'.format(title), '{}'.format(body)])
else:
# отправляем уведомление пользователю, который запустил nohang
Popen(['notify-send', '--icon=dialog-warning', '{}'.format(title), '{}'.format(body)])
def send_notify_etc(pid, name, command):
title = 'NOHANG TRIGGERED'
body = 'Victim is process <b>{}</b>, <b>{}</b>\nExecute the command:\n<b>{}</b>'.format(
pid, name.replace('&', '*'), command.replace('&', '*'))
if root:
# отправляем уведомление всем залогиненным пользователям
b = root_notify_env()
if len(b) > 0:
for i in b:
username, display_env, dbus_env = i[0], i[1], i[2]
Popen(['sudo', '-u', username, 'env', display_env,
dbus_env, 'notify-send', '--icon=dialog-warning',
'{}'.format(title), '{}'.format(body)])
else:
# отправляем уведомление пользователю, который запустил nohang
Popen(['notify-send', '--icon=dialog-warning', '{}'.format(title), '{}'.format(body)])
def sleep_after_send_signal(signal):
if signal is signal.SIGKILL:
if print_sleep_periods:
print(' sleep', min_delay_after_sigkill)
sleep(min_delay_after_sigterm)
else:
if print_sleep_periods:
print(' sleep', min_delay_after_sigterm)
sleep(min_delay_after_sigterm)
def find_victim_and_send_signal(signal):
if decrease_oom_score_adj and root:
func_decrease_oom_score_adj(oom_score_adj_max)
pid_badness_list = []
if regex_matching:
for pid in os.listdir('/proc'):
if pid[0].isdecimal() is not True:
continue
try:
badness = int(rline1('/proc/' + pid + '/oom_score'))
name = pid_to_name(pid)
if fullmatch(avoid_regex, name) is not None:
badness = int(badness / avoid_factor)
if fullmatch(prefer_regex, name) is not None:
badness = int((badness + 1) * prefer_factor)
except FileNotFoundError:
badness = 0
except ProcessLookupError:
badness = 0
pid_badness_list.append((pid, badness))
else:
for pid in os.listdir('/proc'):
if pid[0].isdecimal() is not True:
continue
try:
badness = int(rline1('/proc/' + pid + '/oom_score'))
except FileNotFoundError:
badness = 0
except ProcessLookupError:
badness = 0
pid_badness_list.append((pid, badness))
# получаем отсортированный по badness список пар (pid, badness)
pid_tuple_list = sorted(
pid_badness_list, key=itemgetter(1), reverse=True)[0]
# получаем максимальный badness
victim_badness = pid_tuple_list[1]
if victim_badness >= min_badness:
# пытаемся отправить сигнал найденной жертве
pid = pid_tuple_list[0]
name = pid_to_name(pid)
# находим VmRSS и VmSwap процесса, которому попытаемся послать сигнал
try:
with open('/proc/' + pid + '/status') as f:
for n, line in enumerate(f):
if n is vm_rss_index:
vm_rss = kib_to_mib(int(
line.split('\t')[1][:-4]))
continue
if n is vm_swap_index:
vm_swap = kib_to_mib(int(
line.split('\t')[1][:-4]))
break
except FileNotFoundError:
vm_rss = 0
vm_swap = 0
except ProcessLookupError:
vm_rss = 0
vm_swap = 0
except IndexError:
vm_rss = 0
vm_swap = 0
except ValueError:
vm_rss = 0
vm_swap = 0
if name in etc_dict and signal is signal.SIGTERM:
command = etc_dict[name]
exit_status = os.system(etc_dict[name])
response_time = time() - time0
etc_info = ' Finding the process with the highest badness\n Victim is {}, pid: {}, badness: {}, VmRSS: {} MiB, VmSwap: {} MiB\n Execute the command: {}\n Exit status: {}; response time: {} ms'.format(name, pid, badness, vm_rss, vm_swap, command, exit_status, round(response_time * 1000))
print(mem_info)
print(etc_info)
if gui_notifications:
send_notify_etc(pid, name, command)
else:
try:
os.kill(int(pid), signal)
response_time = time() - time0
send_result = 'signal received; response time: {} ms'.format(round(response_time * 1000))
if gui_notifications:
send_notify(signal, name, pid)
except FileNotFoundError:
response_time = time() - time0
send_result = 'no such process; response time: {} ms'.format(round(response_time * 1000))
except ProcessLookupError:
response_time = time() - time0
send_result = 'no such process; response time: {} ms'.format(round(response_time * 1000))
preventing_oom_message = ' Finding the process with the highest badness\n Victim is {}, pid: {}, badness: {}, VmRSS: {} MiB, VmSwap: {} MiB\n Sending {} to the victim; {}'.format(name, pid, badness, vm_rss, vm_swap, sig_dict[signal], send_result)
print(mem_info)
print(preventing_oom_message)
else:
response_time = time() - time0
victim_badness_is_too_small = ' victim badness {} < min_badness {}; nothing to do; response time: {} ms'.format(victim_badness, min_badness, round(response_time * 1000))
print(victim_badness_is_too_small)
stdout.flush()
sleep_after_send_signal(signal)
def sleep_after_check_mem():
# задание периода сна в зависимости от рейтов и уровней доступной памяти
t_mem = mem_available / rate_mem
t_swap = swap_free / rate_swap
t_zram = (mem_total - mem_used_zram) / rate_zram
t_mem_swap = t_mem + t_swap
t_mem_zram = t_mem + t_zram
if t_mem_swap <= t_mem_zram:
t = t_mem_swap
else:
t = t_mem_zram
try:
if print_sleep_periods:
print('sleep', round(t, 2), ' (t_mem={}, t_swap={}, t_zram={})'.format(
round(t_mem, 2), round(t_swap, 2), round(t_zram, 2)))
sleep(t)
except KeyboardInterrupt:
exit()
##########################################################################
# поиск позиций и mem_total
with open('/proc/meminfo') as file:
mem_list = file.readlines()
mem_list_names = []
for s in mem_list:
mem_list_names.append(s.split(':')[0])
if mem_list_names[2] != 'MemAvailable':
print('Your Linux kernel is too old, Linux 3.14+ requie\nExit')
exit()
swap_total_index = mem_list_names.index('SwapTotal')
swap_free_index = swap_total_index + 1
mem_total = int(mem_list[0].split(':')[1].strip(' kB\n'))
with open('/proc/self/status') as file:
status_list = file.readlines()
# список имен из /proc/*/status для дальнейшего поиска позиций VmRSS and VmSwap
status_names = []
for s in status_list:
status_names.append(s.split(':')[0])
vm_rss_index = status_names.index('VmRSS')
vm_swap_index = status_names.index('VmSwap')
##########################################################################
# получение пути к конфигу
# парсинг аргументов командной строки
parser = ArgumentParser()
parser.add_argument(
'-c',
'--config',
help="""path to the config file, default values:
./nohang.conf, /etc/nohang/nohang.conf""",
default=None,
type=str
)
args = parser.parse_args()
arg_config = args.config
if arg_config is None:
config = None
for i in default_configs:
if os.path.exists(i):
config = i
break
if config is None:
print('По дефолтным путям конфиг не найден\n', conf_err_mess)
exit()
else:
if os.path.exists(arg_config):
config = arg_config
else:
print("File {} doesn't exists{}".format(arg_config, conf_err_mess))
exit()
print('The path to the config:', config)
##########################################################################
# parsing the config with obtaining the parameters dictionary
# conf_parameters_dict
# conf_restart_dict
try:
with open(config) as f:
# dictionary with config options
config_dict = dict()
# dictionary with names and commands for the parameter execute_the_command
etc_dict = dict()
for line in f:
a = line.startswith('#')
b = line.startswith('\n')
c = line.startswith('\t')
d = line.startswith(' ')
etc = line.startswith('$ETC')
if not a and not b and not c and not d and not etc:
a = line.split('=')
config_dict[a[0].strip()] = a[1].strip()
if etc:
a = line[4:].split('///')
etc_name = a[0].strip()
etc_command = a[1].strip()
if len(etc_name) > 15:
print('Invalid config, the length of the process name must not exceed 15 characters\nExit')
exit()
etc_dict[etc_name] = etc_command
except PermissionError:
print('PermissionError', conf_err_mess)
exit()
except UnicodeDecodeError:
print('UnicodeDecodeError', conf_err_mess)
exit()
except IsADirectoryError:
print('IsADirectoryError', conf_err_mess)
exit()
except IndexError:
print('IndexError', conf_err_mess)
exit()
##########################################################################
# extracting parameters from the dictionary
# check for all necessary parameters
# validation of all parameters
print_config = conf_parse_bool('print_config')
print_mem_check_results = conf_parse_bool('print_mem_check_results')
print_sleep_periods = conf_parse_bool('print_sleep_periods')
realtime_ionice = conf_parse_bool('realtime_ionice')
if 'realtime_ionice_classdata' in config_dict:
realtime_ionice_classdata = string_to_int_convert_test(
config_dict['realtime_ionice_classdata'])
if realtime_ionice_classdata is None:
print('Invalid value of the "realtime_ionice_classdata" parameter.')
print('Valid values are integers from the range [0; 7].')
print('Exit')
exit()
if realtime_ionice_classdata < 0 or realtime_ionice_classdata > 7:
print('Invalid value of the "realtime_ionice_classdata" parameter.')
print('Valid values are integers from the range [0; 7].')
print('Exit')
exit()
else:
print('All the necessary parameters must be in the config')
print('There is no "realtime_ionice_classdata" parameter in the config')
exit()
mlockall = conf_parse_bool('mlockall')
if 'niceness' in config_dict:
niceness = string_to_int_convert_test(config_dict['niceness'])
if niceness is None:
print('Invalid niceness value, not integer\nExit')
exit()
if niceness < -20 or niceness > 19:
print('Недопустимое значение niceness\nExit')
exit()
else:
print('niceness not in config\nExit')
exit()
if 'oom_score_adj' in config_dict:
oom_score_adj = string_to_int_convert_test(
config_dict['oom_score_adj'])
if oom_score_adj is None:
print('Invalid oom_score_adj value, not integer\nExit')
exit()
if oom_score_adj < -1000 or oom_score_adj > 1000:
print('Недопустимое значение oom_score_adj\nExit')
exit()
else:
print('oom_score_adj not in config\nExit')
exit()
if 'rate_mem' in config_dict:
rate_mem = string_to_float_convert_test(config_dict['rate_mem'])
if rate_mem is None:
print('Invalid rate_mem value, not float\nExit')
exit()
if rate_mem <= 0:
print('rate_mem должен быть положительным\nExit')
exit()
else:
print('rate_mem not in config\nExit')
exit()
if 'rate_swap' in config_dict:
rate_swap = string_to_float_convert_test(config_dict['rate_swap'])
if rate_swap is None:
print('Invalid rate_swap value, not float\nExit')
exit()
if rate_swap <= 0:
print('rate_swap должен быть положительным\nExit')
exit()
else:
print('rate_swap not in config\nExit')
exit()
if 'rate_zram' in config_dict:
rate_zram = string_to_float_convert_test(config_dict['rate_zram'])
if rate_zram is None:
print('Invalid rate_zram value, not float\nExit')
exit()
if rate_zram <= 0:
print('rate_zram должен быть положительным\nExit')
exit()
else:
print('rate_zram not in config\nExit')
exit()
if 'mem_min_sigterm' in config_dict:
mem_min_sigterm = config_dict['mem_min_sigterm']
if mem_min_sigterm.endswith('%'):
# отбрасываем процент, получаем число
mem_min_sigterm_percent = mem_min_sigterm[:-1].strip()
# далее флоат тест
mem_min_sigterm_percent = string_to_float_convert_test(mem_min_sigterm_percent)
if mem_min_sigterm_percent is None:
print('Invalid mem_min_sigterm value, not float\nExit')
exit()
# окончательная валидация
if mem_min_sigterm_percent < 0 or mem_min_sigterm_percent > 100:
print('mem_min_sigterm, выраженный в процентах, должен быть быть в диапазоне [0; 100]\nExit')
exit()
# mem_min_sigterm_percent это теперь чистое валидное флоат число процентов, можно переводить в кб
mem_min_sigterm_kb = mem_min_sigterm_percent / 100 * mem_total
mem_min_sigterm_mb = round(mem_min_sigterm_kb / 1024)
elif mem_min_sigterm.endswith('M'):
mem_min_sigterm_mb = string_to_float_convert_test(mem_min_sigterm[:-1].strip())
if mem_min_sigterm_mb is None:
print('Invalid mem_min_sigterm value, not float\nExit')
exit()
mem_min_sigterm_kb = mem_min_sigterm_mb * 1024
if mem_min_sigterm_kb > mem_total:
print('mem_min_sigterm value не должен быть больше MemTotal ({} MiB)\nExit'.format(round(mem_total / 1024)))
exit()
mem_min_sigterm_percent = mem_min_sigterm_kb / mem_total * 100
else:
print('Конфиг инвалид, для mem_min_sigterm неверно указаны единицы измерения\nExit')
exit()
else:
print('mem_min_sigterm not in config\nExit')
exit()
if 'mem_min_sigkill' in config_dict:
mem_min_sigkill = config_dict['mem_min_sigkill']
if mem_min_sigkill.endswith('%'):
# отбрасываем процент, получаем число
mem_min_sigkill_percent = mem_min_sigkill[:-1].strip()
# далее флоат тест
mem_min_sigkill_percent = string_to_float_convert_test(mem_min_sigkill_percent)
if mem_min_sigkill_percent is None:
print('Invalid mem_min_sigkill value, not float\nExit')
exit()
# окончательная валидация
if mem_min_sigkill_percent < 0 or mem_min_sigkill_percent > 100:
print('mem_min_sigkill, выраженный в процентах, должен быть быть в диапазоне [0; 100]\nExit')
exit()
# mem_min_sigterm_percent это теперь чистое валидное флоат число процентов, можно переводить в кб
mem_min_sigkill_kb = mem_min_sigkill_percent / 100 * mem_total
mem_min_sigkill_mb = round(mem_min_sigkill_kb / 1024)
elif mem_min_sigkill.endswith('M'):
mem_min_sigkill_mb = string_to_float_convert_test(mem_min_sigkill[:-1].strip())
if mem_min_sigkill_mb is None:
print('Invalid mem_min_sigkill value, not float\nExit')
exit()
mem_min_sigkill_kb = mem_min_sigkill_mb * 1024
if mem_min_sigkill_kb > mem_total:
print('mem_min_sigkill value не должен быть больше MemTotal ({} MiB)\nExit'.format(round(mem_total / 1024)))
exit()
mem_min_sigkill_percent = mem_min_sigkill_kb / mem_total * 100
else:
print('Конфиг инвалид, для mem_min_sigkill неверно указаны единицы измерения\nExit')
exit()
else:
print('mem_min_sigkill not in config\nExit')
exit()
# НУЖНА ВАЛИДАЦИЯ НА МЕСТЕ!
if 'swap_min_sigterm' in config_dict:
swap_min_sigterm = config_dict['swap_min_sigterm']
else:
print('swap_min_sigterm not in config\nExit')
exit()
# НУЖНА ВАЛИДАЦИЯ НА МЕСТЕ!
if 'swap_min_sigkill' in config_dict:
swap_min_sigkill = config_dict['swap_min_sigkill']
else:
print('swap_min_sigkill not in config\nExit')
exit()
if 'zram_max_sigterm' in config_dict:
zram_max_sigterm = config_dict['zram_max_sigterm']
if zram_max_sigterm.endswith('%'):
# отбрасываем процент, получаем число
zram_max_sigterm_percent = zram_max_sigterm[:-1].strip()
# далее флоат тест
zram_max_sigterm_percent = string_to_float_convert_test(zram_max_sigterm_percent)
if zram_max_sigterm_percent is None:
print('Invalid zram_max_sigterm value, not float\nExit')
exit()
# окончательная валидация
if zram_max_sigterm_percent < 0 or zram_max_sigterm_percent > 100:
print('zram_max_sigterm, выраженный в процентах, должен быть быть в диапазоне [0; 100]\nExit')
exit()
# zram_max_sigterm_percent это теперь чистое валидное флоат число процентов, можно переводить в кб
zram_max_sigterm_kb = zram_max_sigterm_percent / 100 * mem_total
zram_max_sigterm_mb = round(zram_max_sigterm_kb / 1024)
elif zram_max_sigterm.endswith('M'):
zram_max_sigterm_mb = string_to_float_convert_test(zram_max_sigterm[:-1].strip())
if zram_max_sigterm_mb is None:
print('Invalid zram_max_sigterm value, not float\nExit')
exit()
zram_max_sigterm_kb = zram_max_sigterm_mb * 1024
if zram_max_sigterm_kb > mem_total:
print('zram_max_sigterm value не должен быть больше MemTotal ({} MiB)\nExit'.format(round(mem_total / 1024)))
exit()
zram_max_sigterm_percent = zram_max_sigterm_kb / mem_total * 100
else:
print('Конфиг инвалид, для zram_max_sigterm неверно указаны единицы измерения\nExit')
exit()
else:
print('zram_max_sigterm not in config\nExit')
exit()
if 'zram_max_sigkill' in config_dict:
zram_max_sigkill = config_dict['zram_max_sigkill']
if zram_max_sigkill.endswith('%'):
# отбрасываем процент, получаем число
zram_max_sigkill_percent = zram_max_sigkill[:-1].strip()
# далее флоат тест
zram_max_sigkill_percent = string_to_float_convert_test(zram_max_sigkill_percent)
if zram_max_sigkill_percent is None:
print('Invalid zram_max_sigkill value, not float\nExit')
exit()
# окончательная валидация
if zram_max_sigkill_percent < 0 or zram_max_sigkill_percent > 100:
print('zram_max_sigkill, выраженный в процентах, должен быть быть в диапазоне [0; 100]\nExit')
exit()
# zram_max_sigkill_percent это теперь чистое валидное флоат число процентов, можно переводить в кб
zram_max_sigkill_kb = zram_max_sigkill_percent / 100 * mem_total
zram_max_sigkill_mb = round(zram_max_sigkill_kb / 1024)
elif zram_max_sigkill.endswith('M'):
zram_max_sigkill_mb = string_to_float_convert_test(zram_max_sigkill[:-1].strip())
if zram_max_sigkill_mb is None:
print('Invalid zram_max_sigkill value, not float\nExit')
exit()
zram_max_sigkill_kb = zram_max_sigkill_mb * 1024
if zram_max_sigkill_kb > mem_total:
print('zram_max_sigkill value не должен быть больше MemTotal ({} MiB)\nExit'.format(round(mem_total / 1024)))
exit()
zram_max_sigkill_percent = zram_max_sigkill_kb / mem_total * 100
else:
print('Конфиг инвалид, для zram_max_sigkill неверно указаны единицы измерения\nExit')
exit()
else:
print('zram_max_sigkill not in config\nExit')
exit()
if 'min_delay_after_sigterm' in config_dict:
min_delay_after_sigterm = string_to_float_convert_test(
config_dict['min_delay_after_sigterm'])
if min_delay_after_sigterm is None:
print('Invalid min_delay_after_sigterm value, not float\nExit')
exit()
if min_delay_after_sigterm < 0:
print('min_delay_after_sigterm должен быть неотрицательным\nExit')
exit()
else:
print('min_delay_after_sigterm not in config\nExit')
exit()
if 'min_delay_after_sigkill' in config_dict:
min_delay_after_sigkill = string_to_float_convert_test(
config_dict['min_delay_after_sigkill'])
if min_delay_after_sigkill is None:
print('Invalid min_delay_after_sigkill value, not float\nExit')
exit()
if min_delay_after_sigkill < 0:
print('min_delay_after_sigkill должен быть неотрицательным\nExit')
exit()
else:
print('min_delay_after_sigkill not in config\nExit')
exit()
if 'min_badness' in config_dict:
min_badness = string_to_int_convert_test(
config_dict['min_badness'])
if min_badness is None:
print('Invalid min_badness value, not integer\nExit')
exit()
if min_badness < 0 or min_badness > 1000:
print('Недопустимое значение min_badness\nExit')
exit()
else:
print('min_badness not in config\nExit')
exit()
decrease_oom_score_adj = conf_parse_bool('decrease_oom_score_adj')
if 'oom_score_adj_max' in config_dict:
oom_score_adj_max = string_to_int_convert_test(
config_dict['oom_score_adj_max'])
if oom_score_adj_max is None:
print('Invalid oom_score_adj_max value, not integer\nExit')
exit()
if oom_score_adj_max < 0 or oom_score_adj_max > 1000:
print('Недопустимое значение oom_score_adj_max\nExit')
exit()
else:
print('oom_score_adj_max not in config\nExit')
exit()
if 'gui_notifications' in config_dict:
gui_notifications = config_dict['gui_notifications']
if gui_notifications == 'True':
gui_notifications = True
from subprocess import Popen, PIPE
elif gui_notifications == 'False':
gui_notifications = False
else:
print('Invalid gui_notifications value {} (shoul' \
'd be True or False)\nExit'.format(
gui_notifications))
exit()
else:
print('gui_notifications not in config\nExit')
exit()
regex_matching = conf_parse_bool('regex_matching')
if regex_matching:
from re import fullmatch
prefer_regex = conf_parse_string('prefer_regex')
if 'prefer_factor' in config_dict:
prefer_factor = string_to_float_convert_test(config_dict['prefer_factor'])
if prefer_factor is None:
print('Invalid prefer_factor value, not float\nExit')
exit()
if prefer_factor < 1 and prefer_factor > 1000:
print('prefer_factor должен быть в диапазоне [1; 1000]\nExit')
exit()
else:
print('prefer_factor not in config\nExit')
exit()
avoid_regex = conf_parse_string('avoid_regex')
if 'avoid_factor' in config_dict:
avoid_factor = string_to_float_convert_test(config_dict['avoid_factor'])
if avoid_factor is None:
print('Invalid avoid_factor value, not float\nExit')
exit()
if avoid_factor < 1 and avoid_factor > 1000:
print('avoid_factor должен быть в диапазоне [1; 1000]\nExit')
exit()
else:
print('avoid_factor not in config\nExit')
exit()
gui_low_memory_warnings = conf_parse_bool('gui_low_memory_warnings')
if 'min_time_between_warnings' in config_dict:
min_time_between_warnings = string_to_float_convert_test(
config_dict['min_time_between_warnings'])
if min_time_between_warnings is None:
print('Invalid min_time_between_warnings value, not float\nExit')
exit()
if min_time_between_warnings < 1 or min_time_between_warnings > 300:
print('Недопустимое значение min_time_between_warnings, должно быть в диапазоне [1; 300]\nExit')
exit()
else:
print('min_time_between_warnings not in config\nExit')
exit()
if 'mem_min_warnings' in config_dict:
mem_min_warnings = config_dict['mem_min_warnings']
if mem_min_warnings.endswith('%'):
# отбрасываем процент, получаем число
mem_min_warnings_percent = mem_min_warnings[:-1].strip()
# далее флоат тест
mem_min_warnings_percent = string_to_float_convert_test(mem_min_warnings_percent)
if mem_min_warnings_percent is None:
print('Invalid mem_min_warnings value, not float\nExit')
exit()
# окончательная валидация
if mem_min_warnings_percent < 0 or mem_min_warnings_percent > 100:
print('mem_min_warnings, выраженный в процентах, должен быть быть в диапазоне [0; 100]\nExit')
exit()
# mem_min_warnings_percent это теперь чистое валидное флоат число процентов, можно переводить в кб
mem_min_warnings_kb = mem_min_warnings_percent / 100 * mem_total
mem_min_warnings_mb = round(mem_min_warnings_kb / 1024)
elif mem_min_warnings.endswith('M'):
mem_min_warnings_mb = string_to_float_convert_test(mem_min_warnings[:-1].strip())
if mem_min_warnings_mb is None:
print('Invalid mem_min_warnings value, not float\nExit')
exit()
mem_min_warnings_kb = mem_min_warnings_mb * 1024
if mem_min_warnings_kb > mem_total:
print('mem_min_warnings value не должен быть больше MemTotal ({} MiB)\nExit'.format(round(mem_total / 1024)))
exit()
mem_min_warnings_percent = mem_min_warnings_kb / mem_total * 100
else:
print('Конфиг инвалид, для mem_min_warnings неверно указаны единицы измерения\nExit')
exit()
else:
print('mem_min_warnings not in config\nExit')
exit()
# НА МЕСТЕ!!!
if 'swap_min_warnings' in config_dict:
swap_min_warnings = config_dict['swap_min_warnings']
else:
print('swap_min_warnings not in config\nExit')
exit()
if 'zram_max_warnings' in config_dict:
zram_max_warnings = config_dict['zram_max_warnings']
if zram_max_warnings.endswith('%'):
# отбрасываем процент, получаем число
zram_max_warnings_percent = zram_max_warnings[:-1].strip()
# далее флоат тест
zram_max_warnings_percent = string_to_float_convert_test(zram_max_warnings_percent)
if zram_max_warnings_percent is None:
print('Invalid zram_max_warnings value, not float\nExit')
exit()
# окончательная валидация
if zram_max_warnings_percent < 0 or zram_max_warnings_percent > 100:
print('zram_max_warnings, выраженный в процентах, должен быть быть в диапазоне [0; 100]\nExit')
exit()
# zram_max_warnings_percent это теперь чистое валидное флоат число процентов, можно переводить в кб
zram_max_warnings_kb = zram_max_warnings_percent / 100 * mem_total
zram_max_warnings_mb = round(zram_max_warnings_kb / 1024)
elif zram_max_warnings.endswith('M'):
zram_max_warnings_mb = string_to_float_convert_test(zram_max_warnings[:-1].strip())
if zram_max_warnings_mb is None:
print('Invalid zram_max_warnings value, not float\nExit')
exit()
zram_max_warnings_kb = zram_max_warnings_mb * 1024
if zram_max_warnings_kb > mem_total:
print('zram_max_warnings value не должен быть больше MemTotal ({} MiB)\nExit'.format(round(mem_total / 1024)))
exit()
zram_max_warnings_percent = zram_max_warnings_kb / mem_total * 100
else:
print('Конфиг инвалид, для zram_max_warnings неверно указаны единицы измерения\nExit')
exit()
else:
print('zram_max_warnings not in config\nExit')
exit()
execute_the_command = conf_parse_bool('execute_the_command')
##########################################################################
# получение уровней в кибибайтах
# возвращает число килобайт при задании в конфиге абсолютного значения,
# или кортеж с числом процентов
def sig_level_to_kb_swap(string):
if string.endswith('%'):
return float(string[:-1].strip()), True
elif string.endswith('M'):
return float(string[:-1].strip()) * 1024
else:
print('Конфиг инвалид, где-то неверно указаны единицы измерения\nExit')
exit()
# получаем число килобайт или кортеж с процентами
swap_min_sigterm_swap = sig_level_to_kb_swap(swap_min_sigterm)
swap_min_sigkill_swap = sig_level_to_kb_swap(swap_min_sigkill)
swap_min_warnings_swap = sig_level_to_kb_swap(swap_min_warnings)
if isinstance(swap_min_sigterm_swap, tuple):
swap_term_is_percent = True
swap_min_sigterm_percent = swap_min_sigterm_swap[0]
else:
swap_term_is_percent = False
swap_min_sigterm_kb = swap_min_sigterm_swap
if isinstance(swap_min_sigkill_swap, tuple):
swap_kill_is_percent = True
swap_min_sigkill_percent = swap_min_sigkill_swap[0]
else:
swap_kill_is_percent = False
swap_min_sigkill_kb = swap_min_sigkill_swap
if isinstance(swap_min_warnings_swap, tuple):
swap_warn_is_percent = True
swap_min_warnings_percent = swap_min_warnings_swap[0]
else:
swap_warn_is_percent = False
swap_min_warnings_kb = swap_min_warnings_swap
##########################################################################
# self-defense
# повышаем приоритет
try:
os.nice(niceness)
niceness_result = 'OK'
except PermissionError:
niceness_result = 'Fail'
pass
# возможность запрета самоубийства
try:
with open('/proc/self/oom_score_adj', 'w') as file:
file.write('{}\n'.format(oom_score_adj))
oom_score_adj_result = 'OK'
except PermissionError:
pass
oom_score_adj_result = 'Fail'
except OSError:
oom_score_adj_result = 'Fail'
pass
# запрет своппинга процесса
if mlockall:
from ctypes import CDLL
result = CDLL('libc.so.6', use_errno=True).mlockall(3)
if result is 0:
mla_res = 'OK'
else:
mla_res = 'Fail'
else:
mla_res = ''
self_uid = os.geteuid()
self_pid = os.getpid()
if self_uid == 0:
root = True
decrease_res = 'OK'
else:
root = False
decrease_res = 'Impossible'
if root and realtime_ionice:
os.system('ionice -c 1 -n {} -p {}'.format(
realtime_ionice_classdata, self_pid))
##########################################################################
if print_config:
print('\n1. Memory levels to respond to as an OOM threat\n[displaying these options need fix]\n')
print('mem_min_sigterm: {} MiB, {} %'.format(
round(mem_min_sigterm_mb), round(mem_min_sigterm_percent, 1)))
print('mem_min_sigkill: {} MiB, {} %'.format(
round(mem_min_sigkill_mb), round(mem_min_sigkill_percent, 1)))
print('swap_min_sigterm: {}'.format(swap_min_sigterm))
print('swap_min_sigkill: {}'.format(swap_min_sigkill))
print('zram_max_sigterm: {} MiB, {} %'.format(
round(zram_max_sigterm_mb), round(zram_max_sigterm_percent, 1)))
print('zram_max_sigkill: {} MiB, {} %'.format(
round(zram_max_sigkill_mb), round(zram_max_sigkill_percent, 1)))
print('\n2. The frequency of checking the level of available memory (and CPU usage)\n')
print('rate_mem: {}'.format(rate_mem))
print('rate_swap: {}'.format(rate_swap))
print('rate_zram: {}'.format(rate_zram))
print('\n3. The prevention of killing innocent victims\n')
print('min_delay_after_sigterm: {}'.format(min_delay_after_sigterm))
print('min_delay_after_sigkill: {}'.format(min_delay_after_sigkill))
print('min_badness: {}'.format(min_badness))
# False (OK) - OK не нужен когда фолс
print('decrease_oom_score_adj: {} ({})'.format(
decrease_oom_score_adj, decrease_res
))
if decrease_oom_score_adj:
print('oom_score_adj_max: {}'.format(oom_score_adj_max))
print('\n4. Impact on the badness of processes via matching their names\nwith regular expressions\n')
print('regex_matching: {}'.format(regex_matching))
if regex_matching:
print('prefer_regex: {}'.format(prefer_regex))
print('prefer_factor: {}'.format(prefer_factor))
print('avoid_regex: {}'.format(avoid_regex))
print('avoid_factor: {}'.format(avoid_factor))
print('\n5. The execution of a specific command instead of sending the\nSIGTERM signal\n')
print('execute_the_command: {}'.format(execute_the_command))
if execute_the_command:
print('\nPROCESS NAME COMMAND TO EXECUTE')
for key in etc_dict:
print('{} {}'.format(key.ljust(15), etc_dict[key]))
print('\n6. GUI notifications:\n- OOM prevention results and\n- low memory warnings\n')
print('gui_notifications: {}'.format(gui_notifications))
print('gui_low_memory_warnings: {}'.format(gui_low_memory_warnings))
if gui_low_memory_warnings:
print('min_time_between_warnings: {}'.format(min_time_between_warnings))
print('mem_min_warnings: {} MiB, {} %'.format(
round(mem_min_warnings_mb), round(mem_min_warnings_percent, 1)))
print('swap_min_warnings: {}'.format(swap_min_warnings))
print('zram_max_warnings: {} MiB, {} %'.format(
round(zram_max_warnings_mb), round(zram_max_warnings_percent, 1)))
print('\n7. Preventing the slowing down of the program\n[displaying these options need fix]\n')
print('mlockall: {} ({})'.format(mlockall, mla_res))
print('niceness: {} ({})'.format(
niceness, niceness_result
))
print('oom_score_adj: {} ({})'.format(
oom_score_adj, oom_score_adj_result
))
print('realtime_ionice: {} ({})'.format(realtime_ionice, ''))
if realtime_ionice:
print('realtime_ionice_classdata: {}'.format(realtime_ionice_classdata))
print('\n8. Output verbosity\n')
print('print_config: {}'.format(print_config))
print('print_mem_check_results: {}'.format(print_mem_check_results))
print('print_sleep_periods: {}\n'.format(print_sleep_periods))
##########################################################################
# for calculating the column width when printing mem and zram
mem_len = len(str(round(mem_total / 1024.0)))
rate_mem = rate_mem * 1048576
rate_swap = rate_swap * 1048576
rate_zram = rate_zram * 1048576
warn_time_now = 0
warn_time_delta = 1000
warn_timer = 0
print('Monitoring started!')
##########################################################################
while True:
# find mem_available, swap_total, swap_free
with open('/proc/meminfo') as f:
for n, line in enumerate(f):
if n is 2:
mem_available = int(line.split(':')[1].strip(' kB\n'))
continue
if n is swap_total_index:
swap_total = int(line.split(':')[1].strip(' kB\n'))
continue
if n is swap_free_index:
swap_free = int(line.split(':')[1].strip(' kB\n'))
break
# if swap_min_sigkill is set in percent
if swap_kill_is_percent:
swap_min_sigkill_kb = swap_total * swap_min_sigkill_percent / 100.0
if swap_term_is_percent:
swap_min_sigterm_kb = swap_total * swap_min_sigterm_percent / 100.0
if swap_warn_is_percent:
swap_min_warnings_kb = swap_total * swap_min_warnings_percent / 100.0
# find MemUsedZram
disksize_sum = 0
mem_used_total_sum = 0
for dev in os.listdir('/sys/block'):
if dev.startswith('zram'):
stat = zram_stat(dev)
disksize_sum += int(stat[0])
mem_used_total_sum += int(stat[1])
mem_used_zram = (
mem_used_total_sum + disksize_sum * zram_disksize_factor
) / 1024.0
if print_mem_check_results:
# для рассчета ширины столбца свопа
swap_len = len(str(round(swap_total / 1024.0)))
# печать размеров доступной памяти
if swap_total == 0 and mem_used_zram == 0:
print('MemAvail: {} M, {} %'.format(
human(mem_available, mem_len),
just_percent_mem(mem_available / mem_total)))
elif swap_total > 0 and mem_used_zram == 0:
print('MemAvail: {} M, {} % | SwapFree: {} M, {} %'.format(
human(mem_available, mem_len),
just_percent_mem(mem_available / mem_total),
human(swap_free, swap_len),
just_percent_swap(swap_free / (swap_total + 0.1))))
else:
print('MemAvail: {} M, {} % | SwapFree: {} M, {} % | Mem' \
'UsedZram: {} M, {} %'.format(
human(mem_available, mem_len),
just_percent_mem(mem_available / mem_total),
human(swap_free, swap_len),
just_percent_swap(swap_free / (swap_total + 0.1)),
human(mem_used_zram, mem_len),
just_percent_mem(mem_used_zram / mem_total)))
# если swap_min_sigkill задан в абсолютной величине и Swap_total = 0
if swap_total > swap_min_sigkill_kb:
swap_sigkill_pc = percent(swap_min_sigkill_kb / (swap_total + 0.1))
else:
swap_sigkill_pc = '-'
if swap_total > swap_min_sigterm_kb:
swap_sigterm_pc = percent(swap_min_sigterm_kb / (swap_total + 0.1))
else:
# СТОИТ ПЕЧАТАТЬ СВОП ТОЛЬКО ПРИ SwapTotal > 0
swap_sigterm_pc = '-'
# проверка превышения порогов
# порог превышен - пытаемся предотвратить OOM
# пороги не превышены - спим
# MEM SWAP KILL
if mem_available <= mem_min_sigkill_kb and swap_free <= swap_min_sigkill_kb:
time0 = time()
mem_info = 'TRIGGERED!\n MemAvailable ({} MiB, {} %) < mem_min_sigkill ({} MiB, {} %)\n Swa' \
'pFree ({} MiB, {} %) < swap_min_sigkill ({} MiB, {} %)'.format(
kib_to_mib(mem_available),
percent(mem_available / mem_total),
kib_to_mib(mem_min_sigkill_kb),
percent(mem_min_sigkill_kb / mem_total),
kib_to_mib(swap_free),
percent(swap_free / (swap_total + 0.1)),
kib_to_mib(swap_min_sigkill_kb),
swap_sigkill_pc)
find_victim_and_send_signal(signal.SIGKILL)
# ZRAM KILL
elif mem_used_zram >= zram_max_sigkill_kb:
time0 = time()
mem_info = 'TRIGGERED!\n MemUsedZram ({} MiB, {} %) > zram_max_sigkill ({} MiB, {} %)'.format(
kib_to_mib(mem_used_zram),
percent(mem_used_zram / mem_total),
kib_to_mib(zram_max_sigkill_kb),
percent(zram_max_sigkill_kb / mem_total))
find_victim_and_send_signal(signal.SIGKILL)
# MEM SWAP TERM
elif mem_available <= mem_min_sigterm_kb and swap_free <= swap_min_sigterm_kb:
time0 = time()
mem_info = 'TRIGGERED!\n MemAvailable ({} MiB, {} %) < mem_min_sigterm ({} MiB, {} %)\n Sw' \
'apFree ({} MiB, {} %) < swap_min_sigterm ({} MiB, {} %)'.format(
kib_to_mib(mem_available),
percent(mem_available / mem_total),
kib_to_mib(mem_min_sigterm_kb),
#percent(mem_min_sigterm_kb / mem_total),
# ОКРУГЛЯТЬ НА МЕСТЕ ВЫШЕ
round(mem_min_sigterm_percent, 1),
kib_to_mib(swap_free),
percent(swap_free / (swap_total + 0.1)),
kib_to_mib(swap_min_sigterm_kb),
swap_sigterm_pc)
find_victim_and_send_signal(signal.SIGTERM)
# ZRAM TERM
elif mem_used_zram >= zram_max_sigterm_kb:
time0 = time()
mem_info = 'TRIGGERED!\n MemUsedZram ({} MiB, {} %) > zram_max_sigter' \
'm ({} M, {} %)'.format(
kib_to_mib(mem_used_zram),
percent(mem_used_zram / mem_total),
kib_to_mib(zram_max_sigterm_kb),
percent(zram_max_sigterm_kb / mem_total))
find_victim_and_send_signal(signal.SIGTERM)
# LOW MEMORY WARNINGS
elif gui_low_memory_warnings and gui_notifications:
if mem_available < mem_min_warnings_kb and swap_free < swap_min_warnings_kb + 0.1 or mem_used_zram > zram_max_warnings_kb:
warn_time_delta = time() - warn_time_now
warn_time_now = time()
warn_timer += warn_time_delta
if warn_timer > min_time_between_warnings:
send_notify_warn()
warn_timer = 0
sleep_after_check_mem()
# SLEEP BETWEEN MEM CHECKS
else:
stdout.flush()
sleep_after_check_mem()