diff --git a/n11 b/n11 deleted file mode 100755 index 9195777..0000000 --- a/n11 +++ /dev/null @@ -1,1754 +0,0 @@ -#!/usr/bin/env python3 -"""A daemon that prevents OOM in Linux systems.""" -import os -from time import sleep, time -from operator import itemgetter -from sys import stdout -from signal import SIGKILL, SIGTERM, SIGSTOP, SIGCONT - -start_time = time() - -sig_dict = {SIGKILL: 'SIGKILL', - SIGTERM: 'SIGTERM'} - -self_pid = str(os.getpid()) - -self_uid = os.geteuid() - -if self_uid == 0: - root = True -else: - root = False - -wait_time = 3 - -# todo: make config option -max_sleep_time = 2 - -# todo: make config option -min_sleep_time = 0.1 - -notify_helper_path = '/usr/sbin/nohang_notify_helper' - -psi_path = '/proc/pressure/memory' - -psi_support = os.path.exists(psi_path) - -HR = '~' * 79 - -# todo: make config option -print_total_stat = True - - -stop_cont = False -stop_cont_warn = False - - -########################################################################## - -# define functions - - - -def pid_to_state(pid): - return rline1('/proc/' + pid + '/stat').rpartition(')')[2][1] - -def stop(): - #print() - #print('Stop running processes...') - t1 = time() - t2 = time() - stopped_list = [] - for pid in os.listdir('/proc')[::-1]: - # only directories whose names consist only of numbers, except /proc/1/ - if pid[0].isdecimal() is False or pid is '1' or pid == self_pid: - continue - try: - oom_score_r = int(rline1('/proc/' + pid + '/oom_score')) - if oom_score_r > 9: - uid_r = pid_to_uid(pid) - #print('PID: {}, State: {}, oom_score {}'.format(pid, pid_to_state(pid), oom_score_r)) - if uid_r != '0': - stopped_list.append(pid) - print('Send SIGSTOP to {}, {}, {}...'.format( - pid, pid_to_name(pid), pid_to_cmdline(pid)[:40])) - os.kill(int(pid), SIGSTOP) - t2 = time() - except FileNotFoundError: - continue - except ProcessLookupError: - continue - print('Stop time:', t2 - t1) - stdout.flush() - - return stopped_list - -def cont(stopped_list): - print() - print('Continue stopped processes...') - t1 = time() - if len(stopped_list) > 0: - for pid in stopped_list: - print('Send SIGCONT to', [pid], pid_to_name(pid)) - try: - os.kill(int(pid), SIGCONT) - except FileNotFoundError: - continue - except ProcessLookupError: - continue - t2 = time() - print('All cont time: ', t2 - t1) - - - -def update_stat_dict_and_print(key): - - if key not in stat_dict: - - stat_dict.update({key: 1}) - - else: - - new_value = stat_dict[key] + 1 - stat_dict.update({key: new_value}) - - if print_total_stat: - - stats_msg = '{}\n\033[4mThe following corrective actions have been implemented in the last {}:\033[0m'.format( - HR, format_time(time() - start_time)) - - for i in stat_dict: - stats_msg += '\n- {}: {}'.format(i, stat_dict[i]) - - print(stats_msg) - - -def psi_mem_some_avg_total(): - if psi_support: - return float(rline1(psi_path).rpartition('=')[2]) - - -def psi_mem_some_avg10(): - return float(rline1(psi_path).split(' ')[1].split('=')[1]) - - -def check_mem(): - """find mem_available""" - return int(rline1('/proc/meminfo').split(':')[1].strip(' kB\n')) - - -def check_mem_and_swap(): - """find mem_available, swap_total, swap_free""" - with open('/proc/meminfo') as f: - for n, line in enumerate(f): - if n is 2: - mem_available = int(line.split(':')[1].strip(' kB\n')) - continue - if n is swap_total_index: - swap_total = int(line.split(':')[1].strip(' kB\n')) - continue - if n is swap_free_index: - swap_free = int(line.split(':')[1].strip(' kB\n')) - break - return mem_available, swap_total, swap_free - - -def check_zram(): - """find MemUsedZram""" - disksize_sum = 0 - mem_used_total_sum = 0 - - for dev in os.listdir('/sys/block'): - if dev.startswith('zram'): - stat = zram_stat(dev) - disksize_sum += int(stat[0]) - mem_used_total_sum += int(stat[1]) - - ZRAM_DISKSIZE_FACTOR = 0.0042 - # Означает, что при задани zram disksize = 1 GiB доступная память - # уменьшится на 0.0042 GiB. - # Найден экспериментально, требует уточнения с разными ядрами и архитектурами. - # На небольших дисксайзах (до гигабайта) может быть больше, до 0.0045. - # Создатель модуля zram утверждает, что ZRAM_DISKSIZE_FACTOR доожен быть 0.001: - # ("zram uses about 0.1% of the size of the disk" - # - https://www.kernel.org/doc/Documentation/blockdev/zram.txt), - # но это утверждение противоречит опытным данным. - # ZRAM_DISKSIZE_FACTOR = deltaMemAvailavle / disksize - # found experimentally - - return (mem_used_total_sum + disksize_sum * ZRAM_DISKSIZE_FACTOR) / 1024.0 - - -def format_time(t): - t = int(t) - if t < 60: - return '{} sec'.format(t) - elif t >= 60 and t < 3600: - m = t // 60 - s = t % 60 - return '{} min {} sec'.format(m, s) - else: - h = t // 3600 - s0 = t - h * 3600 - m = s0 // 60 - s = s0 % 60 - return '{} h {} min {} sec'.format(h, m, s) - - -def string_to_float_convert_test(string): - """Try to interprete string values as floats.""" - try: - return float(string) - except ValueError: - return None - - -def string_to_int_convert_test(string): - """Try to interpret string values as integers.""" - try: - return int(string) - except ValueError: - return None - - -def conf_parse_string(param): - """ - Get string parameters from the config dict. - - param: config_dict key - returns config_dict[param].strip() - """ - if param in config_dict: - return config_dict[param].strip() - else: - print('All the necessary parameters must be in the config') - print('There is no "{}" parameter in the config'.format(param)) - exit() - - -def conf_parse_bool(param): - """ - Get bool parameters from the config_dict. - - param: config_dicst key - returns bool - """ - if param in config_dict: - param_str = config_dict[param] - if param_str == 'True': - return True - elif param_str == 'False': - return False - else: - print('Invalid value of the "{}" parameter.'.format(param_str)) - print('Valid values are True and False.') - print('Exit') - exit() - else: - print('All the necessary parameters must be in the config') - print('There is no "{}" parameter in the config'.format(param_str)) - exit() - - -def rline1(path): - """read 1st line from path.""" - with open(path) as f: - for line in f: - return line[:-1] - - -def kib_to_mib(num): - """Convert KiB values to MiB values.""" - return round(num / 1024.0) - - -def percent(num): - """Interprete mum as percentage.""" - return round(num * 100, 1) - - -def just_percent_mem(num): - """convert num to percent and justify""" - return str(round(num * 100, 1)).rjust(4, ' ') - - -def just_percent_swap(num): - return str(round(num * 100, 1)).rjust(5, ' ') - - -def human(num, lenth): - """Convert KiB values to MiB values with right alignment""" - return str(round(num / 1024)).rjust(lenth, ' ') - - -def zram_stat(zram_id): - """ - Get zram state. - - zram_id: str zram block-device id - returns bytes diskcize, str mem_used_total - """ - try: - disksize = rline1('/sys/block/' + zram_id + '/disksize') - except FileNotFoundError: - return '0', '0' - if disksize == ['0\n']: - return '0', '0' - try: - mm_stat = rline1('/sys/block/' + zram_id + '/mm_stat').split(' ') - mm_stat_list = [] - for i in mm_stat: - if i != '': - mm_stat_list.append(i) - mem_used_total = mm_stat_list[2] - except FileNotFoundError: - mem_used_total = rline1('/sys/block/' + zram_id + '/mem_used_total') - return disksize, mem_used_total # BYTES, str - - -def pid_to_name(pid): - """ - Get process name by pid. - - pid: str pid of required process - returns string process_name - """ - try: - with open('/proc/' + pid + '/status') as f: - f.seek(6) - for line in f: - return line[:-1] - except FileNotFoundError: - return '' - except ProcessLookupError: - return '' - except UnicodeDecodeError: - with open('/proc/' + pid + '/status', 'rb') as f: - f.seek(6) - return f.read(15).decode( - 'utf-8', 'ignore').partition('\n')[0] - - -def pid_to_cmdline(pid): - """ - Get process cmdline by pid. - - pid: str pid of required process - returns string cmdline - """ - with open('/proc/' + pid + '/cmdline') as f: - return f.read().replace('\x00', ' ').rstrip() - - - -def pid_to_uid(pid): - '''return euid''' - try: - with open('/proc/' + pid + '/status') as f: - for n, line in enumerate(f): - if n is uid_index: - return line.split('\t')[2] - except UnicodeDecodeError: - with open('/proc/' + pid + '/status', 'rb') as f: - f_list = f.read().decode('utf-8', 'ignore').split('\n') - return f_list[uid_index].split('\t')[2] - - -def notify_send_wait(title, body): - '''GUI notifications with UID != 0''' - with Popen(['notify-send', '--icon=dialog-warning', title, body]) as proc: - try: - proc.wait(timeout=wait_time) - except TimeoutExpired: - proc.kill() - print('TimeoutExpired: notify-send {} {}'.format(title, body)) - - -def notify_helper(title, body): - '''GUI notification with UID = 0''' - with Popen([notify_helper_path, title, body]) as proc: - try: - proc.wait(timeout=wait_time) - except TimeoutExpired: - proc.kill() - print( - 'TimeoutExpired: nohang_notify_helper: {} {}'.format( - title, body)) - - -def send_notify_warn(): - """ - Look for process with maximum 'badness' and warn user with notification. - (implement Low memory warnings) - """ - - if stop_cont_warn: - stopped_list = stop() - - # find process with max badness - fat_tuple = fattest() - pid = fat_tuple[0] - name = pid_to_name(pid) - - if mem_used_zram > 0: - low_mem_percent = '{}% {}% {}%'.format( - round(mem_available / mem_total * 100), - round(swap_free / (swap_total + 0.1) * 100), - round(mem_used_zram / mem_total * 100)) - elif swap_free > 0: - low_mem_percent = '{}% {}%'.format( - round(mem_available / mem_total * 100), - round(swap_free / (swap_total + 0.1) * 100)) - else: - low_mem_percent = '{}%'.format( - round(mem_available / mem_total * 100)) - - # title = 'Low memory: {}'.format(low_mem_percent) - title = 'Low memory' - - body = 'Hog: {} [{}]'.format( - name.replace( - # symbol '&' can break notifications in some themes, - # therefore it is replaced by '*' - '&', '*'), - pid - ) - - if root: # If nohang was started by root - # send notification to all active users with special script - notify_helper(title, body) - else: # Or by regular user - # send notification to user that runs this nohang - notify_send_wait(title, body) - - if stop_cont_warn: - cont(stopped_list) - - -def send_notify(signal, name, pid): - """ - Notificate about OOM Preventing. - - signal: key for notify_sig_dict - name: str process name - pid: str process pid - """ - title = 'Hang prevention' - body = '{} {} [{}]'.format( - notify_sig_dict[signal], - name.replace( - # symbol '&' can break notifications in some themes, - # therefore it is replaced by '*' - '&', '*'), - pid - ) - if root: - # send notification to all active users with notify-send - notify_helper(title, body) - else: - # send notification to user that runs this nohang - notify_send_wait(title, body) - - -def send_notify_etc(pid, name, command): - """ - Notificate about OOM Preventing. - - command: str command that will be executed - name: str process name - pid: str process pid - """ - title = 'Hang prevention' - body = 'Victim is process {} [{}]\nExecute the command:\n{}'.format( - name.replace('&', '*'), - pid, - command.replace('&', '*') - ) - if root: - # send notification to all active users with notify-send - notify_helper(title, body) - else: - # send notification to user that runs this nohang - notify_send_wait(title, body) - - -def sleep_after_send_signal(signal): - """ - Sleeping after signal was sent. - - signal: sent signal - """ - if signal is SIGKILL: - if print_sleep_periods: - print(' sleep', min_delay_after_sigkill) - sleep(min_delay_after_sigkill) - else: - if print_sleep_periods: - print(' sleep', min_delay_after_sigterm) - sleep(min_delay_after_sigterm) - - - - - - - -def get_pid_list(): - """ - Find pid list expect kthreads and zombies - """ - pid_list = [] - for pid in os.listdir('/proc'): - if os.path.exists('/proc/' + pid + '/exe') is True: - pid_list.append(pid) - return pid_list - - - -pid_list = get_pid_list() - -def get_non_decimal_pids(): - non_decimal_list = [] - for pid in pid_list: - if pid[0].isdecimal() is False: - non_decimal_list.append(pid) - return non_decimal_list - - -#print(get_non_decimal_pids()) - - - -def fattest(): - """ - Find the process with highest badness and its badness adjustment - Return pid and badness - - -> find_mem_hog() or find_victim() or find_worst_process() - """ - - pid_list = get_pid_list() - - pid_list.remove(self_pid) - - if '1' in pid_list: - pid_list.remove('1') - - non_decimal_list = get_non_decimal_pids() - - for i in non_decimal_list: - pid_list.remove(i) - - pid_badness_list = [] - - for pid in pid_list: - - # find and modify badness (if it needs) - try: - - badness = int(rline1('/proc/' + pid + '/oom_score')) - - if decrease_oom_score_adj: - oom_score_adj = int(rline1('/proc/' + pid + '/oom_score_adj')) - if badness > oom_score_adj_max and oom_score_adj > 0: - badness = badness - oom_score_adj + oom_score_adj_max - - if regex_matching: - name = pid_to_name(pid) - for re_tup in processname_re_list: - if search(re_tup[1], name) is not None: - if pid_to_cmdline(pid) == '': - # skip kthreads - continue - badness += int(re_tup[0]) - - if re_match_cmdline: - cmdline = pid_to_cmdline(pid) - if cmdline == '': - # skip kthreads - continue - for re_tup in cmdline_re_list: - if search(re_tup[1], cmdline) is not None: - badness += int(re_tup[0]) - - if re_match_uid: - uid = pid_to_uid(pid) - for re_tup in uid_re_list: - if search(re_tup[1], uid) is not None: - if pid_to_cmdline(pid) == '': - # skip kthreads - continue - badness += int(re_tup[0]) - - - except FileNotFoundError: - continue - except ProcessLookupError: - continue - pid_badness_list.append((pid, badness)) - - # Make list of (pid, badness) tuples, sorted by 'badness' values - #print(pid_badness_list) - pid_tuple_list = sorted( - pid_badness_list, - key=itemgetter(1), - reverse=True - )[0] - - pid = pid_tuple_list[0] - - # Get maximum 'badness' value - victim_badness = pid_tuple_list[1] - - #print(pid_badness_list) - - - return pid, victim_badness - - - - -def find_victim_and_send_signal(signal): - """ - Find victim with highest badness and send SIGTERM/SIGKILL - - -> implement_corrective_action() - """ - - - if stop_cont: - stopped_list = stop() - - - pid, victim_badness = fattest() - name = pid_to_name(pid) - - if victim_badness >= min_badness: - - # Try to send signal to found victim - - # Get VmRSS and VmSwap and cmdline of victim process - # and try to send a signal - try: - - with open('/proc/' + pid + '/status') as f: - for n, line in enumerate(f): - - if n is uid_index: - uid = line.split('\t')[1] - continue - - if n is vm_size_index: - vm_size = kib_to_mib(int(line.split('\t')[1][:-4])) - continue - - if n is vm_rss_index: - vm_rss = kib_to_mib(int(line.split('\t')[1][:-4])) - continue - - if detailed_rss: - - if n is anon_index: - anon_rss = kib_to_mib( - int(line.split('\t')[1][:-4])) - continue - - if n is file_index: - file_rss = kib_to_mib( - int(line.split('\t')[1][:-4])) - continue - - if n is shmem_index: - shmem_rss = kib_to_mib( - int(line.split('\t')[1][:-4])) - continue - - if n is vm_swap_index: - vm_swap = kib_to_mib(int(line.split('\t')[1][:-4])) - break - - cmdline = pid_to_cmdline(pid) - oom_score = rline1('/proc/' + pid + '/oom_score') - oom_score_adj = rline1('/proc/' + pid + '/oom_score_adj') - - except FileNotFoundError: - print(mem_info) - print('The victim died in the search process: FileNotFoundError') - update_stat_dict_and_print( - 'The victim died in the search process: FileNotFoundError') - return None - except ProcessLookupError: - print(mem_info) - print('The victim died in the search process: ProcessLookupError') - update_stat_dict_and_print( - 'The victim died in the search process: ProcessLookupError') - return None - except UnicodeDecodeError: - - # тут надо снова все исключ обработать - - with open('/proc/' + pid + '/status', 'rb') as f: - f_list = f.read().decode('utf-8', 'ignore').split('\n') - - for i in range(len(f_list)): - if i is ppid_index: - ppid = f_list[i].split('\t')[2] - - - for i in range(len(f_list)): - if i is uid_index: - uid = f_list[i].split('\t')[2] - - - - - if i is vm_size_index: - vm_size = kib_to_mib( - int(f_list[i].split('\t')[1][:-3])) - - if i is vm_rss_index: - vm_rss = kib_to_mib(int(f_list[i].split('\t')[1][:-3])) - - if detailed_rss: - - if i is anon_index: - anon_rss = kib_to_mib( - int(f_list[i].split('\t')[1][:-3])) - - if i is file_index: - file_rss = kib_to_mib( - int(f_list[i].split('\t')[1][:-3])) - - if i is shmem_index: - shmem_rss = kib_to_mib( - int(f_list[i].split('\t')[1][:-3])) - - if i is vm_swap_index: - vm_swap = kib_to_mib( - int(f_list[i].split('\t')[1][:-3])) - - with open('/proc/' + pid + '/cmdline') as file: - cmdline = file.readlines()[0].replace('\x00', ' ') - - oom_score = rline1('/proc/' + pid + '/oom_score') - oom_score_adj = rline1('/proc/' + pid + '/oom_score_adj') - - except IndexError: - print(mem_info) - print('The victim died in the search process: IndexError') - update_stat_dict_and_print( - 'The victim died in the search process: IndexError') - return None - except ValueError: - print(mem_info) - print('The victim died in the search process: ValueError') - update_stat_dict_and_print( - 'The victim died in the search process: ValueError') - return None - - len_vm = len(str(vm_size)) - - - - realpath = os.path.realpath('/proc/' + pid + '/exe') - state = pid_to_state(pid) - - - - if detailed_rss: - - victim_info = '\033[4mFound a process with highest badness:\033[0m' \ - '\n Name: \033[33m{}\033[0m' \ - '\n State: \033[33m{}\033[0m' \ - '\n PID: \033[33m{}\033[0m' \ - '\n UID: \033[33m{}\033[0m' \ - '\n badness: \033[33m{}\033[0m, ' \ - 'oom_score: \033[33m{}\033[0m, ' \ - 'oom_score_adj: \033[33m{}\033[0m' \ - '\n VmSize: \033[33m{}\033[0m MiB' \ - '\n VmRSS: \033[33m{}\033[0m MiB (' \ - 'Anon: \033[33m{}\033[0m MiB, ' \ - 'File: \033[33m{}\033[0m MiB, ' \ - 'Shmem: \033[33m{}\033[0m MiB)' \ - '\n VmSwap: \033[33m{}\033[0m MiB' \ - '\n realpath: \033[33m{}\033[0m' \ - '\n cmdline: \033[33m{}\033[0m'.format( - name, - state, - pid, - uid, - victim_badness, - oom_score, - oom_score_adj, - vm_size, - str(vm_rss).rjust(len_vm), - anon_rss, - file_rss, - shmem_rss, - str(vm_swap).rjust(len_vm), - realpath, - cmdline - ) - else: - - - # нахер такое ветвление - - victim_info = '\033[4mFound a process with highest badness:\033[0m' \ - '\n Name: \033[33m{}\033[0m' \ - '\n PID: \033[33m{}\033[0m' \ - '\n UID: \033[33m{}\033[0m' \ - '\n Badness: \033[33m{}\033[0m, ' \ - 'oom_score: \033[33m{}\033[0m, ' \ - 'oom_score_adj: \033[33m{}\033[0m' \ - '\n VmSize: \033[33m{}\033[0m MiB' \ - '\n VmRSS: \033[33m{}\033[0m MiB' \ - '\n VmSwap: \033[33m{}\033[0m MiB' \ - '\n CmdLine: \033[33m{}\033[0m'.format( - name, - pid, - uid, - victim_badness, - oom_score, - oom_score_adj, - vm_size, - str(vm_rss).rjust(len_vm), - str(vm_swap).rjust(len_vm), - cmdline) - - if execute_the_command and signal is SIGTERM and name in etc_dict: - command = etc_dict[name] - exit_status = os.system(etc_dict[name].replace( - '$PID', pid).replace('$NAME', pid_to_name(pid))) - if exit_status == 0: - exit_status = '\033[32m0\033[0m' - else: - exit_status = '\033[31m{}\033[0m'.format(exit_status) - - response_time = time() - time0 - - etc_info = '{}' \ - '\n\033[4mImplement corrective action:\033[0m\n Run the command: \033[4m{}\033[0m' \ - '\n Exit status: {}; response time: {} ms'.format( - victim_info, command.replace( - '$PID', pid).replace('$NAME', pid_to_name(pid)), exit_status, - round(response_time * 1000)) - - print(mem_info) - print(etc_info) - - key = "Run the command '\033[35m{}\033[0m'".format(command) - update_stat_dict_and_print(key) - - if gui_notifications: - send_notify_etc( - pid, - name, - command.replace('$PID', pid).replace('$NAME', pid_to_name(pid))) - - else: - - try: - - - m = check_mem_and_swap() - ma = round(int(m[0]) / 1024.0) - sf = round(int(m[2]) / 1024.0) - print('\nMemory status before sending a signal:\nMemA' - 'v: {} MiB, SwFree: {} MiB'.format(ma, sf)) - - - if stop_cont: - os.kill(int(pid), SIGCONT) - - - os.kill(int(pid), signal) - response_time = time() - time0 - send_result = '\033[32mOK\033[0m; response time: {} ms'.format( - round(response_time * 1000)) - - preventing_oom_message = '{}' \ - '\n\033[4mImplement a corrective action:\033[0m\n ' \ - 'Sending \033[4m{}\033[0m to the victim; {}'.format( - victim_info, sig_dict[signal], send_result) - - key = 'Send \033[35m{}\033[0m to \033[35m{}\033[0m'.format( - sig_dict[signal], name) - - if gui_notifications: - send_notify(signal, name, pid) - - except FileNotFoundError: - response_time = time() - time0 - send_result = 'no such process; response time: {} ms'.format( - round(response_time * 1000)) - key = 'The victim died in the search process: FileNotFoundError' - except ProcessLookupError: - response_time = time() - time0 - send_result = 'no such process; response time: {} ms'.format( - round(response_time * 1000)) - key = 'The victim died in the search process: ProcessLookupError' - - print(mem_info) - print(preventing_oom_message) - - update_stat_dict_and_print(key) - - else: - - response_time = time() - time0 - print(mem_info) - victim_badness_is_too_small = 'victim badness {} < min_b' \ - 'adness {}; nothing to do; response time: {} ms'.format( - victim_badness, - min_badness, - round(response_time * 1000)) - - print(victim_badness_is_too_small) - - # update stat_dict - key = 'victim badness < min_badness' - update_stat_dict_and_print(key) - - - if stop_cont: - cont(stopped_list) - - sleep_after_send_signal(signal) - - -def sleep_after_check_mem(): - """Specify sleep times depends on rates and avialable memory.""" - - if mem_min_sigkill_kb < mem_min_sigterm_kb: - mem_point = mem_available - mem_min_sigterm_kb - else: - mem_point = mem_available - mem_min_sigkill_kb - - if swap_min_sigkill_kb < swap_min_sigterm_kb: - swap_point = swap_free - swap_min_sigterm_kb - else: - swap_point = swap_free - swap_min_sigkill_kb - - t_mem = mem_point / rate_mem - t_swap = swap_point / rate_swap - t_zram = (mem_total * 0.9 - mem_used_zram) / rate_zram - - t_mem_swap = t_mem + t_swap - t_mem_zram = t_mem + t_zram - - if t_mem_swap <= t_mem_zram: - t = t_mem_swap - else: - t = t_mem_zram - - if t > max_sleep_time: - t = max_sleep_time - elif t < min_sleep_time: - t = min_sleep_time - else: - pass - - try: - if print_sleep_periods: - print('sleep', round(t, 2)) - # ' (t_mem={}, t_swap={}, t_zram={})'.format( - # round(t_mem, 2), - # round(t_swap, 2), - # round(t_zram, 2))) - stdout.flush() - sleep(t) - except KeyboardInterrupt: - exit() - - -def calculate_percent(arg_key): - """ - parse conf dict - Calculate mem_min_KEY_percent. - - Try use this one) - arg_key: str key for config_dict - returns int mem_min_percent or NoneType if got some error - """ - - if arg_key in config_dict: - mem_min = config_dict[arg_key] - - if mem_min.endswith('%'): - # truncate percents, so we have a number - mem_min_percent = mem_min[:-1].strip() - # then 'float test' - mem_min_percent = string_to_float_convert_test(mem_min_percent) - if mem_min_percent is None: - print('Invalid {} value, not float\nExit'.format(arg_key)) - exit() - # Final validations... - if mem_min_percent < 0 or mem_min_percent > 100: - print( - '{}, as percents value, out of range [0; 100]\nExit'.format(arg_key)) - exit() - - # mem_min_sigterm_percent is clean and valid float percentage. Can - # translate into Kb - mem_min_kb = mem_min_percent / 100 * mem_total - mem_min_mb = round(mem_min_kb / 1024) - - elif mem_min.endswith('M'): - mem_min_mb = string_to_float_convert_test(mem_min[:-1].strip()) - if mem_min_mb is None: - print('Invalid {} value, not float\nExit'.format(arg_key)) - exit() - mem_min_kb = mem_min_mb * 1024 - if mem_min_kb > mem_total: - print( - '{} value can not be greater then MemTotal ({} MiB)\nExit'.format( - arg_key, round( - mem_total / 1024))) - exit() - mem_min_percent = mem_min_kb / mem_total * 100 - - else: - print('Invalid {} units in config.\n Exit'.format(arg_key)) - mem_min_percent = None - - else: - print('{} not in config\nExit'.format(arg_key)) - mem_min_percent = None - - return mem_min_kb, mem_min_mb, mem_min_percent - - -########################################################################## - - -# find mem_total -# find positions of SwapFree and SwapTotal in /proc/meminfo - -with open('/proc/meminfo') as f: - mem_list = f.readlines() - -mem_list_names = [] -for s in mem_list: - mem_list_names.append(s.split(':')[0]) - -if mem_list_names[2] != 'MemAvailable': - print('Your Linux kernel is too old, Linux 3.14+ requied\nExit') - exit() - -swap_total_index = mem_list_names.index('SwapTotal') -swap_free_index = swap_total_index + 1 - -mem_total = int(mem_list[0].split(':')[1].strip(' kB\n')) - -# Get names from /proc/*/status to be able to get VmRSS and VmSwap values - -with open('/proc/self/status') as file: - status_list = file.readlines() - -status_names = [] -for s in status_list: - status_names.append(s.split(':')[0]) - -ppid_index = status_names.index('PPid') -vm_size_index = status_names.index('VmSize') -vm_rss_index = status_names.index('VmRSS') -vm_swap_index = status_names.index('VmSwap') -uid_index = status_names.index('Uid') - -try: - anon_index = status_names.index('RssAnon') - file_index = status_names.index('RssFile') - shmem_index = status_names.index('RssShmem') - detailed_rss = True - # print(detailed_rss, 'detailed_rss') -except ValueError: - detailed_rss = False - # print('It is not Linux 4.5+') - -########################################################################## - -''' -# Configurations - -cd = os.getcwd() - -''' - -config = '/etc/nohang/nohang.conf' - -# config = 'nohang.conf' - -print('Config:', config) - - -########################################################################## - -# parsing the config with obtaining the parameters dictionary - -# conf_parameters_dict -# conf_restart_dict - -# dictionary with config options -config_dict = dict() - -processname_re_list = [] -cmdline_re_list = [] -uid_re_list = [] - -# dictionary with names and commands for the parameter -# execute_the_command -# тут тоже список нужен, а не словарь -etc_dict = dict() - -# will store corrective actions stat -stat_dict = dict() - -try: - with open(config) as f: - - for line in f: - - a = line.startswith('#') - b = line.startswith('\n') - c = line.startswith('\t') - d = line.startswith(' ') - - etc = line.startswith('$ETC') - - if not a and not b and not c and not d and not etc: - a = line.partition('=') - config_dict[a[0].strip()] = a[2].strip() - - if etc: - a = line[4:].split('///') - etc_name = a[0].strip() - etc_command = a[1].strip() - if len(etc_name) > 15: - print('Invalid config, the length of the process ' - 'name must not exceed 15 characters\nExit') - exit() - etc_dict[etc_name] = etc_command - - # NEED VALIDATION! - if line.startswith('@PROCESSNAME_RE'): - a = line.partition('@PROCESSNAME_RE')[ - 2].strip(' \n').partition('///') - processname_re_list.append((a[0].strip(' '), a[2].strip(' '))) - - if line.startswith('@CMDLINE_RE'): - a = line.partition('@CMDLINE_RE')[2].strip( - ' \n').partition('///') - cmdline_re_list.append((a[0].strip(' '), a[2].strip(' '))) - - if line.startswith('@UID_RE'): - a = line.partition('@UID_RE')[2].strip(' \n').partition('///') - uid_re_list.append((a[0].strip(' '), a[2].strip(' '))) - -except PermissionError: - print('PermissionError', conf_err_mess) - exit() -except UnicodeDecodeError: - print('UnicodeDecodeError', conf_err_mess) - exit() -except IsADirectoryError: - print('IsADirectoryError', conf_err_mess) - exit() -except IndexError: - print('IndexError', conf_err_mess) - exit() - -# print(processname_re_list) -# print(cmdline_re_list) -# print(uid_re_list) - -########################################################################## - - -# extracting parameters from the dictionary -# check for all necessary parameters -# validation of all parameters - -print_config = conf_parse_bool('print_config') -print_mem_check_results = conf_parse_bool('print_mem_check_results') -print_sleep_periods = conf_parse_bool('print_sleep_periods') -gui_low_memory_warnings = conf_parse_bool('gui_low_memory_warnings') -gui_notifications = conf_parse_bool('gui_notifications') -decrease_oom_score_adj = conf_parse_bool('decrease_oom_score_adj') -execute_the_command = conf_parse_bool('execute_the_command') - - -ignore_psi = conf_parse_bool('ignore_psi') - -regex_matching = conf_parse_bool('regex_matching') - -re_match_cmdline = conf_parse_bool('re_match_cmdline') - -re_match_uid = conf_parse_bool('re_match_uid') - -if regex_matching or re_match_cmdline or re_match_uid: - from re import search - import sre_constants - -mem_min_sigterm_kb, mem_min_sigterm_mb, mem_min_sigterm_percent = calculate_percent( - 'mem_min_sigterm') -mem_min_sigkill_kb, mem_min_sigkill_mb, mem_min_sigkill_percent = calculate_percent( - 'mem_min_sigkill') - -zram_max_sigterm_kb, zram_max_sigterm_mb, zram_max_sigterm_percent = calculate_percent( - 'zram_max_sigterm') -zram_max_sigkill_kb, zram_max_sigkill_mb, zram_max_sigkill_percent = calculate_percent( - 'zram_max_sigkill') - -mem_min_warnings_kb, mem_min_warnings_mb, mem_min_warnings_percent = calculate_percent( - 'mem_min_warnings') -zram_max_warnings_kb, zram_max_warnings_mb, zram_max_warnings_percent = calculate_percent( - 'zram_max_warnings') - - -if 'rate_mem' in config_dict: - rate_mem = string_to_float_convert_test(config_dict['rate_mem']) - if rate_mem is None: - print('Invalid rate_mem value, not float\nExit') - exit() - if rate_mem <= 0: - print('rate_mem MUST be > 0\nExit') - exit() -else: - print('rate_mem not in config\nExit') - exit() - - -if 'rate_swap' in config_dict: - rate_swap = string_to_float_convert_test(config_dict['rate_swap']) - if rate_swap is None: - print('Invalid rate_swap value, not float\nExit') - exit() - if rate_swap <= 0: - print('rate_swap MUST be > 0\nExit') - exit() -else: - print('rate_swap not in config\nExit') - exit() - - -if 'rate_zram' in config_dict: - rate_zram = string_to_float_convert_test(config_dict['rate_zram']) - if rate_zram is None: - print('Invalid rate_zram value, not float\nExit') - exit() - if rate_zram <= 0: - print('rate_zram MUST be > 0\nExit') - exit() -else: - print('rate_zram not in config\nExit') - exit() - - -if 'swap_min_sigterm' in config_dict: - swap_min_sigterm = config_dict['swap_min_sigterm'] -else: - print('swap_min_sigterm not in config\nExit') - exit() - - -if 'swap_min_sigkill' in config_dict: - swap_min_sigkill = config_dict['swap_min_sigkill'] -else: - print('swap_min_sigkill not in config\nExit') - exit() - - -if 'min_delay_after_sigterm' in config_dict: - min_delay_after_sigterm = string_to_float_convert_test( - config_dict['min_delay_after_sigterm']) - if min_delay_after_sigterm is None: - print('Invalid min_delay_after_sigterm value, not float\nExit') - exit() - if min_delay_after_sigterm < 0: - print('min_delay_after_sigterm must be positiv\nExit') - exit() -else: - print('min_delay_after_sigterm not in config\nExit') - exit() - - -if 'min_delay_after_sigkill' in config_dict: - min_delay_after_sigkill = string_to_float_convert_test( - config_dict['min_delay_after_sigkill']) - if min_delay_after_sigkill is None: - print('Invalid min_delay_after_sigkill value, not float\nExit') - exit() - if min_delay_after_sigkill < 0: - print('min_delay_after_sigkill must be positive\nExit') - exit() -else: - print('min_delay_after_sigkill not in config\nExit') - exit() - - -if 'psi_avg10_sleep_time' in config_dict: - psi_avg10_sleep_time = string_to_float_convert_test( - config_dict['psi_avg10_sleep_time']) - if psi_avg10_sleep_time is None: - print('Invalid psi_avg10_sleep_time value, not float\nExit') - exit() - if psi_avg10_sleep_time < 0: - print('psi_avg10_sleep_time must be positive\nExit') - exit() -else: - print('psi_avg10_sleep_time not in config\nExit') - exit() - - -if 'sigkill_psi_avg10' in config_dict: - sigkill_psi_avg10 = string_to_float_convert_test( - config_dict['sigkill_psi_avg10']) - if sigkill_psi_avg10 is None: - print('Invalid sigkill_psi_avg10 value, not float\nExit') - exit() - if sigkill_psi_avg10 < 0 or sigkill_psi_avg10 > 100: - print('sigkill_psi_avg10 must be in the range [0; 100]\nExit') - exit() -else: - print('sigkill_psi_avg10 not in config\nExit') - exit() - - -if 'sigterm_psi_avg10' in config_dict: - sigterm_psi_avg10 = string_to_float_convert_test( - config_dict['sigterm_psi_avg10']) - if sigterm_psi_avg10 is None: - print('Invalid sigterm_psi_avg10 value, not float\nExit') - exit() - if sigterm_psi_avg10 < 0 or sigterm_psi_avg10 > 100: - print('sigterm_psi_avg10 must be in the range [0; 100]\nExit') - exit() -else: - print('sigterm_psi_avg10 not in config\nExit') - exit() - - -if 'min_badness' in config_dict: - min_badness = string_to_int_convert_test( - config_dict['min_badness']) - if min_badness is None: - print('Invalid min_badness value, not integer\nExit') - exit() - if min_badness < 0 or min_badness > 1000: - print('Invalud min_badness value\nExit') - exit() -else: - print('min_badness not in config\nExit') - exit() - - -if 'oom_score_adj_max' in config_dict: - oom_score_adj_max = string_to_int_convert_test( - config_dict['oom_score_adj_max']) - if oom_score_adj_max is None: - print('Invalid oom_score_adj_max value, not integer\nExit') - exit() - if oom_score_adj_max < 0 or oom_score_adj_max > 1000: - print('Invalid oom_score_adj_max value\nExit') - exit() -else: - print('oom_score_adj_max not in config\nExit') - exit() - - -if 'min_time_between_warnings' in config_dict: - min_time_between_warnings = string_to_float_convert_test( - config_dict['min_time_between_warnings']) - if min_time_between_warnings is None: - print('Invalid min_time_between_warnings value, not float\nExit') - exit() - if min_time_between_warnings < 1 or min_time_between_warnings > 300: - print('min_time_between_warnings value out of range [1; 300]\nExit') - exit() -else: - print('min_time_between_warnings not in config\nExit') - exit() - - -if 'swap_min_warnings' in config_dict: - swap_min_warnings = config_dict['swap_min_warnings'] -else: - print('swap_min_warnings not in config\nExit') - exit() - - -########################################################################## - - -# Get KiB levels if it's possible. - -# получ кб. если не кб - то процент. Если процент - находим кб ниже на -# основе полученного своптотал и процентов. - - -def get_swap_threshold_tuple(string): - # re (Num %, True) or (Num KiB, False) - """Returns KiB value if abs val was set in config, or tuple with %""" - # return tuple with abs and bool: (abs %, True) or (abs MiB, False) - - if string.endswith('%'): - valid = string_to_float_convert_test(string[:-1]) - if valid is None: - print('somewhere swap unit is not float_%') - exit() - - value = float(string[:-1].strip()) - if value < 0 or value > 100: - print('invalid value, must be from the range[0; 100] %') - exit() - - return value, True - - elif string.endswith('M'): - valid = string_to_float_convert_test(string[:-1]) - if valid is None: - print('somewhere swap unit is not float_M') - exit() - - value = float(string[:-1].strip()) * 1024 - if value < 0: - print('invalid unit in config (negative value)') - exit() - - return value, False - - else: - print('Invalid config file. There are invalid units somewhere\nExit') - exit() - - -swap_min_sigterm_tuple = get_swap_threshold_tuple(swap_min_sigterm) -swap_min_sigkill_tuple = get_swap_threshold_tuple(swap_min_sigkill) -swap_min_warnings_tuple = get_swap_threshold_tuple(swap_min_warnings) - - -swap_term_is_percent = swap_min_sigterm_tuple[1] -if swap_term_is_percent: - swap_min_sigterm_percent = swap_min_sigterm_tuple[0] -else: - swap_min_sigterm_kb = swap_min_sigterm_tuple[0] - - -swap_kill_is_percent = swap_min_sigkill_tuple[1] -if swap_kill_is_percent: - swap_min_sigkill_percent = swap_min_sigkill_tuple[0] -else: - swap_min_sigkill_kb = swap_min_sigkill_tuple[0] - - -swap_warn_is_percent = swap_min_warnings_tuple[1] -if swap_warn_is_percent: - swap_min_warnings_percent = swap_min_warnings_tuple[0] -else: - swap_min_warnings_kb = swap_min_warnings_tuple[0] - - -########################################################################## - -if print_config: - - print( - '\n1. Memory levels to respond to as an OOM threat\n[displaying these options need fix]\n') - - print('mem_min_sigterm: {} MiB, {} %'.format( - round(mem_min_sigterm_mb), round(mem_min_sigterm_percent, 1))) - print('mem_min_sigkill: {} MiB, {} %'.format( - round(mem_min_sigkill_mb), round(mem_min_sigkill_percent, 1))) - - print('swap_min_sigterm: {}'.format(swap_min_sigterm)) - print('swap_min_sigkill: {}'.format(swap_min_sigkill)) - - print('zram_max_sigterm: {} MiB, {} %'.format( - round(zram_max_sigterm_mb), round(zram_max_sigterm_percent, 1))) - print('zram_max_sigkill: {} MiB, {} %'.format( - round(zram_max_sigkill_mb), round(zram_max_sigkill_percent, 1))) - - print('\n2. The frequency of checking the level of available memory (and CPU usage)\n') - print('rate_mem: {}'.format(rate_mem)) - print('rate_swap: {}'.format(rate_swap)) - print('rate_zram: {}'.format(rate_zram)) - - print('\n3. The prevention of killing innocent victims\n') - print('min_delay_after_sigterm: {}'.format(min_delay_after_sigterm)) - print('min_delay_after_sigkill: {}'.format(min_delay_after_sigkill)) - print('min_badness: {}'.format(min_badness)) - - # False (OK) - OK не нужен когда фолс - print('decrease_oom_score_adj: {}'.format( - decrease_oom_score_adj - )) - if decrease_oom_score_adj: - print('oom_score_adj_max: {}'.format(oom_score_adj_max)) - - print('\n4. Impact on the badness of processes via matching their' - ' names, cmdlines ir UIDs with regular expressions\n') - - print('(todo)') - - print('\n5. The execution of a specific command instead of sending the\nSIGTERM signal\n') - print('execute_the_command: {}'.format(execute_the_command)) - if execute_the_command: - print('\nPROCESS NAME COMMAND TO EXECUTE') - for key in etc_dict: - print('{} {}'.format(key.ljust(15), etc_dict[key])) - - print('\n6. GUI notifications:\n- OOM prevention results and\n- low memory warnings\n') - print('gui_notifications: {}'.format(gui_notifications)) - - print('gui_low_memory_warnings: {}'.format(gui_low_memory_warnings)) - if gui_low_memory_warnings: - print('min_time_between_warnings: {}'.format(min_time_between_warnings)) - - print('mem_min_warnings: {} MiB, {} %'.format( - round(mem_min_warnings_mb), round(mem_min_warnings_percent, 1))) - - print('swap_min_warnings: {}'.format(swap_min_warnings)) - - print('zram_max_warnings: {} MiB, {} %'.format( - round(zram_max_warnings_mb), round(zram_max_warnings_percent, 1))) - - print('\n7. Output verbosity\n') - print('print_config: {}'.format(print_config)) - print('print_mem_check_results: {}'.format(print_mem_check_results)) - print('print_sleep_periods: {}\n'.format(print_sleep_periods)) - - -########################################################################## - - -# for calculating the column width when printing mem and zram -mem_len = len(str(round(mem_total / 1024.0))) - -if gui_notifications or gui_low_memory_warnings: - from subprocess import Popen, TimeoutExpired - notify_sig_dict = {SIGKILL: 'Killing', - SIGTERM: 'Terminating'} - -rate_mem = rate_mem * 1048576 -rate_swap = rate_swap * 1048576 -rate_zram = rate_zram * 1048576 - -warn_time_now = 0 -warn_time_delta = 1000 -warn_timer = 0 - -print('Monitoring started!') - -stdout.flush() - -sigterm_psi = sigterm_psi_avg10 -sigkill_psi = sigkill_psi_avg10 -psi_min_sleep_time_after_action = psi_avg10_sleep_time - - -########################################################################## - -if psi_support and not ignore_psi: - kill_psi_t0 = time() + psi_avg10_sleep_time - term_psi_t0 = time() + psi_avg10_sleep_time - -avg_value = '' - - -while True: - - if psi_support and not ignore_psi: - - avg10 = psi_mem_some_avg10() - - if print_mem_check_results: - avg_value = 'PSI mem some avg10: {} | '.format(str(avg10).rjust(6)) - - if avg10 >= sigkill_psi and time() - kill_psi_t0 >= psi_min_sleep_time_after_action: - time0 = time() - mem_info = 'avg ({}) > sigkill_psi ({})'.format(avg10, sigkill_psi) - find_victim_and_send_signal(SIGKILL) - kill_psi_t0 = time() - elif avg10 >= sigterm_psi and time() - term_psi_t0 >= psi_min_sleep_time_after_action: - time0 = time() - mem_info = 'avg ({}) > sigterm_psi ({})'.format(avg10, sigterm_psi) - find_victim_and_send_signal(SIGTERM) - term_psi_t0 = time() - else: - pass - - mem_available, swap_total, swap_free = check_mem_and_swap() - - # если метры - получаем киб выше и сразу. см. - - # if swap_min_sigkill is set in percent - if swap_kill_is_percent: - swap_min_sigkill_kb = swap_total * swap_min_sigkill_percent / 100.0 - - if swap_term_is_percent: - swap_min_sigterm_kb = swap_total * swap_min_sigterm_percent / 100.0 - - if swap_warn_is_percent: - swap_min_warnings_kb = swap_total * swap_min_warnings_percent / 100.0 - - # в общем случае для работы нужны килобайты. Если в процентах задано - - # находим КБ тут, после получения своптотал. - - mem_used_zram = check_zram() - - if print_mem_check_results: - - # Calculate 'swap-column' width - swap_len = len(str(round(swap_total / 1024.0))) - - # Output avialable mem sizes - if swap_total == 0 and mem_used_zram == 0: - print('{}MemAvail: {} M, {} %'.format( - avg_value, - human(mem_available, mem_len), - just_percent_mem(mem_available / mem_total))) - - elif swap_total > 0 and mem_used_zram == 0: - print('{}MemAvail: {} M, {} % | SwapFree: {} M, {} %'.format( - avg_value, - human(mem_available, mem_len), - just_percent_mem(mem_available / mem_total), - human(swap_free, swap_len), - just_percent_swap(swap_free / (swap_total + 0.1)))) - - else: - print('{}MemAvail: {} M, {} % | SwapFree: {} M, {} % | Mem' - 'UsedZram: {} M, {} %'.format( - avg_value, - human(mem_available, mem_len), - just_percent_mem(mem_available / mem_total), - human(swap_free, swap_len), - just_percent_swap(swap_free / (swap_total + 0.1)), - human(mem_used_zram, mem_len), - just_percent_mem(mem_used_zram / mem_total))) - - # если swap_min_sigkill задан в абсолютной величине и Swap_total = 0 - if swap_total > swap_min_sigkill_kb: # If swap_min_sigkill is absolute - swap_sigkill_pc = percent(swap_min_sigkill_kb / (swap_total + 0.1)) - else: - swap_sigkill_pc = '-' - - if swap_total > swap_min_sigterm_kb: - swap_sigterm_pc = percent(swap_min_sigterm_kb / (swap_total + 0.1)) - else: - - # печатать так: SwapTotal = 0, ignore swapspace - swap_sigterm_pc = '-' - - # это для печати меминфо. Все переработать нахрен. - - # далее пошло ветвление - - # MEM SWAP KILL - if mem_available <= mem_min_sigkill_kb and \ - swap_free <= swap_min_sigkill_kb: - time0 = time() - - mem_info = '{}\n\033[4mMemory status that r' \ - 'equires corrective actions:' \ - '\033[0m\n MemAvailable [{} MiB, {} %] <= mem_min_sig' \ - 'kill [{} MiB, {} %]\n SwapFree [{} MiB, {} %] <= swa' \ - 'p_min_sigkill [{} MiB, {} %]'.format( - HR, - kib_to_mib(mem_available), - percent(mem_available / mem_total), - kib_to_mib(mem_min_sigkill_kb), - percent(mem_min_sigkill_kb / mem_total), - kib_to_mib(swap_free), - percent(swap_free / (swap_total + 0.1)), - kib_to_mib(swap_min_sigkill_kb), - swap_sigkill_pc) - - find_victim_and_send_signal(SIGKILL) - kill_psi_t0 = time() - term_psi_t0 = time() - - # ZRAM KILL - elif mem_used_zram >= zram_max_sigkill_kb: - time0 = time() - - mem_info = '{}\n\033[4mMemory statu' \ - 's that requires corrective actions:' \ - '\033[0m\n MemUsedZram [{} MiB, {} %] >= zram_max_sig' \ - 'kill [{} MiB, {} %]'.format( - HR, - kib_to_mib(mem_used_zram), - percent(mem_used_zram / mem_total), - kib_to_mib(zram_max_sigkill_kb), - percent(zram_max_sigkill_kb / mem_total)) - - find_victim_and_send_signal(SIGKILL) - kill_psi_t0 = time() - term_psi_t0 = time() - - # MEM SWAP TERM - elif mem_available <= mem_min_sigterm_kb and \ - swap_free <= swap_min_sigterm_kb: - - time0 = time() - - mem_info = '{}\n\033[4mMemory status tha' \ - 't requires corrective actions:' \ - '\033[0m\n MemAvailable [{} MiB, {} %] <= mem_min_sig' \ - 'term [{} MiB, {} %]\n SwapFree [{} MiB, {} %] <= swa' \ - 'p_min_sigterm [{} MiB, {} %]'.format( - HR, - kib_to_mib(mem_available), - percent(mem_available / mem_total), - kib_to_mib(mem_min_sigterm_kb), - # percent(mem_min_sigterm_kb / mem_total), - # ОКРУГЛЯТЬ НА МЕСТЕ ВЫШЕ (или не выше, хз) - round(mem_min_sigterm_percent, 1), - kib_to_mib(swap_free), - percent(swap_free / (swap_total + 0.1)), - kib_to_mib(swap_min_sigterm_kb), - swap_sigterm_pc) - - find_victim_and_send_signal(SIGTERM) - kill_psi_t0 = time() - term_psi_t0 = time() - - # ZRAM TERM - elif mem_used_zram >= zram_max_sigterm_kb: - time0 = time() - - mem_info = '{}\n\033[4mMemory status that r' \ - 'equires corrective actions:' \ - '\033[0m\n MemUsedZram [{} MiB, {} %] >= ' \ - 'zram_max_sigterm [{} M, {} %]'.format( - HR, - kib_to_mib(mem_used_zram), - percent(mem_used_zram / mem_total), - kib_to_mib(zram_max_sigterm_kb), - percent(zram_max_sigterm_kb / mem_total)) - - find_victim_and_send_signal(SIGTERM) - kill_psi_t0 = time() - term_psi_t0 = time() - - # LOW MEMORY WARNINGS - elif gui_low_memory_warnings: - - if mem_available <= mem_min_warnings_kb and \ - swap_free <= swap_min_warnings_kb + 0.1 or \ - mem_used_zram >= zram_max_warnings_kb: - warn_time_delta = time() - warn_time_now - warn_time_now = time() - warn_timer += warn_time_delta - if warn_timer > min_time_between_warnings: - t0 = time() - send_notify_warn() - print(time() - t0, 'send notify warning time') - warn_timer = 0 - - sleep_after_check_mem() - - # SLEEP BETWEEN MEM CHECKS - else: - sleep_after_check_mem() diff --git a/n4 b/n4 deleted file mode 100755 index d9bf67d..0000000 --- a/n4 +++ /dev/null @@ -1,1766 +0,0 @@ -#!/usr/bin/env python3 -"""A daemon that prevents OOM in Linux systems.""" -import os -from time import sleep, time -from operator import itemgetter -from sys import stdout -from signal import SIGKILL, SIGTERM, SIGSTOP, SIGCONT - -start_time = time() - -sig_dict = {SIGKILL: 'SIGKILL', - SIGTERM: 'SIGTERM'} - -self_pid = str(os.getpid()) - -self_uid = os.geteuid() - -if self_uid == 0: - root = True -else: - root = False - -wait_time = 3 - -# todo: make config option -max_sleep_time = 2 - -# todo: make config option -min_sleep_time = 0.1 - -notify_helper_path = '/usr/sbin/nohang_notify_helper' - -psi_path = '/proc/pressure/memory' - -psi_support = os.path.exists(psi_path) - -HR = '~' * 79 - -# todo: make config option -print_total_stat = True - - -stop_cont = False -stop_cont_warn = False - - - - -# print(os.path.realpath('/proc/29758/exe')) - - - - - -########################################################################## - -# define functions - - - - -def pid_to_state(pid): - return rline1('/proc/' + pid + '/stat').rpartition(')')[2][1] - -def stop(): - #print() - #print('Stop running processes...') - t1 = time() - t2 = time() - stopped_list = [] - for pid in os.listdir('/proc')[::-1]: - # only directories whose names consist only of numbers, except /proc/1/ - if pid[0].isdecimal() is False or pid is '1' or pid == self_pid: - continue - try: - oom_score_r = int(rline1('/proc/' + pid + '/oom_score')) - if oom_score_r > 9: - uid_r = pid_to_uid(pid) - #print('PID: {}, State: {}, oom_score {}'.format(pid, pid_to_state(pid), oom_score_r)) - if uid_r != '0': - stopped_list.append(pid) - print('Send SIGSTOP to {}, {}, {}...'.format( - pid, pid_to_name(pid), pid_to_cmdline(pid)[:40])) - os.kill(int(pid), SIGSTOP) - t2 = time() - except FileNotFoundError: - continue - except ProcessLookupError: - continue - print('Stop time:', t2 - t1) - stdout.flush() - - return stopped_list - -def cont(stopped_list): - print() - print('Continue stopped processes...') - t1 = time() - if len(stopped_list) > 0: - for pid in stopped_list: - print('Send SIGCONT to', [pid], pid_to_name(pid)) - try: - os.kill(int(pid), SIGCONT) - except FileNotFoundError: - continue - except ProcessLookupError: - continue - t2 = time() - print('All cont time: ', t2 - t1) - - - - - - - - - -def update_stat_dict_and_print(key): - - if key not in stat_dict: - - stat_dict.update({key: 1}) - - else: - - new_value = stat_dict[key] + 1 - stat_dict.update({key: new_value}) - - if print_total_stat: - - stats_msg = '{}\n\033[4mThe following corrective actions have been implemented in the last {}:\033[0m'.format( - HR, format_time(time() - start_time)) - - for i in stat_dict: - stats_msg += '\n- {}: {}'.format(i, stat_dict[i]) - - print(stats_msg) - - -def psi_mem_some_avg_total(): - if psi_support: - return float(rline1(psi_path).rpartition('=')[2]) - - -def psi_mem_some_avg10(): - return float(rline1(psi_path).split(' ')[1].split('=')[1]) - - -def check_mem(): - """find mem_available""" - return int(rline1('/proc/meminfo').split(':')[1].strip(' kB\n')) - - -def check_mem_and_swap(): - """find mem_available, swap_total, swap_free""" - with open('/proc/meminfo') as f: - for n, line in enumerate(f): - if n is 2: - mem_available = int(line.split(':')[1].strip(' kB\n')) - continue - if n is swap_total_index: - swap_total = int(line.split(':')[1].strip(' kB\n')) - continue - if n is swap_free_index: - swap_free = int(line.split(':')[1].strip(' kB\n')) - break - return mem_available, swap_total, swap_free - - -def check_zram(): - """find MemUsedZram""" - disksize_sum = 0 - mem_used_total_sum = 0 - - for dev in os.listdir('/sys/block'): - if dev.startswith('zram'): - stat = zram_stat(dev) - disksize_sum += int(stat[0]) - mem_used_total_sum += int(stat[1]) - - ZRAM_DISKSIZE_FACTOR = 0.0042 - # Означает, что при задани zram disksize = 1 GiB доступная память - # уменьшится на 0.0042 GiB. - # Найден экспериментально, требует уточнения с разными ядрами и архитектурами. - # На небольших дисксайзах (до гигабайта) может быть больше, до 0.0045. - # Создатель модуля zram утверждает, что ZRAM_DISKSIZE_FACTOR доожен быть 0.001: - # ("zram uses about 0.1% of the size of the disk" - # - https://www.kernel.org/doc/Documentation/blockdev/zram.txt), - # но это утверждение противоречит опытным данным. - # ZRAM_DISKSIZE_FACTOR = deltaMemAvailavle / disksize - # found experimentally - - return (mem_used_total_sum + disksize_sum * ZRAM_DISKSIZE_FACTOR) / 1024.0 - - -def format_time(t): - t = int(t) - if t < 60: - return '{} sec'.format(t) - elif t >= 60 and t < 3600: - m = t // 60 - s = t % 60 - return '{} min {} sec'.format(m, s) - else: - h = t // 3600 - s0 = t - h * 3600 - m = s0 // 60 - s = s0 % 60 - return '{} h {} min {} sec'.format(h, m, s) - - -def string_to_float_convert_test(string): - """Try to interprete string values as floats.""" - try: - return float(string) - except ValueError: - return None - - -def string_to_int_convert_test(string): - """Try to interpret string values as integers.""" - try: - return int(string) - except ValueError: - return None - - -def conf_parse_string(param): - """ - Get string parameters from the config dict. - - param: config_dict key - returns config_dict[param].strip() - """ - if param in config_dict: - return config_dict[param].strip() - else: - print('All the necessary parameters must be in the config') - print('There is no "{}" parameter in the config'.format(param)) - exit() - - -def conf_parse_bool(param): - """ - Get bool parameters from the config_dict. - - param: config_dicst key - returns bool - """ - if param in config_dict: - param_str = config_dict[param] - if param_str == 'True': - return True - elif param_str == 'False': - return False - else: - print('Invalid value of the "{}" parameter.'.format(param_str)) - print('Valid values are True and False.') - print('Exit') - exit() - else: - print('All the necessary parameters must be in the config') - print('There is no "{}" parameter in the config'.format(param_str)) - exit() - - -def rline1(path): - """read 1st line from path.""" - with open(path) as f: - for line in f: - return line[:-1] - - -def kib_to_mib(num): - """Convert KiB values to MiB values.""" - return round(num / 1024.0) - - -def percent(num): - """Interprete mum as percentage.""" - return round(num * 100, 1) - - -def just_percent_mem(num): - """convert num to percent and justify""" - return str(round(num * 100, 1)).rjust(4, ' ') - - -def just_percent_swap(num): - return str(round(num * 100, 1)).rjust(5, ' ') - - -def human(num, lenth): - """Convert KiB values to MiB values with right alignment""" - return str(round(num / 1024)).rjust(lenth, ' ') - - -def zram_stat(zram_id): - """ - Get zram state. - - zram_id: str zram block-device id - returns bytes diskcize, str mem_used_total - """ - try: - disksize = rline1('/sys/block/' + zram_id + '/disksize') - except FileNotFoundError: - return '0', '0' - if disksize == ['0\n']: - return '0', '0' - try: - mm_stat = rline1('/sys/block/' + zram_id + '/mm_stat').split(' ') - mm_stat_list = [] - for i in mm_stat: - if i != '': - mm_stat_list.append(i) - mem_used_total = mm_stat_list[2] - except FileNotFoundError: - mem_used_total = rline1('/sys/block/' + zram_id + '/mem_used_total') - return disksize, mem_used_total # BYTES, str - - -def pid_to_name(pid): - """ - Get process name by pid. - - pid: str pid of required process - returns string process_name - """ - try: - with open('/proc/' + pid + '/status') as f: - f.seek(6) - for line in f: - return line[:-1] - except FileNotFoundError: - return '' - except ProcessLookupError: - return '' - except UnicodeDecodeError: - with open('/proc/' + pid + '/status', 'rb') as f: - f.seek(6) - return f.read(15).decode( - 'utf-8', 'ignore').partition('\n')[0] - - -def pid_to_cmdline(pid): - """ - Get process cmdline by pid. - - pid: str pid of required process - returns string cmdline - """ - with open('/proc/' + pid + '/cmdline') as f: - return f.read().replace('\x00', ' ').rstrip() - - - -def pid_to_environ(pid): - """ - Get process cmdline by pid. - - pid: str pid of required process - returns string cmdline - """ - with open('/proc/' + pid + '/environ') as f: - return f.read().replace('\x00', '\n').rstrip() - - - - - - - - - - - - - - -def pid_to_uid(pid): - '''return euid''' - try: - with open('/proc/' + pid + '/status') as f: - for n, line in enumerate(f): - if n is uid_index: - return line.split('\t')[2] - except UnicodeDecodeError: - with open('/proc/' + pid + '/status', 'rb') as f: - f_list = f.read().decode('utf-8', 'ignore').split('\n') - return f_list[uid_index].split('\t')[2] - - -def notify_send_wait(title, body): - '''GUI notifications with UID != 0''' - with Popen(['notify-send', '--icon=dialog-warning', title, body]) as proc: - try: - proc.wait(timeout=wait_time) - except TimeoutExpired: - proc.kill() - print('TimeoutExpired: notify-send {} {}'.format(title, body)) - - -def notify_helper(title, body): - '''GUI notification with UID = 0''' - with Popen([notify_helper_path, title, body]) as proc: - try: - proc.wait(timeout=wait_time) - except TimeoutExpired: - proc.kill() - print( - 'TimeoutExpired: nohang_notify_helper: {} {}'.format( - title, body)) - - -def send_notify_warn(): - """ - Look for process with maximum 'badness' and warn user with notification. - (implement Low memory warnings) - """ - - if stop_cont_warn: - stopped_list = stop() - - # find process with max badness - fat_tuple = fattest() - pid = fat_tuple[0] - name = pid_to_name(pid) - - if mem_used_zram > 0: - low_mem_percent = '{}% {}% {}%'.format( - round(mem_available / mem_total * 100), - round(swap_free / (swap_total + 0.1) * 100), - round(mem_used_zram / mem_total * 100)) - elif swap_free > 0: - low_mem_percent = '{}% {}%'.format( - round(mem_available / mem_total * 100), - round(swap_free / (swap_total + 0.1) * 100)) - else: - low_mem_percent = '{}%'.format( - round(mem_available / mem_total * 100)) - - # title = 'Low memory: {}'.format(low_mem_percent) - title = 'Low memory' - - body = 'Hog: {} [{}]'.format( - name.replace( - # symbol '&' can break notifications in some themes, - # therefore it is replaced by '*' - '&', '*'), - pid - ) - - if root: # If nohang was started by root - # send notification to all active users with special script - notify_helper(title, body) - else: # Or by regular user - # send notification to user that runs this nohang - notify_send_wait(title, body) - - if stop_cont_warn: - cont(stopped_list) - - -def send_notify(signal, name, pid): - """ - Notificate about OOM Preventing. - - signal: key for notify_sig_dict - name: str process name - pid: str process pid - """ - title = 'Hang prevention' - body = '{} {} [{}]'.format( - notify_sig_dict[signal], - name.replace( - # symbol '&' can break notifications in some themes, - # therefore it is replaced by '*' - '&', '*'), - pid - ) - if root: - # send notification to all active users with notify-send - notify_helper(title, body) - else: - # send notification to user that runs this nohang - notify_send_wait(title, body) - - -def send_notify_etc(pid, name, command): - """ - Notificate about OOM Preventing. - - command: str command that will be executed - name: str process name - pid: str process pid - """ - title = 'Hang prevention' - body = 'Victim is process {} [{}]\nExecute the command:\n{}'.format( - name.replace('&', '*'), - pid, - command.replace('&', '*') - ) - if root: - # send notification to all active users with notify-send - notify_helper(title, body) - else: - # send notification to user that runs this nohang - notify_send_wait(title, body) - - -def sleep_after_send_signal(signal): - """ - Sleeping after signal was sent. - - signal: sent signal - """ - if signal is SIGKILL: - if print_sleep_periods: - print(' sleep', min_delay_after_sigkill) - sleep(min_delay_after_sigkill) - else: - if print_sleep_periods: - print(' sleep', min_delay_after_sigterm) - sleep(min_delay_after_sigterm) - - -def fattest(): - """ - Find the process with highest badness and its badness adjustment - Return pid and badness - - -> find_mem_hog() or find_victim() - """ - - pid_badness_list = [] - - for pid in os.listdir('/proc'): - # only directories whose names consist only of numbers, except /proc/1/ - #if pid[0].isdecimal() is False: - # continue - - - if pid[0].isdecimal() is False or pid is '1' or pid == self_pid: - continue - - - - - - x = os.path.exists('/proc/' + pid + '/exe') - if x is False: - continue - - #print(x) - # в таком случае в список попадет self и selfpid - # их можно отдельно вычленить - - - # find and modify badness (if it needs) - try: - - - #realpath = os.path.realpath('/proc/' + pid + '/exe') - #print(pid, pid_to_name(pid), realpath) - - - badness = int(rline1('/proc/' + pid + '/oom_score')) - #state = pid_to_state(pid) - name = pid_to_name(pid) - - - - - except FileNotFoundError: - continue - except ProcessLookupError: - continue - pid_badness_list.append((pid, badness)) - - # Make list of (pid, badness) tuples, sorted by 'badness' values - pid_tuple_list = sorted( - pid_badness_list, - key=itemgetter(1), - reverse=True - )[0] - - pid = pid_tuple_list[0] - - # Get maximum 'badness' value - victim_badness = pid_tuple_list[1] - - - - #print(pid_badness_list) - - - - - - - return pid, victim_badness - - - -t0 = time() -x = fattest() -t1 = time() - - -print(t1 - t0) - - - -print(x) - -exit() - - - - -def find_victim_and_send_signal(signal): - """ - Find victim with highest badness and send SIGTERM/SIGKILL - - -> implement_corrective_action() - """ - - - if stop_cont: - stopped_list = stop() - - - - pid, victim_badness = fattest() - name = pid_to_name(pid) - - if victim_badness >= min_badness: - - # Try to send signal to found victim - - # Get VmRSS and VmSwap and cmdline of victim process - # and try to send a signal - try: - - with open('/proc/' + pid + '/status') as f: - for n, line in enumerate(f): - - if n is uid_index: - uid = line.split('\t')[1] - continue - - if n is vm_size_index: - vm_size = kib_to_mib(int(line.split('\t')[1][:-4])) - continue - - if n is vm_rss_index: - vm_rss = kib_to_mib(int(line.split('\t')[1][:-4])) - continue - - if detailed_rss: - - if n is anon_index: - anon_rss = kib_to_mib( - int(line.split('\t')[1][:-4])) - continue - - if n is file_index: - file_rss = kib_to_mib( - int(line.split('\t')[1][:-4])) - continue - - if n is shmem_index: - shmem_rss = kib_to_mib( - int(line.split('\t')[1][:-4])) - continue - - if n is vm_swap_index: - vm_swap = kib_to_mib(int(line.split('\t')[1][:-4])) - break - - with open('/proc/' + pid + '/cmdline') as file: - cmdline = file.readlines()[0].replace('\x00', ' ') - - oom_score = rline1('/proc/' + pid + '/oom_score') - oom_score_adj = rline1('/proc/' + pid + '/oom_score_adj') - - except FileNotFoundError: - print(mem_info) - print('The victim died in the search process: FileNotFoundError') - update_stat_dict_and_print( - 'The victim died in the search process: FileNotFoundError') - return None - except ProcessLookupError: - print(mem_info) - print('The victim died in the search process: ProcessLookupError') - update_stat_dict_and_print( - 'The victim died in the search process: ProcessLookupError') - return None - except UnicodeDecodeError: - - # тут надо снова все исключ обработать - - with open('/proc/' + pid + '/status', 'rb') as f: - f_list = f.read().decode('utf-8', 'ignore').split('\n') - - for i in range(len(f_list)): - if i is ppid_index: - ppid = f_list[i].split('\t')[2] - - - for i in range(len(f_list)): - if i is uid_index: - uid = f_list[i].split('\t')[2] - - - - - if i is vm_size_index: - vm_size = kib_to_mib( - int(f_list[i].split('\t')[1][:-3])) - - if i is vm_rss_index: - vm_rss = kib_to_mib(int(f_list[i].split('\t')[1][:-3])) - - if detailed_rss: - - if i is anon_index: - anon_rss = kib_to_mib( - int(f_list[i].split('\t')[1][:-3])) - - if i is file_index: - file_rss = kib_to_mib( - int(f_list[i].split('\t')[1][:-3])) - - if i is shmem_index: - shmem_rss = kib_to_mib( - int(f_list[i].split('\t')[1][:-3])) - - if i is vm_swap_index: - vm_swap = kib_to_mib( - int(f_list[i].split('\t')[1][:-3])) - - with open('/proc/' + pid + '/cmdline') as file: - cmdline = file.readlines()[0].replace('\x00', ' ') - - oom_score = rline1('/proc/' + pid + '/oom_score') - oom_score_adj = rline1('/proc/' + pid + '/oom_score_adj') - - except IndexError: - print(mem_info) - print('The victim died in the search process: IndexError') - update_stat_dict_and_print( - 'The victim died in the search process: IndexError') - return None - except ValueError: - print(mem_info) - print('The victim died in the search process: ValueError') - update_stat_dict_and_print( - 'The victim died in the search process: ValueError') - return None - - len_vm = len(str(vm_size)) - - if detailed_rss: - - environ = pid_to_environ(pid) - - victim_info = '\033[4mFound a process with highest badness:\033[0m' \ - '\n Name: \033[33m{}\033[0m' \ - '\n PID: \033[33m{}\033[0m' \ - '\n UID: \033[33m{}\033[0m' \ - '\n badness: \033[33m{}\033[0m, ' \ - 'oom_score: \033[33m{}\033[0m, ' \ - 'oom_score_adj: \033[33m{}\033[0m' \ - '\n VmSize: \033[33m{}\033[0m MiB' \ - '\n VmRSS: \033[33m{}\033[0m MiB (' \ - 'Anon: \033[33m{}\033[0m MiB, ' \ - 'File: \033[33m{}\033[0m MiB, ' \ - 'Shmem: \033[33m{}\033[0m MiB)' \ - '\n VmSwap: \033[33m{}\033[0m MiB' \ - '\n environ:\n\033[33m{}\033[0m' \ - '\n cmdline: \033[33m{}\033[0m'.format( - name, - pid, - uid, - victim_badness, - oom_score, - oom_score_adj, - vm_size, - str(vm_rss).rjust(len_vm), - anon_rss, - file_rss, - shmem_rss, - str(vm_swap).rjust(len_vm), - environ, - cmdline - ) - else: - - - # нахер такое ветвление - - victim_info = '\033[4mFound a process with highest badness:\033[0m' \ - '\n Name: \033[33m{}\033[0m' \ - '\n PID: \033[33m{}\033[0m' \ - '\n UID: \033[33m{}\033[0m' \ - '\n Badness: \033[33m{}\033[0m, ' \ - 'oom_score: \033[33m{}\033[0m, ' \ - 'oom_score_adj: \033[33m{}\033[0m' \ - '\n VmSize: \033[33m{}\033[0m MiB' \ - '\n VmRSS: \033[33m{}\033[0m MiB' \ - '\n VmSwap: \033[33m{}\033[0m MiB' \ - '\n CmdLine: \033[33m{}\033[0m'.format( - name, - pid, - uid, - victim_badness, - oom_score, - oom_score_adj, - vm_size, - str(vm_rss).rjust(len_vm), - str(vm_swap).rjust(len_vm), - cmdline) - - if execute_the_command and signal is SIGTERM and name in etc_dict: - command = etc_dict[name] - exit_status = os.system(etc_dict[name].replace( - '$PID', pid).replace('$NAME', pid_to_name(pid))) - if exit_status == 0: - exit_status = '\033[32m0\033[0m' - else: - exit_status = '\033[31m{}\033[0m'.format(exit_status) - - response_time = time() - time0 - - etc_info = '{}' \ - '\n\033[4mImplement corrective action:\033[0m\n Run the command: \033[4m{}\033[0m' \ - '\n Exit status: {}; response time: {} ms'.format( - victim_info, command.replace( - '$PID', pid).replace('$NAME', pid_to_name(pid)), exit_status, - round(response_time * 1000)) - - print(mem_info) - print(etc_info) - - key = "Run the command '\033[35m{}\033[0m'".format(command) - update_stat_dict_and_print(key) - - if gui_notifications: - send_notify_etc( - pid, - name, - command.replace('$PID', pid).replace('$NAME', pid_to_name(pid))) - - else: - - try: - - - - - m = check_mem_and_swap() - ma = round(int(m[0]) / 1024.0) - sf = round(int(m[2]) / 1024.0) - print('\nMemory status before sending a signal:\nMemA' - 'v: {} MiB, SwFree: {} MiB'.format(ma, sf)) - - - - if stop_cont: - os.kill(int(pid), SIGCONT) - - - os.kill(int(pid), signal) - response_time = time() - time0 - send_result = '\033[32mOK\033[0m; response time: {} ms'.format( - round(response_time * 1000)) - - preventing_oom_message = '{}' \ - '\n\033[4mImplement a corrective action:\033[0m\n ' \ - 'Sending \033[4m{}\033[0m to the victim; {}'.format( - victim_info, sig_dict[signal], send_result) - - key = 'Send \033[35m{}\033[0m to \033[35m{}\033[0m'.format( - sig_dict[signal], name) - - if gui_notifications: - send_notify(signal, name, pid) - - except FileNotFoundError: - response_time = time() - time0 - send_result = 'no such process; response time: {} ms'.format( - round(response_time * 1000)) - key = 'The victim died in the search process: FileNotFoundError' - except ProcessLookupError: - response_time = time() - time0 - send_result = 'no such process; response time: {} ms'.format( - round(response_time * 1000)) - key = 'The victim died in the search process: ProcessLookupError' - - print(mem_info) - print(pid_to_state(pid)) - print(preventing_oom_message) - - update_stat_dict_and_print(key) - - else: - - response_time = time() - time0 - print(mem_info) - victim_badness_is_too_small = 'victim badness {} < min_b' \ - 'adness {}; nothing to do; response time: {} ms'.format( - victim_badness, - min_badness, - round(response_time * 1000)) - - print(victim_badness_is_too_small) - - # update stat_dict - key = 'victim badness < min_badness' - update_stat_dict_and_print(key) - - - if stop_cont: - cont(stopped_list) - - sleep_after_send_signal(signal) - - -def sleep_after_check_mem(): - """Specify sleep times depends on rates and avialable memory.""" - - if mem_min_sigkill_kb < mem_min_sigterm_kb: - mem_point = mem_available - mem_min_sigterm_kb - else: - mem_point = mem_available - mem_min_sigkill_kb - - if swap_min_sigkill_kb < swap_min_sigterm_kb: - swap_point = swap_free - swap_min_sigterm_kb - else: - swap_point = swap_free - swap_min_sigkill_kb - - t_mem = mem_point / rate_mem - t_swap = swap_point / rate_swap - t_zram = (mem_total * 0.9 - mem_used_zram) / rate_zram - - t_mem_swap = t_mem + t_swap - t_mem_zram = t_mem + t_zram - - if t_mem_swap <= t_mem_zram: - t = t_mem_swap - else: - t = t_mem_zram - - if t > max_sleep_time: - t = max_sleep_time - elif t < min_sleep_time: - t = min_sleep_time - else: - pass - - try: - if print_sleep_periods: - print('sleep', round(t, 2)) - # ' (t_mem={}, t_swap={}, t_zram={})'.format( - # round(t_mem, 2), - # round(t_swap, 2), - # round(t_zram, 2))) - stdout.flush() - sleep(t) - except KeyboardInterrupt: - exit() - - -def calculate_percent(arg_key): - """ - parse conf dict - Calculate mem_min_KEY_percent. - - Try use this one) - arg_key: str key for config_dict - returns int mem_min_percent or NoneType if got some error - """ - - if arg_key in config_dict: - mem_min = config_dict[arg_key] - - if mem_min.endswith('%'): - # truncate percents, so we have a number - mem_min_percent = mem_min[:-1].strip() - # then 'float test' - mem_min_percent = string_to_float_convert_test(mem_min_percent) - if mem_min_percent is None: - print('Invalid {} value, not float\nExit'.format(arg_key)) - exit() - # Final validations... - if mem_min_percent < 0 or mem_min_percent > 100: - print( - '{}, as percents value, out of range [0; 100]\nExit'.format(arg_key)) - exit() - - # mem_min_sigterm_percent is clean and valid float percentage. Can - # translate into Kb - mem_min_kb = mem_min_percent / 100 * mem_total - mem_min_mb = round(mem_min_kb / 1024) - - elif mem_min.endswith('M'): - mem_min_mb = string_to_float_convert_test(mem_min[:-1].strip()) - if mem_min_mb is None: - print('Invalid {} value, not float\nExit'.format(arg_key)) - exit() - mem_min_kb = mem_min_mb * 1024 - if mem_min_kb > mem_total: - print( - '{} value can not be greater then MemTotal ({} MiB)\nExit'.format( - arg_key, round( - mem_total / 1024))) - exit() - mem_min_percent = mem_min_kb / mem_total * 100 - - else: - print('Invalid {} units in config.\n Exit'.format(arg_key)) - mem_min_percent = None - - else: - print('{} not in config\nExit'.format(arg_key)) - mem_min_percent = None - - return mem_min_kb, mem_min_mb, mem_min_percent - - -########################################################################## - - -# find mem_total -# find positions of SwapFree and SwapTotal in /proc/meminfo - -with open('/proc/meminfo') as f: - mem_list = f.readlines() - -mem_list_names = [] -for s in mem_list: - mem_list_names.append(s.split(':')[0]) - -if mem_list_names[2] != 'MemAvailable': - print('Your Linux kernel is too old, Linux 3.14+ requied\nExit') - exit() - -swap_total_index = mem_list_names.index('SwapTotal') -swap_free_index = swap_total_index + 1 - -mem_total = int(mem_list[0].split(':')[1].strip(' kB\n')) - -# Get names from /proc/*/status to be able to get VmRSS and VmSwap values - -with open('/proc/self/status') as file: - status_list = file.readlines() - -status_names = [] -for s in status_list: - status_names.append(s.split(':')[0]) - -ppid_index = status_names.index('PPid') -vm_size_index = status_names.index('VmSize') -vm_rss_index = status_names.index('VmRSS') -vm_swap_index = status_names.index('VmSwap') -uid_index = status_names.index('Uid') - -try: - anon_index = status_names.index('RssAnon') - file_index = status_names.index('RssFile') - shmem_index = status_names.index('RssShmem') - detailed_rss = True - # print(detailed_rss, 'detailed_rss') -except ValueError: - detailed_rss = False - # print('It is not Linux 4.5+') - -########################################################################## - -''' -# Configurations - -cd = os.getcwd() - -''' - -config = '/etc/nohang/nohang.conf' - -# config = 'nohang.conf' - -print('Config:', config) - - -########################################################################## - -# parsing the config with obtaining the parameters dictionary - -# conf_parameters_dict -# conf_restart_dict - -# dictionary with config options -config_dict = dict() - -processname_re_list = [] -cmdline_re_list = [] -uid_re_list = [] - -# dictionary with names and commands for the parameter -# execute_the_command -# тут тоже список нужен, а не словарь -etc_dict = dict() - -# will store corrective actions stat -stat_dict = dict() - -try: - with open(config) as f: - - for line in f: - - a = line.startswith('#') - b = line.startswith('\n') - c = line.startswith('\t') - d = line.startswith(' ') - - etc = line.startswith('$ETC') - - if not a and not b and not c and not d and not etc: - a = line.partition('=') - config_dict[a[0].strip()] = a[2].strip() - - if etc: - a = line[4:].split('///') - etc_name = a[0].strip() - etc_command = a[1].strip() - if len(etc_name) > 15: - print('Invalid config, the length of the process ' - 'name must not exceed 15 characters\nExit') - exit() - etc_dict[etc_name] = etc_command - - # NEED VALIDATION! - if line.startswith('@PROCESSNAME_RE'): - a = line.partition('@PROCESSNAME_RE')[ - 2].strip(' \n').partition('///') - processname_re_list.append((a[0].strip(' '), a[2].strip(' '))) - - if line.startswith('@CMDLINE_RE'): - a = line.partition('@CMDLINE_RE')[2].strip( - ' \n').partition('///') - cmdline_re_list.append((a[0].strip(' '), a[2].strip(' '))) - - if line.startswith('@UID_RE'): - a = line.partition('@UID_RE')[2].strip(' \n').partition('///') - uid_re_list.append((a[0].strip(' '), a[2].strip(' '))) - -except PermissionError: - print('PermissionError', conf_err_mess) - exit() -except UnicodeDecodeError: - print('UnicodeDecodeError', conf_err_mess) - exit() -except IsADirectoryError: - print('IsADirectoryError', conf_err_mess) - exit() -except IndexError: - print('IndexError', conf_err_mess) - exit() - -# print(processname_re_list) -# print(cmdline_re_list) -# print(uid_re_list) - -########################################################################## - - -# extracting parameters from the dictionary -# check for all necessary parameters -# validation of all parameters - -print_config = conf_parse_bool('print_config') -print_mem_check_results = conf_parse_bool('print_mem_check_results') -print_sleep_periods = conf_parse_bool('print_sleep_periods') -gui_low_memory_warnings = conf_parse_bool('gui_low_memory_warnings') -gui_notifications = conf_parse_bool('gui_notifications') -decrease_oom_score_adj = conf_parse_bool('decrease_oom_score_adj') -execute_the_command = conf_parse_bool('execute_the_command') - - -ignore_psi = conf_parse_bool('ignore_psi') - -regex_matching = conf_parse_bool('regex_matching') - -re_match_cmdline = conf_parse_bool('re_match_cmdline') - -re_match_uid = conf_parse_bool('re_match_uid') - -if regex_matching or re_match_cmdline or re_match_uid: - from re import search - import sre_constants - -mem_min_sigterm_kb, mem_min_sigterm_mb, mem_min_sigterm_percent = calculate_percent( - 'mem_min_sigterm') -mem_min_sigkill_kb, mem_min_sigkill_mb, mem_min_sigkill_percent = calculate_percent( - 'mem_min_sigkill') - -zram_max_sigterm_kb, zram_max_sigterm_mb, zram_max_sigterm_percent = calculate_percent( - 'zram_max_sigterm') -zram_max_sigkill_kb, zram_max_sigkill_mb, zram_max_sigkill_percent = calculate_percent( - 'zram_max_sigkill') - -mem_min_warnings_kb, mem_min_warnings_mb, mem_min_warnings_percent = calculate_percent( - 'mem_min_warnings') -zram_max_warnings_kb, zram_max_warnings_mb, zram_max_warnings_percent = calculate_percent( - 'zram_max_warnings') - - -if 'rate_mem' in config_dict: - rate_mem = string_to_float_convert_test(config_dict['rate_mem']) - if rate_mem is None: - print('Invalid rate_mem value, not float\nExit') - exit() - if rate_mem <= 0: - print('rate_mem MUST be > 0\nExit') - exit() -else: - print('rate_mem not in config\nExit') - exit() - - -if 'rate_swap' in config_dict: - rate_swap = string_to_float_convert_test(config_dict['rate_swap']) - if rate_swap is None: - print('Invalid rate_swap value, not float\nExit') - exit() - if rate_swap <= 0: - print('rate_swap MUST be > 0\nExit') - exit() -else: - print('rate_swap not in config\nExit') - exit() - - -if 'rate_zram' in config_dict: - rate_zram = string_to_float_convert_test(config_dict['rate_zram']) - if rate_zram is None: - print('Invalid rate_zram value, not float\nExit') - exit() - if rate_zram <= 0: - print('rate_zram MUST be > 0\nExit') - exit() -else: - print('rate_zram not in config\nExit') - exit() - - -if 'swap_min_sigterm' in config_dict: - swap_min_sigterm = config_dict['swap_min_sigterm'] -else: - print('swap_min_sigterm not in config\nExit') - exit() - - -if 'swap_min_sigkill' in config_dict: - swap_min_sigkill = config_dict['swap_min_sigkill'] -else: - print('swap_min_sigkill not in config\nExit') - exit() - - -if 'min_delay_after_sigterm' in config_dict: - min_delay_after_sigterm = string_to_float_convert_test( - config_dict['min_delay_after_sigterm']) - if min_delay_after_sigterm is None: - print('Invalid min_delay_after_sigterm value, not float\nExit') - exit() - if min_delay_after_sigterm < 0: - print('min_delay_after_sigterm must be positiv\nExit') - exit() -else: - print('min_delay_after_sigterm not in config\nExit') - exit() - - -if 'min_delay_after_sigkill' in config_dict: - min_delay_after_sigkill = string_to_float_convert_test( - config_dict['min_delay_after_sigkill']) - if min_delay_after_sigkill is None: - print('Invalid min_delay_after_sigkill value, not float\nExit') - exit() - if min_delay_after_sigkill < 0: - print('min_delay_after_sigkill must be positive\nExit') - exit() -else: - print('min_delay_after_sigkill not in config\nExit') - exit() - - -if 'psi_avg10_sleep_time' in config_dict: - psi_avg10_sleep_time = string_to_float_convert_test( - config_dict['psi_avg10_sleep_time']) - if psi_avg10_sleep_time is None: - print('Invalid psi_avg10_sleep_time value, not float\nExit') - exit() - if psi_avg10_sleep_time < 0: - print('psi_avg10_sleep_time must be positive\nExit') - exit() -else: - print('psi_avg10_sleep_time not in config\nExit') - exit() - - -if 'sigkill_psi_avg10' in config_dict: - sigkill_psi_avg10 = string_to_float_convert_test( - config_dict['sigkill_psi_avg10']) - if sigkill_psi_avg10 is None: - print('Invalid sigkill_psi_avg10 value, not float\nExit') - exit() - if sigkill_psi_avg10 < 0 or sigkill_psi_avg10 > 100: - print('sigkill_psi_avg10 must be in the range [0; 100]\nExit') - exit() -else: - print('sigkill_psi_avg10 not in config\nExit') - exit() - - -if 'sigterm_psi_avg10' in config_dict: - sigterm_psi_avg10 = string_to_float_convert_test( - config_dict['sigterm_psi_avg10']) - if sigterm_psi_avg10 is None: - print('Invalid sigterm_psi_avg10 value, not float\nExit') - exit() - if sigterm_psi_avg10 < 0 or sigterm_psi_avg10 > 100: - print('sigterm_psi_avg10 must be in the range [0; 100]\nExit') - exit() -else: - print('sigterm_psi_avg10 not in config\nExit') - exit() - - -if 'min_badness' in config_dict: - min_badness = string_to_int_convert_test( - config_dict['min_badness']) - if min_badness is None: - print('Invalid min_badness value, not integer\nExit') - exit() - if min_badness < 0 or min_badness > 1000: - print('Invalud min_badness value\nExit') - exit() -else: - print('min_badness not in config\nExit') - exit() - - -if 'oom_score_adj_max' in config_dict: - oom_score_adj_max = string_to_int_convert_test( - config_dict['oom_score_adj_max']) - if oom_score_adj_max is None: - print('Invalid oom_score_adj_max value, not integer\nExit') - exit() - if oom_score_adj_max < 0 or oom_score_adj_max > 1000: - print('Invalid oom_score_adj_max value\nExit') - exit() -else: - print('oom_score_adj_max not in config\nExit') - exit() - - -if 'min_time_between_warnings' in config_dict: - min_time_between_warnings = string_to_float_convert_test( - config_dict['min_time_between_warnings']) - if min_time_between_warnings is None: - print('Invalid min_time_between_warnings value, not float\nExit') - exit() - if min_time_between_warnings < 1 or min_time_between_warnings > 300: - print('min_time_between_warnings value out of range [1; 300]\nExit') - exit() -else: - print('min_time_between_warnings not in config\nExit') - exit() - - -if 'swap_min_warnings' in config_dict: - swap_min_warnings = config_dict['swap_min_warnings'] -else: - print('swap_min_warnings not in config\nExit') - exit() - - -########################################################################## - - -# Get KiB levels if it's possible. - -# получ кб. если не кб - то процент. Если процент - находим кб ниже на -# основе полученного своптотал и процентов. - - -def get_swap_threshold_tuple(string): - # re (Num %, True) or (Num KiB, False) - """Returns KiB value if abs val was set in config, or tuple with %""" - # return tuple with abs and bool: (abs %, True) or (abs MiB, False) - - if string.endswith('%'): - valid = string_to_float_convert_test(string[:-1]) - if valid is None: - print('somewhere swap unit is not float_%') - exit() - - value = float(string[:-1].strip()) - if value < 0 or value > 100: - print('invalid value, must be from the range[0; 100] %') - exit() - - return value, True - - elif string.endswith('M'): - valid = string_to_float_convert_test(string[:-1]) - if valid is None: - print('somewhere swap unit is not float_M') - exit() - - value = float(string[:-1].strip()) * 1024 - if value < 0: - print('invalid unit in config (negative value)') - exit() - - return value, False - - else: - print('Invalid config file. There are invalid units somewhere\nExit') - exit() - - -swap_min_sigterm_tuple = get_swap_threshold_tuple(swap_min_sigterm) -swap_min_sigkill_tuple = get_swap_threshold_tuple(swap_min_sigkill) -swap_min_warnings_tuple = get_swap_threshold_tuple(swap_min_warnings) - - -swap_term_is_percent = swap_min_sigterm_tuple[1] -if swap_term_is_percent: - swap_min_sigterm_percent = swap_min_sigterm_tuple[0] -else: - swap_min_sigterm_kb = swap_min_sigterm_tuple[0] - - -swap_kill_is_percent = swap_min_sigkill_tuple[1] -if swap_kill_is_percent: - swap_min_sigkill_percent = swap_min_sigkill_tuple[0] -else: - swap_min_sigkill_kb = swap_min_sigkill_tuple[0] - - -swap_warn_is_percent = swap_min_warnings_tuple[1] -if swap_warn_is_percent: - swap_min_warnings_percent = swap_min_warnings_tuple[0] -else: - swap_min_warnings_kb = swap_min_warnings_tuple[0] - - -########################################################################## - -if print_config: - - print( - '\n1. Memory levels to respond to as an OOM threat\n[displaying these options need fix]\n') - - print('mem_min_sigterm: {} MiB, {} %'.format( - round(mem_min_sigterm_mb), round(mem_min_sigterm_percent, 1))) - print('mem_min_sigkill: {} MiB, {} %'.format( - round(mem_min_sigkill_mb), round(mem_min_sigkill_percent, 1))) - - print('swap_min_sigterm: {}'.format(swap_min_sigterm)) - print('swap_min_sigkill: {}'.format(swap_min_sigkill)) - - print('zram_max_sigterm: {} MiB, {} %'.format( - round(zram_max_sigterm_mb), round(zram_max_sigterm_percent, 1))) - print('zram_max_sigkill: {} MiB, {} %'.format( - round(zram_max_sigkill_mb), round(zram_max_sigkill_percent, 1))) - - print('\n2. The frequency of checking the level of available memory (and CPU usage)\n') - print('rate_mem: {}'.format(rate_mem)) - print('rate_swap: {}'.format(rate_swap)) - print('rate_zram: {}'.format(rate_zram)) - - print('\n3. The prevention of killing innocent victims\n') - print('min_delay_after_sigterm: {}'.format(min_delay_after_sigterm)) - print('min_delay_after_sigkill: {}'.format(min_delay_after_sigkill)) - print('min_badness: {}'.format(min_badness)) - - # False (OK) - OK не нужен когда фолс - print('decrease_oom_score_adj: {}'.format( - decrease_oom_score_adj - )) - if decrease_oom_score_adj: - print('oom_score_adj_max: {}'.format(oom_score_adj_max)) - - print('\n4. Impact on the badness of processes via matching their' - ' names, cmdlines ir UIDs with regular expressions\n') - - print('(todo)') - - print('\n5. The execution of a specific command instead of sending the\nSIGTERM signal\n') - print('execute_the_command: {}'.format(execute_the_command)) - if execute_the_command: - print('\nPROCESS NAME COMMAND TO EXECUTE') - for key in etc_dict: - print('{} {}'.format(key.ljust(15), etc_dict[key])) - - print('\n6. GUI notifications:\n- OOM prevention results and\n- low memory warnings\n') - print('gui_notifications: {}'.format(gui_notifications)) - - print('gui_low_memory_warnings: {}'.format(gui_low_memory_warnings)) - if gui_low_memory_warnings: - print('min_time_between_warnings: {}'.format(min_time_between_warnings)) - - print('mem_min_warnings: {} MiB, {} %'.format( - round(mem_min_warnings_mb), round(mem_min_warnings_percent, 1))) - - print('swap_min_warnings: {}'.format(swap_min_warnings)) - - print('zram_max_warnings: {} MiB, {} %'.format( - round(zram_max_warnings_mb), round(zram_max_warnings_percent, 1))) - - print('\n7. Output verbosity\n') - print('print_config: {}'.format(print_config)) - print('print_mem_check_results: {}'.format(print_mem_check_results)) - print('print_sleep_periods: {}\n'.format(print_sleep_periods)) - - -########################################################################## - - -# for calculating the column width when printing mem and zram -mem_len = len(str(round(mem_total / 1024.0))) - -if gui_notifications or gui_low_memory_warnings: - from subprocess import Popen, TimeoutExpired - notify_sig_dict = {SIGKILL: 'Killing', - SIGTERM: 'Terminating'} - -rate_mem = rate_mem * 1048576 -rate_swap = rate_swap * 1048576 -rate_zram = rate_zram * 1048576 - -warn_time_now = 0 -warn_time_delta = 1000 -warn_timer = 0 - -print('Monitoring started!') - -stdout.flush() - -sigterm_psi = sigterm_psi_avg10 -sigkill_psi = sigkill_psi_avg10 -psi_min_sleep_time_after_action = psi_avg10_sleep_time - - -########################################################################## - -if psi_support and not ignore_psi: - kill_psi_t0 = time() + psi_avg10_sleep_time - term_psi_t0 = time() + psi_avg10_sleep_time - -avg_value = '' - - -while True: - - if psi_support and not ignore_psi: - - avg10 = psi_mem_some_avg10() - - if print_mem_check_results: - avg_value = 'PSI mem some avg10: {} | '.format(str(avg10).rjust(6)) - - if avg10 >= sigkill_psi and time() - kill_psi_t0 >= psi_min_sleep_time_after_action: - time0 = time() - mem_info = 'avg ({}) > sigkill_psi ({})'.format(avg10, sigkill_psi) - find_victim_and_send_signal(SIGKILL) - kill_psi_t0 = time() - elif avg10 >= sigterm_psi and time() - term_psi_t0 >= psi_min_sleep_time_after_action: - time0 = time() - mem_info = 'avg ({}) > sigterm_psi ({})'.format(avg10, sigterm_psi) - find_victim_and_send_signal(SIGTERM) - term_psi_t0 = time() - else: - pass - - mem_available, swap_total, swap_free = check_mem_and_swap() - - # если метры - получаем киб выше и сразу. см. - - # if swap_min_sigkill is set in percent - if swap_kill_is_percent: - swap_min_sigkill_kb = swap_total * swap_min_sigkill_percent / 100.0 - - if swap_term_is_percent: - swap_min_sigterm_kb = swap_total * swap_min_sigterm_percent / 100.0 - - if swap_warn_is_percent: - swap_min_warnings_kb = swap_total * swap_min_warnings_percent / 100.0 - - # в общем случае для работы нужны килобайты. Если в процентах задано - - # находим КБ тут, после получения своптотал. - - mem_used_zram = check_zram() - - if print_mem_check_results: - - # Calculate 'swap-column' width - swap_len = len(str(round(swap_total / 1024.0))) - - # Output avialable mem sizes - if swap_total == 0 and mem_used_zram == 0: - print('{}MemAvail: {} M, {} %'.format( - avg_value, - human(mem_available, mem_len), - just_percent_mem(mem_available / mem_total))) - - elif swap_total > 0 and mem_used_zram == 0: - print('{}MemAvail: {} M, {} % | SwapFree: {} M, {} %'.format( - avg_value, - human(mem_available, mem_len), - just_percent_mem(mem_available / mem_total), - human(swap_free, swap_len), - just_percent_swap(swap_free / (swap_total + 0.1)))) - - else: - print('{}MemAvail: {} M, {} % | SwapFree: {} M, {} % | Mem' - 'UsedZram: {} M, {} %'.format( - avg_value, - human(mem_available, mem_len), - just_percent_mem(mem_available / mem_total), - human(swap_free, swap_len), - just_percent_swap(swap_free / (swap_total + 0.1)), - human(mem_used_zram, mem_len), - just_percent_mem(mem_used_zram / mem_total))) - - # если swap_min_sigkill задан в абсолютной величине и Swap_total = 0 - if swap_total > swap_min_sigkill_kb: # If swap_min_sigkill is absolute - swap_sigkill_pc = percent(swap_min_sigkill_kb / (swap_total + 0.1)) - else: - swap_sigkill_pc = '-' - - if swap_total > swap_min_sigterm_kb: - swap_sigterm_pc = percent(swap_min_sigterm_kb / (swap_total + 0.1)) - else: - - # печатать так: SwapTotal = 0, ignore swapspace - swap_sigterm_pc = '-' - - # это для печати меминфо. Все переработать нахрен. - - # далее пошло ветвление - - # MEM SWAP KILL - if mem_available <= mem_min_sigkill_kb and \ - swap_free <= swap_min_sigkill_kb: - time0 = time() - - mem_info = '{}\n\033[4mMemory status that r' \ - 'equires corrective actions:' \ - '\033[0m\n MemAvailable [{} MiB, {} %] <= mem_min_sig' \ - 'kill [{} MiB, {} %]\n SwapFree [{} MiB, {} %] <= swa' \ - 'p_min_sigkill [{} MiB, {} %]'.format( - HR, - kib_to_mib(mem_available), - percent(mem_available / mem_total), - kib_to_mib(mem_min_sigkill_kb), - percent(mem_min_sigkill_kb / mem_total), - kib_to_mib(swap_free), - percent(swap_free / (swap_total + 0.1)), - kib_to_mib(swap_min_sigkill_kb), - swap_sigkill_pc) - - find_victim_and_send_signal(SIGKILL) - kill_psi_t0 = time() - term_psi_t0 = time() - - # ZRAM KILL - elif mem_used_zram >= zram_max_sigkill_kb: - time0 = time() - - mem_info = '{}\n\033[4mMemory statu' \ - 's that requires corrective actions:' \ - '\033[0m\n MemUsedZram [{} MiB, {} %] >= zram_max_sig' \ - 'kill [{} MiB, {} %]'.format( - HR, - kib_to_mib(mem_used_zram), - percent(mem_used_zram / mem_total), - kib_to_mib(zram_max_sigkill_kb), - percent(zram_max_sigkill_kb / mem_total)) - - find_victim_and_send_signal(SIGKILL) - kill_psi_t0 = time() - term_psi_t0 = time() - - # MEM SWAP TERM - elif mem_available <= mem_min_sigterm_kb and \ - swap_free <= swap_min_sigterm_kb: - - time0 = time() - - mem_info = '{}\n\033[4mMemory status tha' \ - 't requires corrective actions:' \ - '\033[0m\n MemAvailable [{} MiB, {} %] <= mem_min_sig' \ - 'term [{} MiB, {} %]\n SwapFree [{} MiB, {} %] <= swa' \ - 'p_min_sigterm [{} MiB, {} %]'.format( - HR, - kib_to_mib(mem_available), - percent(mem_available / mem_total), - kib_to_mib(mem_min_sigterm_kb), - # percent(mem_min_sigterm_kb / mem_total), - # ОКРУГЛЯТЬ НА МЕСТЕ ВЫШЕ (или не выше, хз) - round(mem_min_sigterm_percent, 1), - kib_to_mib(swap_free), - percent(swap_free / (swap_total + 0.1)), - kib_to_mib(swap_min_sigterm_kb), - swap_sigterm_pc) - - find_victim_and_send_signal(SIGTERM) - kill_psi_t0 = time() - term_psi_t0 = time() - - # ZRAM TERM - elif mem_used_zram >= zram_max_sigterm_kb: - time0 = time() - - mem_info = '{}\n\033[4mMemory status that r' \ - 'equires corrective actions:' \ - '\033[0m\n MemUsedZram [{} MiB, {} %] >= ' \ - 'zram_max_sigterm [{} M, {} %]'.format( - HR, - kib_to_mib(mem_used_zram), - percent(mem_used_zram / mem_total), - kib_to_mib(zram_max_sigterm_kb), - percent(zram_max_sigterm_kb / mem_total)) - - find_victim_and_send_signal(SIGTERM) - kill_psi_t0 = time() - term_psi_t0 = time() - - # LOW MEMORY WARNINGS - elif gui_low_memory_warnings: - - if mem_available <= mem_min_warnings_kb and \ - swap_free <= swap_min_warnings_kb + 0.1 or \ - mem_used_zram >= zram_max_warnings_kb: - warn_time_delta = time() - warn_time_now - warn_time_now = time() - warn_timer += warn_time_delta - if warn_timer > min_time_between_warnings: - t0 = time() - send_notify_warn() - print(time() - t0, 'send notify warning time') - warn_timer = 0 - - sleep_after_check_mem() - - # SLEEP BETWEEN MEM CHECKS - else: - sleep_after_check_mem() diff --git a/n6 b/n6 deleted file mode 100755 index 68479e0..0000000 --- a/n6 +++ /dev/null @@ -1,1779 +0,0 @@ -#!/usr/bin/env python3 -"""A daemon that prevents OOM in Linux systems.""" -import os -from time import sleep, time -from operator import itemgetter -from sys import stdout -from signal import SIGKILL, SIGTERM, SIGSTOP, SIGCONT - -start_time = time() - -sig_dict = {SIGKILL: 'SIGKILL', - SIGTERM: 'SIGTERM'} - -self_pid = str(os.getpid()) - -self_uid = os.geteuid() - -if self_uid == 0: - root = True -else: - root = False - -wait_time = 3 - -# todo: make config option -max_sleep_time = 2 - -# todo: make config option -min_sleep_time = 0.1 - -notify_helper_path = '/usr/sbin/nohang_notify_helper' - -psi_path = '/proc/pressure/memory' - -psi_support = os.path.exists(psi_path) - -HR = '~' * 79 - -# todo: make config option -print_total_stat = True - - -stop_cont = False -stop_cont_warn = False - - - - -# print(os.path.realpath('/proc/29758/exe')) - - - - - -########################################################################## - -# define functions - - - - -def pid_to_state(pid): - return rline1('/proc/' + pid + '/stat').rpartition(')')[2][1] - -def stop(): - #print() - #print('Stop running processes...') - t1 = time() - t2 = time() - stopped_list = [] - for pid in os.listdir('/proc')[::-1]: - # only directories whose names consist only of numbers, except /proc/1/ - if pid[0].isdecimal() is False or pid is '1' or pid == self_pid: - continue - try: - oom_score_r = int(rline1('/proc/' + pid + '/oom_score')) - if oom_score_r > 9: - uid_r = pid_to_uid(pid) - #print('PID: {}, State: {}, oom_score {}'.format(pid, pid_to_state(pid), oom_score_r)) - if uid_r != '0': - stopped_list.append(pid) - print('Send SIGSTOP to {}, {}, {}...'.format( - pid, pid_to_name(pid), pid_to_cmdline(pid)[:40])) - os.kill(int(pid), SIGSTOP) - t2 = time() - except FileNotFoundError: - continue - except ProcessLookupError: - continue - print('Stop time:', t2 - t1) - stdout.flush() - - return stopped_list - -def cont(stopped_list): - print() - print('Continue stopped processes...') - t1 = time() - if len(stopped_list) > 0: - for pid in stopped_list: - print('Send SIGCONT to', [pid], pid_to_name(pid)) - try: - os.kill(int(pid), SIGCONT) - except FileNotFoundError: - continue - except ProcessLookupError: - continue - t2 = time() - print('All cont time: ', t2 - t1) - - - - - - - - - -def update_stat_dict_and_print(key): - - if key not in stat_dict: - - stat_dict.update({key: 1}) - - else: - - new_value = stat_dict[key] + 1 - stat_dict.update({key: new_value}) - - if print_total_stat: - - stats_msg = '{}\n\033[4mThe following corrective actions have been implemented in the last {}:\033[0m'.format( - HR, format_time(time() - start_time)) - - for i in stat_dict: - stats_msg += '\n- {}: {}'.format(i, stat_dict[i]) - - print(stats_msg) - - -def psi_mem_some_avg_total(): - if psi_support: - return float(rline1(psi_path).rpartition('=')[2]) - - -def psi_mem_some_avg10(): - return float(rline1(psi_path).split(' ')[1].split('=')[1]) - - -def check_mem(): - """find mem_available""" - return int(rline1('/proc/meminfo').split(':')[1].strip(' kB\n')) - - -def check_mem_and_swap(): - """find mem_available, swap_total, swap_free""" - with open('/proc/meminfo') as f: - for n, line in enumerate(f): - if n is 2: - mem_available = int(line.split(':')[1].strip(' kB\n')) - continue - if n is swap_total_index: - swap_total = int(line.split(':')[1].strip(' kB\n')) - continue - if n is swap_free_index: - swap_free = int(line.split(':')[1].strip(' kB\n')) - break - return mem_available, swap_total, swap_free - - -def check_zram(): - """find MemUsedZram""" - disksize_sum = 0 - mem_used_total_sum = 0 - - for dev in os.listdir('/sys/block'): - if dev.startswith('zram'): - stat = zram_stat(dev) - disksize_sum += int(stat[0]) - mem_used_total_sum += int(stat[1]) - - ZRAM_DISKSIZE_FACTOR = 0.0042 - # Означает, что при задани zram disksize = 1 GiB доступная память - # уменьшится на 0.0042 GiB. - # Найден экспериментально, требует уточнения с разными ядрами и архитектурами. - # На небольших дисксайзах (до гигабайта) может быть больше, до 0.0045. - # Создатель модуля zram утверждает, что ZRAM_DISKSIZE_FACTOR доожен быть 0.001: - # ("zram uses about 0.1% of the size of the disk" - # - https://www.kernel.org/doc/Documentation/blockdev/zram.txt), - # но это утверждение противоречит опытным данным. - # ZRAM_DISKSIZE_FACTOR = deltaMemAvailavle / disksize - # found experimentally - - return (mem_used_total_sum + disksize_sum * ZRAM_DISKSIZE_FACTOR) / 1024.0 - - -def format_time(t): - t = int(t) - if t < 60: - return '{} sec'.format(t) - elif t >= 60 and t < 3600: - m = t // 60 - s = t % 60 - return '{} min {} sec'.format(m, s) - else: - h = t // 3600 - s0 = t - h * 3600 - m = s0 // 60 - s = s0 % 60 - return '{} h {} min {} sec'.format(h, m, s) - - -def string_to_float_convert_test(string): - """Try to interprete string values as floats.""" - try: - return float(string) - except ValueError: - return None - - -def string_to_int_convert_test(string): - """Try to interpret string values as integers.""" - try: - return int(string) - except ValueError: - return None - - -def conf_parse_string(param): - """ - Get string parameters from the config dict. - - param: config_dict key - returns config_dict[param].strip() - """ - if param in config_dict: - return config_dict[param].strip() - else: - print('All the necessary parameters must be in the config') - print('There is no "{}" parameter in the config'.format(param)) - exit() - - -def conf_parse_bool(param): - """ - Get bool parameters from the config_dict. - - param: config_dicst key - returns bool - """ - if param in config_dict: - param_str = config_dict[param] - if param_str == 'True': - return True - elif param_str == 'False': - return False - else: - print('Invalid value of the "{}" parameter.'.format(param_str)) - print('Valid values are True and False.') - print('Exit') - exit() - else: - print('All the necessary parameters must be in the config') - print('There is no "{}" parameter in the config'.format(param_str)) - exit() - - -def rline1(path): - """read 1st line from path.""" - with open(path) as f: - for line in f: - return line[:-1] - - -def kib_to_mib(num): - """Convert KiB values to MiB values.""" - return round(num / 1024.0) - - -def percent(num): - """Interprete mum as percentage.""" - return round(num * 100, 1) - - -def just_percent_mem(num): - """convert num to percent and justify""" - return str(round(num * 100, 1)).rjust(4, ' ') - - -def just_percent_swap(num): - return str(round(num * 100, 1)).rjust(5, ' ') - - -def human(num, lenth): - """Convert KiB values to MiB values with right alignment""" - return str(round(num / 1024)).rjust(lenth, ' ') - - -def zram_stat(zram_id): - """ - Get zram state. - - zram_id: str zram block-device id - returns bytes diskcize, str mem_used_total - """ - try: - disksize = rline1('/sys/block/' + zram_id + '/disksize') - except FileNotFoundError: - return '0', '0' - if disksize == ['0\n']: - return '0', '0' - try: - mm_stat = rline1('/sys/block/' + zram_id + '/mm_stat').split(' ') - mm_stat_list = [] - for i in mm_stat: - if i != '': - mm_stat_list.append(i) - mem_used_total = mm_stat_list[2] - except FileNotFoundError: - mem_used_total = rline1('/sys/block/' + zram_id + '/mem_used_total') - return disksize, mem_used_total # BYTES, str - - -def pid_to_name(pid): - """ - Get process name by pid. - - pid: str pid of required process - returns string process_name - """ - try: - with open('/proc/' + pid + '/status') as f: - f.seek(6) - for line in f: - return line[:-1] - except FileNotFoundError: - return '' - except ProcessLookupError: - return '' - except UnicodeDecodeError: - with open('/proc/' + pid + '/status', 'rb') as f: - f.seek(6) - return f.read(15).decode( - 'utf-8', 'ignore').partition('\n')[0] - - -def pid_to_cmdline(pid): - """ - Get process cmdline by pid. - - pid: str pid of required process - returns string cmdline - """ - with open('/proc/' + pid + '/cmdline') as f: - return f.read().replace('\x00', ' ').rstrip() - - - -def pid_to_environ(pid): - """ - Get process cmdline by pid. - - pid: str pid of required process - returns string cmdline - """ - with open('/proc/' + pid + '/environ') as f: - return f.read().replace('\x00', '\n').rstrip() - - - - - - - - - - - - - - -def pid_to_uid(pid): - '''return euid''' - try: - with open('/proc/' + pid + '/status') as f: - for n, line in enumerate(f): - if n is uid_index: - return line.split('\t')[2] - except UnicodeDecodeError: - with open('/proc/' + pid + '/status', 'rb') as f: - f_list = f.read().decode('utf-8', 'ignore').split('\n') - return f_list[uid_index].split('\t')[2] - - -def notify_send_wait(title, body): - '''GUI notifications with UID != 0''' - with Popen(['notify-send', '--icon=dialog-warning', title, body]) as proc: - try: - proc.wait(timeout=wait_time) - except TimeoutExpired: - proc.kill() - print('TimeoutExpired: notify-send {} {}'.format(title, body)) - - -def notify_helper(title, body): - '''GUI notification with UID = 0''' - with Popen([notify_helper_path, title, body]) as proc: - try: - proc.wait(timeout=wait_time) - except TimeoutExpired: - proc.kill() - print( - 'TimeoutExpired: nohang_notify_helper: {} {}'.format( - title, body)) - - -def send_notify_warn(): - """ - Look for process with maximum 'badness' and warn user with notification. - (implement Low memory warnings) - """ - - if stop_cont_warn: - stopped_list = stop() - - # find process with max badness - fat_tuple = fattest() - pid = fat_tuple[0] - name = pid_to_name(pid) - - if mem_used_zram > 0: - low_mem_percent = '{}% {}% {}%'.format( - round(mem_available / mem_total * 100), - round(swap_free / (swap_total + 0.1) * 100), - round(mem_used_zram / mem_total * 100)) - elif swap_free > 0: - low_mem_percent = '{}% {}%'.format( - round(mem_available / mem_total * 100), - round(swap_free / (swap_total + 0.1) * 100)) - else: - low_mem_percent = '{}%'.format( - round(mem_available / mem_total * 100)) - - # title = 'Low memory: {}'.format(low_mem_percent) - title = 'Low memory' - - body = 'Hog: {} [{}]'.format( - name.replace( - # symbol '&' can break notifications in some themes, - # therefore it is replaced by '*' - '&', '*'), - pid - ) - - if root: # If nohang was started by root - # send notification to all active users with special script - notify_helper(title, body) - else: # Or by regular user - # send notification to user that runs this nohang - notify_send_wait(title, body) - - if stop_cont_warn: - cont(stopped_list) - - -def send_notify(signal, name, pid): - """ - Notificate about OOM Preventing. - - signal: key for notify_sig_dict - name: str process name - pid: str process pid - """ - title = 'Hang prevention' - body = '{} {} [{}]'.format( - notify_sig_dict[signal], - name.replace( - # symbol '&' can break notifications in some themes, - # therefore it is replaced by '*' - '&', '*'), - pid - ) - if root: - # send notification to all active users with notify-send - notify_helper(title, body) - else: - # send notification to user that runs this nohang - notify_send_wait(title, body) - - -def send_notify_etc(pid, name, command): - """ - Notificate about OOM Preventing. - - command: str command that will be executed - name: str process name - pid: str process pid - """ - title = 'Hang prevention' - body = 'Victim is process {} [{}]\nExecute the command:\n{}'.format( - name.replace('&', '*'), - pid, - command.replace('&', '*') - ) - if root: - # send notification to all active users with notify-send - notify_helper(title, body) - else: - # send notification to user that runs this nohang - notify_send_wait(title, body) - - -def sleep_after_send_signal(signal): - """ - Sleeping after signal was sent. - - signal: sent signal - """ - if signal is SIGKILL: - if print_sleep_periods: - print(' sleep', min_delay_after_sigkill) - sleep(min_delay_after_sigkill) - else: - if print_sleep_periods: - print(' sleep', min_delay_after_sigterm) - sleep(min_delay_after_sigterm) - - - - - - - - - - - -def get_pid_list(): - """ - Find pid list expect kthreads and zombies - """ - pid_list = [] - for pid in os.listdir('/proc'): - if os.path.exists('/proc/' + pid + '/exe') is True: - pid_list.append(pid) - return pid_list - - - -pid_list = get_pid_list() - -def get_non_decimal_pids(): - non_decimal_list = [] - for pid in pid_list: - if pid[0].isdecimal() is False: - non_decimal_list.append(pid) - return non_decimal_list - - -#print(get_non_decimal_pids()) - - - -def fattest(): - """ - Find the process with highest badness and its badness adjustment - Return pid and badness - - -> find_mem_hog() or find_victim() - """ - - pid_list = get_pid_list() - - pid_list.remove(self_pid) - pid_list.remove('1') - - nd = get_non_decimal_pids() - for i in nd: - pid_list.remove(i) - - pid_badness_list = [] - - - for pid in pid_list: - - # find and modify badness (if it needs) - try: - - - #realpath = os.path.realpath('/proc/' + pid + '/exe') - #print(pid, pid_to_name(pid), realpath) - - - badness = int(rline1('/proc/' + pid + '/oom_score')) - #state = pid_to_state(pid) - name = pid_to_name(pid) - #print(pid, name, badness) - - - - except FileNotFoundError: - continue - except ProcessLookupError: - continue - pid_badness_list.append((pid, badness)) - - # Make list of (pid, badness) tuples, sorted by 'badness' values - #print(pid_badness_list) - pid_tuple_list = sorted( - pid_badness_list, - key=itemgetter(1), - reverse=True - )[0] - - pid = pid_tuple_list[0] - - # Get maximum 'badness' value - victim_badness = pid_tuple_list[1] - - - - #print(pid_badness_list) - - - - - - - return pid, victim_badness - - - - - - -def find_victim_and_send_signal(signal): - """ - Find victim with highest badness and send SIGTERM/SIGKILL - - -> implement_corrective_action() - """ - - - if stop_cont: - stopped_list = stop() - - - - pid, victim_badness = fattest() - name = pid_to_name(pid) - - if victim_badness >= min_badness: - - # Try to send signal to found victim - - # Get VmRSS and VmSwap and cmdline of victim process - # and try to send a signal - try: - - with open('/proc/' + pid + '/status') as f: - for n, line in enumerate(f): - - if n is uid_index: - uid = line.split('\t')[1] - continue - - if n is vm_size_index: - vm_size = kib_to_mib(int(line.split('\t')[1][:-4])) - continue - - if n is vm_rss_index: - vm_rss = kib_to_mib(int(line.split('\t')[1][:-4])) - continue - - if detailed_rss: - - if n is anon_index: - anon_rss = kib_to_mib( - int(line.split('\t')[1][:-4])) - continue - - if n is file_index: - file_rss = kib_to_mib( - int(line.split('\t')[1][:-4])) - continue - - if n is shmem_index: - shmem_rss = kib_to_mib( - int(line.split('\t')[1][:-4])) - continue - - if n is vm_swap_index: - vm_swap = kib_to_mib(int(line.split('\t')[1][:-4])) - break - - with open('/proc/' + pid + '/cmdline') as file: - cmdline = file.readlines()[0].replace('\x00', ' ') - - oom_score = rline1('/proc/' + pid + '/oom_score') - oom_score_adj = rline1('/proc/' + pid + '/oom_score_adj') - - except FileNotFoundError: - print(mem_info) - print('The victim died in the search process: FileNotFoundError') - update_stat_dict_and_print( - 'The victim died in the search process: FileNotFoundError') - return None - except ProcessLookupError: - print(mem_info) - print('The victim died in the search process: ProcessLookupError') - update_stat_dict_and_print( - 'The victim died in the search process: ProcessLookupError') - return None - except UnicodeDecodeError: - - # тут надо снова все исключ обработать - - with open('/proc/' + pid + '/status', 'rb') as f: - f_list = f.read().decode('utf-8', 'ignore').split('\n') - - for i in range(len(f_list)): - if i is ppid_index: - ppid = f_list[i].split('\t')[2] - - - for i in range(len(f_list)): - if i is uid_index: - uid = f_list[i].split('\t')[2] - - - - - if i is vm_size_index: - vm_size = kib_to_mib( - int(f_list[i].split('\t')[1][:-3])) - - if i is vm_rss_index: - vm_rss = kib_to_mib(int(f_list[i].split('\t')[1][:-3])) - - if detailed_rss: - - if i is anon_index: - anon_rss = kib_to_mib( - int(f_list[i].split('\t')[1][:-3])) - - if i is file_index: - file_rss = kib_to_mib( - int(f_list[i].split('\t')[1][:-3])) - - if i is shmem_index: - shmem_rss = kib_to_mib( - int(f_list[i].split('\t')[1][:-3])) - - if i is vm_swap_index: - vm_swap = kib_to_mib( - int(f_list[i].split('\t')[1][:-3])) - - with open('/proc/' + pid + '/cmdline') as file: - cmdline = file.readlines()[0].replace('\x00', ' ') - - oom_score = rline1('/proc/' + pid + '/oom_score') - oom_score_adj = rline1('/proc/' + pid + '/oom_score_adj') - - except IndexError: - print(mem_info) - print('The victim died in the search process: IndexError') - update_stat_dict_and_print( - 'The victim died in the search process: IndexError') - return None - except ValueError: - print(mem_info) - print('The victim died in the search process: ValueError') - update_stat_dict_and_print( - 'The victim died in the search process: ValueError') - return None - - len_vm = len(str(vm_size)) - - if detailed_rss: - - environ = pid_to_environ(pid) - - victim_info = '\033[4mFound a process with highest badness:\033[0m' \ - '\n Name: \033[33m{}\033[0m' \ - '\n PID: \033[33m{}\033[0m' \ - '\n UID: \033[33m{}\033[0m' \ - '\n badness: \033[33m{}\033[0m, ' \ - 'oom_score: \033[33m{}\033[0m, ' \ - 'oom_score_adj: \033[33m{}\033[0m' \ - '\n VmSize: \033[33m{}\033[0m MiB' \ - '\n VmRSS: \033[33m{}\033[0m MiB (' \ - 'Anon: \033[33m{}\033[0m MiB, ' \ - 'File: \033[33m{}\033[0m MiB, ' \ - 'Shmem: \033[33m{}\033[0m MiB)' \ - '\n VmSwap: \033[33m{}\033[0m MiB' \ - '\n environ:\n\033[33m{}\033[0m' \ - '\n cmdline: \033[33m{}\033[0m'.format( - name, - pid, - uid, - victim_badness, - oom_score, - oom_score_adj, - vm_size, - str(vm_rss).rjust(len_vm), - anon_rss, - file_rss, - shmem_rss, - str(vm_swap).rjust(len_vm), - environ, - cmdline - ) - else: - - - # нахер такое ветвление - - victim_info = '\033[4mFound a process with highest badness:\033[0m' \ - '\n Name: \033[33m{}\033[0m' \ - '\n PID: \033[33m{}\033[0m' \ - '\n UID: \033[33m{}\033[0m' \ - '\n Badness: \033[33m{}\033[0m, ' \ - 'oom_score: \033[33m{}\033[0m, ' \ - 'oom_score_adj: \033[33m{}\033[0m' \ - '\n VmSize: \033[33m{}\033[0m MiB' \ - '\n VmRSS: \033[33m{}\033[0m MiB' \ - '\n VmSwap: \033[33m{}\033[0m MiB' \ - '\n CmdLine: \033[33m{}\033[0m'.format( - name, - pid, - uid, - victim_badness, - oom_score, - oom_score_adj, - vm_size, - str(vm_rss).rjust(len_vm), - str(vm_swap).rjust(len_vm), - cmdline) - - if execute_the_command and signal is SIGTERM and name in etc_dict: - command = etc_dict[name] - exit_status = os.system(etc_dict[name].replace( - '$PID', pid).replace('$NAME', pid_to_name(pid))) - if exit_status == 0: - exit_status = '\033[32m0\033[0m' - else: - exit_status = '\033[31m{}\033[0m'.format(exit_status) - - response_time = time() - time0 - - etc_info = '{}' \ - '\n\033[4mImplement corrective action:\033[0m\n Run the command: \033[4m{}\033[0m' \ - '\n Exit status: {}; response time: {} ms'.format( - victim_info, command.replace( - '$PID', pid).replace('$NAME', pid_to_name(pid)), exit_status, - round(response_time * 1000)) - - print(mem_info) - print(etc_info) - - key = "Run the command '\033[35m{}\033[0m'".format(command) - update_stat_dict_and_print(key) - - if gui_notifications: - send_notify_etc( - pid, - name, - command.replace('$PID', pid).replace('$NAME', pid_to_name(pid))) - - else: - - try: - - - - - m = check_mem_and_swap() - ma = round(int(m[0]) / 1024.0) - sf = round(int(m[2]) / 1024.0) - print('\nMemory status before sending a signal:\nMemA' - 'v: {} MiB, SwFree: {} MiB'.format(ma, sf)) - - - - if stop_cont: - os.kill(int(pid), SIGCONT) - - - os.kill(int(pid), signal) - response_time = time() - time0 - send_result = '\033[32mOK\033[0m; response time: {} ms'.format( - round(response_time * 1000)) - - preventing_oom_message = '{}' \ - '\n\033[4mImplement a corrective action:\033[0m\n ' \ - 'Sending \033[4m{}\033[0m to the victim; {}'.format( - victim_info, sig_dict[signal], send_result) - - key = 'Send \033[35m{}\033[0m to \033[35m{}\033[0m'.format( - sig_dict[signal], name) - - if gui_notifications: - send_notify(signal, name, pid) - - except FileNotFoundError: - response_time = time() - time0 - send_result = 'no such process; response time: {} ms'.format( - round(response_time * 1000)) - key = 'The victim died in the search process: FileNotFoundError' - except ProcessLookupError: - response_time = time() - time0 - send_result = 'no such process; response time: {} ms'.format( - round(response_time * 1000)) - key = 'The victim died in the search process: ProcessLookupError' - - print(mem_info) - print(pid_to_state(pid)) - print(preventing_oom_message) - - update_stat_dict_and_print(key) - - else: - - response_time = time() - time0 - print(mem_info) - victim_badness_is_too_small = 'victim badness {} < min_b' \ - 'adness {}; nothing to do; response time: {} ms'.format( - victim_badness, - min_badness, - round(response_time * 1000)) - - print(victim_badness_is_too_small) - - # update stat_dict - key = 'victim badness < min_badness' - update_stat_dict_and_print(key) - - - if stop_cont: - cont(stopped_list) - - sleep_after_send_signal(signal) - - -def sleep_after_check_mem(): - """Specify sleep times depends on rates and avialable memory.""" - - if mem_min_sigkill_kb < mem_min_sigterm_kb: - mem_point = mem_available - mem_min_sigterm_kb - else: - mem_point = mem_available - mem_min_sigkill_kb - - if swap_min_sigkill_kb < swap_min_sigterm_kb: - swap_point = swap_free - swap_min_sigterm_kb - else: - swap_point = swap_free - swap_min_sigkill_kb - - t_mem = mem_point / rate_mem - t_swap = swap_point / rate_swap - t_zram = (mem_total * 0.9 - mem_used_zram) / rate_zram - - t_mem_swap = t_mem + t_swap - t_mem_zram = t_mem + t_zram - - if t_mem_swap <= t_mem_zram: - t = t_mem_swap - else: - t = t_mem_zram - - if t > max_sleep_time: - t = max_sleep_time - elif t < min_sleep_time: - t = min_sleep_time - else: - pass - - try: - if print_sleep_periods: - print('sleep', round(t, 2)) - # ' (t_mem={}, t_swap={}, t_zram={})'.format( - # round(t_mem, 2), - # round(t_swap, 2), - # round(t_zram, 2))) - stdout.flush() - sleep(t) - except KeyboardInterrupt: - exit() - - -def calculate_percent(arg_key): - """ - parse conf dict - Calculate mem_min_KEY_percent. - - Try use this one) - arg_key: str key for config_dict - returns int mem_min_percent or NoneType if got some error - """ - - if arg_key in config_dict: - mem_min = config_dict[arg_key] - - if mem_min.endswith('%'): - # truncate percents, so we have a number - mem_min_percent = mem_min[:-1].strip() - # then 'float test' - mem_min_percent = string_to_float_convert_test(mem_min_percent) - if mem_min_percent is None: - print('Invalid {} value, not float\nExit'.format(arg_key)) - exit() - # Final validations... - if mem_min_percent < 0 or mem_min_percent > 100: - print( - '{}, as percents value, out of range [0; 100]\nExit'.format(arg_key)) - exit() - - # mem_min_sigterm_percent is clean and valid float percentage. Can - # translate into Kb - mem_min_kb = mem_min_percent / 100 * mem_total - mem_min_mb = round(mem_min_kb / 1024) - - elif mem_min.endswith('M'): - mem_min_mb = string_to_float_convert_test(mem_min[:-1].strip()) - if mem_min_mb is None: - print('Invalid {} value, not float\nExit'.format(arg_key)) - exit() - mem_min_kb = mem_min_mb * 1024 - if mem_min_kb > mem_total: - print( - '{} value can not be greater then MemTotal ({} MiB)\nExit'.format( - arg_key, round( - mem_total / 1024))) - exit() - mem_min_percent = mem_min_kb / mem_total * 100 - - else: - print('Invalid {} units in config.\n Exit'.format(arg_key)) - mem_min_percent = None - - else: - print('{} not in config\nExit'.format(arg_key)) - mem_min_percent = None - - return mem_min_kb, mem_min_mb, mem_min_percent - - -########################################################################## - - -# find mem_total -# find positions of SwapFree and SwapTotal in /proc/meminfo - -with open('/proc/meminfo') as f: - mem_list = f.readlines() - -mem_list_names = [] -for s in mem_list: - mem_list_names.append(s.split(':')[0]) - -if mem_list_names[2] != 'MemAvailable': - print('Your Linux kernel is too old, Linux 3.14+ requied\nExit') - exit() - -swap_total_index = mem_list_names.index('SwapTotal') -swap_free_index = swap_total_index + 1 - -mem_total = int(mem_list[0].split(':')[1].strip(' kB\n')) - -# Get names from /proc/*/status to be able to get VmRSS and VmSwap values - -with open('/proc/self/status') as file: - status_list = file.readlines() - -status_names = [] -for s in status_list: - status_names.append(s.split(':')[0]) - -ppid_index = status_names.index('PPid') -vm_size_index = status_names.index('VmSize') -vm_rss_index = status_names.index('VmRSS') -vm_swap_index = status_names.index('VmSwap') -uid_index = status_names.index('Uid') - -try: - anon_index = status_names.index('RssAnon') - file_index = status_names.index('RssFile') - shmem_index = status_names.index('RssShmem') - detailed_rss = True - # print(detailed_rss, 'detailed_rss') -except ValueError: - detailed_rss = False - # print('It is not Linux 4.5+') - -########################################################################## - -''' -# Configurations - -cd = os.getcwd() - -''' - -config = '/etc/nohang/nohang.conf' - -# config = 'nohang.conf' - -print('Config:', config) - - -########################################################################## - -# parsing the config with obtaining the parameters dictionary - -# conf_parameters_dict -# conf_restart_dict - -# dictionary with config options -config_dict = dict() - -processname_re_list = [] -cmdline_re_list = [] -uid_re_list = [] - -# dictionary with names and commands for the parameter -# execute_the_command -# тут тоже список нужен, а не словарь -etc_dict = dict() - -# will store corrective actions stat -stat_dict = dict() - -try: - with open(config) as f: - - for line in f: - - a = line.startswith('#') - b = line.startswith('\n') - c = line.startswith('\t') - d = line.startswith(' ') - - etc = line.startswith('$ETC') - - if not a and not b and not c and not d and not etc: - a = line.partition('=') - config_dict[a[0].strip()] = a[2].strip() - - if etc: - a = line[4:].split('///') - etc_name = a[0].strip() - etc_command = a[1].strip() - if len(etc_name) > 15: - print('Invalid config, the length of the process ' - 'name must not exceed 15 characters\nExit') - exit() - etc_dict[etc_name] = etc_command - - # NEED VALIDATION! - if line.startswith('@PROCESSNAME_RE'): - a = line.partition('@PROCESSNAME_RE')[ - 2].strip(' \n').partition('///') - processname_re_list.append((a[0].strip(' '), a[2].strip(' '))) - - if line.startswith('@CMDLINE_RE'): - a = line.partition('@CMDLINE_RE')[2].strip( - ' \n').partition('///') - cmdline_re_list.append((a[0].strip(' '), a[2].strip(' '))) - - if line.startswith('@UID_RE'): - a = line.partition('@UID_RE')[2].strip(' \n').partition('///') - uid_re_list.append((a[0].strip(' '), a[2].strip(' '))) - -except PermissionError: - print('PermissionError', conf_err_mess) - exit() -except UnicodeDecodeError: - print('UnicodeDecodeError', conf_err_mess) - exit() -except IsADirectoryError: - print('IsADirectoryError', conf_err_mess) - exit() -except IndexError: - print('IndexError', conf_err_mess) - exit() - -# print(processname_re_list) -# print(cmdline_re_list) -# print(uid_re_list) - -########################################################################## - - -# extracting parameters from the dictionary -# check for all necessary parameters -# validation of all parameters - -print_config = conf_parse_bool('print_config') -print_mem_check_results = conf_parse_bool('print_mem_check_results') -print_sleep_periods = conf_parse_bool('print_sleep_periods') -gui_low_memory_warnings = conf_parse_bool('gui_low_memory_warnings') -gui_notifications = conf_parse_bool('gui_notifications') -decrease_oom_score_adj = conf_parse_bool('decrease_oom_score_adj') -execute_the_command = conf_parse_bool('execute_the_command') - - -ignore_psi = conf_parse_bool('ignore_psi') - -regex_matching = conf_parse_bool('regex_matching') - -re_match_cmdline = conf_parse_bool('re_match_cmdline') - -re_match_uid = conf_parse_bool('re_match_uid') - -if regex_matching or re_match_cmdline or re_match_uid: - from re import search - import sre_constants - -mem_min_sigterm_kb, mem_min_sigterm_mb, mem_min_sigterm_percent = calculate_percent( - 'mem_min_sigterm') -mem_min_sigkill_kb, mem_min_sigkill_mb, mem_min_sigkill_percent = calculate_percent( - 'mem_min_sigkill') - -zram_max_sigterm_kb, zram_max_sigterm_mb, zram_max_sigterm_percent = calculate_percent( - 'zram_max_sigterm') -zram_max_sigkill_kb, zram_max_sigkill_mb, zram_max_sigkill_percent = calculate_percent( - 'zram_max_sigkill') - -mem_min_warnings_kb, mem_min_warnings_mb, mem_min_warnings_percent = calculate_percent( - 'mem_min_warnings') -zram_max_warnings_kb, zram_max_warnings_mb, zram_max_warnings_percent = calculate_percent( - 'zram_max_warnings') - - -if 'rate_mem' in config_dict: - rate_mem = string_to_float_convert_test(config_dict['rate_mem']) - if rate_mem is None: - print('Invalid rate_mem value, not float\nExit') - exit() - if rate_mem <= 0: - print('rate_mem MUST be > 0\nExit') - exit() -else: - print('rate_mem not in config\nExit') - exit() - - -if 'rate_swap' in config_dict: - rate_swap = string_to_float_convert_test(config_dict['rate_swap']) - if rate_swap is None: - print('Invalid rate_swap value, not float\nExit') - exit() - if rate_swap <= 0: - print('rate_swap MUST be > 0\nExit') - exit() -else: - print('rate_swap not in config\nExit') - exit() - - -if 'rate_zram' in config_dict: - rate_zram = string_to_float_convert_test(config_dict['rate_zram']) - if rate_zram is None: - print('Invalid rate_zram value, not float\nExit') - exit() - if rate_zram <= 0: - print('rate_zram MUST be > 0\nExit') - exit() -else: - print('rate_zram not in config\nExit') - exit() - - -if 'swap_min_sigterm' in config_dict: - swap_min_sigterm = config_dict['swap_min_sigterm'] -else: - print('swap_min_sigterm not in config\nExit') - exit() - - -if 'swap_min_sigkill' in config_dict: - swap_min_sigkill = config_dict['swap_min_sigkill'] -else: - print('swap_min_sigkill not in config\nExit') - exit() - - -if 'min_delay_after_sigterm' in config_dict: - min_delay_after_sigterm = string_to_float_convert_test( - config_dict['min_delay_after_sigterm']) - if min_delay_after_sigterm is None: - print('Invalid min_delay_after_sigterm value, not float\nExit') - exit() - if min_delay_after_sigterm < 0: - print('min_delay_after_sigterm must be positiv\nExit') - exit() -else: - print('min_delay_after_sigterm not in config\nExit') - exit() - - -if 'min_delay_after_sigkill' in config_dict: - min_delay_after_sigkill = string_to_float_convert_test( - config_dict['min_delay_after_sigkill']) - if min_delay_after_sigkill is None: - print('Invalid min_delay_after_sigkill value, not float\nExit') - exit() - if min_delay_after_sigkill < 0: - print('min_delay_after_sigkill must be positive\nExit') - exit() -else: - print('min_delay_after_sigkill not in config\nExit') - exit() - - -if 'psi_avg10_sleep_time' in config_dict: - psi_avg10_sleep_time = string_to_float_convert_test( - config_dict['psi_avg10_sleep_time']) - if psi_avg10_sleep_time is None: - print('Invalid psi_avg10_sleep_time value, not float\nExit') - exit() - if psi_avg10_sleep_time < 0: - print('psi_avg10_sleep_time must be positive\nExit') - exit() -else: - print('psi_avg10_sleep_time not in config\nExit') - exit() - - -if 'sigkill_psi_avg10' in config_dict: - sigkill_psi_avg10 = string_to_float_convert_test( - config_dict['sigkill_psi_avg10']) - if sigkill_psi_avg10 is None: - print('Invalid sigkill_psi_avg10 value, not float\nExit') - exit() - if sigkill_psi_avg10 < 0 or sigkill_psi_avg10 > 100: - print('sigkill_psi_avg10 must be in the range [0; 100]\nExit') - exit() -else: - print('sigkill_psi_avg10 not in config\nExit') - exit() - - -if 'sigterm_psi_avg10' in config_dict: - sigterm_psi_avg10 = string_to_float_convert_test( - config_dict['sigterm_psi_avg10']) - if sigterm_psi_avg10 is None: - print('Invalid sigterm_psi_avg10 value, not float\nExit') - exit() - if sigterm_psi_avg10 < 0 or sigterm_psi_avg10 > 100: - print('sigterm_psi_avg10 must be in the range [0; 100]\nExit') - exit() -else: - print('sigterm_psi_avg10 not in config\nExit') - exit() - - -if 'min_badness' in config_dict: - min_badness = string_to_int_convert_test( - config_dict['min_badness']) - if min_badness is None: - print('Invalid min_badness value, not integer\nExit') - exit() - if min_badness < 0 or min_badness > 1000: - print('Invalud min_badness value\nExit') - exit() -else: - print('min_badness not in config\nExit') - exit() - - -if 'oom_score_adj_max' in config_dict: - oom_score_adj_max = string_to_int_convert_test( - config_dict['oom_score_adj_max']) - if oom_score_adj_max is None: - print('Invalid oom_score_adj_max value, not integer\nExit') - exit() - if oom_score_adj_max < 0 or oom_score_adj_max > 1000: - print('Invalid oom_score_adj_max value\nExit') - exit() -else: - print('oom_score_adj_max not in config\nExit') - exit() - - -if 'min_time_between_warnings' in config_dict: - min_time_between_warnings = string_to_float_convert_test( - config_dict['min_time_between_warnings']) - if min_time_between_warnings is None: - print('Invalid min_time_between_warnings value, not float\nExit') - exit() - if min_time_between_warnings < 1 or min_time_between_warnings > 300: - print('min_time_between_warnings value out of range [1; 300]\nExit') - exit() -else: - print('min_time_between_warnings not in config\nExit') - exit() - - -if 'swap_min_warnings' in config_dict: - swap_min_warnings = config_dict['swap_min_warnings'] -else: - print('swap_min_warnings not in config\nExit') - exit() - - -########################################################################## - - -# Get KiB levels if it's possible. - -# получ кб. если не кб - то процент. Если процент - находим кб ниже на -# основе полученного своптотал и процентов. - - -def get_swap_threshold_tuple(string): - # re (Num %, True) or (Num KiB, False) - """Returns KiB value if abs val was set in config, or tuple with %""" - # return tuple with abs and bool: (abs %, True) or (abs MiB, False) - - if string.endswith('%'): - valid = string_to_float_convert_test(string[:-1]) - if valid is None: - print('somewhere swap unit is not float_%') - exit() - - value = float(string[:-1].strip()) - if value < 0 or value > 100: - print('invalid value, must be from the range[0; 100] %') - exit() - - return value, True - - elif string.endswith('M'): - valid = string_to_float_convert_test(string[:-1]) - if valid is None: - print('somewhere swap unit is not float_M') - exit() - - value = float(string[:-1].strip()) * 1024 - if value < 0: - print('invalid unit in config (negative value)') - exit() - - return value, False - - else: - print('Invalid config file. There are invalid units somewhere\nExit') - exit() - - -swap_min_sigterm_tuple = get_swap_threshold_tuple(swap_min_sigterm) -swap_min_sigkill_tuple = get_swap_threshold_tuple(swap_min_sigkill) -swap_min_warnings_tuple = get_swap_threshold_tuple(swap_min_warnings) - - -swap_term_is_percent = swap_min_sigterm_tuple[1] -if swap_term_is_percent: - swap_min_sigterm_percent = swap_min_sigterm_tuple[0] -else: - swap_min_sigterm_kb = swap_min_sigterm_tuple[0] - - -swap_kill_is_percent = swap_min_sigkill_tuple[1] -if swap_kill_is_percent: - swap_min_sigkill_percent = swap_min_sigkill_tuple[0] -else: - swap_min_sigkill_kb = swap_min_sigkill_tuple[0] - - -swap_warn_is_percent = swap_min_warnings_tuple[1] -if swap_warn_is_percent: - swap_min_warnings_percent = swap_min_warnings_tuple[0] -else: - swap_min_warnings_kb = swap_min_warnings_tuple[0] - - -########################################################################## - -if print_config: - - print( - '\n1. Memory levels to respond to as an OOM threat\n[displaying these options need fix]\n') - - print('mem_min_sigterm: {} MiB, {} %'.format( - round(mem_min_sigterm_mb), round(mem_min_sigterm_percent, 1))) - print('mem_min_sigkill: {} MiB, {} %'.format( - round(mem_min_sigkill_mb), round(mem_min_sigkill_percent, 1))) - - print('swap_min_sigterm: {}'.format(swap_min_sigterm)) - print('swap_min_sigkill: {}'.format(swap_min_sigkill)) - - print('zram_max_sigterm: {} MiB, {} %'.format( - round(zram_max_sigterm_mb), round(zram_max_sigterm_percent, 1))) - print('zram_max_sigkill: {} MiB, {} %'.format( - round(zram_max_sigkill_mb), round(zram_max_sigkill_percent, 1))) - - print('\n2. The frequency of checking the level of available memory (and CPU usage)\n') - print('rate_mem: {}'.format(rate_mem)) - print('rate_swap: {}'.format(rate_swap)) - print('rate_zram: {}'.format(rate_zram)) - - print('\n3. The prevention of killing innocent victims\n') - print('min_delay_after_sigterm: {}'.format(min_delay_after_sigterm)) - print('min_delay_after_sigkill: {}'.format(min_delay_after_sigkill)) - print('min_badness: {}'.format(min_badness)) - - # False (OK) - OK не нужен когда фолс - print('decrease_oom_score_adj: {}'.format( - decrease_oom_score_adj - )) - if decrease_oom_score_adj: - print('oom_score_adj_max: {}'.format(oom_score_adj_max)) - - print('\n4. Impact on the badness of processes via matching their' - ' names, cmdlines ir UIDs with regular expressions\n') - - print('(todo)') - - print('\n5. The execution of a specific command instead of sending the\nSIGTERM signal\n') - print('execute_the_command: {}'.format(execute_the_command)) - if execute_the_command: - print('\nPROCESS NAME COMMAND TO EXECUTE') - for key in etc_dict: - print('{} {}'.format(key.ljust(15), etc_dict[key])) - - print('\n6. GUI notifications:\n- OOM prevention results and\n- low memory warnings\n') - print('gui_notifications: {}'.format(gui_notifications)) - - print('gui_low_memory_warnings: {}'.format(gui_low_memory_warnings)) - if gui_low_memory_warnings: - print('min_time_between_warnings: {}'.format(min_time_between_warnings)) - - print('mem_min_warnings: {} MiB, {} %'.format( - round(mem_min_warnings_mb), round(mem_min_warnings_percent, 1))) - - print('swap_min_warnings: {}'.format(swap_min_warnings)) - - print('zram_max_warnings: {} MiB, {} %'.format( - round(zram_max_warnings_mb), round(zram_max_warnings_percent, 1))) - - print('\n7. Output verbosity\n') - print('print_config: {}'.format(print_config)) - print('print_mem_check_results: {}'.format(print_mem_check_results)) - print('print_sleep_periods: {}\n'.format(print_sleep_periods)) - - -########################################################################## - - -# for calculating the column width when printing mem and zram -mem_len = len(str(round(mem_total / 1024.0))) - -if gui_notifications or gui_low_memory_warnings: - from subprocess import Popen, TimeoutExpired - notify_sig_dict = {SIGKILL: 'Killing', - SIGTERM: 'Terminating'} - -rate_mem = rate_mem * 1048576 -rate_swap = rate_swap * 1048576 -rate_zram = rate_zram * 1048576 - -warn_time_now = 0 -warn_time_delta = 1000 -warn_timer = 0 - -print('Monitoring started!') - -stdout.flush() - -sigterm_psi = sigterm_psi_avg10 -sigkill_psi = sigkill_psi_avg10 -psi_min_sleep_time_after_action = psi_avg10_sleep_time - - -########################################################################## - -if psi_support and not ignore_psi: - kill_psi_t0 = time() + psi_avg10_sleep_time - term_psi_t0 = time() + psi_avg10_sleep_time - -avg_value = '' - - -while True: - - if psi_support and not ignore_psi: - - avg10 = psi_mem_some_avg10() - - if print_mem_check_results: - avg_value = 'PSI mem some avg10: {} | '.format(str(avg10).rjust(6)) - - if avg10 >= sigkill_psi and time() - kill_psi_t0 >= psi_min_sleep_time_after_action: - time0 = time() - mem_info = 'avg ({}) > sigkill_psi ({})'.format(avg10, sigkill_psi) - find_victim_and_send_signal(SIGKILL) - kill_psi_t0 = time() - elif avg10 >= sigterm_psi and time() - term_psi_t0 >= psi_min_sleep_time_after_action: - time0 = time() - mem_info = 'avg ({}) > sigterm_psi ({})'.format(avg10, sigterm_psi) - find_victim_and_send_signal(SIGTERM) - term_psi_t0 = time() - else: - pass - - mem_available, swap_total, swap_free = check_mem_and_swap() - - # если метры - получаем киб выше и сразу. см. - - # if swap_min_sigkill is set in percent - if swap_kill_is_percent: - swap_min_sigkill_kb = swap_total * swap_min_sigkill_percent / 100.0 - - if swap_term_is_percent: - swap_min_sigterm_kb = swap_total * swap_min_sigterm_percent / 100.0 - - if swap_warn_is_percent: - swap_min_warnings_kb = swap_total * swap_min_warnings_percent / 100.0 - - # в общем случае для работы нужны килобайты. Если в процентах задано - - # находим КБ тут, после получения своптотал. - - mem_used_zram = check_zram() - - if print_mem_check_results: - - # Calculate 'swap-column' width - swap_len = len(str(round(swap_total / 1024.0))) - - # Output avialable mem sizes - if swap_total == 0 and mem_used_zram == 0: - print('{}MemAvail: {} M, {} %'.format( - avg_value, - human(mem_available, mem_len), - just_percent_mem(mem_available / mem_total))) - - elif swap_total > 0 and mem_used_zram == 0: - print('{}MemAvail: {} M, {} % | SwapFree: {} M, {} %'.format( - avg_value, - human(mem_available, mem_len), - just_percent_mem(mem_available / mem_total), - human(swap_free, swap_len), - just_percent_swap(swap_free / (swap_total + 0.1)))) - - else: - print('{}MemAvail: {} M, {} % | SwapFree: {} M, {} % | Mem' - 'UsedZram: {} M, {} %'.format( - avg_value, - human(mem_available, mem_len), - just_percent_mem(mem_available / mem_total), - human(swap_free, swap_len), - just_percent_swap(swap_free / (swap_total + 0.1)), - human(mem_used_zram, mem_len), - just_percent_mem(mem_used_zram / mem_total))) - - # если swap_min_sigkill задан в абсолютной величине и Swap_total = 0 - if swap_total > swap_min_sigkill_kb: # If swap_min_sigkill is absolute - swap_sigkill_pc = percent(swap_min_sigkill_kb / (swap_total + 0.1)) - else: - swap_sigkill_pc = '-' - - if swap_total > swap_min_sigterm_kb: - swap_sigterm_pc = percent(swap_min_sigterm_kb / (swap_total + 0.1)) - else: - - # печатать так: SwapTotal = 0, ignore swapspace - swap_sigterm_pc = '-' - - # это для печати меминфо. Все переработать нахрен. - - # далее пошло ветвление - - # MEM SWAP KILL - if mem_available <= mem_min_sigkill_kb and \ - swap_free <= swap_min_sigkill_kb: - time0 = time() - - mem_info = '{}\n\033[4mMemory status that r' \ - 'equires corrective actions:' \ - '\033[0m\n MemAvailable [{} MiB, {} %] <= mem_min_sig' \ - 'kill [{} MiB, {} %]\n SwapFree [{} MiB, {} %] <= swa' \ - 'p_min_sigkill [{} MiB, {} %]'.format( - HR, - kib_to_mib(mem_available), - percent(mem_available / mem_total), - kib_to_mib(mem_min_sigkill_kb), - percent(mem_min_sigkill_kb / mem_total), - kib_to_mib(swap_free), - percent(swap_free / (swap_total + 0.1)), - kib_to_mib(swap_min_sigkill_kb), - swap_sigkill_pc) - - find_victim_and_send_signal(SIGKILL) - kill_psi_t0 = time() - term_psi_t0 = time() - - # ZRAM KILL - elif mem_used_zram >= zram_max_sigkill_kb: - time0 = time() - - mem_info = '{}\n\033[4mMemory statu' \ - 's that requires corrective actions:' \ - '\033[0m\n MemUsedZram [{} MiB, {} %] >= zram_max_sig' \ - 'kill [{} MiB, {} %]'.format( - HR, - kib_to_mib(mem_used_zram), - percent(mem_used_zram / mem_total), - kib_to_mib(zram_max_sigkill_kb), - percent(zram_max_sigkill_kb / mem_total)) - - find_victim_and_send_signal(SIGKILL) - kill_psi_t0 = time() - term_psi_t0 = time() - - # MEM SWAP TERM - elif mem_available <= mem_min_sigterm_kb and \ - swap_free <= swap_min_sigterm_kb: - - time0 = time() - - mem_info = '{}\n\033[4mMemory status tha' \ - 't requires corrective actions:' \ - '\033[0m\n MemAvailable [{} MiB, {} %] <= mem_min_sig' \ - 'term [{} MiB, {} %]\n SwapFree [{} MiB, {} %] <= swa' \ - 'p_min_sigterm [{} MiB, {} %]'.format( - HR, - kib_to_mib(mem_available), - percent(mem_available / mem_total), - kib_to_mib(mem_min_sigterm_kb), - # percent(mem_min_sigterm_kb / mem_total), - # ОКРУГЛЯТЬ НА МЕСТЕ ВЫШЕ (или не выше, хз) - round(mem_min_sigterm_percent, 1), - kib_to_mib(swap_free), - percent(swap_free / (swap_total + 0.1)), - kib_to_mib(swap_min_sigterm_kb), - swap_sigterm_pc) - - find_victim_and_send_signal(SIGTERM) - kill_psi_t0 = time() - term_psi_t0 = time() - - # ZRAM TERM - elif mem_used_zram >= zram_max_sigterm_kb: - time0 = time() - - mem_info = '{}\n\033[4mMemory status that r' \ - 'equires corrective actions:' \ - '\033[0m\n MemUsedZram [{} MiB, {} %] >= ' \ - 'zram_max_sigterm [{} M, {} %]'.format( - HR, - kib_to_mib(mem_used_zram), - percent(mem_used_zram / mem_total), - kib_to_mib(zram_max_sigterm_kb), - percent(zram_max_sigterm_kb / mem_total)) - - find_victim_and_send_signal(SIGTERM) - kill_psi_t0 = time() - term_psi_t0 = time() - - # LOW MEMORY WARNINGS - elif gui_low_memory_warnings: - - if mem_available <= mem_min_warnings_kb and \ - swap_free <= swap_min_warnings_kb + 0.1 or \ - mem_used_zram >= zram_max_warnings_kb: - warn_time_delta = time() - warn_time_now - warn_time_now = time() - warn_timer += warn_time_delta - if warn_timer > min_time_between_warnings: - t0 = time() - send_notify_warn() - print(time() - t0, 'send notify warning time') - warn_timer = 0 - - sleep_after_check_mem() - - # SLEEP BETWEEN MEM CHECKS - else: - sleep_after_check_mem()