From ecbe7d6f50ca70a1e84a6cba864617570cbf0cea Mon Sep 17 00:00:00 2001 From: Alexey Avramov Date: Thu, 6 Jun 2019 14:08:03 +0900 Subject: [PATCH] check mem after finding a victim --- README.md | 1 - nohang | 351 ++++++++++++++++++++++++++++++++++++++++-------------- nohang.1 | 2 - 3 files changed, 262 insertions(+), 92 deletions(-) diff --git a/README.md b/README.md index eae798c..51476c9 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,6 @@ usage: nohang [-h] [-v] [-t] [-p] [-c CONFIG] optional arguments: -h, --help show this help message and exit -v, --version print version - -t, --test print some tests -p, --print-proc-table print table of processes with their badness values -c CONFIG, --config CONFIG diff --git a/nohang b/nohang index d79e91f..578a280 100755 --- a/nohang +++ b/nohang @@ -15,19 +15,6 @@ from signal import signal, SIGKILL, SIGTERM, SIGINT, SIGQUIT, SIGHUP # define functions -''' -def self_rss(): - """ - """ - return pid_to_status(self_pid)[5] - - -def print_self_rss(): - """ - """ - log('Self RSS: {} MiB'.format(self_rss())) -''' - def get_swap_threshold_tuple(string): # re (Num %, True) or (Num KiB, False) @@ -204,46 +191,6 @@ def print_version(): exit() -def test(): - """ - """ - print('\n(This option is not ready to use!)\n') - - print(version) - print(argv) - - hr = '==================================' - print(hr) - print("uptime()") - print(uptime()) - - print(hr) - print("os.uname()") - print(os.uname()) - - print(hr) - print("pid_to_starttime('self')") - print(pid_to_starttime('self')) - - print(hr) - print("get_victim_id('self')") - print(get_victim_id('self')) - - print(hr) - print("errprint('test')") - print(errprint('test')) - - print(hr) - print("mlockall()") - print(mlockall()) - - print(hr) - print("pid_to_state('2')") - print(pid_to_state('2')) - - exit() - - def pid_to_cgroup_v1(pid): """ """ @@ -1170,7 +1117,7 @@ def find_victim_info(pid, victim_badness, name): vm_swap = kib_to_mib(int(line.split('\t')[1][:-4])) break - cmdline = pid_to_cmdline(pid) + cmdline = pid_to_cmdline(pid) # make it optional! oom_score = rline1('/proc/' + pid + '/oom_score') oom_score_adj = rline1('/proc/' + pid + '/oom_score_adj') @@ -1225,7 +1172,7 @@ def find_victim_info(pid, victim_badness, name): vm_swap = kib_to_mib( int(f_list[i].split('\t')[1][:-3])) - cmdline = pid_to_cmdline(pid) + cmdline = pid_to_cmdline(pid) # make it optional! oom_score = rline1('/proc/' + pid + '/oom_score') oom_score_adj = rline1('/proc/' + pid + '/oom_score_adj') @@ -1310,26 +1257,155 @@ def find_victim_info(pid, victim_badness, name): victim_cgroup_v1, victim_cgroup_v2, realpath, - cmdline, + cmdline, # make it optional! victim_lifetime) return victim_info -def implement_corrective_action(signal): + + + + + + + + + + + + +def check_mem_swap_ex(): + """ + Check: is mem and swap threshold exceeded? + Return: None, (SIGTERM, meminfo), (SIGKILL, meminfo) + """ + + mem_available, swap_total, swap_free = check_mem_and_swap() + + # if swap_min_sigkill is set in percent + if swap_kill_is_percent: + swap_min_sigkill_kb = swap_total * swap_min_sigkill_percent / 100.0 + + if swap_term_is_percent: + swap_min_sigterm_kb = swap_total * swap_min_sigterm_percent / 100.0 + + if swap_warn_is_percent: + swap_min_warnings_kb = swap_total * swap_min_warnings_percent / 100.0 + + if swap_total > swap_min_sigkill_kb: + swap_sigkill_pc = percent(swap_min_sigkill_kb / (swap_total + 0.1)) + else: + swap_sigkill_pc = '-' + + if swap_total > swap_min_sigterm_kb: + swap_sigterm_pc = percent(swap_min_sigterm_kb / (swap_total + 0.1)) + else: + swap_sigterm_pc = '-' + + if (mem_available <= mem_min_sigkill_kb and + swap_free <= swap_min_sigkill_kb): + + mem_info = 'Hard threshold exceeded\nMemory status that requ' \ + 'ires corrective actions:' \ + '\n MemAvailable [{} MiB, {} %] <= mem_min_sig' \ + 'kill [{} MiB, {} %]\n SwapFree [{} MiB, {} %] <= swa' \ + 'p_min_sigkill [{} MiB, {} %]'.format( + kib_to_mib(mem_available), + percent(mem_available / mem_total), + kib_to_mib(mem_min_sigkill_kb), + percent(mem_min_sigkill_kb / mem_total), + kib_to_mib(swap_free), + percent(swap_free / (swap_total + 0.1)), + kib_to_mib(swap_min_sigkill_kb), + swap_sigkill_pc) + + return SIGKILL, mem_info + + if (mem_available <= mem_min_sigterm_kb and + swap_free <= swap_min_sigterm_kb): + + mem_info = 'Soft threshold exceeded\nMemory status that requi' \ + 'res corrective actions:' \ + '\n MemAvailable [{} MiB, {} %] <= mem_min_sig' \ + 'term [{} MiB, {} %]\n SwapFree [{} MiB, {} %] <= swa' \ + 'p_min_sigterm [{} MiB, {} %]'.format( + kib_to_mib(mem_available), + percent(mem_available / mem_total), + kib_to_mib(mem_min_sigterm_kb), + round(mem_min_sigterm_percent, 1), + kib_to_mib(swap_free), + percent(swap_free / (swap_total + 0.1)), + kib_to_mib(swap_min_sigterm_kb), + swap_sigterm_pc) + + return SIGTERM, mem_info + + return None, None + + + + + + + + + + + + + + + + + +def is_victim_alive(pid): + """ + Проверка статуса жертвы: + 1 - жива + 0 - полное исчезновение + 2 - умирает, освобождает память, зомби + """ + exe_exists = os.path.exists('/proc/{}/exe'.format(pid)) + if exe_exists: + return 1 + statm_exists = os.path.exists('/proc/{}/statm'.format(pid)) + if statm_exists: + return 2 + else: + return 0 + + + + + + + + + +def implement_corrective_action(signal, mem_info): """ Find victim with highest badness and send SIGTERM/SIGKILL """ - time0 = time() + + + + # Ёбаная запутанная фция. Распутать всё нахуй. Выделить части в отдельн фции. + # Разбить саму фцию на части. Сделать ее структуру простой и понятной. + + + time0 = time() # начало корр действия. Для вычисл времени действия. + # выходим из фции, если для SIGTERM порога не превышено время # min_delay_after_sigterm и спим в течение over_sleep + # если хард порог превышен - идем дальше. if signal is SIGTERM: dt = time() - actions_time_dict['action_handled'][0] if dt < min_delay_after_sigterm: - print('min_delay_after_sigterm IS NOT EXCEEDED ({} < {})'.format( + log('min_delay_after_sigterm IS NOT EXCEEDED ({} < {})'.format( round(dt, 3), min_delay_after_sigterm)) if print_sleep_periods: @@ -1340,7 +1416,7 @@ def implement_corrective_action(signal): return None # время задержки между действиями не истекло else: - print('min_delay_after_sigterm IS EXCEEDED, it is time to action') + log('min_delay_after_sigterm IS EXCEEDED, it is time to action') """ @@ -1360,7 +1436,26 @@ def implement_corrective_action(signal): log(mem_info) + # ищем жертву с ее бэднес. pid, victim_badness, name = find_victim(print_proc_table) + # sleep(0.1) + + new_signal, mem_info = check_mem_swap_ex() + + + + #log(new_signal) + #log(mem_info) + + if new_signal is None: + log('Thresholds is not exceeded now') + return None + + if new_signal is not signal: + log(mem_info) + signal = new_signal + + #log(mem_info) if victim_badness >= min_badness: @@ -1427,7 +1522,9 @@ def implement_corrective_action(signal): soft_match = True break - if soft_match: + + + if soft_match: # ПЕРЕОПРЕДЕЛЕНИЕ МЯГКОГО КОРРЕКТИРУЮЩЕГО ДЕЙСТВИЯ # todo: make new func m = check_mem_and_swap() @@ -1482,6 +1579,14 @@ def implement_corrective_action(signal): else: # обычное действие через сигнал + + + + # вот тут поработать. Тут ебаный цикл. Нахуй его. + + + + try: os.kill(int(pid), signal) @@ -1489,18 +1594,15 @@ def implement_corrective_action(signal): response_time = kill_timestamp - time0 while True: - exe_exists = os.path.exists('/proc/{}/exe'.format(pid)) - rss = pid_to_rss(pid) + victim_alive = is_victim_alive(pid) dt = time() - kill_timestamp - log('Victim VmRSS: {} KiB'.format(rss)) - if not exe_exists or rss == 0 or dt > 0.01: + if victim_alive == 2 or dt > 0.02: # print(dt) break - sleep(0.001) + sleep(0.002) - if dt > 0.01: - log('Timer (value = 0.01 sec) expired; seems' - ' like the victim handles signal') + if dt > 0.02: + log('Timer (value = 0.02 sec) expired; victim does not respond on action in 0.02 sec') actions_time_dict['action_handled'] = [ time(), get_victim_id(pid)] @@ -1515,20 +1617,24 @@ def implement_corrective_action(signal): log('Process exited (VmRSS = 0) in {} sec'.format( round(dt, 5))) - if signal is SIGKILL or not exe_exists or rss == 0: + + + + if signal is SIGKILL or victim_alive == 2: + # жертва умирает от SIGKILL. Дожидаемся ее полной смерти. while True: - sleep(0.001) - # рсс не важен когда путь не существует. Проверяй - # просто существование пид. + sleep(0.002) rss = pid_to_rss(pid) - if rss is None: + if rss is None: # процесс исчез break t1 = time() kill_duration = t1 - kill_timestamp log('The victim died in {} sec'.format( round(kill_duration, 3))) + + mem_available, swap_total, swap_free = check_mem_and_swap() ma_mib = int(mem_available) / 1024.0 @@ -1577,11 +1683,13 @@ def implement_corrective_action(signal): try: log(preventing_oom_message) - except UnboundLocalError: + except UnboundLocalError: # какой позор preventing_oom_message = key update_stat_dict_and_print(key) + + # нехуй делать, бэднес жертвы слишком мал else: response_time = time() - time0 @@ -1609,6 +1717,22 @@ def implement_corrective_action(signal): print('##################################################################') + + + + + + + + + + + + + + + + def sleep_after_check_mem(): """Specify sleep times depends on rates and avialable memory.""" @@ -1743,7 +1867,57 @@ def calculate_percent(arg_key): return mem_min_kb, mem_min_mb, mem_min_percent + + + + + + + + + + + + + + + + ########################################################################## + + + + +victim_dict = dict() +victim_id = None +actions_time_dict = dict() +actions_time_dict['action_handled'] = [time(), victim_id] +# print(actions_time_dict) + + + + + + + + + + + + + + + + + + + + + + + + + start_time = time() @@ -1754,7 +1928,6 @@ help_mess = """usage: nohang [-h] [-v] [-t] [-p] [-c CONFIG] optional arguments: -h, --help show this help message and exit -v, --version print version - -t, --test print some tests -p, --print-proc-table print table of processes with their badness values -c CONFIG, --config CONFIG @@ -1794,13 +1967,15 @@ else: notify_helper_path = '/usr/sbin/nohang_notify_helper' -victim_dict = dict() -victim_id = None -actions_time_dict = dict() -actions_time_dict['action_handled'] = [time(), victim_id] -# print(actions_time_dict) + + + + + + + # will store corrective actions stat @@ -1837,8 +2012,6 @@ elif len(argv) == 2: exit() elif argv[1] == '--version' or argv[1] == '-v': print_version() - elif argv[1] == '--test' or argv[1] == '-t': - test() elif argv[1] == '--print-proc-table' or argv[1] == '-p': print_proc_table_flag = True if os.path.exists('./nohang.conf'): @@ -2861,7 +3034,7 @@ while True: kib_to_mib(swap_min_sigkill_kb), swap_sigkill_pc) - implement_corrective_action(SIGKILL) + implement_corrective_action(SIGKILL, mem_info) psi_t0 = time() continue @@ -2877,7 +3050,7 @@ while True: kib_to_mib(zram_max_sigkill_kb), percent(zram_max_sigkill_kb / mem_total)) - implement_corrective_action(SIGKILL) + implement_corrective_action(SIGKILL, mem_info) psi_t0 = time() continue @@ -2912,7 +3085,7 @@ while True: round(psi_kill_exceeded_timer, 1) ) - implement_corrective_action(SIGKILL) + implement_corrective_action(SIGKILL, mem_info) psi_t0 = time() continue @@ -2937,7 +3110,7 @@ while True: kib_to_mib(swap_min_sigterm_kb), swap_sigterm_pc) - implement_corrective_action(SIGTERM) + implement_corrective_action(SIGTERM, mem_info) psi_t0 = time() continue @@ -2952,7 +3125,7 @@ while True: kib_to_mib(zram_max_sigterm_kb), percent(zram_max_sigterm_kb / mem_total)) - implement_corrective_action(SIGTERM) + implement_corrective_action(SIGTERM, mem_info) psi_t0 = time() continue @@ -2985,7 +3158,7 @@ while True: round(psi_term_exceeded_timer, 1) ) - implement_corrective_action(SIGTERM) + implement_corrective_action(SIGTERM, mem_info) psi_t0 = time() continue diff --git a/nohang.1 b/nohang.1 index bda7e00..08533a5 100644 --- a/nohang.1 +++ b/nohang.1 @@ -38,8 +38,6 @@ Nohang is a highly configurable daemon for Linux which is able to correctly prev -v, --version print version - -t, --test print some tests - -p, --print-proc-table print table of processes with their badness values