fix UnboundLocalError, break output

This commit is contained in:
Alexey Avramov 2019-07-03 23:22:41 +09:00
parent 96da870eca
commit 1ae54155be

170
nohang
View File

@ -248,6 +248,11 @@ def get_victim_id(pid):
def pid_to_state(pid): def pid_to_state(pid):
""" Handle FNF error! (BTW it already handled in find_victim_info()) """ Handle FNF error! (BTW it already handled in find_victim_info())
also handle UDErr
МОЖНО ЧИТАТЬ ТОЛЬКО НАЧАЛО ФАЙЛА
""" """
return rline1('/proc/' + pid + '/stat').rpartition(')')[2][1] return rline1('/proc/' + pid + '/stat').rpartition(')')[2][1]
@ -1314,7 +1319,8 @@ def check_mem_swap_ex():
kib_to_mib(swap_min_sigkill_kb), kib_to_mib(swap_min_sigkill_kb),
swap_sigkill_pc) swap_sigkill_pc)
return SIGKILL, mem_info, mem_available, swap_min_sigkill_kb, swap_min_sigterm_kb, swap_free, swap_total return (SIGKILL, mem_info, mem_available, swap_min_sigkill_kb,
swap_min_sigterm_kb, swap_free, swap_total)
if (mem_available <= mem_min_sigterm_kb and if (mem_available <= mem_min_sigterm_kb and
swap_free <= swap_min_sigterm_kb): swap_free <= swap_min_sigterm_kb):
@ -1333,15 +1339,18 @@ def check_mem_swap_ex():
kib_to_mib(swap_min_sigterm_kb), kib_to_mib(swap_min_sigterm_kb),
swap_sigterm_pc) swap_sigterm_pc)
return SIGTERM, mem_info, mem_available, swap_min_sigkill_kb, swap_min_sigterm_kb, swap_free, swap_total return (SIGTERM, mem_info, mem_available, swap_min_sigkill_kb,
swap_min_sigterm_kb, swap_free, swap_total)
if gui_low_memory_warnings: if gui_low_memory_warnings:
if (mem_available <= mem_min_warnings_kb and swap_free <= if (mem_available <= mem_min_warnings_kb and swap_free <=
swap_min_warnings_kb + 0.1): swap_min_warnings_kb + 0.1):
return 'WARN', None, mem_available, swap_min_sigkill_kb, swap_min_sigterm_kb, swap_free, swap_total return ('WARN', None, mem_available, swap_min_sigkill_kb,
swap_min_sigterm_kb, swap_free, swap_total)
return None, None, mem_available, swap_min_sigkill_kb, swap_min_sigterm_kb, swap_free, swap_total return (None, None, mem_available, swap_min_sigkill_kb,
swap_min_sigterm_kb, swap_free, swap_total)
def check_zram_ex(): def check_zram_ex():
@ -1364,9 +1373,9 @@ def check_zram_ex():
if mem_used_zram >= zram_max_sigterm_kb: if mem_used_zram >= zram_max_sigterm_kb:
mem_info = 'Memory status that require' \ mem_info = 'Memory status that requires corrective actions (soft th' \
's corrective actions (soft threshold exceeded):\n MemUsedZram [{} MiB, {} %] >= zra' \ 'reshold exceeded):\n MemUsedZram [{} MiB, {} %] >= zram_max_s' \
'm_max_sigterm [{} M, {} %]'.format( 'igterm [{} M, {} %]'.format(
kib_to_mib(mem_used_zram), kib_to_mib(mem_used_zram),
percent(mem_used_zram / mem_total), percent(mem_used_zram / mem_total),
kib_to_mib(zram_max_sigterm_kb), kib_to_mib(zram_max_sigterm_kb),
@ -1430,7 +1439,8 @@ def check_psi_ex(psi_t0, psi_kill_exceeded_timer, psi_term_exceeded_timer, x0):
round(psi_kill_exceeded_timer, 1) round(psi_kill_exceeded_timer, 1)
) )
return SIGKILL, mem_info, psi_t0, psi_kill_exceeded_timer, psi_term_exceeded_timer, x0 return (SIGKILL, mem_info, psi_t0, psi_kill_exceeded_timer,
psi_term_exceeded_timer, x0)
if psi_avg_value >= sigterm_psi_threshold: if psi_avg_value >= sigterm_psi_threshold:
sigterm_psi_exceeded = True sigterm_psi_exceeded = True
@ -1460,14 +1470,17 @@ def check_psi_ex(psi_t0, psi_kill_exceeded_timer, psi_term_exceeded_timer, x0):
round(psi_term_exceeded_timer, 1) round(psi_term_exceeded_timer, 1)
) )
return SIGTERM, mem_info, psi_t0, psi_kill_exceeded_timer, psi_term_exceeded_timer, x0 return (SIGTERM, mem_info, psi_t0, psi_kill_exceeded_timer,
psi_term_exceeded_timer, x0)
if gui_low_memory_warnings: if gui_low_memory_warnings:
if psi_avg_value >= psi_avg_warnings: if psi_avg_value >= psi_avg_warnings:
return 'WARN', None, psi_t0, psi_kill_exceeded_timer, psi_term_exceeded_timer, x0 return ('WARN', None, psi_t0, psi_kill_exceeded_timer,
psi_term_exceeded_timer, x0)
return None, None, psi_t0, psi_kill_exceeded_timer, psi_term_exceeded_timer, x0 return (None, None, psi_t0, psi_kill_exceeded_timer,
psi_term_exceeded_timer, x0)
def is_victim_alive(pid): def is_victim_alive(pid):
@ -1501,6 +1514,35 @@ def implement_corrective_action(
time0 = time() time0 = time()
# d.pop("A1")
# print('++++++++++++++++++++++++')
for i in v_dict:
pid1 = i.split('_pid')[1]
print([pid1])
vi1 = get_victim_id(pid1)
print([vi1])
if vi1 == '':
print('pop:', i)
v_dict.pop(i)
a_dict['any'] -= min_delay_after_sigterm
# Старая жертва умерла, сброс таймера
# На самом деле сброс можно делать либо только если все старые
# жертвы умерли и словарь опустошился, либо хз
# Это трудно протестировать.
if threshold is SIGTERM: if threshold is SIGTERM:
dt = time() - a_dict['any'] dt = time() - a_dict['any']
@ -1535,7 +1577,8 @@ def implement_corrective_action(
psi_term_exceeded_timer, x0) = check_psi_ex( psi_term_exceeded_timer, x0) = check_psi_ex(
psi_t0, psi_kill_exceeded_timer, psi_term_exceeded_timer, x0) psi_t0, psi_kill_exceeded_timer, psi_term_exceeded_timer, x0)
if (masf_threshold is SIGKILL or zram_threshold is SIGKILL or psi_threshold is SIGKILL): if (masf_threshold is SIGKILL or zram_threshold is SIGKILL or
psi_threshold is SIGKILL):
new_threshold = SIGKILL new_threshold = SIGKILL
mem_info_list = [] mem_info_list = []
@ -1549,7 +1592,8 @@ def implement_corrective_action(
if psi_threshold is SIGKILL or psi_threshold is SIGTERM: if psi_threshold is SIGKILL or psi_threshold is SIGTERM:
mem_info_list.append(psi_info) mem_info_list.append(psi_info)
elif (masf_threshold is SIGTERM or zram_threshold is SIGTERM or psi_threshold is SIGTERM): elif (masf_threshold is SIGTERM or zram_threshold is SIGTERM or
psi_threshold is SIGTERM):
new_threshold = SIGTERM new_threshold = SIGTERM
mem_info_list = [] mem_info_list = []
@ -1578,6 +1622,12 @@ def implement_corrective_action(
vwd = None # Victim Will Die vwd = None # Victim Will Die
if victim_badness >= min_badness: if victim_badness >= min_badness:
log('Try to implement a corrective action...') log('Try to implement a corrective action...')
@ -1590,8 +1640,9 @@ def implement_corrective_action(
'victim will get SIGKILL') 'victim will get SIGKILL')
threshold = SIGKILL threshold = SIGKILL
else: else:
log('max_post_sigterm_victim_lifetime IS NOT EXCEEDED ({} < {})'.format( log('max_post_sigterm_victim_lifetime IS NOT EXCEEDED ('
round(dt, 1), max_post_sigterm_victim_lifetime)) '{} < {})'.format(round(
dt, 1), max_post_sigterm_victim_lifetime))
if print_sleep_periods: if print_sleep_periods:
log('Sleep {} sec (over_sleep)'.format(over_sleep)) log('Sleep {} sec (over_sleep)'.format(over_sleep))
@ -1637,6 +1688,25 @@ def implement_corrective_action(
response_time = time() - time0 response_time = time() - time0
preventing_oom_message = 'Implement a corrective act' \
'ion:\n Run the command: {}' \
'\n Exit status: {}; total response ' \
'time: {} ms'.format(
cmd,
exit_status,
round(response_time * 1000))
log(preventing_oom_message)
key = "Run the command '{}'".format(cmd)
update_stat_dict_and_print(key)
if gui_notifications: if gui_notifications:
send_notify_etc(pid, name, cmd) send_notify_etc(pid, name, cmd)
@ -1665,14 +1735,16 @@ def implement_corrective_action(
response_time = time() - time0 response_time = time() - time0
send_result = 'no such process; response time: {} ms'.format( send_result = 'no such process; response time: {} ms'.format(
round(response_time * 1000)) round(response_time * 1000))
key = 'The victim died in the search process: FileNotFoundError' key = 'The victim died in the search process: ' \
'FileNotFoundError'
except ProcessLookupError: except ProcessLookupError:
vwd = True vwd = True
success = False success = False
response_time = time() - time0 response_time = time() - time0
send_result = 'no such process; response time: {} ms'.format( send_result = 'no such process; response time: {} ms'.format(
round(response_time * 1000)) round(response_time * 1000))
key = 'The victim died in the search process: ProcessLookupError' key = 'The victim died in the search process: ' \
'ProcessLookupError'
try: try:
log(preventing_oom_message) log(preventing_oom_message)
@ -1680,17 +1752,31 @@ def implement_corrective_action(
except UnboundLocalError: except UnboundLocalError:
preventing_oom_message = key preventing_oom_message = key
if vwd: if vwd:
pass
"""
a_dict['hard'] = a_dict['any'] = time() a_dict['hard'] = a_dict['any'] = time()
if victim_id not in v_dict: if victim_id not in v_dict:
v_dict[victim_id] = dict() v_dict[victim_id] = dict()
v_dict[victim_id]['hard'] = v_dict[victim_id]['any'] = time() v_dict[victim_id]['hard'] = v_dict[victim_id]['any'] = time()
"""
else: else:
a_dict['soft'] = a_dict['any'] = time() a_dict['soft'] = a_dict['any'] = time()
if victim_id not in v_dict: if victim_id not in v_dict:
v_dict[victim_id] = dict() v_dict[victim_id] = dict()
v_dict[victim_id]['soft'] = v_dict[victim_id]['any'] = time() v_dict[victim_id]['soft'] = v_dict[victim_id]['any'] = time()
response_time = time() - time0 response_time = time() - time0
log('success: ' + str(success)) log('success: ' + str(success))
@ -1699,19 +1785,28 @@ def implement_corrective_action(
kill_timestamp = time() kill_timestamp = time()
while True:
# ПОЧЕМУ по 2 раза отслеживаем? НАХУЙ ТАК ЖИТЬ
while True: # тест на чувствительность
victim_alive = is_victim_alive(pid) victim_alive = is_victim_alive(pid)
dt = time() - a_dict['any'] dt = time() - a_dict['any']
if victim_alive == 2 or dt > 0.05: if victim_alive == 2 or dt > 0.05:
break break
sleep(0.005) sleep(0.005)
if dt > 0.05: if dt > 0.05:
log('Timer (value = 0.05 sec) expired; victim does not respond on action in 0.05 sec') log('Timer (value = 0.05 sec) expired; victim does not respond'
' on action in 0.05 sec')
else: else:
log('Process exited (VmRSS = 0) in {} sec'.format( log('Process exited (VmRSS = 0) in {} sec'.format(
round(dt, 5))) round(dt, 5)))
v_dict.pop(victim_id)
# непррерываемый цикл
if threshold is SIGKILL or victim_alive == 2: if threshold is SIGKILL or victim_alive == 2:
# жертва умирает от SIGKILL. Дожидаемся ее полной смерти. # жертва умирает от SIGKILL. Дожидаемся ее полной смерти.
# Сброс таймеа. Готовность к новым мягким # Сброс таймеа. Готовность к новым мягким
@ -1731,6 +1826,19 @@ def implement_corrective_action(
round(kill_duration, 3))) round(kill_duration, 3)))
vwd = True vwd = True
if victim_id in v_dict:
v_dict.pop(victim_id)
psi_t0 = time() psi_t0 = time()
# КОНЕЦ ОТСЛЕЖИВАНИЯ # КОНЕЦ ОТСЛЕЖИВАНИЯ
@ -2689,16 +2797,12 @@ else:
if max_sleep < min_sleep: if max_sleep < min_sleep:
errprint( errprint('min_sleep value must not exceed max_sleep value.\nExit')
'max_sleep value must not exceed min_sleep value.\nExit'
)
exit(1) exit(1)
if min_sleep < over_sleep: if min_sleep < over_sleep:
errprint( errprint('over_sleep value must not exceed min_sleep value.\nExit')
'min_sleep value must not exceed over_sleep value.\nExit'
)
exit(1) exit(1)
@ -2886,7 +2990,8 @@ if print_config:
log(' zram_max_warnings: {} MiB, {} %'.format( log(' zram_max_warnings: {} MiB, {} %'.format(
round(zram_max_warnings_mb), round(zram_max_warnings_percent, 1))) round(zram_max_warnings_mb), round(zram_max_warnings_percent, 1)))
log(' psi_avg_warnings: {}'.format(psi_avg_warnings)) log(' psi_avg_warnings: {}'.format(psi_avg_warnings))
log(' min_time_between_warnings: {} sec'.format(min_time_between_warnings)) log(' min_time_between_warnings: {} sec'.format(
min_time_between_warnings))
log('8. Verbosity') log('8. Verbosity')
@ -2982,8 +3087,6 @@ threshold = None
mem_info = None mem_info = None
#print(x0, 'x0')
CHECK_PSI = False CHECK_PSI = False
if psi_support and not ignore_psi: if psi_support and not ignore_psi:
CHECK_PSI = True CHECK_PSI = True
@ -3095,7 +3198,8 @@ while True:
) )
) )
if masf_threshold is SIGKILL or zram_threshold is SIGKILL or psi_threshold is SIGKILL: if (masf_threshold is SIGKILL or zram_threshold is SIGKILL or
psi_threshold is SIGKILL):
threshold = SIGKILL threshold = SIGKILL
mem_info_list = [] mem_info_list = []
@ -3118,7 +3222,8 @@ while True:
x0, psi_threshold, zram_threshold, zram_info, psi_info) x0, psi_threshold, zram_threshold, zram_info, psi_info)
continue continue
if masf_threshold is SIGTERM or zram_threshold is SIGTERM or psi_threshold is SIGTERM: if (masf_threshold is SIGTERM or zram_threshold is SIGTERM or
psi_threshold is SIGTERM):
threshold = SIGTERM threshold = SIGTERM
mem_info_list = [] mem_info_list = []
@ -3143,7 +3248,8 @@ while True:
if gui_low_memory_warnings: if gui_low_memory_warnings:
if masf_threshold == 'WARN' or zram_threshold == 'WARN' or psi_threshold == 'WARN': if (masf_threshold == 'WARN' or zram_threshold == 'WARN' or
psi_threshold == 'WARN'):
warn_time_delta = time() - warn_time_now warn_time_delta = time() - warn_time_now
warn_time_now = time() warn_time_now = time()