fix UnboundLocalError, break output
This commit is contained in:
parent
96da870eca
commit
1ae54155be
172
nohang
172
nohang
@ -248,6 +248,11 @@ def get_victim_id(pid):
|
|||||||
|
|
||||||
def pid_to_state(pid):
|
def pid_to_state(pid):
|
||||||
""" Handle FNF error! (BTW it already handled in find_victim_info())
|
""" Handle FNF error! (BTW it already handled in find_victim_info())
|
||||||
|
also handle UDErr
|
||||||
|
|
||||||
|
МОЖНО ЧИТАТЬ ТОЛЬКО НАЧАЛО ФАЙЛА
|
||||||
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
return rline1('/proc/' + pid + '/stat').rpartition(')')[2][1]
|
return rline1('/proc/' + pid + '/stat').rpartition(')')[2][1]
|
||||||
|
|
||||||
@ -1314,7 +1319,8 @@ def check_mem_swap_ex():
|
|||||||
kib_to_mib(swap_min_sigkill_kb),
|
kib_to_mib(swap_min_sigkill_kb),
|
||||||
swap_sigkill_pc)
|
swap_sigkill_pc)
|
||||||
|
|
||||||
return SIGKILL, mem_info, mem_available, swap_min_sigkill_kb, swap_min_sigterm_kb, swap_free, swap_total
|
return (SIGKILL, mem_info, mem_available, swap_min_sigkill_kb,
|
||||||
|
swap_min_sigterm_kb, swap_free, swap_total)
|
||||||
|
|
||||||
if (mem_available <= mem_min_sigterm_kb and
|
if (mem_available <= mem_min_sigterm_kb and
|
||||||
swap_free <= swap_min_sigterm_kb):
|
swap_free <= swap_min_sigterm_kb):
|
||||||
@ -1333,15 +1339,18 @@ def check_mem_swap_ex():
|
|||||||
kib_to_mib(swap_min_sigterm_kb),
|
kib_to_mib(swap_min_sigterm_kb),
|
||||||
swap_sigterm_pc)
|
swap_sigterm_pc)
|
||||||
|
|
||||||
return SIGTERM, mem_info, mem_available, swap_min_sigkill_kb, swap_min_sigterm_kb, swap_free, swap_total
|
return (SIGTERM, mem_info, mem_available, swap_min_sigkill_kb,
|
||||||
|
swap_min_sigterm_kb, swap_free, swap_total)
|
||||||
|
|
||||||
if gui_low_memory_warnings:
|
if gui_low_memory_warnings:
|
||||||
|
|
||||||
if (mem_available <= mem_min_warnings_kb and swap_free <=
|
if (mem_available <= mem_min_warnings_kb and swap_free <=
|
||||||
swap_min_warnings_kb + 0.1):
|
swap_min_warnings_kb + 0.1):
|
||||||
return 'WARN', None, mem_available, swap_min_sigkill_kb, swap_min_sigterm_kb, swap_free, swap_total
|
return ('WARN', None, mem_available, swap_min_sigkill_kb,
|
||||||
|
swap_min_sigterm_kb, swap_free, swap_total)
|
||||||
|
|
||||||
return None, None, mem_available, swap_min_sigkill_kb, swap_min_sigterm_kb, swap_free, swap_total
|
return (None, None, mem_available, swap_min_sigkill_kb,
|
||||||
|
swap_min_sigterm_kb, swap_free, swap_total)
|
||||||
|
|
||||||
|
|
||||||
def check_zram_ex():
|
def check_zram_ex():
|
||||||
@ -1364,9 +1373,9 @@ def check_zram_ex():
|
|||||||
|
|
||||||
if mem_used_zram >= zram_max_sigterm_kb:
|
if mem_used_zram >= zram_max_sigterm_kb:
|
||||||
|
|
||||||
mem_info = 'Memory status that require' \
|
mem_info = 'Memory status that requires corrective actions (soft th' \
|
||||||
's corrective actions (soft threshold exceeded):\n MemUsedZram [{} MiB, {} %] >= zra' \
|
'reshold exceeded):\n MemUsedZram [{} MiB, {} %] >= zram_max_s' \
|
||||||
'm_max_sigterm [{} M, {} %]'.format(
|
'igterm [{} M, {} %]'.format(
|
||||||
kib_to_mib(mem_used_zram),
|
kib_to_mib(mem_used_zram),
|
||||||
percent(mem_used_zram / mem_total),
|
percent(mem_used_zram / mem_total),
|
||||||
kib_to_mib(zram_max_sigterm_kb),
|
kib_to_mib(zram_max_sigterm_kb),
|
||||||
@ -1430,7 +1439,8 @@ def check_psi_ex(psi_t0, psi_kill_exceeded_timer, psi_term_exceeded_timer, x0):
|
|||||||
round(psi_kill_exceeded_timer, 1)
|
round(psi_kill_exceeded_timer, 1)
|
||||||
)
|
)
|
||||||
|
|
||||||
return SIGKILL, mem_info, psi_t0, psi_kill_exceeded_timer, psi_term_exceeded_timer, x0
|
return (SIGKILL, mem_info, psi_t0, psi_kill_exceeded_timer,
|
||||||
|
psi_term_exceeded_timer, x0)
|
||||||
|
|
||||||
if psi_avg_value >= sigterm_psi_threshold:
|
if psi_avg_value >= sigterm_psi_threshold:
|
||||||
sigterm_psi_exceeded = True
|
sigterm_psi_exceeded = True
|
||||||
@ -1460,14 +1470,17 @@ def check_psi_ex(psi_t0, psi_kill_exceeded_timer, psi_term_exceeded_timer, x0):
|
|||||||
round(psi_term_exceeded_timer, 1)
|
round(psi_term_exceeded_timer, 1)
|
||||||
)
|
)
|
||||||
|
|
||||||
return SIGTERM, mem_info, psi_t0, psi_kill_exceeded_timer, psi_term_exceeded_timer, x0
|
return (SIGTERM, mem_info, psi_t0, psi_kill_exceeded_timer,
|
||||||
|
psi_term_exceeded_timer, x0)
|
||||||
|
|
||||||
if gui_low_memory_warnings:
|
if gui_low_memory_warnings:
|
||||||
|
|
||||||
if psi_avg_value >= psi_avg_warnings:
|
if psi_avg_value >= psi_avg_warnings:
|
||||||
return 'WARN', None, psi_t0, psi_kill_exceeded_timer, psi_term_exceeded_timer, x0
|
return ('WARN', None, psi_t0, psi_kill_exceeded_timer,
|
||||||
|
psi_term_exceeded_timer, x0)
|
||||||
|
|
||||||
return None, None, psi_t0, psi_kill_exceeded_timer, psi_term_exceeded_timer, x0
|
return (None, None, psi_t0, psi_kill_exceeded_timer,
|
||||||
|
psi_term_exceeded_timer, x0)
|
||||||
|
|
||||||
|
|
||||||
def is_victim_alive(pid):
|
def is_victim_alive(pid):
|
||||||
@ -1501,6 +1514,35 @@ def implement_corrective_action(
|
|||||||
|
|
||||||
time0 = time()
|
time0 = time()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# d.pop("A1")
|
||||||
|
# print('++++++++++++++++++++++++')
|
||||||
|
|
||||||
|
for i in v_dict:
|
||||||
|
pid1 = i.split('_pid')[1]
|
||||||
|
print([pid1])
|
||||||
|
vi1 = get_victim_id(pid1)
|
||||||
|
print([vi1])
|
||||||
|
|
||||||
|
if vi1 == '':
|
||||||
|
print('pop:', i)
|
||||||
|
v_dict.pop(i)
|
||||||
|
a_dict['any'] -= min_delay_after_sigterm
|
||||||
|
# Старая жертва умерла, сброс таймера
|
||||||
|
# На самом деле сброс можно делать либо только если все старые
|
||||||
|
# жертвы умерли и словарь опустошился, либо хз
|
||||||
|
# Это трудно протестировать.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if threshold is SIGTERM:
|
if threshold is SIGTERM:
|
||||||
|
|
||||||
dt = time() - a_dict['any']
|
dt = time() - a_dict['any']
|
||||||
@ -1535,7 +1577,8 @@ def implement_corrective_action(
|
|||||||
psi_term_exceeded_timer, x0) = check_psi_ex(
|
psi_term_exceeded_timer, x0) = check_psi_ex(
|
||||||
psi_t0, psi_kill_exceeded_timer, psi_term_exceeded_timer, x0)
|
psi_t0, psi_kill_exceeded_timer, psi_term_exceeded_timer, x0)
|
||||||
|
|
||||||
if (masf_threshold is SIGKILL or zram_threshold is SIGKILL or psi_threshold is SIGKILL):
|
if (masf_threshold is SIGKILL or zram_threshold is SIGKILL or
|
||||||
|
psi_threshold is SIGKILL):
|
||||||
|
|
||||||
new_threshold = SIGKILL
|
new_threshold = SIGKILL
|
||||||
mem_info_list = []
|
mem_info_list = []
|
||||||
@ -1549,7 +1592,8 @@ def implement_corrective_action(
|
|||||||
if psi_threshold is SIGKILL or psi_threshold is SIGTERM:
|
if psi_threshold is SIGKILL or psi_threshold is SIGTERM:
|
||||||
mem_info_list.append(psi_info)
|
mem_info_list.append(psi_info)
|
||||||
|
|
||||||
elif (masf_threshold is SIGTERM or zram_threshold is SIGTERM or psi_threshold is SIGTERM):
|
elif (masf_threshold is SIGTERM or zram_threshold is SIGTERM or
|
||||||
|
psi_threshold is SIGTERM):
|
||||||
|
|
||||||
new_threshold = SIGTERM
|
new_threshold = SIGTERM
|
||||||
mem_info_list = []
|
mem_info_list = []
|
||||||
@ -1578,6 +1622,12 @@ def implement_corrective_action(
|
|||||||
|
|
||||||
vwd = None # Victim Will Die
|
vwd = None # Victim Will Die
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if victim_badness >= min_badness:
|
if victim_badness >= min_badness:
|
||||||
|
|
||||||
log('Try to implement a corrective action...')
|
log('Try to implement a corrective action...')
|
||||||
@ -1590,8 +1640,9 @@ def implement_corrective_action(
|
|||||||
'victim will get SIGKILL')
|
'victim will get SIGKILL')
|
||||||
threshold = SIGKILL
|
threshold = SIGKILL
|
||||||
else:
|
else:
|
||||||
log('max_post_sigterm_victim_lifetime IS NOT EXCEEDED ({} < {})'.format(
|
log('max_post_sigterm_victim_lifetime IS NOT EXCEEDED ('
|
||||||
round(dt, 1), max_post_sigterm_victim_lifetime))
|
'{} < {})'.format(round(
|
||||||
|
dt, 1), max_post_sigterm_victim_lifetime))
|
||||||
|
|
||||||
if print_sleep_periods:
|
if print_sleep_periods:
|
||||||
log('Sleep {} sec (over_sleep)'.format(over_sleep))
|
log('Sleep {} sec (over_sleep)'.format(over_sleep))
|
||||||
@ -1637,6 +1688,25 @@ def implement_corrective_action(
|
|||||||
|
|
||||||
response_time = time() - time0
|
response_time = time() - time0
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
preventing_oom_message = 'Implement a corrective act' \
|
||||||
|
'ion:\n Run the command: {}' \
|
||||||
|
'\n Exit status: {}; total response ' \
|
||||||
|
'time: {} ms'.format(
|
||||||
|
cmd,
|
||||||
|
exit_status,
|
||||||
|
round(response_time * 1000))
|
||||||
|
|
||||||
|
log(preventing_oom_message)
|
||||||
|
|
||||||
|
key = "Run the command '{}'".format(cmd)
|
||||||
|
update_stat_dict_and_print(key)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if gui_notifications:
|
if gui_notifications:
|
||||||
send_notify_etc(pid, name, cmd)
|
send_notify_etc(pid, name, cmd)
|
||||||
|
|
||||||
@ -1665,14 +1735,16 @@ def implement_corrective_action(
|
|||||||
response_time = time() - time0
|
response_time = time() - time0
|
||||||
send_result = 'no such process; response time: {} ms'.format(
|
send_result = 'no such process; response time: {} ms'.format(
|
||||||
round(response_time * 1000))
|
round(response_time * 1000))
|
||||||
key = 'The victim died in the search process: FileNotFoundError'
|
key = 'The victim died in the search process: ' \
|
||||||
|
'FileNotFoundError'
|
||||||
except ProcessLookupError:
|
except ProcessLookupError:
|
||||||
vwd = True
|
vwd = True
|
||||||
success = False
|
success = False
|
||||||
response_time = time() - time0
|
response_time = time() - time0
|
||||||
send_result = 'no such process; response time: {} ms'.format(
|
send_result = 'no such process; response time: {} ms'.format(
|
||||||
round(response_time * 1000))
|
round(response_time * 1000))
|
||||||
key = 'The victim died in the search process: ProcessLookupError'
|
key = 'The victim died in the search process: ' \
|
||||||
|
'ProcessLookupError'
|
||||||
|
|
||||||
try:
|
try:
|
||||||
log(preventing_oom_message)
|
log(preventing_oom_message)
|
||||||
@ -1680,17 +1752,31 @@ def implement_corrective_action(
|
|||||||
except UnboundLocalError:
|
except UnboundLocalError:
|
||||||
preventing_oom_message = key
|
preventing_oom_message = key
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if vwd:
|
if vwd:
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
"""
|
||||||
a_dict['hard'] = a_dict['any'] = time()
|
a_dict['hard'] = a_dict['any'] = time()
|
||||||
if victim_id not in v_dict:
|
if victim_id not in v_dict:
|
||||||
v_dict[victim_id] = dict()
|
v_dict[victim_id] = dict()
|
||||||
v_dict[victim_id]['hard'] = v_dict[victim_id]['any'] = time()
|
v_dict[victim_id]['hard'] = v_dict[victim_id]['any'] = time()
|
||||||
|
"""
|
||||||
else:
|
else:
|
||||||
a_dict['soft'] = a_dict['any'] = time()
|
a_dict['soft'] = a_dict['any'] = time()
|
||||||
if victim_id not in v_dict:
|
if victim_id not in v_dict:
|
||||||
v_dict[victim_id] = dict()
|
v_dict[victim_id] = dict()
|
||||||
v_dict[victim_id]['soft'] = v_dict[victim_id]['any'] = time()
|
v_dict[victim_id]['soft'] = v_dict[victim_id]['any'] = time()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
response_time = time() - time0
|
response_time = time() - time0
|
||||||
|
|
||||||
log('success: ' + str(success))
|
log('success: ' + str(success))
|
||||||
@ -1699,19 +1785,28 @@ def implement_corrective_action(
|
|||||||
|
|
||||||
kill_timestamp = time()
|
kill_timestamp = time()
|
||||||
|
|
||||||
while True:
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# ПОЧЕМУ по 2 раза отслеживаем? НАХУЙ ТАК ЖИТЬ
|
||||||
|
|
||||||
|
while True: # тест на чувствительность
|
||||||
victim_alive = is_victim_alive(pid)
|
victim_alive = is_victim_alive(pid)
|
||||||
dt = time() - a_dict['any']
|
dt = time() - a_dict['any']
|
||||||
if victim_alive == 2 or dt > 0.05:
|
if victim_alive == 2 or dt > 0.05:
|
||||||
break
|
break
|
||||||
sleep(0.005)
|
sleep(0.005)
|
||||||
|
|
||||||
if dt > 0.05:
|
if dt > 0.05:
|
||||||
log('Timer (value = 0.05 sec) expired; victim does not respond on action in 0.05 sec')
|
log('Timer (value = 0.05 sec) expired; victim does not respond'
|
||||||
|
' on action in 0.05 sec')
|
||||||
else:
|
else:
|
||||||
log('Process exited (VmRSS = 0) in {} sec'.format(
|
log('Process exited (VmRSS = 0) in {} sec'.format(
|
||||||
round(dt, 5)))
|
round(dt, 5)))
|
||||||
|
|
||||||
|
v_dict.pop(victim_id)
|
||||||
|
|
||||||
|
# непррерываемый цикл
|
||||||
if threshold is SIGKILL or victim_alive == 2:
|
if threshold is SIGKILL or victim_alive == 2:
|
||||||
# жертва умирает от SIGKILL. Дожидаемся ее полной смерти.
|
# жертва умирает от SIGKILL. Дожидаемся ее полной смерти.
|
||||||
# Сброс таймеа. Готовность к новым мягким
|
# Сброс таймеа. Готовность к новым мягким
|
||||||
@ -1731,6 +1826,19 @@ def implement_corrective_action(
|
|||||||
round(kill_duration, 3)))
|
round(kill_duration, 3)))
|
||||||
vwd = True
|
vwd = True
|
||||||
|
|
||||||
|
if victim_id in v_dict:
|
||||||
|
v_dict.pop(victim_id)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
psi_t0 = time()
|
psi_t0 = time()
|
||||||
|
|
||||||
# КОНЕЦ ОТСЛЕЖИВАНИЯ
|
# КОНЕЦ ОТСЛЕЖИВАНИЯ
|
||||||
@ -2571,7 +2679,7 @@ if 'extra_table_info' in config_dict:
|
|||||||
extra_table_info != 'cgroup_v2' and
|
extra_table_info != 'cgroup_v2' and
|
||||||
extra_table_info != 'cmdline' and
|
extra_table_info != 'cmdline' and
|
||||||
extra_table_info != 'environ' and
|
extra_table_info != 'environ' and
|
||||||
extra_table_info != 'realpath'):
|
extra_table_info != 'realpath'):
|
||||||
|
|
||||||
errprint('Invalid config: invalid extra_table_info value\nExit')
|
errprint('Invalid config: invalid extra_table_info value\nExit')
|
||||||
exit(1)
|
exit(1)
|
||||||
@ -2689,16 +2797,12 @@ else:
|
|||||||
|
|
||||||
|
|
||||||
if max_sleep < min_sleep:
|
if max_sleep < min_sleep:
|
||||||
errprint(
|
errprint('min_sleep value must not exceed max_sleep value.\nExit')
|
||||||
'max_sleep value must not exceed min_sleep value.\nExit'
|
|
||||||
)
|
|
||||||
exit(1)
|
exit(1)
|
||||||
|
|
||||||
|
|
||||||
if min_sleep < over_sleep:
|
if min_sleep < over_sleep:
|
||||||
errprint(
|
errprint('over_sleep value must not exceed min_sleep value.\nExit')
|
||||||
'min_sleep value must not exceed over_sleep value.\nExit'
|
|
||||||
)
|
|
||||||
exit(1)
|
exit(1)
|
||||||
|
|
||||||
|
|
||||||
@ -2886,7 +2990,8 @@ if print_config:
|
|||||||
log(' zram_max_warnings: {} MiB, {} %'.format(
|
log(' zram_max_warnings: {} MiB, {} %'.format(
|
||||||
round(zram_max_warnings_mb), round(zram_max_warnings_percent, 1)))
|
round(zram_max_warnings_mb), round(zram_max_warnings_percent, 1)))
|
||||||
log(' psi_avg_warnings: {}'.format(psi_avg_warnings))
|
log(' psi_avg_warnings: {}'.format(psi_avg_warnings))
|
||||||
log(' min_time_between_warnings: {} sec'.format(min_time_between_warnings))
|
log(' min_time_between_warnings: {} sec'.format(
|
||||||
|
min_time_between_warnings))
|
||||||
|
|
||||||
log('8. Verbosity')
|
log('8. Verbosity')
|
||||||
|
|
||||||
@ -2982,8 +3087,6 @@ threshold = None
|
|||||||
mem_info = None
|
mem_info = None
|
||||||
|
|
||||||
|
|
||||||
#print(x0, 'x0')
|
|
||||||
|
|
||||||
CHECK_PSI = False
|
CHECK_PSI = False
|
||||||
if psi_support and not ignore_psi:
|
if psi_support and not ignore_psi:
|
||||||
CHECK_PSI = True
|
CHECK_PSI = True
|
||||||
@ -3095,7 +3198,8 @@ while True:
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
if masf_threshold is SIGKILL or zram_threshold is SIGKILL or psi_threshold is SIGKILL:
|
if (masf_threshold is SIGKILL or zram_threshold is SIGKILL or
|
||||||
|
psi_threshold is SIGKILL):
|
||||||
|
|
||||||
threshold = SIGKILL
|
threshold = SIGKILL
|
||||||
mem_info_list = []
|
mem_info_list = []
|
||||||
@ -3118,7 +3222,8 @@ while True:
|
|||||||
x0, psi_threshold, zram_threshold, zram_info, psi_info)
|
x0, psi_threshold, zram_threshold, zram_info, psi_info)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if masf_threshold is SIGTERM or zram_threshold is SIGTERM or psi_threshold is SIGTERM:
|
if (masf_threshold is SIGTERM or zram_threshold is SIGTERM or
|
||||||
|
psi_threshold is SIGTERM):
|
||||||
|
|
||||||
threshold = SIGTERM
|
threshold = SIGTERM
|
||||||
mem_info_list = []
|
mem_info_list = []
|
||||||
@ -3143,7 +3248,8 @@ while True:
|
|||||||
|
|
||||||
if gui_low_memory_warnings:
|
if gui_low_memory_warnings:
|
||||||
|
|
||||||
if masf_threshold == 'WARN' or zram_threshold == 'WARN' or psi_threshold == 'WARN':
|
if (masf_threshold == 'WARN' or zram_threshold == 'WARN' or
|
||||||
|
psi_threshold == 'WARN'):
|
||||||
|
|
||||||
warn_time_delta = time() - warn_time_now
|
warn_time_delta = time() - warn_time_now
|
||||||
warn_time_now = time()
|
warn_time_now = time()
|
||||||
|
Loading…
Reference in New Issue
Block a user