github/nohang
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add print_proc_table, add min_mem_report_interval

Browse Source
This commit is contained in:
Alexey Avramov 2019-02-17 04:51:53 +09:00
parent 3b5eafdaeb
commit 62f4948b39
1 changed files with 102 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
nohang
View File

@ -76,12 +76,14 @@ psi_path = '/proc/pressure/memory'
psi_support = os.path.exists(psi_path) psi_support = os.path.exists(psi_path)
HR = '~' * 79 HR = ''
# todo: make config option # todo: make config option
print_total_stat = True print_total_stat = True
debug = False print_proc_table = False
min_mem_report_interval = 10
########################################################################## ##########################################################################
@ -133,7 +135,7 @@ def update_stat_dict_and_print(key):
if print_total_stat: if print_total_stat:
stats_msg = '{}\n\033[4mThe following corrective actions have been implemented in the last {}:\033[0m'.format( stats_msg = '{}\n\033[4mWhat happened in the last {}:\033[0m'.format(
HR, format_time(time() - start_time)) HR, format_time(time() - start_time))
for i in stat_dict: for i in stat_dict:
@ -153,7 +155,7 @@ def psi_mem_some_avg10():
def check_mem(): def check_mem():
"""find mem_available""" """find mem_available"""
return int(rline1('/proc/meminfo').split(':')[1].strip(' kB\n')) return int(rline1('/proc/meminfo').split(':')[1][:-4])
def check_mem_and_swap(): def check_mem_and_swap():
@ -161,13 +163,13 @@ def check_mem_and_swap():
with open('/proc/meminfo') as f: with open('/proc/meminfo') as f:
for n, line in enumerate(f): for n, line in enumerate(f):
if n is 2: if n is 2:
mem_available = int(line.split(':')[1].strip(' kB\n')) mem_available = int(line.split(':')[1][:-4])
continue continue
if n is swap_total_index: if n is swap_total_index:
swap_total = int(line.split(':')[1].strip(' kB\n')) swap_total = int(line.split(':')[1][:-4])
continue continue
if n is swap_free_index: if n is swap_free_index:
swap_free = int(line.split(':')[1].strip(' kB\n')) swap_free = int(line.split(':')[1][:-4])
break break
return mem_available, swap_total, swap_free return mem_available, swap_total, swap_free
@ -359,7 +361,6 @@ def pid_to_cmdline(pid):
return f.read().replace('\x00', ' ').rstrip() return f.read().replace('\x00', ' ').rstrip()
def pid_to_uid(pid): def pid_to_uid(pid):
'''return euid''' '''return euid'''
try: try:
@ -373,9 +374,6 @@ def pid_to_uid(pid):
return f_list[uid_index].split('\t')[2] return f_list[uid_index].split('\t')[2]
def notify_send_wait(title, body): def notify_send_wait(title, body):
'''GUI notifications with UID != 0''' '''GUI notifications with UID != 0'''
with Popen(['notify-send', '--icon=dialog-warning', title, body]) as proc: with Popen(['notify-send', '--icon=dialog-warning', title, body]) as proc:
@ -399,9 +397,6 @@ def notify_helper(title, body):
title, body)) title, body))
def send_notify_warn(): def send_notify_warn():
""" """
Look for process with maximum 'badness' and warn user with notification. Look for process with maximum 'badness' and warn user with notification.
@ -431,11 +426,6 @@ def send_notify_warn():
title = 'Low memory' title = 'Low memory'
''' '''
''' '''
body = 'Next victim: {}[{}]'.format( body = 'Next victim: {}[{}]'.format(
name.replace( name.replace(
@ -460,7 +450,6 @@ def send_notify_warn():
notify_send_wait(title, body) notify_send_wait(title, body)
''' '''
b = """{} 'Low memory' 'MemAvail: {}%\nSwapFree: {}%' &""".format( b = """{} 'Low memory' 'MemAvail: {}%\nSwapFree: {}%' &""".format(
notify_helper_path, notify_helper_path,
round(mem_available / mem_total * 100), round(mem_available / mem_total * 100),
@ -473,9 +462,6 @@ def send_notify_warn():
print('t:', t1 - t0) print('t:', t1 - t0)
def send_notify(signal, name, pid): def send_notify(signal, name, pid):
""" """
Notificate about OOM Preventing. Notificate about OOM Preventing.
@ -569,6 +555,8 @@ def fattest():
-> find_mem_hog() or find_victim() or find_worst_process() -> find_mem_hog() or find_victim() or find_worst_process()
""" """
ft1 = time()
pid_list = get_pid_list() pid_list = get_pid_list()
pid_list.remove(self_pid) pid_list.remove(self_pid)
@ -584,14 +572,13 @@ def fattest():
pid_badness_list = [] pid_badness_list = []
if debug: if print_proc_table:
print(' PID badness Name eUID') print(' PID badness Name eUID')
print('------- ------- --------------- ----------') print('------- ------- --------------- ----------')
for pid in pid_list: for pid in pid_list:
# find and modify badness (if it needs) # find and modify badness (if it needs)
try: try:
@ -620,9 +607,7 @@ def fattest():
if search(re_tup[1], uid) is not None: if search(re_tup[1], uid) is not None:
badness += int(re_tup[0]) badness += int(re_tup[0])
if print_proc_table:
if debug:
print('{} {} {} {}'.format( print('{} {} {} {}'.format(
pid.rjust(7), pid.rjust(7),
str(badness).rjust(7), str(badness).rjust(7),
@ -631,8 +616,6 @@ def fattest():
) )
) )
except FileNotFoundError: except FileNotFoundError:
continue continue
except ProcessLookupError: except ProcessLookupError:
@ -652,11 +635,18 @@ def fattest():
# Get maximum 'badness' value # Get maximum 'badness' value
victim_badness = pid_tuple_list[1] victim_badness = pid_tuple_list[1]
print(
'\nWorst process (PID: {}, Name: {}, badness: {}) found in {} ms'.format(
pid,
pid_to_name(pid),
victim_badness,
round((time() - ft1) * 1000)
)
)
return pid, victim_badness return pid, victim_badness
def find_victim_and_send_signal(signal): def find_victim_and_send_signal(signal):
""" """
Find victim with highest badness and send SIGTERM/SIGKILL Find victim with highest badness and send SIGTERM/SIGKILL
@ -925,7 +915,7 @@ def find_victim_and_send_signal(signal):
preventing_oom_message = '{}' \ preventing_oom_message = '{}' \
'\n\033[4mImplement a corrective action:\033[0m\n ' \ '\n\033[4mImplement a corrective action:\033[0m\n ' \
'Sending \033[4m{}\033[0m to the victim; {}'.format( 'Send \033[4m{}\033[0m to the victim; {}'.format(
victim_info, sig_dict[signal], send_result) victim_info, sig_dict[signal], send_result)
key = 'Send \033[35m{}\033[0m to \033[35m{}\033[0m'.format( key = 'Send \033[35m{}\033[0m to \033[35m{}\033[0m'.format(
@ -938,12 +928,12 @@ def find_victim_and_send_signal(signal):
response_time = time() - time0 response_time = time() - time0
send_result = 'no such process; response time: {} ms'.format( send_result = 'no such process; response time: {} ms'.format(
round(response_time * 1000)) round(response_time * 1000))
key = 'The victim died in the search process: FileNotFoundError' key = '\033[33mFileNotFoundError\033[0m (the victim died in the search process): '
except ProcessLookupError: except ProcessLookupError:
response_time = time() - time0 response_time = time() - time0
send_result = 'no such process; response time: {} ms'.format( send_result = 'no such process; response time: {} ms'.format(
round(response_time * 1000)) round(response_time * 1000))
key = 'The victim died in the search process: ProcessLookupError' key = '\033[33mProcessLookupError\033[0m (the victim died in the search process): '
print(mem_info) print(mem_info)
print(preventing_oom_message) print(preventing_oom_message)
@ -1087,7 +1077,7 @@ def calculate_percent(arg_key):
########################################################################## ##########################################################################
# Lock all memory # Try to lock all memory
mlockall() mlockall()
@ -1113,7 +1103,7 @@ if mem_list_names[2] != 'MemAvailable':
swap_total_index = mem_list_names.index('SwapTotal') swap_total_index = mem_list_names.index('SwapTotal')
swap_free_index = swap_total_index + 1 swap_free_index = swap_total_index + 1
mem_total = int(mem_list[0].split(':')[1].strip(' kB\n')) mem_total = int(mem_list[0].split(':')[1][:-4])
# Get names from /proc/*/status to be able to get VmRSS and VmSwap values # Get names from /proc/*/status to be able to get VmRSS and VmSwap values
@ -1601,6 +1591,7 @@ if gui_notifications or gui_low_memory_warnings:
notify_sig_dict = {SIGKILL: 'Killing', notify_sig_dict = {SIGKILL: 'Killing',
SIGTERM: 'Terminating'} SIGTERM: 'Terminating'}
rate_mem = rate_mem * 1048576 rate_mem = rate_mem * 1048576
rate_swap = rate_swap * 1048576 rate_swap = rate_swap * 1048576
rate_zram = rate_zram * 1048576 rate_zram = rate_zram * 1048576
@ -1613,7 +1604,7 @@ warn_time_now = 0
warn_time_delta = 1000 warn_time_delta = 1000
warn_timer = 0 warn_timer = 0
if debug: if print_proc_table:
print() print()
fattest() fattest()
print() print()
@ -1636,6 +1627,14 @@ if psi_support and not ignore_psi:
avg_value = '' avg_value = ''
if print_mem_check_results:
# to find delta mem
wt2 = 0
new_mem = 0
# init mem report interval
report0 = 0
while True: while True:
@ -1662,6 +1661,7 @@ while True:
mem_available, swap_total, swap_free = check_mem_and_swap() mem_available, swap_total, swap_free = check_mem_and_swap()
# print(mem_available, swap_total, swap_free) # print(mem_available, swap_total, swap_free)
# если метры - получаем киб выше и сразу. см. # если метры - получаем киб выше и сразу. см.
@ -1683,34 +1683,69 @@ while True:
if print_mem_check_results: if print_mem_check_results:
wt1 = time()
delta = (mem_available + swap_free) - new_mem
t_cycle = wt1 - wt2
report_delta = wt1 - report0
if report_delta >= min_mem_report_interval:
mem_report = True
new_mem = mem_available + swap_free
report0 = wt1
else:
mem_report = False
wt2 = time()
if mem_report:
speed = delta / 1024.0 / report_delta
speed_info = ' | ΔMem, M/s: {}'.format(
str(round(speed)).rjust(5)
)
# Calculate 'swap-column' width # Calculate 'swap-column' width
swap_len = len(str(round(swap_total / 1024.0))) swap_len = len(str(round(swap_total / 1024.0)))
# Output avialable mem sizes # Output avialable mem sizes
if swap_total == 0 and mem_used_zram == 0: if swap_total == 0 and mem_used_zram == 0:
print('{}MemAvail: {} M, {} %'.format( print('{}MemAvail: {} M, {} %{}'.format(
avg_value, avg_value,
human(mem_available, mem_len), human(mem_available, mem_len),
just_percent_mem(mem_available / mem_total))) just_percent_mem(mem_available / mem_total),
speed_info
)
)
elif swap_total > 0 and mem_used_zram == 0: elif swap_total > 0 and mem_used_zram == 0:
print('{}MemAvail: {} M, {} % | SwapFree: {} M, {} %'.format( print('{}MemAvail: {} M, {} % | SwapFree: {} M, {} %{}'.format(
avg_value, avg_value,
human(mem_available, mem_len), human(mem_available, mem_len),
just_percent_mem(mem_available / mem_total), just_percent_mem(mem_available / mem_total),
human(swap_free, swap_len), human(swap_free, swap_len),
just_percent_swap(swap_free / (swap_total + 0.1)))) just_percent_swap(swap_free / (swap_total + 0.1)),
speed_info)
)
else: else:
print('{}MemAvail: {} M, {} % | SwapFree: {} M, {} % | Mem' print('{}MemAvail: {} M, {} % | SwapFree: {} M, {} % | Mem'
'UsedZram: {} M, {} %'.format( 'UsedZram: {} M, {} %{}'.format(
avg_value, avg_value,
human(mem_available, mem_len), human(mem_available, mem_len),
just_percent_mem(mem_available / mem_total), just_percent_mem(mem_available / mem_total),
human(swap_free, swap_len), human(swap_free, swap_len),
just_percent_swap(swap_free / (swap_total + 0.1)), just_percent_swap(swap_free / (swap_total + 0.1)),
human(mem_used_zram, mem_len), human(mem_used_zram, mem_len),
just_percent_mem(mem_used_zram / mem_total))) just_percent_mem(mem_used_zram / mem_total),
speed_info
)
)
# если swap_min_sigkill задан в абсолютной величине и Swap_total = 0 # если swap_min_sigkill задан в абсолютной величине и Swap_total = 0
if swap_total > swap_min_sigkill_kb: # If swap_min_sigkill is absolute if swap_total > swap_min_sigkill_kb: # If swap_min_sigkill is absolute
@ -1813,8 +1848,11 @@ while True:
percent(zram_max_sigterm_kb / mem_total)) percent(zram_max_sigterm_kb / mem_total))
find_victim_and_send_signal(SIGTERM) find_victim_and_send_signal(SIGTERM)
# сделать одно время для обоих уровней.
kill_psi_t0 = time() kill_psi_t0 = time()
term_psi_t0 = time() term_psi_t0 = time()
# -> psi_t0
# LOW MEMORY WARNINGS # LOW MEMORY WARNINGS
elif gui_low_memory_warnings: elif gui_low_memory_warnings:
@ -1835,4 +1873,5 @@ while True:
# SLEEP BETWEEN MEM CHECKS # SLEEP BETWEEN MEM CHECKS
else: else:
sleep_after_check_mem() sleep_after_check_mem()