add print_proc_table, add min_mem_report_interval
This commit is contained in:
Alexey Avramov
parent
3b5eafdaeb
commit
62f4948b39
127
nohang
127
nohang
@ -76,12 +76,14 @@ psi_path = '/proc/pressure/memory'
|
||||
|
||||
psi_support = os.path.exists(psi_path)
|
||||
|
||||
HR = '~' * 79
|
||||
HR = ''
|
||||
|
||||
# todo: make config option
|
||||
print_total_stat = True
|
||||
|
||||
debug = False
|
||||
print_proc_table = False
|
||||
|
||||
min_mem_report_interval = 10
|
||||
|
||||
|
||||
##########################################################################
|
||||
@ -133,7 +135,7 @@ def update_stat_dict_and_print(key):
|
||||
|
||||
if print_total_stat:
|
||||
|
||||
stats_msg = '{}\n\033[4mThe following corrective actions have been implemented in the last {}:\033[0m'.format(
|
||||
stats_msg = '{}\n\033[4mWhat happened in the last {}:\033[0m'.format(
|
||||
HR, format_time(time() - start_time))
|
||||
|
||||
for i in stat_dict:
|
||||
@ -153,7 +155,7 @@ def psi_mem_some_avg10():
|
||||
|
||||
def check_mem():
|
||||
"""find mem_available"""
|
||||
return int(rline1('/proc/meminfo').split(':')[1].strip(' kB\n'))
|
||||
return int(rline1('/proc/meminfo').split(':')[1][:-4])
|
||||
|
||||
|
||||
def check_mem_and_swap():
|
||||
@ -161,13 +163,13 @@ def check_mem_and_swap():
|
||||
with open('/proc/meminfo') as f:
|
||||
for n, line in enumerate(f):
|
||||
if n is 2:
|
||||
mem_available = int(line.split(':')[1].strip(' kB\n'))
|
||||
mem_available = int(line.split(':')[1][:-4])
|
||||
continue
|
||||
if n is swap_total_index:
|
||||
swap_total = int(line.split(':')[1].strip(' kB\n'))
|
||||
swap_total = int(line.split(':')[1][:-4])
|
||||
continue
|
||||
if n is swap_free_index:
|
||||
swap_free = int(line.split(':')[1].strip(' kB\n'))
|
||||
swap_free = int(line.split(':')[1][:-4])
|
||||
break
|
||||
return mem_available, swap_total, swap_free
|
||||
|
||||
@ -359,7 +361,6 @@ def pid_to_cmdline(pid):
|
||||
return f.read().replace('\x00', ' ').rstrip()
|
||||
|
||||
|
||||
|
||||
def pid_to_uid(pid):
|
||||
'''return euid'''
|
||||
try:
|
||||
@ -373,9 +374,6 @@ def pid_to_uid(pid):
|
||||
return f_list[uid_index].split('\t')[2]
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def notify_send_wait(title, body):
|
||||
'''GUI notifications with UID != 0'''
|
||||
with Popen(['notify-send', '--icon=dialog-warning', title, body]) as proc:
|
||||
@ -399,9 +397,6 @@ def notify_helper(title, body):
|
||||
title, body))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def send_notify_warn():
|
||||
"""
|
||||
Look for process with maximum 'badness' and warn user with notification.
|
||||
@ -431,11 +426,6 @@ def send_notify_warn():
|
||||
title = 'Low memory'
|
||||
'''
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
'''
|
||||
body = 'Next victim: {}[{}]'.format(
|
||||
name.replace(
|
||||
@ -460,7 +450,6 @@ def send_notify_warn():
|
||||
notify_send_wait(title, body)
|
||||
'''
|
||||
|
||||
|
||||
b = """{} 'Low memory' 'MemAvail: {}%\nSwapFree: {}%' &""".format(
|
||||
notify_helper_path,
|
||||
round(mem_available / mem_total * 100),
|
||||
@ -473,9 +462,6 @@ def send_notify_warn():
|
||||
print('t:', t1 - t0)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def send_notify(signal, name, pid):
|
||||
"""
|
||||
Notificate about OOM Preventing.
|
||||
@ -569,6 +555,8 @@ def fattest():
|
||||
-> find_mem_hog() or find_victim() or find_worst_process()
|
||||
"""
|
||||
|
||||
ft1 = time()
|
||||
|
||||
pid_list = get_pid_list()
|
||||
|
||||
pid_list.remove(self_pid)
|
||||
@ -584,14 +572,13 @@ def fattest():
|
||||
pid_badness_list = []
|
||||
|
||||
|
||||
if debug:
|
||||
if print_proc_table:
|
||||
print(' PID badness Name eUID')
|
||||
print('------- ------- --------------- ----------')
|
||||
|
||||
|
||||
for pid in pid_list:
|
||||
|
||||
|
||||
# find and modify badness (if it needs)
|
||||
try:
|
||||
|
||||
@ -620,9 +607,7 @@ def fattest():
|
||||
if search(re_tup[1], uid) is not None:
|
||||
badness += int(re_tup[0])
|
||||
|
||||
|
||||
|
||||
if debug:
|
||||
if print_proc_table:
|
||||
print('{} {} {} {}'.format(
|
||||
pid.rjust(7),
|
||||
str(badness).rjust(7),
|
||||
@ -631,8 +616,6 @@ def fattest():
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
except ProcessLookupError:
|
||||
@ -652,11 +635,18 @@ def fattest():
|
||||
# Get maximum 'badness' value
|
||||
victim_badness = pid_tuple_list[1]
|
||||
|
||||
print(
|
||||
'\nWorst process (PID: {}, Name: {}, badness: {}) found in {} ms'.format(
|
||||
pid,
|
||||
pid_to_name(pid),
|
||||
victim_badness,
|
||||
round((time() - ft1) * 1000)
|
||||
)
|
||||
)
|
||||
|
||||
return pid, victim_badness
|
||||
|
||||
|
||||
|
||||
|
||||
def find_victim_and_send_signal(signal):
|
||||
"""
|
||||
Find victim with highest badness and send SIGTERM/SIGKILL
|
||||
@ -925,7 +915,7 @@ def find_victim_and_send_signal(signal):
|
||||
|
||||
preventing_oom_message = '{}' \
|
||||
'\n\033[4mImplement a corrective action:\033[0m\n ' \
|
||||
'Sending \033[4m{}\033[0m to the victim; {}'.format(
|
||||
'Send \033[4m{}\033[0m to the victim; {}'.format(
|
||||
victim_info, sig_dict[signal], send_result)
|
||||
|
||||
key = 'Send \033[35m{}\033[0m to \033[35m{}\033[0m'.format(
|
||||
@ -938,12 +928,12 @@ def find_victim_and_send_signal(signal):
|
||||
response_time = time() - time0
|
||||
send_result = 'no such process; response time: {} ms'.format(
|
||||
round(response_time * 1000))
|
||||
key = 'The victim died in the search process: FileNotFoundError'
|
||||
key = '\033[33mFileNotFoundError\033[0m (the victim died in the search process): '
|
||||
except ProcessLookupError:
|
||||
response_time = time() - time0
|
||||
send_result = 'no such process; response time: {} ms'.format(
|
||||
round(response_time * 1000))
|
||||
key = 'The victim died in the search process: ProcessLookupError'
|
||||
key = '\033[33mProcessLookupError\033[0m (the victim died in the search process): '
|
||||
|
||||
print(mem_info)
|
||||
print(preventing_oom_message)
|
||||
@ -1087,7 +1077,7 @@ def calculate_percent(arg_key):
|
||||
|
||||
##########################################################################
|
||||
|
||||
# Lock all memory
|
||||
# Try to lock all memory
|
||||
|
||||
|
||||
mlockall()
|
||||
@ -1113,7 +1103,7 @@ if mem_list_names[2] != 'MemAvailable':
|
||||
swap_total_index = mem_list_names.index('SwapTotal')
|
||||
swap_free_index = swap_total_index + 1
|
||||
|
||||
mem_total = int(mem_list[0].split(':')[1].strip(' kB\n'))
|
||||
mem_total = int(mem_list[0].split(':')[1][:-4])
|
||||
|
||||
# Get names from /proc/*/status to be able to get VmRSS and VmSwap values
|
||||
|
||||
@ -1601,6 +1591,7 @@ if gui_notifications or gui_low_memory_warnings:
|
||||
notify_sig_dict = {SIGKILL: 'Killing',
|
||||
SIGTERM: 'Terminating'}
|
||||
|
||||
|
||||
rate_mem = rate_mem * 1048576
|
||||
rate_swap = rate_swap * 1048576
|
||||
rate_zram = rate_zram * 1048576
|
||||
@ -1613,7 +1604,7 @@ warn_time_now = 0
|
||||
warn_time_delta = 1000
|
||||
warn_timer = 0
|
||||
|
||||
if debug:
|
||||
if print_proc_table:
|
||||
print()
|
||||
fattest()
|
||||
print()
|
||||
@ -1636,6 +1627,14 @@ if psi_support and not ignore_psi:
|
||||
avg_value = ''
|
||||
|
||||
|
||||
if print_mem_check_results:
|
||||
|
||||
# to find delta mem
|
||||
wt2 = 0
|
||||
new_mem = 0
|
||||
|
||||
# init mem report interval
|
||||
report0 = 0
|
||||
|
||||
|
||||
while True:
|
||||
@ -1662,6 +1661,7 @@ while True:
|
||||
|
||||
mem_available, swap_total, swap_free = check_mem_and_swap()
|
||||
|
||||
|
||||
# print(mem_available, swap_total, swap_free)
|
||||
|
||||
# если метры - получаем киб выше и сразу. см.
|
||||
@ -1683,34 +1683,69 @@ while True:
|
||||
|
||||
if print_mem_check_results:
|
||||
|
||||
wt1 = time()
|
||||
|
||||
delta = (mem_available + swap_free) - new_mem
|
||||
|
||||
t_cycle = wt1 - wt2
|
||||
|
||||
report_delta = wt1 - report0
|
||||
|
||||
if report_delta >= min_mem_report_interval:
|
||||
|
||||
mem_report = True
|
||||
new_mem = mem_available + swap_free
|
||||
|
||||
report0 = wt1
|
||||
|
||||
else:
|
||||
mem_report = False
|
||||
|
||||
wt2 = time()
|
||||
|
||||
if mem_report:
|
||||
|
||||
speed = delta / 1024.0 / report_delta
|
||||
speed_info = ' | ΔMem, M/s: {}'.format(
|
||||
str(round(speed)).rjust(5)
|
||||
)
|
||||
|
||||
# Calculate 'swap-column' width
|
||||
swap_len = len(str(round(swap_total / 1024.0)))
|
||||
|
||||
# Output avialable mem sizes
|
||||
if swap_total == 0 and mem_used_zram == 0:
|
||||
print('{}MemAvail: {} M, {} %'.format(
|
||||
print('{}MemAvail: {} M, {} %{}'.format(
|
||||
avg_value,
|
||||
human(mem_available, mem_len),
|
||||
just_percent_mem(mem_available / mem_total)))
|
||||
just_percent_mem(mem_available / mem_total),
|
||||
speed_info
|
||||
)
|
||||
)
|
||||
|
||||
elif swap_total > 0 and mem_used_zram == 0:
|
||||
print('{}MemAvail: {} M, {} % | SwapFree: {} M, {} %'.format(
|
||||
print('{}MemAvail: {} M, {} % | SwapFree: {} M, {} %{}'.format(
|
||||
avg_value,
|
||||
human(mem_available, mem_len),
|
||||
just_percent_mem(mem_available / mem_total),
|
||||
human(swap_free, swap_len),
|
||||
just_percent_swap(swap_free / (swap_total + 0.1))))
|
||||
just_percent_swap(swap_free / (swap_total + 0.1)),
|
||||
speed_info)
|
||||
)
|
||||
|
||||
else:
|
||||
print('{}MemAvail: {} M, {} % | SwapFree: {} M, {} % | Mem'
|
||||
'UsedZram: {} M, {} %'.format(
|
||||
'UsedZram: {} M, {} %{}'.format(
|
||||
avg_value,
|
||||
human(mem_available, mem_len),
|
||||
just_percent_mem(mem_available / mem_total),
|
||||
human(swap_free, swap_len),
|
||||
just_percent_swap(swap_free / (swap_total + 0.1)),
|
||||
human(mem_used_zram, mem_len),
|
||||
just_percent_mem(mem_used_zram / mem_total)))
|
||||
just_percent_mem(mem_used_zram / mem_total),
|
||||
speed_info
|
||||
)
|
||||
)
|
||||
|
||||
# если swap_min_sigkill задан в абсолютной величине и Swap_total = 0
|
||||
if swap_total > swap_min_sigkill_kb: # If swap_min_sigkill is absolute
|
||||
@ -1813,8 +1848,11 @@ while True:
|
||||
percent(zram_max_sigterm_kb / mem_total))
|
||||
|
||||
find_victim_and_send_signal(SIGTERM)
|
||||
|
||||
# сделать одно время для обоих уровней.
|
||||
kill_psi_t0 = time()
|
||||
term_psi_t0 = time()
|
||||
# -> psi_t0
|
||||
|
||||
# LOW MEMORY WARNINGS
|
||||
elif gui_low_memory_warnings:
|
||||
@ -1835,4 +1873,5 @@ while True:
|
||||
|
||||
# SLEEP BETWEEN MEM CHECKS
|
||||
else:
|
||||
|
||||
sleep_after_check_mem()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user