github/nohang
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix re matching and deduplicate code

Browse Source
This commit is contained in:
Alexey Avramov 2018-12-12 15:43:34 +09:00
parent 68f1af9aef
commit 8d5de95698
3 changed files with 99 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

226
nohang
View File

@ -221,8 +221,8 @@ def send_notify_warn():
""" """
# find process with max badness # find process with max badness
fat_tuple = fattest() fat_tuple = fattest()
pid = fat_tuple[1] pid = fat_tuple[0]
name = fat_tuple[0] name = pid_to_name(pid)
if mem_used_zram > 0: if mem_used_zram > 0:
low_mem_percent = '{}% {}% {}%'.format( low_mem_percent = '{}% {}% {}%'.format(
@ -325,17 +325,14 @@ def sleep_after_send_signal(signal):
sleep(min_delay_after_sigterm) sleep(min_delay_after_sigterm)
def find_victim_and_send_signal(signal): def fattest():
""" """Find the 'fattest' process, return pid and badness"""
Find victim with highest badness and send SIGTERM/SIGKILL
"""
print()
pid_badness_list = [] pid_badness_list = []
for pid in os.listdir('/proc'): for pid in os.listdir('/proc'):
# only directories whose names consist only of numbers, except /proc/1/ # only directories whose names consist only of numbers, except /proc/1/
if pid[0].isdecimal() is False or pid == '1': if pid[0].isdecimal() is False or pid is '1':
continue continue
# find and modify badness (if it needs) # find and modify badness (if it needs)
@ -344,74 +341,82 @@ def find_victim_and_send_signal(signal):
if decrease_oom_score_adj: if decrease_oom_score_adj:
oom_score_adj = int(rline1('/proc/' + pid + '/oom_score_adj')) oom_score_adj = int(rline1('/proc/' + pid + '/oom_score_adj'))
if badness > oom_score_adj_max: if badness > oom_score_adj_max and oom_score_adj > 0:
badness = badness - oom_score_adj + oom_score_adj_max badness = badness - oom_score_adj + oom_score_adj_max
if regex_matching: if regex_matching:
name = pid_to_name(pid) name = pid_to_name(pid)
cmdline = pid_to_cmdline(pid)
uid = pid_to_uid(pid)
# skip kthreads
if cmdline == '':
continue
#print([uid], [name], [cmdline])
if search(avoid_regex, name) is not None: if search(avoid_regex, name) is not None:
if pid_to_cmdline(pid) == '':
# skip kthreads
continue
badness = int(badness / avoid_factor) badness = int(badness / avoid_factor)
print(' Name matches with avoid_regex \033[33m{}\033[0m: \033[33m{}\033[0m, CmdLine: {}'.format(
avoid_regex, name, cmdline))
if search(prefer_regex, name) is not None: if search(prefer_regex, name) is not None:
if pid_to_cmdline(pid) == '':
# skip kthreads
continue
badness = int((badness + 1) * prefer_factor) badness = int((badness + 1) * prefer_factor)
print(' Name matches with prefer_regex \033[33m{}\033[0m: \033[33m{}\033[0m, CmdLine: {}'.format(
prefer_regex, name, cmdline)) if re_match_cmdline:
cmdline = pid_to_cmdline(pid)
if cmdline == '':
# skip kthreads
continue
if search(avoid_re_cmdline, cmdline) is not None: if search(avoid_re_cmdline, cmdline) is not None:
badness = int(badness / avoid_cmd_factor) badness = int(badness / avoid_cmd_factor)
print(' Cmdline matches with avoid_re_cmdline \033[33m{}\033[0m: \033[33m{}\033[0m, Name: {}'.format(
avoid_re_cmdline, cmdline, name))
if search(prefer_re_cmdline, cmdline) is not None: if search(prefer_re_cmdline, cmdline) is not None:
badness = int((badness + 1) * prefer_cmd_factor) badness = int((badness + 1) * prefer_cmd_factor)
print(' Cmdline matches with prefer_re_cmdline \033[33m{}\033[0m: \033[33m{}\033[0m, Name: {}'.format(
prefer_re_cmdline, cmdline, name))
if re_match_uid:
uid = pid_to_uid(pid)
if search(avoid_re_uid, uid) is not None: if search(avoid_re_uid, uid) is not None:
if pid_to_cmdline(pid) == '':
# skip kthreads
continue
badness = int(badness / avoid_uid_factor) badness = int(badness / avoid_uid_factor)
print(' UID matches with avoid_re_uid \033[33m{}\033[0m: \033[33m{}\033[0m, Name: {}'.format(
avoid_re_uid, uid, name))
if search(prefer_re_uid, uid) is not None: if search(prefer_re_uid, uid) is not None:
if pid_to_cmdline(pid) == '':
# skip kthreads
continue
badness = int((badness + 1) * prefer_uid_factor) badness = int((badness + 1) * prefer_uid_factor)
print(' UID matches with prefer_re_uid \033[33m{}\033[0m: \033[33m{}\033[0m, Name: {}'.format(
prefer_re_uid, uid, name))
except FileNotFoundError: except FileNotFoundError:
badness = 0 continue
except ProcessLookupError: except ProcessLookupError:
badness = 0 continue
pid_badness_list.append((pid, badness)) pid_badness_list.append((pid, badness))
# Make list of (pid, badness) tuples, sorted by 'badness' values # Make list of (pid, badness) tuples, sorted by 'badness' values
pid_tuple_list = sorted( pid_tuple_list = sorted(
pid_badness_list, key=itemgetter(1), reverse=True)[0] pid_badness_list, key=itemgetter(1), reverse=True)[0]
pid = pid_tuple_list[0]
# Get maximum 'badness' value # Get maximum 'badness' value
victim_badness = pid_tuple_list[1] victim_badness = pid_tuple_list[1]
return pid, victim_badness
def find_victim_and_send_signal(signal):
"""
Find victim with highest badness and send SIGTERM/SIGKILL
"""
print()
pid, victim_badness = fattest()
name = pid_to_name(pid)
if victim_badness >= min_badness: if victim_badness >= min_badness:
# Try to send signal to found victim # Try to send signal to found victim
pid = pid_tuple_list[0]
name = pid_to_name(pid)
# Get VmRSS and VmSwap and cmdline of victim process and try to send # Get VmRSS and VmSwap and cmdline of victim process and try to send
# signal # signal
try: try:
@ -467,17 +472,22 @@ def find_victim_and_send_signal(signal):
pass pass
if detailed_rss: if detailed_rss:
print('anon file shmem, MiB:', anon_rss, file_rss, shmem_rss) victim_info = ' Found the victim with highest badness:' \
'\n Name: \033[33m{}\033[0m' \
if execute_the_command and signal is SIGTERM and name in etc_dict: '\n PID: \033[33m{}\033[0m' \
command = etc_dict[name] '\n UID: \033[33m{}\033[0m' \
exit_status = os.system(etc_dict[name]) '\n Badness: \033[33m{}\033[0m' \
response_time = time() - time0 '\n VmSize: \033[33m{}\033[0m MiB' \
'\n VmRSS: \033[33m{}\033[0m MiB' \
# todo: mem_info, victim_info, exe_info, signal_info '\n Anon: \033[33m{}\033[0m MiB' \
# todo: display VmSize, oom_score, oom_score_adj '\n File: \033[33m{}\033[0m MiB' \
'\n Shmem: \033[33m{}\033[0m MiB' \
etc_info = ' Found the victim with highest badness:' \ '\n VmSwap: \033[33m{}\033[0m MiB' \
'\n CmdLine: \033[33m{}\033[0m'.format(
name, pid, uid, victim_badness, vm_size,
vm_rss, anon_rss, file_rss, shmem_rss, vm_swap, cmdline)
else:
victim_info = ' Found the victim with highest badness:' \
'\n Name: \033[33m{}\033[0m' \ '\n Name: \033[33m{}\033[0m' \
'\n PID: \033[33m{}\033[0m' \ '\n PID: \033[33m{}\033[0m' \
'\n UID: \033[33m{}\033[0m' \ '\n UID: \033[33m{}\033[0m' \
@ -485,11 +495,21 @@ def find_victim_and_send_signal(signal):
'\n VmSize: \033[33m{}\033[0m MiB' \ '\n VmSize: \033[33m{}\033[0m MiB' \
'\n VmRSS: \033[33m{}\033[0m MiB' \ '\n VmRSS: \033[33m{}\033[0m MiB' \
'\n VmSwap: \033[33m{}\033[0m MiB' \ '\n VmSwap: \033[33m{}\033[0m MiB' \
'\n CmdLine: \033[33m{}\033[0m' \ '\n CmdLine: \033[33m{}\033[0m'.format(
name, pid, uid, victim_badness, vm_size,
vm_rss, vm_swap, cmdline)
if execute_the_command and signal is SIGTERM and name in etc_dict:
command = etc_dict[name]
exit_status = os.system(etc_dict[name])
response_time = time() - time0
# todo: display oom_score, oom_score_adj
etc_info = '{}' \
'\n Execute the command: \033[4m{}\033[0m' \ '\n Execute the command: \033[4m{}\033[0m' \
'\n Exit status: {}; response time: {} ms'.format( '\n Exit status: {}; response time: {} ms'.format(
name, pid, uid, victim_badness, vm_size, vm_rss, vm_swap, victim_info, command, exit_status,
cmdline, command, exit_status,
round(response_time * 1000)) round(response_time * 1000))
print(mem_info) print(mem_info)
@ -517,18 +537,9 @@ def find_victim_and_send_signal(signal):
send_result = 'no such process; response time: {} ms'.format( send_result = 'no such process; response time: {} ms'.format(
round(response_time * 1000)) round(response_time * 1000))
preventing_oom_message = ' Found the process with highest badness:' \ preventing_oom_message = '{}' \
'\n Name: \033[33m{}\033[0m' \
'\n PID: \033[33m{}\033[0m' \
'\n UID: \033[33m{}\033[0m' \
'\n Badness: \033[33m{}\033[0m' \
'\n VmSize: \033[33m{}\033[0m MiB' \
'\n VmRSS: \033[33m{}\033[0m MiB' \
'\n VmSwap: \033[33m{}\033[0m MiB' \
'\n CmdLine: \033[33m{}\033[0m' \
'\n Sending \033[4m{}\033[0m to the victim; {}'.format( '\n Sending \033[4m{}\033[0m to the victim; {}'.format(
name, pid, uid, victim_badness, vm_size, vm_rss, vm_swap, victim_info, sig_dict[signal], send_result)
cmdline, sig_dict[signal], send_result)
print(mem_info) print(mem_info)
print(preventing_oom_message) print(preventing_oom_message)
@ -573,88 +584,6 @@ def sleep_after_check_mem():
exit() exit()
def fattest():
"""Find the 'fattest' process"""
pid_badness_list = []
for pid in os.listdir('/proc'):
# only directories whose names consist only of numbers, except /proc/1/
if pid[0].isdecimal() is False or pid == '1':
continue
# find and modify badness (if it needs)
try:
badness = int(rline1('/proc/' + pid + '/oom_score'))
if decrease_oom_score_adj:
oom_score_adj = int(rline1('/proc/' + pid + '/oom_score_adj'))
if badness > oom_score_adj_max:
badness = badness - oom_score_adj + oom_score_adj_max
if regex_matching:
name = pid_to_name(pid)
cmdline = pid_to_cmdline(pid)
uid = pid_to_uid(pid)
# skip kthreads
if cmdline == '':
continue
#print([uid], [name], [cmdline])
if search(avoid_regex, name) is not None:
badness = int(badness / avoid_factor)
print(' Name matches with avoid_regex \033[33m{}\033[0m: \033[33m{}\033[0m, CmdLine: {}'.format(
avoid_regex, name, cmdline))
if search(prefer_regex, name) is not None:
badness = int((badness + 1) * prefer_factor)
print(' Name matches with prefer_regex \033[33m{}\033[0m: \033[33m{}\033[0m, CmdLine: {}'.format(
prefer_regex, name, cmdline))
if search(avoid_re_cmdline, cmdline) is not None:
badness = int(badness / avoid_cmd_factor)
print(' Cmdline matches with avoid_re_cmdline \033[33m{}\033[0m: \033[33m{}\033[0m, Name: {}'.format(
avoid_re_cmdline, cmdline, name))
if search(prefer_re_cmdline, cmdline) is not None:
badness = int((badness + 1) * prefer_cmd_factor)
print(' Cmdline matches with prefer_re_cmdline \033[33m{}\033[0m: \033[33m{}\033[0m, Name: {}'.format(
prefer_re_cmdline, cmdline, name))
if search(avoid_re_uid, uid) is not None:
badness = int(badness / avoid_uid_factor)
print(' UID matches with avoid_re_uid \033[33m{}\033[0m: \033[33m{}\033[0m, Name: {}'.format(
avoid_re_uid, uid, name))
if search(prefer_re_uid, uid) is not None:
badness = int((badness + 1) * prefer_uid_factor)
print(' UID matches with prefer_re_uid \033[33m{}\033[0m: \033[33m{}\033[0m, Name: {}'.format(
prefer_re_uid, uid, name))
except FileNotFoundError:
badness = 0
except ProcessLookupError:
badness = 0
pid_badness_list.append((pid, badness))
# Make list of (pid, badness) tuples, sorted by 'badness' values
pid_tuple_list = sorted(
pid_badness_list, key=itemgetter(1), reverse=True)[0]
# Get maximum 'badness' value
victim_badness = pid_tuple_list[1]
pid = pid_tuple_list[0]
name = pid_to_name(pid)
return (name, pid)
def calculate_percent(arg_key): def calculate_percent(arg_key):
""" """
parse conf dict parse conf dict
@ -891,7 +820,14 @@ execute_the_command = conf_parse_bool('execute_the_command')
regex_matching = conf_parse_bool('regex_matching') regex_matching = conf_parse_bool('regex_matching')
if regex_matching: re_match_cmdline = conf_parse_bool('re_match_cmdline')
re_match_uid = conf_parse_bool('re_match_uid')
if regex_matching or re_match_cmdline or re_match_uid:
from re import search from re import search
import sre_constants import sre_constants

26
nohang.conf
View File

@ -120,18 +120,23 @@ oom_score_adj_max = 30
##################################################################### #####################################################################
4. Impact on the badness of processes via matching their names Adjusting the choice of the victim
and cmdlines with regular expressions using re.search().
4. Impact on the badness of processes via matching their names,
cmdlines or UIDs with regular expressions using re.search().
See https://en.wikipedia.org/wiki/Regular_expression and See https://en.wikipedia.org/wiki/Regular_expression and
https://en.wikipedia.org/wiki/Perl_Compatible_Regular_Expressions https://en.wikipedia.org/wiki/Perl_Compatible_Regular_Expressions
Enabling this option slows down the search for the victim Enabling this option slows down the search for the victim
because the names or cmdlines of all processes because the names, cmdlines or UIDs of all processes
(except init and kthreads) are compared with the (except init and kthreads) are compared with the
specified regex patterns (in fact slowing down is caused by specified regex patterns (in fact slowing down is caused by
reading all /proc/*/cmdline and /proc/*/status files). reading all /proc/*/cmdline and /proc/*/status files).
Use script `oom-sort` from nohang package to view
names, cmdlines and UIDs of processes.
Valid values are True and False. Valid values are True and False.
regex_matching = False regex_matching = False
@ -140,7 +145,7 @@ regex_matching = False
be calculated by the following formula: be calculated by the following formula:
badness = (oom_score + 1) * prefer_factor badness = (oom_score + 1) * prefer_factor
RE pattern must not be empty! RE patterns must be valid and must not be empty!
Matching process names with RE patterns Matching process names with RE patterns
@ -156,7 +161,7 @@ prefer_factor = 2
# Need more examples # Need more examples
avoid_regex = ^(Xorg|ssd)$ avoid_regex = ^(Xorg|sshd)$
Valid values are floating-point numbers from the range [1; 1000]. Valid values are floating-point numbers from the range [1; 1000].
@ -165,10 +170,11 @@ avoid_factor = 3
Matching cmdlines with RE patterns Matching cmdlines with RE patterns
# re_match_cmdline = True A good option that allows fine adjustment.
# this default pattern for prefer re_match_cmdline = False
# childs of firefox and chromium
# this default pattern for prefer childs of firefox and chromium
prefer_re_cmdline = -childID|--type=renderer prefer_re_cmdline = -childID|--type=renderer
prefer_cmd_factor = 12 prefer_cmd_factor = 12
@ -178,7 +184,9 @@ avoid_cmd_factor = 3
Matching UIDs with RE patterns Matching UIDs with RE patterns
# re_match_uid = True The most slow option
re_match_uid = False
prefer_re_uid = ^()$ prefer_re_uid = ^()$
prefer_uid_factor = 1 prefer_uid_factor = 1

2
nohang_notify_low_mem
View File

@ -103,6 +103,6 @@ if len(b) > 0:
'{}'.format(title), '{}'.format(body) '{}'.format(title), '{}'.format(body)
], env={ ], env={
display_key: display_value, dbus_key: dbus_value display_key: display_value, dbus_key: dbus_value
}).wait(3) })
else: else:
print('Low memory warnings: nobody logged in with GUI. Nothing to do.') print('Low memory warnings: nobody logged in with GUI. Nothing to do.')