fix re matching and deduplicate code
This commit is contained in:
Alexey Avramov
parent
68f1af9aef
commit
8d5de95698
226
nohang
226
nohang
@ -221,8 +221,8 @@ def send_notify_warn():
|
||||
"""
|
||||
# find process with max badness
|
||||
fat_tuple = fattest()
|
||||
pid = fat_tuple[1]
|
||||
name = fat_tuple[0]
|
||||
pid = fat_tuple[0]
|
||||
name = pid_to_name(pid)
|
||||
|
||||
if mem_used_zram > 0:
|
||||
low_mem_percent = '{}% {}% {}%'.format(
|
||||
@ -325,17 +325,14 @@ def sleep_after_send_signal(signal):
|
||||
sleep(min_delay_after_sigterm)
|
||||
|
||||
|
||||
def find_victim_and_send_signal(signal):
|
||||
"""
|
||||
Find victim with highest badness and send SIGTERM/SIGKILL
|
||||
"""
|
||||
print()
|
||||
def fattest():
|
||||
"""Find the 'fattest' process, return pid and badness"""
|
||||
|
||||
pid_badness_list = []
|
||||
|
||||
for pid in os.listdir('/proc'):
|
||||
# only directories whose names consist only of numbers, except /proc/1/
|
||||
if pid[0].isdecimal() is False or pid == '1':
|
||||
if pid[0].isdecimal() is False or pid is '1':
|
||||
continue
|
||||
|
||||
# find and modify badness (if it needs)
|
||||
@ -344,74 +341,82 @@ def find_victim_and_send_signal(signal):
|
||||
|
||||
if decrease_oom_score_adj:
|
||||
oom_score_adj = int(rline1('/proc/' + pid + '/oom_score_adj'))
|
||||
if badness > oom_score_adj_max:
|
||||
if badness > oom_score_adj_max and oom_score_adj > 0:
|
||||
badness = badness - oom_score_adj + oom_score_adj_max
|
||||
|
||||
if regex_matching:
|
||||
|
||||
name = pid_to_name(pid)
|
||||
cmdline = pid_to_cmdline(pid)
|
||||
uid = pid_to_uid(pid)
|
||||
|
||||
# skip kthreads
|
||||
if cmdline == '':
|
||||
continue
|
||||
|
||||
#print([uid], [name], [cmdline])
|
||||
|
||||
if search(avoid_regex, name) is not None:
|
||||
if pid_to_cmdline(pid) == '':
|
||||
# skip kthreads
|
||||
continue
|
||||
badness = int(badness / avoid_factor)
|
||||
print(' Name matches with avoid_regex \033[33m{}\033[0m: \033[33m{}\033[0m, CmdLine: {}'.format(
|
||||
avoid_regex, name, cmdline))
|
||||
|
||||
if search(prefer_regex, name) is not None:
|
||||
if pid_to_cmdline(pid) == '':
|
||||
# skip kthreads
|
||||
continue
|
||||
badness = int((badness + 1) * prefer_factor)
|
||||
print(' Name matches with prefer_regex \033[33m{}\033[0m: \033[33m{}\033[0m, CmdLine: {}'.format(
|
||||
prefer_regex, name, cmdline))
|
||||
|
||||
|
||||
if re_match_cmdline:
|
||||
cmdline = pid_to_cmdline(pid)
|
||||
if cmdline == '':
|
||||
# skip kthreads
|
||||
continue
|
||||
|
||||
if search(avoid_re_cmdline, cmdline) is not None:
|
||||
badness = int(badness / avoid_cmd_factor)
|
||||
print(' Cmdline matches with avoid_re_cmdline \033[33m{}\033[0m: \033[33m{}\033[0m, Name: {}'.format(
|
||||
avoid_re_cmdline, cmdline, name))
|
||||
|
||||
if search(prefer_re_cmdline, cmdline) is not None:
|
||||
badness = int((badness + 1) * prefer_cmd_factor)
|
||||
print(' Cmdline matches with prefer_re_cmdline \033[33m{}\033[0m: \033[33m{}\033[0m, Name: {}'.format(
|
||||
prefer_re_cmdline, cmdline, name))
|
||||
|
||||
if re_match_uid:
|
||||
uid = pid_to_uid(pid)
|
||||
|
||||
if search(avoid_re_uid, uid) is not None:
|
||||
if pid_to_cmdline(pid) == '':
|
||||
# skip kthreads
|
||||
continue
|
||||
badness = int(badness / avoid_uid_factor)
|
||||
print(' UID matches with avoid_re_uid \033[33m{}\033[0m: \033[33m{}\033[0m, Name: {}'.format(
|
||||
avoid_re_uid, uid, name))
|
||||
|
||||
if search(prefer_re_uid, uid) is not None:
|
||||
if pid_to_cmdline(pid) == '':
|
||||
# skip kthreads
|
||||
continue
|
||||
badness = int((badness + 1) * prefer_uid_factor)
|
||||
print(' UID matches with prefer_re_uid \033[33m{}\033[0m: \033[33m{}\033[0m, Name: {}'.format(
|
||||
prefer_re_uid, uid, name))
|
||||
|
||||
except FileNotFoundError:
|
||||
badness = 0
|
||||
continue
|
||||
except ProcessLookupError:
|
||||
badness = 0
|
||||
continue
|
||||
pid_badness_list.append((pid, badness))
|
||||
|
||||
# Make list of (pid, badness) tuples, sorted by 'badness' values
|
||||
pid_tuple_list = sorted(
|
||||
pid_badness_list, key=itemgetter(1), reverse=True)[0]
|
||||
|
||||
pid = pid_tuple_list[0]
|
||||
|
||||
# Get maximum 'badness' value
|
||||
victim_badness = pid_tuple_list[1]
|
||||
|
||||
return pid, victim_badness
|
||||
|
||||
|
||||
def find_victim_and_send_signal(signal):
|
||||
"""
|
||||
Find victim with highest badness and send SIGTERM/SIGKILL
|
||||
"""
|
||||
print()
|
||||
|
||||
pid, victim_badness = fattest()
|
||||
name = pid_to_name(pid)
|
||||
|
||||
if victim_badness >= min_badness:
|
||||
|
||||
# Try to send signal to found victim
|
||||
|
||||
pid = pid_tuple_list[0]
|
||||
|
||||
name = pid_to_name(pid)
|
||||
|
||||
# Get VmRSS and VmSwap and cmdline of victim process and try to send
|
||||
# signal
|
||||
try:
|
||||
@ -467,17 +472,22 @@ def find_victim_and_send_signal(signal):
|
||||
pass
|
||||
|
||||
if detailed_rss:
|
||||
print('anon file shmem, MiB:', anon_rss, file_rss, shmem_rss)
|
||||
|
||||
if execute_the_command and signal is SIGTERM and name in etc_dict:
|
||||
command = etc_dict[name]
|
||||
exit_status = os.system(etc_dict[name])
|
||||
response_time = time() - time0
|
||||
|
||||
# todo: mem_info, victim_info, exe_info, signal_info
|
||||
# todo: display VmSize, oom_score, oom_score_adj
|
||||
|
||||
etc_info = ' Found the victim with highest badness:' \
|
||||
victim_info = ' Found the victim with highest badness:' \
|
||||
'\n Name: \033[33m{}\033[0m' \
|
||||
'\n PID: \033[33m{}\033[0m' \
|
||||
'\n UID: \033[33m{}\033[0m' \
|
||||
'\n Badness: \033[33m{}\033[0m' \
|
||||
'\n VmSize: \033[33m{}\033[0m MiB' \
|
||||
'\n VmRSS: \033[33m{}\033[0m MiB' \
|
||||
'\n Anon: \033[33m{}\033[0m MiB' \
|
||||
'\n File: \033[33m{}\033[0m MiB' \
|
||||
'\n Shmem: \033[33m{}\033[0m MiB' \
|
||||
'\n VmSwap: \033[33m{}\033[0m MiB' \
|
||||
'\n CmdLine: \033[33m{}\033[0m'.format(
|
||||
name, pid, uid, victim_badness, vm_size,
|
||||
vm_rss, anon_rss, file_rss, shmem_rss, vm_swap, cmdline)
|
||||
else:
|
||||
victim_info = ' Found the victim with highest badness:' \
|
||||
'\n Name: \033[33m{}\033[0m' \
|
||||
'\n PID: \033[33m{}\033[0m' \
|
||||
'\n UID: \033[33m{}\033[0m' \
|
||||
@ -485,11 +495,21 @@ def find_victim_and_send_signal(signal):
|
||||
'\n VmSize: \033[33m{}\033[0m MiB' \
|
||||
'\n VmRSS: \033[33m{}\033[0m MiB' \
|
||||
'\n VmSwap: \033[33m{}\033[0m MiB' \
|
||||
'\n CmdLine: \033[33m{}\033[0m' \
|
||||
'\n CmdLine: \033[33m{}\033[0m'.format(
|
||||
name, pid, uid, victim_badness, vm_size,
|
||||
vm_rss, vm_swap, cmdline)
|
||||
|
||||
if execute_the_command and signal is SIGTERM and name in etc_dict:
|
||||
command = etc_dict[name]
|
||||
exit_status = os.system(etc_dict[name])
|
||||
response_time = time() - time0
|
||||
|
||||
# todo: display oom_score, oom_score_adj
|
||||
|
||||
etc_info = '{}' \
|
||||
'\n Execute the command: \033[4m{}\033[0m' \
|
||||
'\n Exit status: {}; response time: {} ms'.format(
|
||||
name, pid, uid, victim_badness, vm_size, vm_rss, vm_swap,
|
||||
cmdline, command, exit_status,
|
||||
victim_info, command, exit_status,
|
||||
round(response_time * 1000))
|
||||
|
||||
print(mem_info)
|
||||
@ -517,18 +537,9 @@ def find_victim_and_send_signal(signal):
|
||||
send_result = 'no such process; response time: {} ms'.format(
|
||||
round(response_time * 1000))
|
||||
|
||||
preventing_oom_message = ' Found the process with highest badness:' \
|
||||
'\n Name: \033[33m{}\033[0m' \
|
||||
'\n PID: \033[33m{}\033[0m' \
|
||||
'\n UID: \033[33m{}\033[0m' \
|
||||
'\n Badness: \033[33m{}\033[0m' \
|
||||
'\n VmSize: \033[33m{}\033[0m MiB' \
|
||||
'\n VmRSS: \033[33m{}\033[0m MiB' \
|
||||
'\n VmSwap: \033[33m{}\033[0m MiB' \
|
||||
'\n CmdLine: \033[33m{}\033[0m' \
|
||||
preventing_oom_message = '{}' \
|
||||
'\n Sending \033[4m{}\033[0m to the victim; {}'.format(
|
||||
name, pid, uid, victim_badness, vm_size, vm_rss, vm_swap,
|
||||
cmdline, sig_dict[signal], send_result)
|
||||
victim_info, sig_dict[signal], send_result)
|
||||
print(mem_info)
|
||||
print(preventing_oom_message)
|
||||
|
||||
@ -573,88 +584,6 @@ def sleep_after_check_mem():
|
||||
exit()
|
||||
|
||||
|
||||
def fattest():
|
||||
"""Find the 'fattest' process"""
|
||||
pid_badness_list = []
|
||||
|
||||
for pid in os.listdir('/proc'):
|
||||
# only directories whose names consist only of numbers, except /proc/1/
|
||||
if pid[0].isdecimal() is False or pid == '1':
|
||||
continue
|
||||
|
||||
# find and modify badness (if it needs)
|
||||
try:
|
||||
badness = int(rline1('/proc/' + pid + '/oom_score'))
|
||||
|
||||
if decrease_oom_score_adj:
|
||||
oom_score_adj = int(rline1('/proc/' + pid + '/oom_score_adj'))
|
||||
if badness > oom_score_adj_max:
|
||||
badness = badness - oom_score_adj + oom_score_adj_max
|
||||
|
||||
if regex_matching:
|
||||
|
||||
name = pid_to_name(pid)
|
||||
cmdline = pid_to_cmdline(pid)
|
||||
uid = pid_to_uid(pid)
|
||||
|
||||
# skip kthreads
|
||||
if cmdline == '':
|
||||
continue
|
||||
|
||||
#print([uid], [name], [cmdline])
|
||||
|
||||
if search(avoid_regex, name) is not None:
|
||||
badness = int(badness / avoid_factor)
|
||||
print(' Name matches with avoid_regex \033[33m{}\033[0m: \033[33m{}\033[0m, CmdLine: {}'.format(
|
||||
avoid_regex, name, cmdline))
|
||||
|
||||
if search(prefer_regex, name) is not None:
|
||||
badness = int((badness + 1) * prefer_factor)
|
||||
print(' Name matches with prefer_regex \033[33m{}\033[0m: \033[33m{}\033[0m, CmdLine: {}'.format(
|
||||
prefer_regex, name, cmdline))
|
||||
|
||||
|
||||
if search(avoid_re_cmdline, cmdline) is not None:
|
||||
badness = int(badness / avoid_cmd_factor)
|
||||
print(' Cmdline matches with avoid_re_cmdline \033[33m{}\033[0m: \033[33m{}\033[0m, Name: {}'.format(
|
||||
avoid_re_cmdline, cmdline, name))
|
||||
|
||||
if search(prefer_re_cmdline, cmdline) is not None:
|
||||
badness = int((badness + 1) * prefer_cmd_factor)
|
||||
print(' Cmdline matches with prefer_re_cmdline \033[33m{}\033[0m: \033[33m{}\033[0m, Name: {}'.format(
|
||||
prefer_re_cmdline, cmdline, name))
|
||||
|
||||
|
||||
if search(avoid_re_uid, uid) is not None:
|
||||
badness = int(badness / avoid_uid_factor)
|
||||
print(' UID matches with avoid_re_uid \033[33m{}\033[0m: \033[33m{}\033[0m, Name: {}'.format(
|
||||
avoid_re_uid, uid, name))
|
||||
|
||||
if search(prefer_re_uid, uid) is not None:
|
||||
badness = int((badness + 1) * prefer_uid_factor)
|
||||
print(' UID matches with prefer_re_uid \033[33m{}\033[0m: \033[33m{}\033[0m, Name: {}'.format(
|
||||
prefer_re_uid, uid, name))
|
||||
|
||||
except FileNotFoundError:
|
||||
badness = 0
|
||||
except ProcessLookupError:
|
||||
badness = 0
|
||||
pid_badness_list.append((pid, badness))
|
||||
|
||||
# Make list of (pid, badness) tuples, sorted by 'badness' values
|
||||
pid_tuple_list = sorted(
|
||||
pid_badness_list, key=itemgetter(1), reverse=True)[0]
|
||||
|
||||
# Get maximum 'badness' value
|
||||
victim_badness = pid_tuple_list[1]
|
||||
|
||||
pid = pid_tuple_list[0]
|
||||
|
||||
name = pid_to_name(pid)
|
||||
|
||||
return (name, pid)
|
||||
|
||||
|
||||
def calculate_percent(arg_key):
|
||||
"""
|
||||
parse conf dict
|
||||
@ -891,7 +820,14 @@ execute_the_command = conf_parse_bool('execute_the_command')
|
||||
|
||||
regex_matching = conf_parse_bool('regex_matching')
|
||||
|
||||
if regex_matching:
|
||||
re_match_cmdline = conf_parse_bool('re_match_cmdline')
|
||||
|
||||
re_match_uid = conf_parse_bool('re_match_uid')
|
||||
|
||||
|
||||
|
||||
|
||||
if regex_matching or re_match_cmdline or re_match_uid:
|
||||
from re import search
|
||||
import sre_constants
|
||||
|
||||
|
||||
26
nohang.conf
26
nohang.conf
@ -120,18 +120,23 @@ oom_score_adj_max = 30
|
||||
|
||||
#####################################################################
|
||||
|
||||
4. Impact on the badness of processes via matching their names
|
||||
and cmdlines with regular expressions using re.search().
|
||||
Adjusting the choice of the victim
|
||||
|
||||
4. Impact on the badness of processes via matching their names,
|
||||
cmdlines or UIDs with regular expressions using re.search().
|
||||
|
||||
See https://en.wikipedia.org/wiki/Regular_expression and
|
||||
https://en.wikipedia.org/wiki/Perl_Compatible_Regular_Expressions
|
||||
|
||||
Enabling this option slows down the search for the victim
|
||||
because the names or cmdlines of all processes
|
||||
because the names, cmdlines or UIDs of all processes
|
||||
(except init and kthreads) are compared with the
|
||||
specified regex patterns (in fact slowing down is caused by
|
||||
reading all /proc/*/cmdline and /proc/*/status files).
|
||||
|
||||
Use script `oom-sort` from nohang package to view
|
||||
names, cmdlines and UIDs of processes.
|
||||
|
||||
Valid values are True and False.
|
||||
|
||||
regex_matching = False
|
||||
@ -140,7 +145,7 @@ regex_matching = False
|
||||
be calculated by the following formula:
|
||||
badness = (oom_score + 1) * prefer_factor
|
||||
|
||||
RE pattern must not be empty!
|
||||
RE patterns must be valid and must not be empty!
|
||||
|
||||
Matching process names with RE patterns
|
||||
|
||||
@ -156,7 +161,7 @@ prefer_factor = 2
|
||||
|
||||
# Need more examples
|
||||
|
||||
avoid_regex = ^(Xorg|ssd)$
|
||||
avoid_regex = ^(Xorg|sshd)$
|
||||
|
||||
Valid values are floating-point numbers from the range [1; 1000].
|
||||
|
||||
@ -165,10 +170,11 @@ avoid_factor = 3
|
||||
|
||||
Matching cmdlines with RE patterns
|
||||
|
||||
# re_match_cmdline = True
|
||||
A good option that allows fine adjustment.
|
||||
|
||||
# this default pattern for prefer
|
||||
# childs of firefox and chromium
|
||||
re_match_cmdline = False
|
||||
|
||||
# this default pattern for prefer childs of firefox and chromium
|
||||
prefer_re_cmdline = -childID|--type=renderer
|
||||
prefer_cmd_factor = 12
|
||||
|
||||
@ -178,7 +184,9 @@ avoid_cmd_factor = 3
|
||||
|
||||
Matching UIDs with RE patterns
|
||||
|
||||
# re_match_uid = True
|
||||
The most slow option
|
||||
|
||||
re_match_uid = False
|
||||
|
||||
prefer_re_uid = ^()$
|
||||
prefer_uid_factor = 1
|
||||
|
||||
@ -103,6 +103,6 @@ if len(b) > 0:
|
||||
'{}'.format(title), '{}'.format(body)
|
||||
], env={
|
||||
display_key: display_value, dbus_key: dbus_value
|
||||
}).wait(3)
|
||||
})
|
||||
else:
|
||||
print('Low memory warnings: nobody logged in with GUI. Nothing to do.')
|
||||
|
||||
Loading…
Reference in New Issue
Block a user