files relocation
This commit is contained in:
Alexey Avramov
276
tools/oom-sort
Executable file
276
tools/oom-sort
Executable file
@@ -0,0 +1,276 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
sort processes by oom_score
|
||||
"""
|
||||
|
||||
|
||||
from operator import itemgetter
|
||||
from os import listdir
|
||||
from argparse import ArgumentParser
|
||||
|
||||
##########################################################################
|
||||
|
||||
|
||||
# define funtcions
|
||||
|
||||
|
||||
def pid_to_oom_score(pid):
|
||||
with open('/proc/' + pid + '/oom_score') as f:
|
||||
return f.readline()[:-1]
|
||||
|
||||
|
||||
def pid_to_oom_score_adj(pid):
|
||||
with open('/proc/' + pid + '/oom_score_adj') as f:
|
||||
return f.readline()[:-1]
|
||||
|
||||
|
||||
def pid_to_cmdline(pid):
|
||||
"""
|
||||
Get process cmdline by pid.
|
||||
|
||||
pid: str pid of required process
|
||||
returns string cmdline
|
||||
"""
|
||||
with open('/proc/' + pid + '/cmdline') as f:
|
||||
return f.read().replace('\x00', ' ').rstrip()
|
||||
|
||||
|
||||
def pid_to_status_units(pid):
|
||||
|
||||
with open('/proc/' + pid + '/status', 'rb') as f:
|
||||
f_list = f.read().decode('utf-8', 'ignore').split('\n')
|
||||
|
||||
for i in range(len(f_list)):
|
||||
|
||||
if i == 1:
|
||||
name = f_list[0].split('\t')[1]
|
||||
|
||||
if i == uid_index:
|
||||
uid = f_list[i].split('\t')[2]
|
||||
|
||||
if i == vm_rss_index:
|
||||
vm_rss = f_list[i].split('\t')[1][:-3]
|
||||
|
||||
if i == vm_swap_index:
|
||||
vm_swap = f_list[i].split('\t')[1][:-3]
|
||||
|
||||
return name, uid, vm_rss, vm_swap
|
||||
|
||||
|
||||
def get_max_pid_len():
|
||||
with open('/proc/sys/kernel/pid_max') as file:
|
||||
for line in file:
|
||||
return len(line.strip())
|
||||
|
||||
|
||||
##########################################################################
|
||||
|
||||
|
||||
sort_dict = {
|
||||
'PID': 0,
|
||||
'oom_score': 1,
|
||||
'oom_score_adj': 2,
|
||||
'cmdline': 3,
|
||||
'Name': 4,
|
||||
'UID': 5,
|
||||
'VmRSS': 6,
|
||||
'VmSwap': 7
|
||||
}
|
||||
|
||||
|
||||
##########################################################################
|
||||
|
||||
|
||||
# parse CLI args
|
||||
|
||||
# @TODO: user input validation
|
||||
|
||||
parser = ArgumentParser()
|
||||
|
||||
parser.add_argument(
|
||||
'--num',
|
||||
'-n',
|
||||
help="""max number of lines; default: 99999""",
|
||||
default=99999,
|
||||
type=str
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--len',
|
||||
'-l',
|
||||
help="""max cmdline length; default: 99999""",
|
||||
default=99999,
|
||||
type=int
|
||||
)
|
||||
|
||||
|
||||
parser.add_argument(
|
||||
'--sort',
|
||||
'-s',
|
||||
help="""sort by unit; default: oom_score""",
|
||||
default='oom_score',
|
||||
type=str
|
||||
)
|
||||
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
display_cmdline = args.len
|
||||
|
||||
num_lines = args.num
|
||||
|
||||
sort_by = args.sort
|
||||
|
||||
|
||||
if sort_by not in sort_dict:
|
||||
print('Invalid -s/--sort value. Valid values are:\nPID\noom_scor'
|
||||
'e [default value]\noom-sore_adj\nUID\nName\ncmdline\nVmR'
|
||||
'SS\nVmSwap')
|
||||
exit()
|
||||
|
||||
|
||||
##########################################################################
|
||||
|
||||
# find VmRSS, VmSwap and UID positions in /proc/*/status for further
|
||||
# searching positions of UID, VmRSS and VmSwap in each process
|
||||
|
||||
with open('/proc/self/status') as file:
|
||||
status_list = file.readlines()
|
||||
|
||||
status_names = []
|
||||
for s in status_list:
|
||||
status_names.append(s.split(':')[0])
|
||||
|
||||
uid_index = status_names.index('Uid')
|
||||
vm_rss_index = status_names.index('VmRSS')
|
||||
vm_swap_index = status_names.index('VmSwap')
|
||||
|
||||
|
||||
##########################################################################
|
||||
|
||||
# get sorted list with pid, oom_score, oom_score_adj, cmdline
|
||||
# get status units: name, uid, rss, swap
|
||||
|
||||
|
||||
oom_list = []
|
||||
|
||||
for pid in listdir('/proc'):
|
||||
|
||||
# skip non-numeric entries and PID 1
|
||||
if pid.isdigit() is False or pid == '1':
|
||||
continue
|
||||
|
||||
try:
|
||||
|
||||
oom_score = pid_to_oom_score(pid)
|
||||
|
||||
oom_score_adj = pid_to_oom_score_adj(pid)
|
||||
|
||||
cmdline = pid_to_cmdline(pid)
|
||||
if cmdline == '':
|
||||
continue
|
||||
|
||||
name, uid, vm_rss, vm_swap = pid_to_status_units(pid)
|
||||
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
|
||||
except ProcessLookupError:
|
||||
continue
|
||||
|
||||
oom_list.append((
|
||||
int(pid), int(oom_score), int(oom_score_adj), cmdline,
|
||||
name, int(uid), int(vm_rss), int(vm_swap)))
|
||||
|
||||
# list sorted by oom_score
|
||||
oom_list_sorted = sorted(
|
||||
oom_list, key=itemgetter(int(sort_dict[sort_by])), reverse=True)
|
||||
|
||||
|
||||
##########################################################################
|
||||
|
||||
# find width of columns
|
||||
|
||||
|
||||
max_pid_len = get_max_pid_len()
|
||||
|
||||
|
||||
max_uid_len = len(str(sorted(
|
||||
oom_list, key=itemgetter(5), reverse=True)[0][5]))
|
||||
|
||||
|
||||
max_vm_rss_len = len(str(round(
|
||||
sorted(oom_list, key=itemgetter(6), reverse=True)[0][6] / 1024.0)))
|
||||
|
||||
if max_vm_rss_len < 5:
|
||||
max_vm_rss_len = 5
|
||||
|
||||
|
||||
##########################################################################
|
||||
|
||||
|
||||
# print output
|
||||
|
||||
if display_cmdline == 0:
|
||||
|
||||
print(
|
||||
'oom_score oom_score_adj{}UID{}PID Name {}VmRSS VmSwap'.format(
|
||||
' ' * (max_uid_len - 2),
|
||||
' ' * (max_pid_len - 2),
|
||||
' ' * max_vm_rss_len
|
||||
)
|
||||
)
|
||||
|
||||
print(
|
||||
'--------- ------------- {} {} --------------- {}-- --------'.format(
|
||||
'-' * max_uid_len,
|
||||
'-' * max_pid_len,
|
||||
'-' * max_vm_rss_len
|
||||
)
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
print(
|
||||
'oom_score oom_score_adj{}UID{}PID Name {}VmRSS VmSwa'
|
||||
'p cmdline'.format(
|
||||
' ' * (max_uid_len - 2),
|
||||
' ' * (max_pid_len - 2),
|
||||
' ' * max_vm_rss_len
|
||||
)
|
||||
)
|
||||
|
||||
print(
|
||||
'--------- ------------- {} {} --------------- {}-- ------'
|
||||
'-- -------'.format(
|
||||
'-' * max_uid_len,
|
||||
'-' * max_pid_len,
|
||||
'-' * max_vm_rss_len
|
||||
)
|
||||
)
|
||||
|
||||
# print processes stats sorted by sort_dict[sort_by]
|
||||
|
||||
for i in oom_list_sorted[:int(num_lines)]:
|
||||
|
||||
pid = i[0]
|
||||
oom_score = i[1]
|
||||
oom_score_adj = i[2]
|
||||
cmdline = i[3]
|
||||
name = i[4]
|
||||
uid = i[5]
|
||||
vm_rss = i[6]
|
||||
vm_swap = i[7]
|
||||
|
||||
print(
|
||||
'{} {} {} {} {} {} M {} M {}'.format(
|
||||
str(oom_score).rjust(9),
|
||||
str(oom_score_adj).rjust(13),
|
||||
str(uid).rjust(max_uid_len),
|
||||
str(pid).rjust(max_pid_len),
|
||||
name.ljust(15),
|
||||
str(round(vm_rss / 1024.0)).rjust(max_vm_rss_len, ' '),
|
||||
str(round(vm_swap / 1024.0)).rjust(6, ' '),
|
||||
cmdline[:display_cmdline]
|
||||
)
|
||||
)
|
||||
5
tools/oom-sort.1
Normal file
5
tools/oom-sort.1
Normal file
@@ -0,0 +1,5 @@
|
||||
.TH oom-sort 1
|
||||
|
||||
.SH NAME
|
||||
|
||||
oom-sort \- sort processes by oom_score
|
||||
263
tools/psi-monitor
Executable file
263
tools/psi-monitor
Executable file
@@ -0,0 +1,263 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
from time import sleep
|
||||
from ctypes import CDLL
|
||||
from argparse import ArgumentParser
|
||||
|
||||
|
||||
###############################################################################
|
||||
|
||||
|
||||
def cgroup2_root():
|
||||
"""
|
||||
"""
|
||||
with open(mounts) as f:
|
||||
for line in f:
|
||||
if cgroup2_separator in line:
|
||||
return line.partition(cgroup2_separator)[0].partition(' ')[2]
|
||||
|
||||
|
||||
def mlockall():
|
||||
"""
|
||||
"""
|
||||
|
||||
MCL_CURRENT = 1
|
||||
MCL_FUTURE = 2
|
||||
MCL_ONFAULT = 4
|
||||
|
||||
libc = CDLL('libc.so.6', use_errno=True)
|
||||
|
||||
result = libc.mlockall(
|
||||
MCL_CURRENT | MCL_FUTURE | MCL_ONFAULT
|
||||
)
|
||||
if result != 0:
|
||||
result = libc.mlockall(
|
||||
MCL_CURRENT | MCL_FUTURE
|
||||
)
|
||||
if result != 0:
|
||||
pass
|
||||
else:
|
||||
pass
|
||||
else:
|
||||
pass
|
||||
|
||||
|
||||
def psi_file_mem_to_metrics(psi_path):
|
||||
"""
|
||||
"""
|
||||
|
||||
with open(psi_path) as f:
|
||||
psi_list = f.readlines()
|
||||
# print(psi_list)
|
||||
some_list, full_list = psi_list[0].split(' '), psi_list[1].split(' ')
|
||||
# print(some_list, full_list)
|
||||
some_avg10 = some_list[1].split('=')[1]
|
||||
some_avg60 = some_list[2].split('=')[1]
|
||||
some_avg300 = some_list[3].split('=')[1]
|
||||
|
||||
full_avg10 = full_list[1].split('=')[1]
|
||||
full_avg60 = full_list[2].split('=')[1]
|
||||
full_avg300 = full_list[3].split('=')[1]
|
||||
|
||||
return (some_avg10, some_avg60, some_avg300,
|
||||
full_avg10, full_avg60, full_avg300)
|
||||
|
||||
|
||||
def psi_file_cpu_to_metrics(psi_path):
|
||||
"""
|
||||
"""
|
||||
|
||||
with open(psi_path) as f:
|
||||
psi_list = f.readlines()
|
||||
# print(psi_list)
|
||||
some_list = psi_list[0].split(' ')
|
||||
# print(some_list, full_list)
|
||||
some_avg10 = some_list[1].split('=')[1]
|
||||
some_avg60 = some_list[2].split('=')[1]
|
||||
some_avg300 = some_list[3].split('=')[1]
|
||||
|
||||
return (some_avg10, some_avg60, some_avg300)
|
||||
|
||||
|
||||
def log(*msg):
|
||||
"""
|
||||
"""
|
||||
|
||||
print(*msg)
|
||||
|
||||
if separate_log:
|
||||
logging.info(*msg)
|
||||
|
||||
###############################################################################
|
||||
|
||||
|
||||
parser = ArgumentParser()
|
||||
|
||||
parser.add_argument(
|
||||
'-t',
|
||||
'--target',
|
||||
help="""target (cgroup2 or SYTSTEM_WIDE)""",
|
||||
default='SYSTEM_WIDE',
|
||||
type=str
|
||||
)
|
||||
|
||||
|
||||
parser.add_argument(
|
||||
'-p',
|
||||
'--period',
|
||||
help="""period in sec""",
|
||||
default=2,
|
||||
type=float
|
||||
)
|
||||
|
||||
|
||||
parser.add_argument(
|
||||
'-l',
|
||||
'--log',
|
||||
help="""path to log file""",
|
||||
default=None,
|
||||
type=str
|
||||
)
|
||||
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
target = args.target
|
||||
|
||||
period = args.period
|
||||
|
||||
log_file = args.log
|
||||
|
||||
|
||||
if log_file is None:
|
||||
separate_log = False
|
||||
else:
|
||||
separate_log = True
|
||||
import logging
|
||||
|
||||
|
||||
def sleeeep():
|
||||
try:
|
||||
sleep(period)
|
||||
except KeyboardInterrupt:
|
||||
log('Exit')
|
||||
exit()
|
||||
|
||||
|
||||
###############################################################################
|
||||
|
||||
if separate_log:
|
||||
logging.basicConfig(
|
||||
filename=log_file,
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s: %(message)s")
|
||||
|
||||
###############################################################################
|
||||
|
||||
|
||||
log('target: {}'.format(target))
|
||||
log('period: {}'.format(period))
|
||||
if log_file is not None:
|
||||
log('log file: {}'.format(log_file))
|
||||
|
||||
|
||||
if not os.path.exists('/proc/pressure'):
|
||||
log('ERROR: /proc/pressure does not exist, PSI is not supported, exit')
|
||||
exit()
|
||||
|
||||
|
||||
if target == 'SYSTEM_WIDE':
|
||||
|
||||
cpu_file = "/proc/pressure/cpu"
|
||||
memory_file = "/proc/pressure/memory"
|
||||
io_file = "/proc/pressure/io"
|
||||
|
||||
else:
|
||||
|
||||
mounts = '/proc/mounts'
|
||||
cgroup2_separator = ' cgroup2 rw,'
|
||||
cgroup2_mountpoint = cgroup2_root()
|
||||
|
||||
if cgroup2_mountpoint is None:
|
||||
|
||||
log('ERROR: unified cgroup hierarchy is not mounted, exit')
|
||||
exit()
|
||||
|
||||
else:
|
||||
|
||||
log('cgroup2 mountpoint: {}'.format(cgroup2_mountpoint))
|
||||
|
||||
cpu_file = cgroup2_mountpoint + target + "/cpu.pressure"
|
||||
memory_file = cgroup2_mountpoint + target + "/memory.pressure"
|
||||
io_file = cgroup2_mountpoint + target + "/io.pressure"
|
||||
|
||||
|
||||
mlockall()
|
||||
|
||||
|
||||
log('Starting psi-monitor, target: {}, period: {}'.format(target, period))
|
||||
log('----------------------------------------------------------------------'
|
||||
'--------------------------------------------')
|
||||
log(' some cpu pressure || some memory pressure | full memory pressure ||'
|
||||
' some io pressure | full io pressure')
|
||||
log('---------------------||----------------------|----------------------||'
|
||||
'----------------------|---------------------')
|
||||
log(' avg10 avg60 avg300 || avg10 avg60 avg300 | avg10 avg60 avg300 ||'
|
||||
' avg10 avg60 avg300 | avg10 avg60 avg300')
|
||||
log('------ ------ ------ || ------ ------ ------ | ------ ------ ------ ||'
|
||||
' ------ ------ ------ | ------ ------ ------')
|
||||
|
||||
|
||||
while True:
|
||||
|
||||
if not os.path.exists(cpu_file):
|
||||
log('ERROR: cpu pressure file does not exist: {}'.format(cpu_file))
|
||||
sleeeep()
|
||||
continue
|
||||
|
||||
if not os.path.exists(memory_file):
|
||||
log('ERROR: memory pressure file does not exist: {}'.format(
|
||||
memory_file))
|
||||
sleeeep()
|
||||
continue
|
||||
|
||||
if not os.path.exists(io_file):
|
||||
log('ERROR: io pressure file does not exist: {}'.format(cpu_file))
|
||||
sleeeep()
|
||||
continue
|
||||
|
||||
(c_some_avg10, c_some_avg60, c_some_avg300
|
||||
) = psi_file_cpu_to_metrics(cpu_file)
|
||||
|
||||
(m_some_avg10, m_some_avg60, m_some_avg300,
|
||||
m_full_avg10, m_full_avg60, m_full_avg300
|
||||
) = psi_file_mem_to_metrics(memory_file)
|
||||
|
||||
(i_some_avg10, i_some_avg60, i_some_avg300,
|
||||
i_full_avg10, i_full_avg60, i_full_avg300
|
||||
) = psi_file_mem_to_metrics(io_file)
|
||||
|
||||
log('{} {} {} || {} {} {} | {} {} {} || {} {} {} | {} {} {}'.format(
|
||||
|
||||
c_some_avg10.rjust(6),
|
||||
c_some_avg60.rjust(6),
|
||||
c_some_avg300.rjust(6),
|
||||
|
||||
m_some_avg10.rjust(6),
|
||||
m_some_avg60.rjust(6),
|
||||
m_some_avg300.rjust(6),
|
||||
m_full_avg10.rjust(6),
|
||||
m_full_avg60.rjust(6),
|
||||
m_full_avg300.rjust(6),
|
||||
|
||||
i_some_avg10.rjust(6),
|
||||
i_some_avg60.rjust(6),
|
||||
i_some_avg300.rjust(6),
|
||||
i_full_avg10.rjust(6),
|
||||
i_full_avg60.rjust(6),
|
||||
i_full_avg300.rjust(6)
|
||||
|
||||
))
|
||||
|
||||
sleeeep()
|
||||
5
tools/psi-monitor.1
Normal file
5
tools/psi-monitor.1
Normal file
@@ -0,0 +1,5 @@
|
||||
.TH psi-monitor 1
|
||||
|
||||
.SH NAME
|
||||
|
||||
psi-monitor \- PSI metrics monitor
|
||||
113
tools/psi-top
Executable file
113
tools/psi-top
Executable file
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
|
||||
|
||||
###############################################################################
|
||||
|
||||
|
||||
def psi_path_to_metrics(psi_path):
|
||||
"""
|
||||
"""
|
||||
with open(psi_path) as f:
|
||||
psi_list = f.readlines()
|
||||
# print(psi_list)
|
||||
some_list, full_list = psi_list[0].split(' '), psi_list[1].split(' ')
|
||||
#print(some_list, full_list)
|
||||
some_avg10 = some_list[1].split('=')[1]
|
||||
some_avg60 = some_list[2].split('=')[1]
|
||||
some_avg300 = some_list[3].split('=')[1]
|
||||
|
||||
full_avg10 = full_list[1].split('=')[1]
|
||||
full_avg60 = full_list[2].split('=')[1]
|
||||
full_avg300 = full_list[3].split('=')[1]
|
||||
|
||||
return (some_avg10, some_avg60, some_avg300,
|
||||
full_avg10, full_avg60, full_avg300)
|
||||
|
||||
|
||||
def cgroup2_root():
|
||||
"""
|
||||
"""
|
||||
with open(mounts) as f:
|
||||
for line in f:
|
||||
if cgroup2_separator in line:
|
||||
return line.partition(cgroup2_separator)[0].partition(' ')[2]
|
||||
|
||||
|
||||
def get_psi_mem_files(cgroup2_path):
|
||||
"""
|
||||
"""
|
||||
path_list = []
|
||||
|
||||
for root, dirs, files in os.walk(cgroup2_path):
|
||||
for file in files:
|
||||
path = os.path.join(root, file)
|
||||
if path.endswith('/memory.pressure'):
|
||||
path_list.append(path)
|
||||
|
||||
return path_list
|
||||
|
||||
|
||||
def psi_path_to_cgroup2(path):
|
||||
"""
|
||||
"""
|
||||
return path.partition(cgroup2_mountpoint)[2][:-16]
|
||||
|
||||
|
||||
###############################################################################
|
||||
|
||||
|
||||
psi_path = '/proc/pressure/memory'
|
||||
mounts = '/proc/mounts'
|
||||
cgroup2_separator = ' cgroup2 rw,'
|
||||
|
||||
cgroup2_mountpoint = cgroup2_root()
|
||||
|
||||
|
||||
if cgroup2_mountpoint is None:
|
||||
print('cgroup2 is not mounted')
|
||||
else:
|
||||
print('cgroup2 mountpoint:', cgroup2_mountpoint)
|
||||
|
||||
|
||||
psi_support = os.path.exists(psi_path)
|
||||
|
||||
if not psi_support:
|
||||
print('PSI is not supported, /proc/pressure/memory does not exist. Exit.')
|
||||
exit(1)
|
||||
|
||||
|
||||
if cgroup2_mountpoint is not None:
|
||||
y = get_psi_mem_files(cgroup2_mountpoint)
|
||||
for path in y:
|
||||
pass # print(psi_path_to_cgroup2(path))
|
||||
|
||||
path_list = get_psi_mem_files(cgroup2_mountpoint)
|
||||
|
||||
print(' avg10 avg60 avg300 avg10 avg60 avg300 cgroup2')
|
||||
print(' ----- ----- ------ ----- ----- ------ ---------')
|
||||
|
||||
(some_avg10, some_avg60, some_avg300, full_avg10, full_avg60, full_avg300
|
||||
) = psi_path_to_metrics(psi_path)
|
||||
|
||||
print('some {} {} {} | full {} {} {} {}'.format(
|
||||
some_avg10.rjust(6),
|
||||
some_avg60.rjust(6),
|
||||
some_avg300.rjust(6),
|
||||
full_avg10.rjust(6),
|
||||
full_avg60.rjust(6),
|
||||
full_avg300.rjust(6), '[SYSTEM_WIDE]'))
|
||||
|
||||
|
||||
for psi_path in path_list:
|
||||
(some_avg10, some_avg60, some_avg300,
|
||||
full_avg10, full_avg60, full_avg300) = psi_path_to_metrics(psi_path)
|
||||
|
||||
print('some {} {} {} | full {} {} {} {}'.format(
|
||||
some_avg10.rjust(6),
|
||||
some_avg60.rjust(6),
|
||||
some_avg300.rjust(6),
|
||||
full_avg10.rjust(6),
|
||||
full_avg60.rjust(6),
|
||||
full_avg300.rjust(6), psi_path_to_cgroup2(psi_path)))
|
||||
5
tools/psi-top.1
Normal file
5
tools/psi-top.1
Normal file
@@ -0,0 +1,5 @@
|
||||
.TH psi-top 1
|
||||
|
||||
.SH NAME
|
||||
|
||||
psi-top \- psi-top
|
||||
Reference in New Issue
Block a user