github/opencas-test-framework
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor disk tools and fs tools

Browse Source 
Signed-off-by: Katarzyna Treder <katarzyna.treder@h-partners.com>
This commit is contained in:
Katarzyna Treder
2024-12-11 18:56:23 +01:00
parent 6dd9c9ca8c
commit fd869a0afc
25 changed files with 179 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
test_tools/blktrace.py
View File

@@ -14,7 +14,8 @@ from datetime import timedelta
from core.test_run import TestRun
from storage_devices.device import Device
from test_utils.filesystem.directory import Directory
from test_tools.os_tools import is_mounted, drop_caches, DropCachesMode
from test_tools.os_tools import drop_caches, DropCachesMode
from test_tools.fs_tools import is_mounted
from type_def.size import Size, Unit
DEBUGFS_MOUNT_POINT = "/sys/kernel/debug"

14
test_tools/disk_finder.py
View File

@@ -7,9 +7,9 @@ import os
import posixpath
from core.test_run import TestRun
from test_tools import disk_utils
from test_tools.disk_utils import get_sysfs_path
from test_tools.fs_utils import check_if_file_exists, readlink
from test_tools import disk_tools
from test_tools.disk_tools import get_sysfs_path
from test_tools.fs_tools import check_if_file_exists, readlink
from connection.utils.output import CmdException
@@ -50,7 +50,7 @@ def discover_hdd_devices(block_devices, devices_res):
for dev in block_devices:
if TestRun.executor.run_expect_success(f"cat /sys/block/{dev}/removable").stdout == "1":
continue # skip removable drives
block_size = disk_utils.get_block_size(dev)
block_size = disk_tools.get_block_size(dev)
if int(block_size) == 4096:
disk_type = 'hdd4k'
else:
@@ -62,7 +62,7 @@ def discover_hdd_devices(block_devices, devices_res):
f"sg_inq /dev/{dev} | grep -i 'serial number'"
).stdout.split(': ')[1].strip(),
"blocksize": block_size,
"size": disk_utils.get_size(dev)})
"size": disk_tools.get_size(dev)})
block_devices.clear()
@@ -99,8 +99,8 @@ def discover_ssd_devices(block_devices, devices_res):
"type": disk_type,
"path": resolve_to_by_id_link(device_path),
"serial": serial_number,
"blocksize": disk_utils.get_block_size(dev),
"size": disk_utils.get_size(dev)})
"blocksize": disk_tools.get_block_size(dev),
"size": disk_tools.get_size(dev)})
block_devices.remove(dev)

67
test_tools/disk_utils.py → test_tools/disk_tools.py
View File

@@ -10,22 +10,17 @@ import time
from enum import Enum
from typing import List
import test_tools.fs_tools
from core.test_run import TestRun
from test_tools import fs_utils
from test_tools.dd import Dd
from test_tools.fs_utils import readlink, parse_ls_output, ls
from connection.utils.output import CmdException
from test_tools.fs_tools import readlink, parse_ls_output, ls, check_if_directory_exists, \
create_directory, wipe_filesystem
from test_tools.udev import Udev
from type_def.size import Size, Unit
SECTOR_SIZE = 512
class Filesystem(Enum):
xfs = 0
ext3 = 1
ext4 = 2
class PartitionTable(Enum):
msdos = 0
gpt = 1
@@ -43,21 +38,6 @@ class PartitionType(Enum):
unknown = 8
def create_filesystem(device, filesystem: Filesystem, force=True, blocksize=None):
TestRun.LOGGER.info(
f"Creating filesystem ({filesystem.name}) on device: {device.path}")
force_param = ' -f ' if filesystem == Filesystem.xfs else ' -F '
force_param = force_param if force else ''
block_size_param = f' -b size={blocksize}' if filesystem == Filesystem.xfs \
else f' -b {blocksize}'
block_size_param = block_size_param if blocksize else ''
cmd = f'mkfs.{filesystem.name} {force_param} {device.path} {block_size_param}'
cmd = re.sub(' +', ' ', cmd)
TestRun.executor.run_expect_success(cmd)
TestRun.LOGGER.info(
f"Successfully created filesystem on device: {device.path}")
def create_partition_table(device, partition_table_type: PartitionTable = PartitionTable.gpt):
TestRun.LOGGER.info(
f"Creating partition table ({partition_table_type.name}) for device: {device.path}")
@@ -267,8 +247,7 @@ def get_first_partition_offset(device, aligned: bool):
def remove_partitions(device):
from test_tools.udev import Udev
if device.is_mounted():
if test_tools.fs_tools.is_mounted(device.path):
device.unmount()
for partition in device.partitions:
@@ -276,7 +255,7 @@ def remove_partitions(device):
TestRun.LOGGER.info(f"Removing partitions from device: {device.path} "
f"({device.get_device_id()}).")
device.wipe_filesystem()
wipe_filesystem(device)
Udev.trigger()
Udev.settle()
output = TestRun.executor.run(f"ls {device.path}* -1")
@@ -287,8 +266,8 @@ def remove_partitions(device):
def mount(device, mount_point, options: [str] = None):
if not fs_utils.check_if_directory_exists(mount_point):
fs_utils.create_directory(mount_point, True)
if not check_if_directory_exists(mount_point):
create_directory(mount_point, True)
TestRun.LOGGER.info(f"Mounting device {device.path} ({device.get_device_id()}) "
f"to {mount_point}.")
cmd = f"mount {device.path} {mount_point}"
@@ -330,15 +309,6 @@ def unit_to_string(unit):
return unit_string.get(unit, "Invalid unit.")
def wipe_filesystem(device, force=True):
TestRun.LOGGER.info(f"Erasing the device: {device.path}")
force_param = ' -f' if force else ''
cmd = f'wipefs -a{force_param} {device.path}'
TestRun.executor.run_expect_success(cmd)
TestRun.LOGGER.info(
f"Successfully wiped device: {device.path}")
def check_if_device_supports_trim(device):
if device.get_device_id().startswith("nvme"):
return True
@@ -351,27 +321,6 @@ def check_if_device_supports_trim(device):
return int(command_output.stdout) > 0
def get_device_filesystem_type(device_id):
cmd = f'lsblk -l -o NAME,FSTYPE | sort | uniq | grep "{device_id} "'
try:
stdout = TestRun.executor.run_expect_success(cmd).stdout
except CmdException:
# unusual devices might not be listed in output (i.e. RAID containers)
if TestRun.executor.run(f"test -b /dev/{device_id}").exit_code != 0:
raise
else:
return None
split_stdout = stdout.strip().split()
if len(split_stdout) <= 1:
return None
else:
try:
return Filesystem[split_stdout[1]]
except KeyError:
TestRun.LOGGER.warning(f"Unrecognized filesystem: {split_stdout[1]}")
return None
def _is_by_id_path(path: str):
"""check if given path already is proper by-id path"""
dev_by_id_dir = "/dev/disk/by-id"

6
test_tools/fio/fio.py
View File

@@ -9,10 +9,10 @@ import uuid
from packaging.version import Version
import test_tools.fio.fio_param
import test_tools.fs_utils
import test_tools.fs_tools
import test_tools.wget
from core.test_run import TestRun
from test_tools import fs_utils
from test_tools import fs_tools
from connection.utils.output import CmdException
@@ -51,7 +51,7 @@ class Fio:
def install(self):
fio_url = f"http://brick.kernel.dk/snaps/fio-{self.min_fio_version}.tar.bz2"
fio_package = test_tools.wget.download_file(fio_url)
fs_utils.uncompress_archive(fio_package)
fs_tools.uncompress_archive(fio_package)
TestRun.executor.run_expect_success(
f"cd {fio_package.parent_dir}/fio-{self.min_fio_version}"
f" && ./configure && make -j && make install"

65
test_tools/fs_utils.py → test_tools/fs_tools.py
View File

@@ -7,17 +7,24 @@
import base64
import math
import re
import textwrap
from collections import namedtuple
from datetime import datetime, timedelta
from enum import Enum, IntFlag
from aenum import IntFlag, Enum
from connection.utils.output import CmdException
from core.test_run import TestRun
from test_tools.dd import Dd
from type_def.size import Size, Unit
class Filesystem(Enum):
xfs = 0
ext3 = 1
ext4 = 2
class Permissions(IntFlag):
r = 4
w = 2
@@ -50,7 +57,7 @@ class PermissionSign(Enum):
set = '='
class FilesPermissions():
class FilesPermissions:
perms_exceptions = {}
def __init__(self, files_list: list):
@@ -393,3 +400,55 @@ def create_random_test_file(target_file_path: str,
dd.run()
file.refresh_item()
return file
def create_filesystem(device, filesystem: Filesystem, force=True, blocksize=None):
TestRun.LOGGER.info(
f"Creating filesystem ({filesystem.name}) on device: {device.path}")
force_param = ' -f ' if filesystem == Filesystem.xfs else ' -F '
force_param = force_param if force else ''
block_size_param = f' -b size={blocksize}' if filesystem == Filesystem.xfs \
else f' -b {blocksize}'
block_size_param = block_size_param if blocksize else ''
cmd = f'mkfs.{filesystem.name} {force_param} {device.path} {block_size_param}'
cmd = re.sub(' +', ' ', cmd)
TestRun.executor.run_expect_success(cmd)
TestRun.LOGGER.info(
f"Successfully created filesystem on device: {device.path}")
def wipe_filesystem(device, force=True):
TestRun.LOGGER.info(f"Erasing the device: {device.path}")
force_param = ' -f' if force else ''
cmd = f'wipefs -a{force_param} {device.path}'
TestRun.executor.run_expect_success(cmd)
TestRun.LOGGER.info(
f"Successfully wiped device: {device.path}")
def get_device_filesystem_type(device_id):
cmd = f'lsblk -l -o NAME,FSTYPE | sort | uniq | grep "{device_id} "'
try:
stdout = TestRun.executor.run_expect_success(cmd).stdout
except CmdException:
# unusual devices might not be listed in output (i.e. RAID containers)
if TestRun.executor.run(f"test -b /dev/{device_id}").exit_code != 0:
raise
else:
return None
split_stdout = stdout.strip().split()
if len(split_stdout) <= 1:
return None
else:
try:
return Filesystem[split_stdout[1]]
except KeyError:
TestRun.LOGGER.warning(f"Unrecognized filesystem: {split_stdout[1]}")
return None
def is_mounted(path: str):
if path is None or path.isspace():
raise Exception("Checked path cannot be empty")
command = f"mount | grep --fixed-strings '{path.rstrip('/')} '"
return TestRun.executor.run(command).exit_code == 0

6
test_tools/fstab.py
View File

@@ -4,11 +4,11 @@
# SPDX-License-Identifier: BSD-3-Clause
#
from test_tools import fs_utils, systemctl
from test_tools import fs_tools, systemctl
def add_mountpoint(device, mount_point, fs_type, mount_now=True):
fs_utils.append_line("/etc/fstab",
fs_tools.append_line("/etc/fstab",
f"{device.path} {mount_point} {fs_type.name} defaults 0 0")
systemctl.reload_daemon()
if mount_now:
@@ -16,5 +16,5 @@ def add_mountpoint(device, mount_point, fs_type, mount_now=True):
def remove_mountpoint(device):
fs_utils.remove_lines("/etc/fstab", device.path)
fs_tools.remove_lines("/etc/fstab", device.path)
systemctl.reload_daemon()

4
test_tools/memory.py
View File

@@ -3,9 +3,9 @@ import math
from connection.utils.output import CmdException
from core.test_run import TestRun
from test_tools.dd import Dd
from test_tools.fs_utils import check_if_directory_exists, create_directory
from test_tools.fs_tools import check_if_directory_exists, create_directory, is_mounted
from test_tools.os_tools import OvercommitMemoryMode, drop_caches, DropCachesMode, \
MEMORY_MOUNT_POINT, is_mounted
MEMORY_MOUNT_POINT
from type_def.size import Size, Unit

11
test_tools/os_tools.py
View File

@@ -14,8 +14,8 @@ from packaging import version
from core.test_run import TestRun
from storage_devices.device import Device
from test_tools.disk_utils import get_sysfs_path
from test_tools.fs_utils import check_if_file_exists
from test_tools.disk_tools import get_sysfs_path
from test_tools.fs_tools import check_if_file_exists, is_mounted
from test_utils.filesystem.file import File
from connection.utils.retry import Retry
@@ -110,13 +110,6 @@ def get_kernel_module_parameter(module_name, parameter):
return File(param_file_path).read()
def is_mounted(path: str):
if path is None or path.isspace():
raise Exception("Checked path cannot be empty")
command = f"mount | grep --fixed-strings '{path.rstrip('/')} '"
return TestRun.executor.run(command).exit_code == 0
def mount_debugfs():
if not is_mounted(DEBUGFS_MOUNT_POINT):
TestRun.executor.run_expect_success(f"mount -t debugfs none {DEBUGFS_MOUNT_POINT}")

10
test_tools/peach_fuzzer/peach_fuzzer.py
View File

@@ -14,8 +14,8 @@ from collections import namedtuple
import test_tools.wget
from core.test_run import TestRun
from test_tools import fs_utils
from test_tools.fs_utils import create_directory, check_if_file_exists, write_file
from test_tools import fs_tools
from test_tools.fs_tools import create_directory, check_if_file_exists, write_file
class PeachFuzzer:
@@ -75,7 +75,7 @@ class PeachFuzzer:
cls._install()
if not cls._is_xml_config_prepared():
TestRun.block("No Peach Fuzzer XML config needed to generate fuzzed values was found!")
fs_utils.remove(cls.fuzzy_output_file, force=True, ignore_errors=True)
fs_tools.remove(cls.fuzzy_output_file, force=True, ignore_errors=True)
TestRun.LOGGER.info(f"Generate {count} unique fuzzed values")
cmd = f"cd {cls.base_dir}; {cls.peach_dir}/peach --range 0,{count - 1} " \
f"--seed {random.randrange(2 ** 32)} {cls.xml_config_file} > " \
@@ -172,7 +172,7 @@ class PeachFuzzer:
"""
if not cls._is_mono_installed():
TestRun.block("Mono is not installed, can't continue with Peach Fuzzer!")
if fs_utils.check_if_directory_exists(posixpath.join(cls.base_dir, cls.peach_dir)):
if fs_tools.check_if_directory_exists(posixpath.join(cls.base_dir, cls.peach_dir)):
return "Peach" in TestRun.executor.run(
f"cd {cls.base_dir} && {cls.peach_dir}/peach --version").stdout.strip()
else:
@@ -197,7 +197,7 @@ class PeachFuzzer:
"""
Check if Peach Fuzzer XML config is present on the DUT
"""
if fs_utils.check_if_file_exists(cls.xml_config_file):
if fs_tools.check_if_file_exists(cls.xml_config_file):
return True
else:
return False