From 622c64c46bb36cfe72001264e9f6d95c8b1c006b Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Tue, 10 Dec 2024 12:18:07 +0100 Subject: [PATCH 01/23] Move size to types Signed-off-by: Katarzyna Treder --- storage_devices/device.py | 2 +- storage_devices/disk.py | 2 +- storage_devices/lvm.py | 2 +- storage_devices/partition.py | 2 +- storage_devices/raid.py | 2 +- storage_devices/ramdisk.py | 2 +- test_tools/blktrace.py | 2 +- test_tools/dd.py | 2 +- test_tools/ddrescue.py | 2 +- test_tools/device_mapper.py | 2 +- test_tools/disk_utils.py | 2 +- test_tools/fio/fio_param.py | 2 +- test_tools/fio/fio_result.py | 2 +- test_tools/fs_utils.py | 2 +- test_tools/iostat.py | 2 +- test_tools/mdadm.py | 2 +- test_utils/filesystem/file.py | 2 +- test_utils/os_utils.py | 2 +- types/__init__.py | 0 {test_utils => types}/size.py | 0 20 files changed, 18 insertions(+), 18 deletions(-) create mode 100644 types/__init__.py rename {test_utils => types}/size.py (100%) diff --git a/storage_devices/device.py b/storage_devices/device.py index 6125b90..5ce9bce 100644 --- a/storage_devices/device.py +++ b/storage_devices/device.py @@ -9,7 +9,7 @@ from core.test_run import TestRun from test_tools import disk_utils, fs_utils from test_tools.disk_utils import get_device_filesystem_type, get_sysfs_path from test_utils.io_stats import IoStats -from test_utils.size import Size, Unit +from types.size import Size, Unit class Device: diff --git a/storage_devices/disk.py b/storage_devices/disk.py index 00bd486..1658abe 100644 --- a/storage_devices/disk.py +++ b/storage_devices/disk.py @@ -16,7 +16,7 @@ from test_tools import disk_utils, fs_utils, nvme_cli from test_utils import disk_finder from test_utils.os_utils import wait from test_utils.output import Output -from test_utils.size import Unit +from types.size import Unit class DiskType(IntEnum): diff --git a/storage_devices/lvm.py b/storage_devices/lvm.py index 3e30528..2eb8e5b 100644 --- a/storage_devices/lvm.py +++ b/storage_devices/lvm.py @@ -13,7 +13,7 @@ from storage_devices.disk import Disk from test_tools.fs_utils import readlink from test_utils.disk_finder import resolve_to_by_id_link, get_system_disks from test_utils.filesystem.symlink import Symlink -from test_utils.size import Size +from types.size import Size lvm_config_path = "/etc/lvm/lvm.conf" filter_prototype_regex = r"^\sfilter\s=\s\[" diff --git a/storage_devices/partition.py b/storage_devices/partition.py index 080ab4c..966e057 100644 --- a/storage_devices/partition.py +++ b/storage_devices/partition.py @@ -5,7 +5,7 @@ from storage_devices.device import Device from test_tools import disk_utils -from test_utils.size import Size +from types.size import Size class Partition(Device): diff --git a/storage_devices/raid.py b/storage_devices/raid.py index 4e3685c..1ac022b 100644 --- a/storage_devices/raid.py +++ b/storage_devices/raid.py @@ -11,7 +11,7 @@ from storage_devices.disk import Disk from test_tools.fs_utils import readlink from test_tools.mdadm import Mdadm from test_utils.disk_finder import resolve_to_by_id_link -from test_utils.size import Size, Unit +from types.size import Size, Unit def get_devices_paths_string(devices: [Device]): diff --git a/storage_devices/ramdisk.py b/storage_devices/ramdisk.py index 011b812..a4513ce 100644 --- a/storage_devices/ramdisk.py +++ b/storage_devices/ramdisk.py @@ -11,7 +11,7 @@ from test_tools import disk_utils from test_tools.fs_utils import ls, parse_ls_output from test_utils.filesystem.symlink import Symlink from test_utils.os_utils import reload_kernel_module, unload_kernel_module, is_kernel_module_loaded -from test_utils.size import Size, Unit +from types.size import Size, Unit class RamDisk(Device): diff --git a/test_tools/blktrace.py b/test_tools/blktrace.py index 494cd0b..f09cd2d 100644 --- a/test_tools/blktrace.py +++ b/test_tools/blktrace.py @@ -15,7 +15,7 @@ from core.test_run import TestRun from storage_devices.device import Device from test_utils.filesystem.directory import Directory from test_utils.os_utils import is_mounted, drop_caches, DropCachesMode -from test_utils.size import Size, Unit +from types.size import Size, Unit DEBUGFS_MOUNT_POINT = "/sys/kernel/debug" PREFIX = "trace_" diff --git a/test_tools/dd.py b/test_tools/dd.py index 1ca51db..e61d255 100644 --- a/test_tools/dd.py +++ b/test_tools/dd.py @@ -4,7 +4,7 @@ # import test_utils.linux_command as linux_comm -import test_utils.size as size +import types.size as size from core.test_run import TestRun diff --git a/test_tools/ddrescue.py b/test_tools/ddrescue.py index dfd3079..799bde5 100644 --- a/test_tools/ddrescue.py +++ b/test_tools/ddrescue.py @@ -4,7 +4,7 @@ # import test_utils.linux_command as linux_comm -import test_utils.size as size +import types.size as size from core.test_run import TestRun diff --git a/test_tools/device_mapper.py b/test_tools/device_mapper.py index ede38df..b44e244 100644 --- a/test_tools/device_mapper.py +++ b/test_tools/device_mapper.py @@ -9,7 +9,7 @@ from core.test_run import TestRun from storage_devices.device import Device from test_utils.disk_finder import resolve_to_by_id_link from test_utils.linux_command import LinuxCommand -from test_utils.size import Size, Unit +from types.size import Size, Unit class DmTarget(Enum): diff --git a/test_tools/disk_utils.py b/test_tools/disk_utils.py index 56edd2a..901a56e 100644 --- a/test_tools/disk_utils.py +++ b/test_tools/disk_utils.py @@ -14,7 +14,7 @@ from test_tools import fs_utils from test_tools.dd import Dd from test_tools.fs_utils import readlink, parse_ls_output, ls from test_utils.output import CmdException -from test_utils.size import Size, Unit +from types.size import Size, Unit SECTOR_SIZE = 512 diff --git a/test_tools/fio/fio_param.py b/test_tools/fio/fio_param.py index 8db911b..ff8409f 100644 --- a/test_tools/fio/fio_param.py +++ b/test_tools/fio/fio_param.py @@ -14,7 +14,7 @@ from core.test_run import TestRun from storage_devices.device import Device from test_tools.fio.fio_result import FioResult from test_utils.linux_command import LinuxCommand -from test_utils.size import Size +from types.size import Size class CpusAllowedPolicy(Enum): diff --git a/test_tools/fio/fio_result.py b/test_tools/fio/fio_result.py index 334e977..1ed4ff4 100644 --- a/test_tools/fio/fio_result.py +++ b/test_tools/fio/fio_result.py @@ -4,7 +4,7 @@ # -from test_utils.size import Size, Unit, UnitPerSecond +from types.size import Size, Unit, UnitPerSecond from test_utils.time import Time diff --git a/test_tools/fs_utils.py b/test_tools/fs_utils.py index 0ed99ff..44c3eb2 100644 --- a/test_tools/fs_utils.py +++ b/test_tools/fs_utils.py @@ -15,7 +15,7 @@ from aenum import IntFlag, Enum from core.test_run import TestRun from test_tools.dd import Dd -from test_utils.size import Size, Unit +from types.size import Size, Unit class Permissions(IntFlag): diff --git a/test_tools/iostat.py b/test_tools/iostat.py index 3e310bf..0abb8cc 100644 --- a/test_tools/iostat.py +++ b/test_tools/iostat.py @@ -5,7 +5,7 @@ from core.test_run import TestRun from storage_devices.device import Device -from test_utils.size import Size, Unit, UnitPerSecond +from types.size import Size, Unit, UnitPerSecond from test_utils.time import Time import csv diff --git a/test_tools/mdadm.py b/test_tools/mdadm.py index db5e60a..5893512 100644 --- a/test_tools/mdadm.py +++ b/test_tools/mdadm.py @@ -7,7 +7,7 @@ import re from core.test_run import TestRun -from test_utils.size import Unit +from types.size import Unit from test_utils.os_utils import Udev diff --git a/test_utils/filesystem/file.py b/test_utils/filesystem/file.py index e8a8130..efa40c1 100644 --- a/test_utils/filesystem/file.py +++ b/test_utils/filesystem/file.py @@ -8,7 +8,7 @@ from datetime import timedelta from test_tools import fs_utils from test_tools.dd import Dd from test_utils.filesystem.fs_item import FsItem -from test_utils.size import Size +from types.size import Size class File(FsItem): diff --git a/test_utils/os_utils.py b/test_utils/os_utils.py index cc7abd5..621a560 100644 --- a/test_utils/os_utils.py +++ b/test_utils/os_utils.py @@ -22,7 +22,7 @@ from test_tools.fs_utils import check_if_directory_exists, create_directory, che from test_utils.filesystem.file import File from test_utils.output import CmdException from test_utils.retry import Retry -from test_utils.size import Size, Unit +from types.size import Size, Unit DEBUGFS_MOUNT_POINT = "/sys/kernel/debug" MEMORY_MOUNT_POINT = "/mnt/memspace" diff --git a/types/__init__.py b/types/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test_utils/size.py b/types/size.py similarity index 100% rename from test_utils/size.py rename to types/size.py From 513f006d17468ead7a768e13f48014e725b9ddd1 Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Tue, 10 Dec 2024 12:19:57 +0100 Subject: [PATCH 02/23] Move Time to types Signed-off-by: Katarzyna Treder --- test_tools/fio/fio_result.py | 2 +- test_tools/iostat.py | 2 +- {test_utils => types}/time.py | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename {test_utils => types}/time.py (100%) diff --git a/test_tools/fio/fio_result.py b/test_tools/fio/fio_result.py index 1ed4ff4..e2912f7 100644 --- a/test_tools/fio/fio_result.py +++ b/test_tools/fio/fio_result.py @@ -5,7 +5,7 @@ from types.size import Size, Unit, UnitPerSecond -from test_utils.time import Time +from types.time import Time class FioResult: diff --git a/test_tools/iostat.py b/test_tools/iostat.py index 0abb8cc..d355bbc 100644 --- a/test_tools/iostat.py +++ b/test_tools/iostat.py @@ -6,7 +6,7 @@ from core.test_run import TestRun from storage_devices.device import Device from types.size import Size, Unit, UnitPerSecond -from test_utils.time import Time +from types.time import Time import csv diff --git a/test_utils/time.py b/types/time.py similarity index 100% rename from test_utils/time.py rename to types/time.py From 161cc7957afb70256cf07a69673dad63889ccbfc Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Tue, 10 Dec 2024 12:29:44 +0100 Subject: [PATCH 03/23] Move output to connection utils Signed-off-by: Katarzyna Treder --- connection/base_executor.py | 2 +- connection/local_executor.py | 2 +- connection/ssh_executor.py | 2 +- connection/utils/__init__.py | 0 {test_utils => connection/utils}/output.py | 0 internal_plugins/scsi_debug/__init__.py | 2 +- log/logger.py | 2 +- storage_devices/disk.py | 2 +- storage_devices/drbd.py | 3 +-- test_tools/disk_utils.py | 2 +- test_tools/fio/fio.py | 2 +- test_tools/linux_packaging.py | 2 +- test_utils/disk_finder.py | 2 +- test_utils/dmesg.py | 2 +- test_utils/git.py | 2 +- test_utils/io_stats.py | 2 +- test_utils/os_utils.py | 2 +- 17 files changed, 15 insertions(+), 16 deletions(-) create mode 100644 connection/utils/__init__.py rename {test_utils => connection/utils}/output.py (100%) diff --git a/connection/base_executor.py b/connection/base_executor.py index 7162fac..3be941c 100644 --- a/connection/base_executor.py +++ b/connection/base_executor.py @@ -8,7 +8,7 @@ import time from datetime import timedelta from core.test_run import TestRun -from test_utils.output import CmdException +from connection.utils.output import CmdException class BaseExecutor: diff --git a/connection/local_executor.py b/connection/local_executor.py index 9c6cad6..b8ba46d 100644 --- a/connection/local_executor.py +++ b/connection/local_executor.py @@ -10,7 +10,7 @@ from datetime import timedelta from connection.base_executor import BaseExecutor from core.test_run import TestRun from test_tools.fs_utils import copy -from test_utils.output import Output, CmdException +from connection.utils.output import Output, CmdException class LocalExecutor(BaseExecutor): diff --git a/connection/ssh_executor.py b/connection/ssh_executor.py index 2415311..173fef5 100644 --- a/connection/ssh_executor.py +++ b/connection/ssh_executor.py @@ -13,7 +13,7 @@ import paramiko from connection.base_executor import BaseExecutor from core.test_run import TestRun, Blocked -from test_utils.output import Output +from connection.utils.output import Output class SshExecutor(BaseExecutor): diff --git a/connection/utils/__init__.py b/connection/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test_utils/output.py b/connection/utils/output.py similarity index 100% rename from test_utils/output.py rename to connection/utils/output.py diff --git a/internal_plugins/scsi_debug/__init__.py b/internal_plugins/scsi_debug/__init__.py index 95f6563..8902c63 100644 --- a/internal_plugins/scsi_debug/__init__.py +++ b/internal_plugins/scsi_debug/__init__.py @@ -7,7 +7,7 @@ from time import sleep from core.test_run_utils import TestRun from storage_devices.device import Device from test_utils import os_utils -from test_utils.output import CmdException +from connection.utils.output import CmdException class ScsiDebug: diff --git a/log/logger.py b/log/logger.py index 0c82be6..9de1ec3 100644 --- a/log/logger.py +++ b/log/logger.py @@ -15,7 +15,7 @@ import portalocker from log.html_log_config import HtmlLogConfig from log.html_log_manager import HtmlLogManager from log.html_presentation_policy import html_policy -from test_utils.output import Output +from connection.utils.output import Output from test_utils.singleton import Singleton diff --git a/storage_devices/disk.py b/storage_devices/disk.py index 1658abe..8a1dc6c 100644 --- a/storage_devices/disk.py +++ b/storage_devices/disk.py @@ -15,7 +15,7 @@ from storage_devices.device import Device from test_tools import disk_utils, fs_utils, nvme_cli from test_utils import disk_finder from test_utils.os_utils import wait -from test_utils.output import Output +from connection.utils.output import Output from types.size import Unit diff --git a/storage_devices/drbd.py b/storage_devices/drbd.py index 66baeda..a8e2ba8 100644 --- a/storage_devices/drbd.py +++ b/storage_devices/drbd.py @@ -3,14 +3,13 @@ # SPDX-License-Identifier: BSD-3-Clause-Clear # -import os import posixpath from core.test_run import TestRun from storage_devices.device import Device from test_tools.drbdadm import Drbdadm from test_utils.filesystem.symlink import Symlink -from test_utils.output import CmdException +from connection.utils.output import CmdException class Drbd(Device): diff --git a/test_tools/disk_utils.py b/test_tools/disk_utils.py index 901a56e..a02437e 100644 --- a/test_tools/disk_utils.py +++ b/test_tools/disk_utils.py @@ -13,7 +13,7 @@ from core.test_run import TestRun from test_tools import fs_utils from test_tools.dd import Dd from test_tools.fs_utils import readlink, parse_ls_output, ls -from test_utils.output import CmdException +from connection.utils.output import CmdException from types.size import Size, Unit SECTOR_SIZE = 512 diff --git a/test_tools/fio/fio.py b/test_tools/fio/fio.py index 393a530..813de89 100644 --- a/test_tools/fio/fio.py +++ b/test_tools/fio/fio.py @@ -13,7 +13,7 @@ import test_tools.fs_utils from core.test_run import TestRun from test_tools import fs_utils from test_utils import os_utils -from test_utils.output import CmdException +from connection.utils.output import CmdException class Fio: diff --git a/test_tools/linux_packaging.py b/test_tools/linux_packaging.py index 3dd882d..8f1940e 100644 --- a/test_tools/linux_packaging.py +++ b/test_tools/linux_packaging.py @@ -9,7 +9,7 @@ import os import re from core.test_run import TestRun -from test_utils.output import CmdException +from connection.utils.output import CmdException class RpmSet: diff --git a/test_utils/disk_finder.py b/test_utils/disk_finder.py index d738fa1..a02bb85 100644 --- a/test_utils/disk_finder.py +++ b/test_utils/disk_finder.py @@ -9,7 +9,7 @@ from core.test_run import TestRun from test_tools import disk_utils from test_tools.fs_utils import check_if_file_exists, readlink from test_utils import os_utils -from test_utils.output import CmdException +from connection.utils.output import CmdException def find_disks(): diff --git a/test_utils/dmesg.py b/test_utils/dmesg.py index f5ce808..50131b3 100644 --- a/test_utils/dmesg.py +++ b/test_utils/dmesg.py @@ -4,7 +4,7 @@ # from core.test_run import TestRun -from test_utils.output import Output +from connection.utils.output import Output def get_dmesg() -> str: diff --git a/test_utils/git.py b/test_utils/git.py index f52ca2e..83ad6f7 100644 --- a/test_utils/git.py +++ b/test_utils/git.py @@ -9,7 +9,7 @@ import posixpath from core.test_run import TestRun from connection.local_executor import LocalExecutor -from test_utils.output import CmdException +from connection.utils.output import CmdException def __get_executor_and_repo_path(from_dut): diff --git a/test_utils/io_stats.py b/test_utils/io_stats.py index 4f0fe63..fb9607d 100644 --- a/test_utils/io_stats.py +++ b/test_utils/io_stats.py @@ -5,7 +5,7 @@ import re from core.test_run import TestRun -from test_utils.output import CmdException +from connection.utils.output import CmdException SYSFS_LINE_FORMAT = r"^(\d+\s+){10,}\d+$" PROCFS_LINE_FORMAT = r"^\d+\s+\d+\s+[\w-]+\s+" + SYSFS_LINE_FORMAT[1:] diff --git a/test_utils/os_utils.py b/test_utils/os_utils.py index 621a560..4a39786 100644 --- a/test_utils/os_utils.py +++ b/test_utils/os_utils.py @@ -20,7 +20,7 @@ from test_tools.dd import Dd from test_tools.disk_utils import get_sysfs_path from test_tools.fs_utils import check_if_directory_exists, create_directory, check_if_file_exists from test_utils.filesystem.file import File -from test_utils.output import CmdException +from connection.utils.output import CmdException from test_utils.retry import Retry from types.size import Size, Unit From f5087073947cb49f587e17a6b823640751c98f9b Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Tue, 10 Dec 2024 12:50:13 +0100 Subject: [PATCH 04/23] Move asynchronous to connection utils Signed-off-by: Katarzyna Treder --- {test_utils => connection/utils}/asynchronous.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {test_utils => connection/utils}/asynchronous.py (100%) diff --git a/test_utils/asynchronous.py b/connection/utils/asynchronous.py similarity index 100% rename from test_utils/asynchronous.py rename to connection/utils/asynchronous.py From 5409fa9a40de5c56eddf79742546f863c0cc6d37 Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Tue, 10 Dec 2024 12:54:18 +0100 Subject: [PATCH 05/23] Move retry to connection utils Signed-off-by: Katarzyna Treder --- {test_utils => connection/utils}/retry.py | 0 test_utils/os_utils.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename {test_utils => connection/utils}/retry.py (100%) diff --git a/test_utils/retry.py b/connection/utils/retry.py similarity index 100% rename from test_utils/retry.py rename to connection/utils/retry.py diff --git a/test_utils/os_utils.py b/test_utils/os_utils.py index 4a39786..61dc2c3 100644 --- a/test_utils/os_utils.py +++ b/test_utils/os_utils.py @@ -21,7 +21,7 @@ from test_tools.disk_utils import get_sysfs_path from test_tools.fs_utils import check_if_directory_exists, create_directory, check_if_file_exists from test_utils.filesystem.file import File from connection.utils.output import CmdException -from test_utils.retry import Retry +from connection.utils.retry import Retry from types.size import Size, Unit DEBUGFS_MOUNT_POINT = "/sys/kernel/debug" From de4305af8c7f270cef906829c88a544ea7269062 Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Tue, 10 Dec 2024 13:24:33 +0100 Subject: [PATCH 06/23] Move singleton to common utils Signed-off-by: Katarzyna Treder --- log/logger.py | 2 +- test_utils/common/__init__.py | 0 test_utils/{ => common}/singleton.py | 0 3 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 test_utils/common/__init__.py rename test_utils/{ => common}/singleton.py (100%) diff --git a/log/logger.py b/log/logger.py index 9de1ec3..5d7cf18 100644 --- a/log/logger.py +++ b/log/logger.py @@ -16,7 +16,7 @@ from log.html_log_config import HtmlLogConfig from log.html_log_manager import HtmlLogManager from log.html_presentation_policy import html_policy from connection.utils.output import Output -from test_utils.singleton import Singleton +from test_utils.common.singleton import Singleton def create_log(log_base_path, test_module, additional_args=None): diff --git a/test_utils/common/__init__.py b/test_utils/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test_utils/singleton.py b/test_utils/common/singleton.py similarity index 100% rename from test_utils/singleton.py rename to test_utils/common/singleton.py From 8a7e848afc0056439d1bd62fae3a9f956d583b77 Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Tue, 10 Dec 2024 13:38:45 +0100 Subject: [PATCH 07/23] Move checksec to scripts Signed-off-by: Katarzyna Treder --- scripts/__init__.py | 0 {test_tools => scripts}/checksec.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 scripts/__init__.py rename {test_tools => scripts}/checksec.sh (100%) diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test_tools/checksec.sh b/scripts/checksec.sh similarity index 100% rename from test_tools/checksec.sh rename to scripts/checksec.sh From f60e90192e61b2f4a9864283de76f31d7b397393 Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Tue, 10 Dec 2024 13:46:25 +0100 Subject: [PATCH 08/23] Move error device to storage devices as separate file Signed-off-by: Katarzyna Treder --- storage_devices/error_device.py | 87 +++++++++++++++++++++++++++++++++ test_tools/device_mapper.py | 81 +----------------------------- 2 files changed, 89 insertions(+), 79 deletions(-) create mode 100644 storage_devices/error_device.py diff --git a/storage_devices/error_device.py b/storage_devices/error_device.py new file mode 100644 index 0000000..0708603 --- /dev/null +++ b/storage_devices/error_device.py @@ -0,0 +1,87 @@ +# +# Copyright(c) 2019-2022 Intel Corporation +# Copyright(c) 2023-2024 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +# + +from core.test_run import TestRun +from storage_devices.device import Device +from test_tools.device_mapper import DmTable, DeviceMapper +from test_utils.disk_finder import resolve_to_by_id_link + + +class ErrorDevice(Device): + def __init__(self, name: str, base_device: Device, table: DmTable = None): + self.device = base_device + self.mapper = DeviceMapper(name) + self.name = name + self.table = DmTable.passthrough_table(base_device) if not table else table + self.active = False + self.start() + self.path = resolve_to_by_id_link(self.mapper.get_path().replace('/dev/', '')) + + @property + def system_path(self): + if self.active: + output = TestRun.executor.run_expect_success(f"realpath {self.mapper.get_path()}") + + return output.stdout + + return None + + @property + def size(self): + if self.active: + return self.table.get_size() + + return None + + def start(self): + self.mapper.create(self.table) + self.active = True + + def stop(self): + self.mapper.remove() + self.active = False + + def change_table(self, table: DmTable, permanent=True): + if self.active: + self.mapper.suspend() + + self.mapper.reload(table) + + self.mapper.resume() + + if permanent: + self.table = table + + def suspend_errors(self): + empty_table = DmTable.passthrough_table(self.device) + TestRun.LOGGER.info(f"Suspending issuing errors for error device '{self.name}'") + + self.change_table(empty_table, False) + + def resume_errors(self): + TestRun.LOGGER.info(f"Resuming issuing errors for error device '{self.name}'") + + self.change_table(self.table, False) + + def suspend(self): + if not self.active: + TestRun.LOGGER.warning( + f"cannot suspend error device '{self.name}'! It's already running" + ) + + self.mapper.suspend() + + self.active = False + + def resume(self): + if self.active: + TestRun.LOGGER.warning( + f"cannot resume error device '{self.name}'! It's already running" + ) + + self.mapper.resume() + + self.active = True diff --git a/test_tools/device_mapper.py b/test_tools/device_mapper.py index b44e244..c2cf0cf 100644 --- a/test_tools/device_mapper.py +++ b/test_tools/device_mapper.py @@ -1,5 +1,6 @@ # -# Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2019-2022 Intel Corporation +# Copyright(c) 2023-2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # @@ -7,7 +8,6 @@ from enum import Enum from core.test_run import TestRun from storage_devices.device import Device -from test_utils.disk_finder import resolve_to_by_id_link from test_utils.linux_command import LinuxCommand from types.size import Size, Unit @@ -250,80 +250,3 @@ class DeviceMapper(LinuxCommand): return TestRun.executor.run_expect_success( f"{self.command_name} reload {self.name} {self.wrap_table(table)}" ) - - -class ErrorDevice(Device): - def __init__(self, name: str, base_device: Device, table: DmTable = None): - self.device = base_device - self.mapper = DeviceMapper(name) - self.name = name - self.table = DmTable.passthrough_table(base_device) if not table else table - self.active = False - self.start() - self.path = resolve_to_by_id_link(self.mapper.get_path().replace('/dev/', '')) - - @property - def system_path(self): - if self.active: - output = TestRun.executor.run_expect_success(f"realpath {self.mapper.get_path()}") - - return output.stdout - - return None - - @property - def size(self): - if self.active: - return self.table.get_size() - - return None - - def start(self): - self.mapper.create(self.table) - self.active = True - - def stop(self): - self.mapper.remove() - self.active = False - - def change_table(self, table: DmTable, permanent=True): - if self.active: - self.mapper.suspend() - - self.mapper.reload(table) - - self.mapper.resume() - - if permanent: - self.table = table - - def suspend_errors(self): - empty_table = DmTable.passthrough_table(self.device) - TestRun.LOGGER.info(f"Suspending issuing errors for error device '{self.name}'") - - self.change_table(empty_table, False) - - def resume_errors(self): - TestRun.LOGGER.info(f"Resuming issuing errors for error device '{self.name}'") - - self.change_table(self.table, False) - - def suspend(self): - if not self.active: - TestRun.LOGGER.warning( - f"cannot suspend error device '{self.name}'! It's already running" - ) - - self.mapper.suspend() - - self.active = False - - def resume(self): - if self.active: - TestRun.LOGGER.warning( - f"cannot resume error device '{self.name}'! It's already running" - ) - - self.mapper.resume() - - self.active = True From a954e47b33de846810bb6d74d14bc3a3a23c1901 Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Tue, 10 Dec 2024 13:57:18 +0100 Subject: [PATCH 09/23] Move linux command and wait method to common tools Signed-off-by: Katarzyna Treder --- storage_devices/disk.py | 2 +- test_tools/common/__init__.py | 0 .../common}/linux_command.py | 0 test_tools/common/wait.py | 20 +++++++++++++++++++ test_tools/dd.py | 2 +- test_tools/ddrescue.py | 2 +- test_tools/device_mapper.py | 2 +- test_tools/fio/fio_param.py | 2 +- test_utils/os_utils.py | 14 +------------ 9 files changed, 26 insertions(+), 18 deletions(-) create mode 100644 test_tools/common/__init__.py rename {test_utils => test_tools/common}/linux_command.py (100%) create mode 100644 test_tools/common/wait.py diff --git a/storage_devices/disk.py b/storage_devices/disk.py index 8a1dc6c..59b60ff 100644 --- a/storage_devices/disk.py +++ b/storage_devices/disk.py @@ -14,7 +14,7 @@ from core.test_run import TestRun from storage_devices.device import Device from test_tools import disk_utils, fs_utils, nvme_cli from test_utils import disk_finder -from test_utils.os_utils import wait +from test_tools.common.wait import wait from connection.utils.output import Output from types.size import Unit diff --git a/test_tools/common/__init__.py b/test_tools/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test_utils/linux_command.py b/test_tools/common/linux_command.py similarity index 100% rename from test_utils/linux_command.py rename to test_tools/common/linux_command.py diff --git a/test_tools/common/wait.py b/test_tools/common/wait.py new file mode 100644 index 0000000..ca7d9f5 --- /dev/null +++ b/test_tools/common/wait.py @@ -0,0 +1,20 @@ +# +# Copyright(c) 2019-2022 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +# + +import time +from datetime import timedelta, datetime + + +def wait(predicate, timeout: timedelta, interval: timedelta = None): + start_time = datetime.now() + result = False + while start_time + timeout > datetime.now(): + result = predicate() + if result: + break + if interval is not None: + time.sleep(interval.total_seconds()) + return result diff --git a/test_tools/dd.py b/test_tools/dd.py index e61d255..e798281 100644 --- a/test_tools/dd.py +++ b/test_tools/dd.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: BSD-3-Clause # -import test_utils.linux_command as linux_comm +import test_tools.common.linux_command as linux_comm import types.size as size from core.test_run import TestRun diff --git a/test_tools/ddrescue.py b/test_tools/ddrescue.py index 799bde5..c7da56e 100644 --- a/test_tools/ddrescue.py +++ b/test_tools/ddrescue.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: BSD-3-Clause # -import test_utils.linux_command as linux_comm +import test_tools.common.linux_command as linux_comm import types.size as size from core.test_run import TestRun diff --git a/test_tools/device_mapper.py b/test_tools/device_mapper.py index c2cf0cf..920022f 100644 --- a/test_tools/device_mapper.py +++ b/test_tools/device_mapper.py @@ -8,7 +8,7 @@ from enum import Enum from core.test_run import TestRun from storage_devices.device import Device -from test_utils.linux_command import LinuxCommand +from test_tools.common.linux_command import LinuxCommand from types.size import Size, Unit diff --git a/test_tools/fio/fio_param.py b/test_tools/fio/fio_param.py index ff8409f..8a14489 100644 --- a/test_tools/fio/fio_param.py +++ b/test_tools/fio/fio_param.py @@ -13,7 +13,7 @@ from connection.base_executor import BaseExecutor from core.test_run import TestRun from storage_devices.device import Device from test_tools.fio.fio_result import FioResult -from test_utils.linux_command import LinuxCommand +from test_tools.common.linux_command import LinuxCommand from types.size import Size diff --git a/test_utils/os_utils.py b/test_utils/os_utils.py index 61dc2c3..992ab25 100644 --- a/test_utils/os_utils.py +++ b/test_utils/os_utils.py @@ -9,7 +9,7 @@ import posixpath import re import time -from datetime import timedelta, datetime +from datetime import timedelta from enum import IntFlag, Enum, IntEnum, StrEnum from packaging import version from typing import List @@ -422,18 +422,6 @@ def kill_all_io(graceful=True): raise Exception(f"Failed to stop vdbench!") -def wait(predicate, timeout: timedelta, interval: timedelta = None): - start_time = datetime.now() - result = False - while start_time + timeout > datetime.now(): - result = predicate() - if result: - break - if interval is not None: - time.sleep(interval.total_seconds()) - return result - - def sync(): TestRun.executor.run_expect_success("sync") From 7f5bbb5240cc22ad5a50ad1231bc3ade34b26334 Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Tue, 10 Dec 2024 14:05:43 +0100 Subject: [PATCH 10/23] Move dmesg to tools Signed-off-by: Katarzyna Treder --- {test_utils => test_tools}/dmesg.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {test_utils => test_tools}/dmesg.py (100%) diff --git a/test_utils/dmesg.py b/test_tools/dmesg.py similarity index 100% rename from test_utils/dmesg.py rename to test_tools/dmesg.py From 2390e4c2af1e64ac2c28d1aa21be87ab84eba392 Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Tue, 10 Dec 2024 14:20:54 +0100 Subject: [PATCH 11/23] Remove generator from test utils Signed-off-by: Katarzyna Treder --- test_utils/generator.py | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 test_utils/generator.py diff --git a/test_utils/generator.py b/test_utils/generator.py deleted file mode 100644 index 8c64ba6..0000000 --- a/test_utils/generator.py +++ /dev/null @@ -1,11 +0,0 @@ -# -# Copyright(c) 2019-2021 Intel Corporation -# SPDX-License-Identifier: BSD-3-Clause -# - -import random -import string - - -def random_string(length: int, chars=string.ascii_letters + string.digits): - return ''.join(random.choice(chars) for i in range(length)) From 324edb0eba157b21374ce730bfb6b85f78fd0126 Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Tue, 10 Dec 2024 14:28:16 +0100 Subject: [PATCH 12/23] Move fstab to tools Signed-off-by: Katarzyna Treder --- {test_utils => test_tools}/fstab.py | 1 + 1 file changed, 1 insertion(+) rename {test_utils => test_tools}/fstab.py (91%) diff --git a/test_utils/fstab.py b/test_tools/fstab.py similarity index 91% rename from test_utils/fstab.py rename to test_tools/fstab.py index fceb375..8d53f33 100644 --- a/test_utils/fstab.py +++ b/test_tools/fstab.py @@ -1,5 +1,6 @@ # # Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # From 40f850c1f9077039ec8f50f713bf75adfc9a0170 Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Tue, 10 Dec 2024 14:31:39 +0100 Subject: [PATCH 13/23] Move git to tools Signed-off-by: Katarzyna Treder --- {test_utils => test_tools}/git.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) rename {test_utils => test_tools}/git.py (94%) diff --git a/test_utils/git.py b/test_tools/git.py similarity index 94% rename from test_utils/git.py rename to test_tools/git.py index 83ad6f7..74c161d 100644 --- a/test_utils/git.py +++ b/test_tools/git.py @@ -97,14 +97,10 @@ def get_commit_hash(version, from_dut: bool = False): return output.stdout -def get_release_tags(forbidden_characters: list = None): +def get_tags(): repo_path = os.path.join(TestRun.usr.working_dir, ".git") output = TestRun.executor.run_expect_success(f"git --git-dir={repo_path} tag").stdout - - if not forbidden_characters: - return output.splitlines() - else: - return [v for v in output.splitlines() if all(c not in v for c in forbidden_characters)] + return output.splitlines() def checkout_version(version): From a6551b032a9f0abc2626c34a07ea42dc31508e66 Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Tue, 10 Dec 2024 14:35:01 +0100 Subject: [PATCH 14/23] Move scsi_debug to tools Signed-off-by: Katarzyna Treder --- {test_utils => test_tools}/scsi_debug.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {test_utils => test_tools}/scsi_debug.py (100%) diff --git a/test_utils/scsi_debug.py b/test_tools/scsi_debug.py similarity index 100% rename from test_utils/scsi_debug.py rename to test_tools/scsi_debug.py From a6848ed8254a7ef90f73be8eea0b62b8f96f15d3 Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Tue, 10 Dec 2024 14:39:38 +0100 Subject: [PATCH 15/23] Remove kedr Signed-off-by: Katarzyna Treder --- test_tools/kedr.py | 129 --------------------------------------------- 1 file changed, 129 deletions(-) delete mode 100644 test_tools/kedr.py diff --git a/test_tools/kedr.py b/test_tools/kedr.py deleted file mode 100644 index cceb43f..0000000 --- a/test_tools/kedr.py +++ /dev/null @@ -1,129 +0,0 @@ -# -# Copyright(c) 2019-2021 Intel Corporation -# SPDX-License-Identifier: BSD-3-Clause -# - -import wget -import os -from enum import Enum - -from core.test_run import TestRun -from test_tools import fs_utils -from test_utils.os_utils import DEBUGFS_MOUNT_POINT - - -KEDR_0_6_URL = "https://github.com/euspectre/kedr/archive/v0.6.tar.gz" -BUILD_DIR = "build" -LEAKS_LOGS_PATH = f"{DEBUGFS_MOUNT_POINT}/kedr_leak_check" -KMALLOC_FAULT_SIMULATION_PATH = "/sys/kernel/debug/kedr_fault_simulation" - - -class KedrProfile(Enum): - MEM_LEAK_CHECK = "leak_check.conf" - FAULT_SIM = "fsim.conf" - - -class Kedr: - @staticmethod - def is_installed(): - return "KEDR version" in TestRun.executor.run("kedr --version").stdout.strip() - - @classmethod - def install(cls): - if cls.is_installed(): - TestRun.LOGGER.info("Kedr is already installed!") - return - - # TODO check if cmake is installed before - # TODO consider using os_utils.download_file() - kedr_archive = wget.download(KEDR_0_6_URL) - - TestRun.executor.rsync_to( - f"\"{kedr_archive}\"", - f"{TestRun.config['working_dir']}") - - kedr_dir = TestRun.executor.run_expect_success( - f"cd {TestRun.config['working_dir']} && " - f"tar -ztf \"{kedr_archive}\" | sed -e 's@/.*@@' | uniq" - ).stdout - - TestRun.executor.run_expect_success( - f"cd {TestRun.config['working_dir']} && " - f"tar -xf \"{kedr_archive}\" && " - f"cd {kedr_dir} && " - f"mkdir -p {BUILD_DIR} && " - f"cd {BUILD_DIR} && " - f"cmake ../sources/ && " - f"make && " - f"make install" - ) - - os.remove(kedr_archive) - TestRun.LOGGER.info("Kedr installed succesfully") - - @classmethod - def is_loaded(cls): - if not cls.is_installed(): - raise Exception("Kedr is not installed!") - - if "KEDR status: loaded" in TestRun.executor.run_expect_success("kedr status").stdout: - return True - else: - return False - - @classmethod - def start(cls, module, profile: KedrProfile = KedrProfile.MEM_LEAK_CHECK): - if not cls.is_installed(): - raise Exception("Kedr is not installed!") - - TestRun.LOGGER.info(f"Starting kedr with {profile} profile") - start_cmd = f"kedr start {module} -f {profile.value}" - TestRun.executor.run_expect_success(start_cmd) - - # TODO extend to scenarios other than kmalloc - def setup_fault_injections(condition: str = "1"): - TestRun.executor.run_expect_success( - f'echo "kmalloc" > {KMALLOC_FAULT_SIMULATION_PATH}/points/kmalloc/current_indicator') - TestRun.executor.run_expect_success( - f'echo "{condition}" > {KMALLOC_FAULT_SIMULATION_PATH}/points/kmalloc/expression') - - @classmethod - def fsim_show_last_fault(cls): - if not cls.is_installed(): - raise Exception("Kedr is not installed!") - - if not cls.is_loaded(): - raise Exception("Kedr is not loaded!") - - return fs_utils.read_file(f"{KMALLOC_FAULT_SIMULATION_PATH}/last_fault") - - @classmethod - def stop(cls): - if not cls.is_installed(): - raise Exception("Kedr is not installed!") - - TestRun.executor.run_expect_success("kedr stop") - - @classmethod - def check_for_mem_leaks(cls, module): - if not cls.is_installed(): - raise Exception("Kedr is not installed!") - - if not cls.is_loaded(): - raise Exception("Kedr is not loaded!") - - if fs_utils.check_if_directory_exists(f"{LEAKS_LOGS_PATH}/{module}"): - logs_path = f"{LEAKS_LOGS_PATH}/{module}" - elif fs_utils.check_if_directory_exists(f"{DEBUGFS_MOUNT_POINT}"): - logs_path = f"{LEAKS_LOGS_PATH}" - else: - raise Exception("Couldn't find kedr logs dir!") - - leaks = fs_utils.read_file(f"{logs_path}/possible_leaks") - frees = fs_utils.read_file(f"{logs_path}/unallocated_frees") - summary = fs_utils.read_file(f"{logs_path}/info") - if leaks or frees: - raise Exception("Memory leaks found!\n" - f"Kedr summary: {summary}\n" - f"Possible memory leaks: {leaks}\n" - f"Unallocated frees: {frees}\n") From ae9b036b4716934406a01f62277558f86a40bf46 Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Tue, 10 Dec 2024 14:46:04 +0100 Subject: [PATCH 16/23] Rename systemd to systemctl and move it to tools Signed-off-by: Katarzyna Treder --- test_tools/fstab.py | 9 ++++----- test_utils/systemd.py => test_tools/systemctl.py | 2 +- test_utils/emergency_escape.py | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) rename test_utils/systemd.py => test_tools/systemctl.py (92%) diff --git a/test_tools/fstab.py b/test_tools/fstab.py index 8d53f33..1b0b669 100644 --- a/test_tools/fstab.py +++ b/test_tools/fstab.py @@ -4,18 +4,17 @@ # SPDX-License-Identifier: BSD-3-Clause # -from test_tools import fs_utils -from test_utils import systemd +from test_tools import fs_utils, systemctl def add_mountpoint(device, mount_point, fs_type, mount_now=True): fs_utils.append_line("/etc/fstab", f"{device.path} {mount_point} {fs_type.name} defaults 0 0") - systemd.reload_daemon() + systemctl.reload_daemon() if mount_now: - systemd.restart_service("local-fs.target") + systemctl.restart_service("local-fs.target") def remove_mountpoint(device): fs_utils.remove_lines("/etc/fstab", device.path) - systemd.reload_daemon() + systemctl.reload_daemon() diff --git a/test_utils/systemd.py b/test_tools/systemctl.py similarity index 92% rename from test_utils/systemd.py rename to test_tools/systemctl.py index dc4bcfd..db31c7f 100644 --- a/test_utils/systemd.py +++ b/test_tools/systemctl.py @@ -1,10 +1,10 @@ # # Copyright(c) 2019-2022 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # from pathlib import Path - from core.test_run import TestRun systemd_service_directory = Path("/usr/lib/systemd/system/") diff --git a/test_utils/emergency_escape.py b/test_utils/emergency_escape.py index a997032..8378ab0 100644 --- a/test_utils/emergency_escape.py +++ b/test_utils/emergency_escape.py @@ -7,7 +7,7 @@ from textwrap import dedent from string import Template from pathlib import Path -from .systemd import enable_service, reload_daemon, systemd_service_directory, disable_service +from test_tools.systemctl import enable_service, reload_daemon, systemd_service_directory from test_tools.fs_utils import ( create_file, write_file, From 7512420e2adbecf2a83847f963f61326d17c77fc Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Wed, 11 Dec 2024 07:54:25 +0100 Subject: [PATCH 17/23] OS tools refactor: - make udev, runlevel, memory and wget separate files - rename os_utils to os_tools and move it to test_tools - remove ModuleRemoveMethod and set modprobe as default - fix regex in is_kernel_module_loaded method - remove get_sys_block_path - move get_block_device_names method to disk tools Signed-off-by: Katarzyna Treder --- internal_plugins/scsi_debug/__init__.py | 8 +- storage_devices/nullblk.py | 5 +- storage_devices/ramdisk.py | 2 +- test_tools/blktrace.py | 2 +- test_tools/disk_utils.py | 13 +- test_tools/fio/fio.py | 4 +- test_tools/initramfs.py | 2 +- test_tools/mdadm.py | 2 +- test_tools/memory.py | 103 +++++ test_tools/os_tools.py | 236 +++++++++++ test_tools/peach_fuzzer/peach_fuzzer.py | 4 +- test_tools/runlevel.py | 109 ++++++ test_tools/systemctl.py | 10 + test_tools/udev.py | 27 ++ test_tools/wget.py | 17 + test_utils/disk_finder.py | 11 +- test_utils/os_utils.py | 496 ------------------------ 17 files changed, 533 insertions(+), 518 deletions(-) create mode 100644 test_tools/memory.py create mode 100644 test_tools/os_tools.py create mode 100644 test_tools/runlevel.py create mode 100644 test_tools/udev.py create mode 100644 test_tools/wget.py delete mode 100644 test_utils/os_utils.py diff --git a/internal_plugins/scsi_debug/__init__.py b/internal_plugins/scsi_debug/__init__.py index 8902c63..f843b7c 100644 --- a/internal_plugins/scsi_debug/__init__.py +++ b/internal_plugins/scsi_debug/__init__.py @@ -6,8 +6,8 @@ from time import sleep from core.test_run_utils import TestRun from storage_devices.device import Device -from test_utils import os_utils from connection.utils.output import CmdException +from test_tools.os_tools import load_kernel_module, is_kernel_module_loaded, unload_kernel_module class ScsiDebug: @@ -24,7 +24,7 @@ class ScsiDebug: def reload(self): self.teardown() sleep(1) - load_output = os_utils.load_kernel_module(self.module_name, self.params) + load_output = load_kernel_module(self.module_name, self.params) if load_output.exit_code != 0: raise CmdException(f"Failed to load {self.module_name} module", load_output) TestRun.LOGGER.info(f"{self.module_name} loaded successfully.") @@ -32,8 +32,8 @@ class ScsiDebug: TestRun.scsi_debug_devices = Device.get_scsi_debug_devices() def teardown(self): - if os_utils.is_kernel_module_loaded(self.module_name): - os_utils.unload_kernel_module(self.module_name) + if is_kernel_module_loaded(self.module_name): + unload_kernel_module(self.module_name) plugin_class = ScsiDebug diff --git a/storage_devices/nullblk.py b/storage_devices/nullblk.py index c55cf5f..559425e 100644 --- a/storage_devices/nullblk.py +++ b/storage_devices/nullblk.py @@ -6,10 +6,9 @@ from core.test_run import TestRun from storage_devices.device import Device from test_tools.fs_utils import ls, parse_ls_output -from test_utils.os_utils import ( +from test_tools.os_tools import ( unload_kernel_module, is_kernel_module_loaded, - ModuleRemoveMethod, reload_kernel_module, ) @@ -37,7 +36,7 @@ class NullBlk(Device): if not is_kernel_module_loaded(cls._module): return TestRun.LOGGER.info("Removing null_blk ") - unload_kernel_module(module_name=cls._module, unload_method=ModuleRemoveMethod.modprobe) + unload_kernel_module(module_name=cls._module) @classmethod def list(cls): diff --git a/storage_devices/ramdisk.py b/storage_devices/ramdisk.py index a4513ce..0baef2f 100644 --- a/storage_devices/ramdisk.py +++ b/storage_devices/ramdisk.py @@ -10,7 +10,7 @@ from storage_devices.device import Device from test_tools import disk_utils from test_tools.fs_utils import ls, parse_ls_output from test_utils.filesystem.symlink import Symlink -from test_utils.os_utils import reload_kernel_module, unload_kernel_module, is_kernel_module_loaded +from test_tools.os_tools import reload_kernel_module, unload_kernel_module, is_kernel_module_loaded from types.size import Size, Unit diff --git a/test_tools/blktrace.py b/test_tools/blktrace.py index f09cd2d..8776acb 100644 --- a/test_tools/blktrace.py +++ b/test_tools/blktrace.py @@ -14,7 +14,7 @@ from datetime import timedelta from core.test_run import TestRun from storage_devices.device import Device from test_utils.filesystem.directory import Directory -from test_utils.os_utils import is_mounted, drop_caches, DropCachesMode +from test_tools.os_tools import is_mounted, drop_caches, DropCachesMode from types.size import Size, Unit DEBUGFS_MOUNT_POINT = "/sys/kernel/debug" diff --git a/test_tools/disk_utils.py b/test_tools/disk_utils.py index a02437e..2bc66eb 100644 --- a/test_tools/disk_utils.py +++ b/test_tools/disk_utils.py @@ -8,6 +8,7 @@ import posixpath import re import time from enum import Enum +from typing import List from core.test_run import TestRun from test_tools import fs_utils @@ -266,7 +267,7 @@ def get_first_partition_offset(device, aligned: bool): def remove_partitions(device): - from test_utils.os_utils import Udev + from test_tools.udev import Udev if device.is_mounted(): device.unmount() @@ -406,3 +407,13 @@ def validate_dev_path(path: str): return path raise ValueError(f'By-id device link {path} is broken.') + + +def get_block_device_names_list(exclude_list: List[int] = None) -> List[str]: + cmd = "lsblk -lo NAME" + if exclude_list is not None: + cmd += f" -e {','.join(str(type_id) for type_id in exclude_list)}" + devices = TestRun.executor.run_expect_success(cmd).stdout + devices_list = devices.splitlines() + devices_list.sort() + return devices_list diff --git a/test_tools/fio/fio.py b/test_tools/fio/fio.py index 813de89..416f422 100644 --- a/test_tools/fio/fio.py +++ b/test_tools/fio/fio.py @@ -10,9 +10,9 @@ import uuid from packaging.version import Version import test_tools.fio.fio_param import test_tools.fs_utils +import test_tools.wget from core.test_run import TestRun from test_tools import fs_utils -from test_utils import os_utils from connection.utils.output import CmdException @@ -50,7 +50,7 @@ class Fio: def install(self): fio_url = f"http://brick.kernel.dk/snaps/fio-{self.min_fio_version}.tar.bz2" - fio_package = os_utils.download_file(fio_url) + fio_package = test_tools.wget.download_file(fio_url) fs_utils.uncompress_archive(fio_package) TestRun.executor.run_expect_success( f"cd {fio_package.parent_dir}/fio-{self.min_fio_version}" diff --git a/test_tools/initramfs.py b/test_tools/initramfs.py index 2a4f259..b1e6a7a 100644 --- a/test_tools/initramfs.py +++ b/test_tools/initramfs.py @@ -4,7 +4,7 @@ # from core.test_run import TestRun -from test_utils.os_utils import get_distro, Distro +from test_tools.os_tools import get_distro, Distro def update(): diff --git a/test_tools/mdadm.py b/test_tools/mdadm.py index 5893512..410ab29 100644 --- a/test_tools/mdadm.py +++ b/test_tools/mdadm.py @@ -8,7 +8,7 @@ import re from core.test_run import TestRun from types.size import Unit -from test_utils.os_utils import Udev +from test_tools.udev import Udev class Mdadm: diff --git a/test_tools/memory.py b/test_tools/memory.py new file mode 100644 index 0000000..8bf782f --- /dev/null +++ b/test_tools/memory.py @@ -0,0 +1,103 @@ +import math + +from connection.utils.output import CmdException +from core.test_run import TestRun +from test_tools.dd import Dd +from test_tools.fs_utils import check_if_directory_exists, create_directory +from test_tools.os_tools import OvercommitMemoryMode, drop_caches, DropCachesMode, \ + MEMORY_MOUNT_POINT, is_mounted +from types.size import Size, Unit + + +def disable_memory_affecting_functions(): + """Disables system functions affecting memory""" + # Don't allow sshd to be killed in case of out-of-memory: + TestRun.executor.run( + "echo '-1000' > /proc/`cat /var/run/sshd.pid`/oom_score_adj" + ) + TestRun.executor.run( + "echo -17 > /proc/`cat /var/run/sshd.pid`/oom_adj" + ) # deprecated + TestRun.executor.run_expect_success( + f"echo {OvercommitMemoryMode.NEVER.value} > /proc/sys/vm/overcommit_memory" + ) + TestRun.executor.run_expect_success("echo '100' > /proc/sys/vm/overcommit_ratio") + TestRun.executor.run_expect_success( + "echo '64 64 32' > /proc/sys/vm/lowmem_reserve_ratio" + ) + TestRun.executor.run_expect_success("swapoff --all") + drop_caches(DropCachesMode.SLAB) + + +def defaultize_memory_affecting_functions(): + """Sets default values to system functions affecting memory""" + TestRun.executor.run_expect_success( + f"echo {OvercommitMemoryMode.DEFAULT.value} > /proc/sys/vm/overcommit_memory" + ) + TestRun.executor.run_expect_success("echo 50 > /proc/sys/vm/overcommit_ratio") + TestRun.executor.run_expect_success( + "echo '256 256 32' > /proc/sys/vm/lowmem_reserve_ratio" + ) + TestRun.executor.run_expect_success("swapon --all") + + +def get_mem_free(): + """Returns free amount of memory in bytes""" + output = TestRun.executor.run_expect_success("free -b") + output = output.stdout.splitlines() + for line in output: + if 'free' in line: + index = line.split().index('free') + 1 # 1st row has 1 element less than following rows + if 'Mem' in line: + mem_line = line.split() + + return Size(int(mem_line[index])) + + +def get_mem_available(): + """Returns amount of available memory from /proc/meminfo""" + cmd = "cat /proc/meminfo | grep MemAvailable | awk '{ print $2 }'" + mem_available = TestRun.executor.run(cmd).stdout + + return Size(int(mem_available), Unit.KibiByte) + + +def get_module_mem_footprint(module_name): + """Returns allocated size of specific module's metadata from /proc/vmallocinfo""" + cmd = f"cat /proc/vmallocinfo | grep {module_name} | awk '{{ print $2 }}' " + output_lines = TestRun.executor.run(cmd).stdout.splitlines() + memory_used = 0 + for line in output_lines: + memory_used += int(line) + + return Size(memory_used) + + +def allocate_memory(size: Size): + """Allocates given amount of memory""" + mount_ramfs() + TestRun.LOGGER.info(f"Allocating {size.get_value(Unit.MiB):0.2f} MiB of memory.") + bs = Size(1, Unit.Blocks512) + dd = ( + Dd() + .block_size(bs) + .count(math.ceil(size / bs)) + .input("/dev/zero") + .output(f"{MEMORY_MOUNT_POINT}/data") + ) + output = dd.run() + if output.exit_code != 0: + raise CmdException("Allocating memory failed.", output) + + +def mount_ramfs(): + """Mounts ramfs to enable allocating memory space""" + if not check_if_directory_exists(MEMORY_MOUNT_POINT): + create_directory(MEMORY_MOUNT_POINT) + if not is_mounted(MEMORY_MOUNT_POINT): + TestRun.executor.run_expect_success(f"mount -t ramfs ramfs {MEMORY_MOUNT_POINT}") + + +def unmount_ramfs(): + """Unmounts ramfs and releases whole space allocated by it in memory""" + TestRun.executor.run_expect_success(f"umount {MEMORY_MOUNT_POINT}") diff --git a/test_tools/os_tools.py b/test_tools/os_tools.py new file mode 100644 index 0000000..f9427be --- /dev/null +++ b/test_tools/os_tools.py @@ -0,0 +1,236 @@ +# +# Copyright(c) 2019-2022 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +# + +import posixpath +import re +import time + +from datetime import timedelta +from enum import IntFlag, Enum, StrEnum +from packaging import version + +from core.test_run import TestRun +from storage_devices.device import Device +from test_tools.disk_utils import get_sysfs_path +from test_tools.fs_utils import check_if_file_exists +from test_utils.filesystem.file import File +from connection.utils.retry import Retry + +DEBUGFS_MOUNT_POINT = "/sys/kernel/debug" +MEMORY_MOUNT_POINT = "/mnt/memspace" + + +class Distro(StrEnum): + UBUNTU = "ubuntu" + DEBIAN = "debian" + REDHAT = "rhel" + OPENEULER = "openeuler" + CENTOS = "centos" + + +class DropCachesMode(IntFlag): + PAGECACHE = 1 + SLAB = 2 + ALL = PAGECACHE | SLAB + + +class OvercommitMemoryMode(Enum): + DEFAULT = 0 + ALWAYS = 1 + NEVER = 2 + + +class SystemManagerType(Enum): + sysv = 0 + systemd = 1 + + +def get_distro(): + output = TestRun.executor.run( + "cat /etc/os-release | grep -e \"^ID=\" | awk -F= '{print$2}' | tr -d '\"'" + ).stdout.lower() + + try: + return Distro(output) + except ValueError: + raise ValueError(f"Could not resolve distro name. Command output: {output}") + + +def drop_caches(level: DropCachesMode = DropCachesMode.ALL): + TestRun.executor.run_expect_success( + f"echo {level.value} > /proc/sys/vm/drop_caches") + + +def get_number_of_processors_from_cpuinfo(): + """Returns number of processors (count) which are listed out in /proc/cpuinfo""" + cmd = f"cat /proc/cpuinfo | grep processor | wc -l" + output = TestRun.executor.run(cmd).stdout + + return int(output) + + +def get_number_of_processes(process_name): + cmd = f"ps aux | grep {process_name} | grep -v grep | wc -l" + output = TestRun.executor.run(cmd).stdout + + return int(output) + + +def get_kernel_version(): + version_string = TestRun.executor.run_expect_success("uname -r").stdout + version_string = version_string.split('-')[0] + return version.Version(version_string) + + +def is_kernel_module_loaded(module_name): + output = TestRun.executor.run(f"lsmod | grep ^{module_name}$") + return output.exit_code == 0 + + +def load_kernel_module(module_name, module_args: {str, str}=None): + cmd = f"modprobe {module_name}" + if module_args is not None: + for key, value in module_args.items(): + cmd += f" {key}={value}" + return TestRun.executor.run(cmd) + + +def unload_kernel_module(module_name): + cmd = f"modprobe -r {module_name}" + return TestRun.executor.run_expect_success(cmd) + + +def get_kernel_module_parameter(module_name, parameter): + param_file_path = f"/sys/module/{module_name}/parameters/{parameter}" + if not check_if_file_exists(param_file_path): + raise FileNotFoundError(f"File {param_file_path} does not exist!") + return File(param_file_path).read() + + +def is_mounted(path: str): + if path is None or path.isspace(): + raise Exception("Checked path cannot be empty") + command = f"mount | grep --fixed-strings '{path.rstrip('/')} '" + return TestRun.executor.run(command).exit_code == 0 + + +def mount_debugfs(): + if not is_mounted(DEBUGFS_MOUNT_POINT): + TestRun.executor.run_expect_success(f"mount -t debugfs none {DEBUGFS_MOUNT_POINT}") + + +def reload_kernel_module(module_name, module_args: {str, str}=None): + if is_kernel_module_loaded(module_name): + unload_kernel_module(module_name) + + Retry.run_while_false( + lambda: load_kernel_module(module_name, module_args).exit_code == 0, + timeout=timedelta(seconds=5) + ) + + +def get_module_path(module_name): + cmd = f"modinfo {module_name}" + + # module path is in second column of first line of `modinfo` output + module_info = TestRun.executor.run_expect_success(cmd).stdout + module_path = module_info.splitlines()[0].split()[1] + + return module_path + + +def get_executable_path(exec_name): + cmd = f"which {exec_name}" + + path = TestRun.executor.run_expect_success(cmd).stdout + + return path + + +def kill_all_io(graceful=True): + if graceful: + # TERM signal should be used in preference to the KILL signal, since a + # process may install a handler for the TERM signal in order to perform + # clean-up steps before terminating in an orderly fashion. + TestRun.executor.run("killall -q --signal TERM dd fio blktrace") + time.sleep(3) + TestRun.executor.run("killall -q --signal TERM dd fio blktrace") + time.sleep(3) + TestRun.executor.run("killall -q --signal KILL dd fio blktrace") + TestRun.executor.run("kill -9 `ps aux | grep -i vdbench.* | awk '{ print $2 }'`") + + if TestRun.executor.run("pgrep -x dd").exit_code == 0: + raise Exception(f"Failed to stop dd!") + if TestRun.executor.run("pgrep -x fio").exit_code == 0: + raise Exception(f"Failed to stop fio!") + if TestRun.executor.run("pgrep -x blktrace").exit_code == 0: + raise Exception(f"Failed to stop blktrace!") + if TestRun.executor.run("pgrep vdbench").exit_code == 0: + raise Exception(f"Failed to stop vdbench!") + + +def sync(): + TestRun.executor.run_expect_success("sync") + + +def get_dut_cpu_number(): + return int(TestRun.executor.run_expect_success("nproc").stdout) + + +def get_dut_cpu_physical_cores(): + """ Get list of CPU numbers that don't share physical cores """ + output = TestRun.executor.run_expect_success("lscpu --all --parse").stdout + + core_list = [] + visited_phys_cores = [] + for line in output.split("\n"): + if "#" in line: + continue + + cpu_no, phys_core_no = line.split(",")[:2] + if phys_core_no not in visited_phys_cores: + core_list.append(cpu_no) + visited_phys_cores.append(phys_core_no) + + return core_list + + +def set_wbt_lat(device: Device, value: int): + if value < 0: + raise ValueError("Write back latency can't be negative number") + + wbt_lat_config_path = posixpath.join( + get_sysfs_path(device.get_device_id()), "queue/wbt_lat_usec" + ) + + return TestRun.executor.run_expect_success(f"echo {value} > {wbt_lat_config_path}") + + +def get_wbt_lat(device: Device): + wbt_lat_config_path = posixpath.join( + get_sysfs_path(device.get_device_id()), "queue/wbt_lat_usec" + ) + + return int(TestRun.executor.run_expect_success(f"cat {wbt_lat_config_path}").stdout) + + +def get_cores_ids_range(numa_node: int): + output = TestRun.executor.run_expect_success(f"lscpu --all --parse").stdout + parse_output = re.findall(r'(\d+),(\d+),(?:\d+),(\d+),,', output, re.I) + + return [element[0] for element in parse_output if int(element[2]) == numa_node] + + +def create_user(username, additional_params=None): + command = "useradd " + if additional_params: + command += "".join([f"-{p} " for p in additional_params]) + command += username + return TestRun.executor.run_expect_success(command) + + +def check_if_user_exists(username): + return TestRun.executor.run(f"id {username}").exit_code == 0 diff --git a/test_tools/peach_fuzzer/peach_fuzzer.py b/test_tools/peach_fuzzer/peach_fuzzer.py index 3301d20..0fabfd2 100644 --- a/test_tools/peach_fuzzer/peach_fuzzer.py +++ b/test_tools/peach_fuzzer/peach_fuzzer.py @@ -12,10 +12,10 @@ import tempfile import lxml.etree as etree from collections import namedtuple +import test_tools.wget from core.test_run import TestRun from test_tools import fs_utils from test_tools.fs_utils import create_directory, check_if_file_exists, write_file -from test_utils import os_utils class PeachFuzzer: @@ -155,7 +155,7 @@ class PeachFuzzer: Install Peach Fuzzer on the DUT """ create_directory(cls.base_dir, True) - peach_archive = os_utils.download_file( + peach_archive = test_tools.wget.download_file( cls.peach_fuzzer_3_0_url, destination_dir=cls.base_dir ) TestRun.executor.run_expect_success( diff --git a/test_tools/runlevel.py b/test_tools/runlevel.py new file mode 100644 index 0000000..f0df1ba --- /dev/null +++ b/test_tools/runlevel.py @@ -0,0 +1,109 @@ +# +# Copyright(c) 2019-2022 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +# + +from enum import IntEnum + +from core.test_run import TestRun +from test_tools.os_tools import SystemManagerType + + +class Runlevel(IntEnum): + """ + Halt the system. + SysV Runlevel: 0 + systemd Target: runlevel0.target, poweroff.target + """ + runlevel0 = 0 + poweroff = runlevel0 + + """ + Single user mode. + SysV Runlevel: 1, s, single + systemd Target: runlevel1.target, rescue.target + """ + runlevel1 = 1 + rescue = runlevel1 + + """ + User-defined/Site-specific runlevels. By default, identical to 3. + SysV Runlevel: 2, 4 + systemd Target: runlevel2.target, runlevel4.target, multi-user.target + """ + runlevel2 = 2 + + """ + Multi-user, non-graphical. Users can usually login via multiple consoles or via the network. + SysV Runlevel: 3 + systemd Target: runlevel3.target, multi-user.target + """ + runlevel3 = 3 + multi_user = runlevel3 + + """ + Multi-user, graphical. Usually has all the services of runlevel 3 plus a graphical login. + SysV Runlevel: 5 + systemd Target: runlevel5.target, graphical.target + """ + runlevel5 = 5 + graphical = runlevel5 + + """ + Reboot + SysV Runlevel: 6 + systemd Target: runlevel6.target, reboot.target + """ + runlevel6 = 6 + reboot = runlevel6 + + """ + Emergency shell + SysV Runlevel: emergency + systemd Target: emergency.target + """ + runlevel7 = 7 + emergency = runlevel7 + + +def get_system_manager(): + output = TestRun.executor.run_expect_success("ps -p 1").stdout + type = output.split('\n')[1].split()[3] + if type == "init": + return SystemManagerType.sysv + elif type == "systemd": + return SystemManagerType.systemd + raise Exception(f"Unknown system manager type ({type}).") + + +def change_runlevel(runlevel: Runlevel): + if runlevel == get_runlevel(): + return + if Runlevel.runlevel0 < runlevel < Runlevel.runlevel6: + system_manager = get_system_manager() + if system_manager == SystemManagerType.systemd: + TestRun.executor.run_expect_success(f"systemctl set-default {runlevel.name}.target") + else: + TestRun.executor.run_expect_success( + f"sed -i 's/^.*id:.*$/id:{runlevel.value}:initdefault: /' /etc/inittab") + TestRun.executor.run_expect_success(f"init {runlevel.value}") + + +def get_runlevel(): + system_manager = get_system_manager() + if system_manager == SystemManagerType.systemd: + result = TestRun.executor.run_expect_success("systemctl get-default") + try: + name = result.stdout.split(".")[0].replace("-", "_") + return Runlevel[name] + except Exception: + raise Exception(f"Cannot parse '{result.output}' to runlevel.") + else: + result = TestRun.executor.run_expect_success("runlevel") + try: + split_output = result.stdout.split() + runlevel = Runlevel(int(split_output[1])) + return runlevel + except Exception: + raise Exception(f"Cannot parse '{result.output}' to runlevel.") diff --git a/test_tools/systemctl.py b/test_tools/systemctl.py index db31c7f..5098208 100644 --- a/test_tools/systemctl.py +++ b/test_tools/systemctl.py @@ -23,3 +23,13 @@ def reload_daemon(): def restart_service(name): TestRun.executor.run_expect_success(f"systemctl restart {name}") + + +def get_service_path(unit_name): + cmd = f"systemctl cat {unit_name}" + + # path is in second column of first line of output + info = TestRun.executor.run_expect_success(cmd).stdout + path = info.splitlines()[0].split()[1] + + return path diff --git a/test_tools/udev.py b/test_tools/udev.py new file mode 100644 index 0000000..541c405 --- /dev/null +++ b/test_tools/udev.py @@ -0,0 +1,27 @@ +# +# Copyright(c) 2019-2022 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +# + +from core.test_run import TestRun + + +class Udev(object): + @staticmethod + def enable(): + TestRun.LOGGER.info("Enabling udev") + TestRun.executor.run_expect_success("udevadm control --start-exec-queue") + + @staticmethod + def disable(): + TestRun.LOGGER.info("Disabling udev") + TestRun.executor.run_expect_success("udevadm control --stop-exec-queue") + + @staticmethod + def trigger(): + TestRun.executor.run_expect_success("udevadm trigger") + + @staticmethod + def settle(): + TestRun.executor.run_expect_success("udevadm settle") diff --git a/test_tools/wget.py b/test_tools/wget.py new file mode 100644 index 0000000..e4df657 --- /dev/null +++ b/test_tools/wget.py @@ -0,0 +1,17 @@ +# +# Copyright(c) 2019-2022 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +# + +from core.test_run import TestRun +from test_utils.filesystem.file import File + + +def download_file(url, destination_dir="/tmp"): + # TODO use wget module instead + command = ("wget --tries=3 --timeout=5 --continue --quiet " + f"--directory-prefix={destination_dir} {url}") + TestRun.executor.run_expect_success(command) + path = f"{destination_dir.rstrip('/')}/{File.get_name(url)}" + return File(path) diff --git a/test_utils/disk_finder.py b/test_utils/disk_finder.py index a02bb85..86cdfaf 100644 --- a/test_utils/disk_finder.py +++ b/test_utils/disk_finder.py @@ -2,13 +2,13 @@ # Copyright(c) 2019-2021 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # - +import os import posixpath from core.test_run import TestRun from test_tools import disk_utils +from test_tools.disk_utils import get_sysfs_path from test_tools.fs_utils import check_if_file_exists, readlink -from test_utils import os_utils from connection.utils.output import CmdException @@ -141,14 +141,13 @@ def get_system_disks(): system_device = TestRun.executor.run_expect_success('mount | grep " / "').stdout.split()[0] readlink_output = readlink(system_device) device_name = readlink_output.split('/')[-1] - sys_block_path = os_utils.get_sys_block_path() used_device_names = __get_slaves(device_name) if not used_device_names: used_device_names = [device_name] disk_names = [] for device_name in used_device_names: - if check_if_file_exists(f'{sys_block_path}/{device_name}/partition'): - parent_device = readlink(f'{sys_block_path}/{device_name}/..').split('/')[-1] + if check_if_file_exists(os.path.join(get_sysfs_path(device_name), "partition")): + parent_device = readlink(os.path.join(get_sysfs_path(device_name), "..")).split('/')[-1] disk_names.append(parent_device) else: disk_names.append(device_name) @@ -159,7 +158,7 @@ def get_system_disks(): def __get_slaves(device_name: str): try: device_names = TestRun.executor.run_expect_success( - f'ls {os_utils.get_sys_block_path()}/{device_name}/slaves').stdout.splitlines() + f"ls {os.path.join(get_sysfs_path(device_name), "slaves")}").stdout.splitlines() except CmdException as e: if "No such file or directory" not in e.output.stderr: raise diff --git a/test_utils/os_utils.py b/test_utils/os_utils.py deleted file mode 100644 index 992ab25..0000000 --- a/test_utils/os_utils.py +++ /dev/null @@ -1,496 +0,0 @@ -# -# Copyright(c) 2019-2022 Intel Corporation -# Copyright(c) 2024 Huawei Technologies Co., Ltd. -# SPDX-License-Identifier: BSD-3-Clause -# - -import math -import posixpath -import re -import time - -from datetime import timedelta -from enum import IntFlag, Enum, IntEnum, StrEnum -from packaging import version -from typing import List - -from core.test_run import TestRun -from storage_devices.device import Device -from test_tools.dd import Dd -from test_tools.disk_utils import get_sysfs_path -from test_tools.fs_utils import check_if_directory_exists, create_directory, check_if_file_exists -from test_utils.filesystem.file import File -from connection.utils.output import CmdException -from connection.utils.retry import Retry -from types.size import Size, Unit - -DEBUGFS_MOUNT_POINT = "/sys/kernel/debug" -MEMORY_MOUNT_POINT = "/mnt/memspace" - - -class Distro(StrEnum): - UBUNTU = "ubuntu" - DEBIAN = "debian" - REDHAT = "rhel" - OPENEULER = "openeuler" - CENTOS = "centos" - - -class DropCachesMode(IntFlag): - PAGECACHE = 1 - SLAB = 2 - ALL = PAGECACHE | SLAB - - -class OvercommitMemoryMode(Enum): - DEFAULT = 0 - ALWAYS = 1 - NEVER = 2 - - -class Runlevel(IntEnum): - """ - Halt the system. - SysV Runlevel: 0 - systemd Target: runlevel0.target, poweroff.target - """ - runlevel0 = 0 - poweroff = runlevel0 - - """ - Single user mode. - SysV Runlevel: 1, s, single - systemd Target: runlevel1.target, rescue.target - """ - runlevel1 = 1 - rescue = runlevel1 - - """ - User-defined/Site-specific runlevels. By default, identical to 3. - SysV Runlevel: 2, 4 - systemd Target: runlevel2.target, runlevel4.target, multi-user.target - """ - runlevel2 = 2 - - """ - Multi-user, non-graphical. Users can usually login via multiple consoles or via the network. - SysV Runlevel: 3 - systemd Target: runlevel3.target, multi-user.target - """ - runlevel3 = 3 - multi_user = runlevel3 - - """ - Multi-user, graphical. Usually has all the services of runlevel 3 plus a graphical login. - SysV Runlevel: 5 - systemd Target: runlevel5.target, graphical.target - """ - runlevel5 = 5 - graphical = runlevel5 - - """ - Reboot - SysV Runlevel: 6 - systemd Target: runlevel6.target, reboot.target - """ - runlevel6 = 6 - reboot = runlevel6 - - """ - Emergency shell - SysV Runlevel: emergency - systemd Target: emergency.target - """ - runlevel7 = 7 - emergency = runlevel7 - - -class SystemManagerType(Enum): - sysv = 0 - systemd = 1 - - -def get_distro(): - output = TestRun.executor.run( - "cat /etc/os-release | grep -e \"^ID=\" | awk -F= '{print$2}' | tr -d '\"'" - ).stdout.lower() - - try: - return Distro(output) - except ValueError: - raise ValueError(f"Could not resolve distro name. Command output: {output}") - - -def get_system_manager(): - output = TestRun.executor.run_expect_success("ps -p 1").stdout - type = output.split('\n')[1].split()[3] - if type == "init": - return SystemManagerType.sysv - elif type == "systemd": - return SystemManagerType.systemd - raise Exception(f"Unknown system manager type ({type}).") - - -def change_runlevel(runlevel: Runlevel): - if runlevel == get_runlevel(): - return - if Runlevel.runlevel0 < runlevel < Runlevel.runlevel6: - system_manager = get_system_manager() - if system_manager == SystemManagerType.systemd: - TestRun.executor.run_expect_success(f"systemctl set-default {runlevel.name}.target") - else: - TestRun.executor.run_expect_success( - f"sed -i 's/^.*id:.*$/id:{runlevel.value}:initdefault: /' /etc/inittab") - TestRun.executor.run_expect_success(f"init {runlevel.value}") - - -def get_runlevel(): - system_manager = get_system_manager() - if system_manager == SystemManagerType.systemd: - result = TestRun.executor.run_expect_success("systemctl get-default") - try: - name = result.stdout.split(".")[0].replace("-", "_") - return Runlevel[name] - except Exception: - raise Exception(f"Cannot parse '{result.output}' to runlevel.") - else: - result = TestRun.executor.run_expect_success("runlevel") - try: - split_output = result.stdout.split() - runlevel = Runlevel(int(split_output[1])) - return runlevel - except Exception: - raise Exception(f"Cannot parse '{result.output}' to runlevel.") - - -class Udev(object): - @staticmethod - def enable(): - TestRun.LOGGER.info("Enabling udev") - TestRun.executor.run_expect_success("udevadm control --start-exec-queue") - - @staticmethod - def disable(): - TestRun.LOGGER.info("Disabling udev") - TestRun.executor.run_expect_success("udevadm control --stop-exec-queue") - - @staticmethod - def trigger(): - TestRun.executor.run_expect_success("udevadm trigger") - - @staticmethod - def settle(): - TestRun.executor.run_expect_success("udevadm settle") - - -def drop_caches(level: DropCachesMode = DropCachesMode.ALL): - TestRun.executor.run_expect_success( - f"echo {level.value} > /proc/sys/vm/drop_caches") - - -def disable_memory_affecting_functions(): - """Disables system functions affecting memory""" - # Don't allow sshd to be killed in case of out-of-memory: - TestRun.executor.run( - "echo '-1000' > /proc/`cat /var/run/sshd.pid`/oom_score_adj" - ) - TestRun.executor.run( - "echo -17 > /proc/`cat /var/run/sshd.pid`/oom_adj" - ) # deprecated - TestRun.executor.run_expect_success( - f"echo {OvercommitMemoryMode.NEVER.value} > /proc/sys/vm/overcommit_memory" - ) - TestRun.executor.run_expect_success("echo '100' > /proc/sys/vm/overcommit_ratio") - TestRun.executor.run_expect_success( - "echo '64 64 32' > /proc/sys/vm/lowmem_reserve_ratio" - ) - TestRun.executor.run_expect_success("swapoff --all") - drop_caches(DropCachesMode.SLAB) - - -def defaultize_memory_affecting_functions(): - """Sets default values to system functions affecting memory""" - TestRun.executor.run_expect_success( - f"echo {OvercommitMemoryMode.DEFAULT.value} > /proc/sys/vm/overcommit_memory" - ) - TestRun.executor.run_expect_success("echo 50 > /proc/sys/vm/overcommit_ratio") - TestRun.executor.run_expect_success( - "echo '256 256 32' > /proc/sys/vm/lowmem_reserve_ratio" - ) - TestRun.executor.run_expect_success("swapon --all") - - -def get_mem_free(): - """Returns free amount of memory in bytes""" - output = TestRun.executor.run_expect_success("free -b") - output = output.stdout.splitlines() - for line in output: - if 'free' in line: - index = line.split().index('free') + 1 # 1st row has 1 element less than following rows - if 'Mem' in line: - mem_line = line.split() - - return Size(int(mem_line[index])) - - -def get_mem_available(): - """Returns amount of available memory from /proc/meminfo""" - cmd = "cat /proc/meminfo | grep MemAvailable | awk '{ print $2 }'" - mem_available = TestRun.executor.run(cmd).stdout - - return Size(int(mem_available), Unit.KibiByte) - - -def get_module_mem_footprint(module_name): - """Returns allocated size of specific module's metadata from /proc/vmallocinfo""" - cmd = f"cat /proc/vmallocinfo | grep {module_name} | awk '{{ print $2 }}' " - output_lines = TestRun.executor.run(cmd).stdout.splitlines() - memory_used = 0 - for line in output_lines: - memory_used += int(line) - - return Size(memory_used) - - -def allocate_memory(size: Size): - """Allocates given amount of memory""" - mount_ramfs() - TestRun.LOGGER.info(f"Allocating {size.get_value(Unit.MiB):0.2f} MiB of memory.") - bs = Size(1, Unit.Blocks512) - dd = ( - Dd() - .block_size(bs) - .count(math.ceil(size / bs)) - .input("/dev/zero") - .output(f"{MEMORY_MOUNT_POINT}/data") - ) - output = dd.run() - if output.exit_code != 0: - raise CmdException("Allocating memory failed.", output) - - -def get_number_of_processors_from_cpuinfo(): - """Returns number of processors (count) which are listed out in /proc/cpuinfo""" - cmd = f"cat /proc/cpuinfo | grep processor | wc -l" - output = TestRun.executor.run(cmd).stdout - - return int(output) - - -def get_number_of_processes(process_name): - cmd = f"ps aux | grep {process_name} | grep -v grep | wc -l" - output = TestRun.executor.run(cmd).stdout - - return int(output) - - -def mount_ramfs(): - """Mounts ramfs to enable allocating memory space""" - if not check_if_directory_exists(MEMORY_MOUNT_POINT): - create_directory(MEMORY_MOUNT_POINT) - if not is_mounted(MEMORY_MOUNT_POINT): - TestRun.executor.run_expect_success(f"mount -t ramfs ramfs {MEMORY_MOUNT_POINT}") - - -def unmount_ramfs(): - """Unmounts ramfs and releases whole space allocated by it in memory""" - TestRun.executor.run_expect_success(f"umount {MEMORY_MOUNT_POINT}") - - -def download_file(url, destination_dir="/tmp"): - # TODO use wget module instead - command = ("wget --tries=3 --timeout=5 --continue --quiet " - f"--directory-prefix={destination_dir} {url}") - TestRun.executor.run_expect_success(command) - path = f"{destination_dir.rstrip('/')}/{File.get_name(url)}" - return File(path) - - -def get_kernel_version(): - version_string = TestRun.executor.run_expect_success("uname -r").stdout - version_string = version_string.split('-')[0] - return version.Version(version_string) - - -class ModuleRemoveMethod(Enum): - rmmod = "rmmod" - modprobe = "modprobe -r" - - -def is_kernel_module_loaded(module_name): - output = TestRun.executor.run(f"lsmod | grep ^{module_name}") - return output.exit_code == 0 - - -def get_sys_block_path(): - sys_block = "/sys/class/block" - if not check_if_directory_exists(sys_block): - sys_block = "/sys/block" - return sys_block - - -def load_kernel_module(module_name, module_args: {str, str}=None): - cmd = f"modprobe {module_name}" - if module_args is not None: - for key, value in module_args.items(): - cmd += f" {key}={value}" - return TestRun.executor.run(cmd) - - -def unload_kernel_module(module_name, unload_method: ModuleRemoveMethod = ModuleRemoveMethod.rmmod): - cmd = f"{unload_method.value} {module_name}" - return TestRun.executor.run_expect_success(cmd) - - -def get_kernel_module_parameter(module_name, parameter): - param_file_path = f"/sys/module/{module_name}/parameters/{parameter}" - if not check_if_file_exists(param_file_path): - raise FileNotFoundError(f"File {param_file_path} does not exist!") - return File(param_file_path).read() - - -def is_mounted(path: str): - if path is None or path.isspace(): - raise Exception("Checked path cannot be empty") - command = f"mount | grep --fixed-strings '{path.rstrip('/')} '" - return TestRun.executor.run(command).exit_code == 0 - - -def mount_debugfs(): - if not is_mounted(DEBUGFS_MOUNT_POINT): - TestRun.executor.run_expect_success(f"mount -t debugfs none {DEBUGFS_MOUNT_POINT}") - - -def reload_kernel_module(module_name, module_args: {str, str}=None, - unload_method: ModuleRemoveMethod = ModuleRemoveMethod.rmmod): - if is_kernel_module_loaded(module_name): - unload_kernel_module(module_name, unload_method) - - Retry.run_while_false( - lambda: load_kernel_module(module_name, module_args).exit_code == 0, - timeout=timedelta(seconds=5) - ) - - -def get_module_path(module_name): - cmd = f"modinfo {module_name}" - - # module path is in second column of first line of `modinfo` output - module_info = TestRun.executor.run_expect_success(cmd).stdout - module_path = module_info.splitlines()[0].split()[1] - - return module_path - - -def get_executable_path(exec_name): - cmd = f"which {exec_name}" - - path = TestRun.executor.run_expect_success(cmd).stdout - - return path - - -def get_udev_service_path(unit_name): - cmd = f"systemctl cat {unit_name}" - - # path is in second column of first line of output - info = TestRun.executor.run_expect_success(cmd).stdout - path = info.splitlines()[0].split()[1] - - return path - - -def kill_all_io(graceful=True): - if graceful: - # TERM signal should be used in preference to the KILL signal, since a - # process may install a handler for the TERM signal in order to perform - # clean-up steps before terminating in an orderly fashion. - TestRun.executor.run("killall -q --signal TERM dd fio blktrace") - time.sleep(3) - TestRun.executor.run("killall -q --signal TERM dd fio blktrace") - time.sleep(3) - TestRun.executor.run("killall -q --signal KILL dd fio blktrace") - TestRun.executor.run("kill -9 `ps aux | grep -i vdbench.* | awk '{ print $2 }'`") - - if TestRun.executor.run("pgrep -x dd").exit_code == 0: - raise Exception(f"Failed to stop dd!") - if TestRun.executor.run("pgrep -x fio").exit_code == 0: - raise Exception(f"Failed to stop fio!") - if TestRun.executor.run("pgrep -x blktrace").exit_code == 0: - raise Exception(f"Failed to stop blktrace!") - if TestRun.executor.run("pgrep vdbench").exit_code == 0: - raise Exception(f"Failed to stop vdbench!") - - -def sync(): - TestRun.executor.run_expect_success("sync") - - -def get_dut_cpu_number(): - return int(TestRun.executor.run_expect_success("nproc").stdout) - - -def get_dut_cpu_physical_cores(): - """ Get list of CPU numbers that don't share physical cores """ - output = TestRun.executor.run_expect_success("lscpu --all --parse").stdout - - core_list = [] - visited_phys_cores = [] - for line in output.split("\n"): - if "#" in line: - continue - - cpu_no, phys_core_no = line.split(",")[:2] - if phys_core_no not in visited_phys_cores: - core_list.append(cpu_no) - visited_phys_cores.append(phys_core_no) - - return core_list - - -def set_wbt_lat(device: Device, value: int): - if value < 0: - raise ValueError("Write back latency can't be negative number") - - wbt_lat_config_path = posixpath.join( - get_sysfs_path(device.get_device_id()), "queue/wbt_lat_usec" - ) - - return TestRun.executor.run_expect_success(f"echo {value} > {wbt_lat_config_path}") - - -def get_wbt_lat(device: Device): - wbt_lat_config_path = posixpath.join( - get_sysfs_path(device.get_device_id()), "queue/wbt_lat_usec" - ) - - return int(TestRun.executor.run_expect_success(f"cat {wbt_lat_config_path}").stdout) - - -def get_cores_ids_range(numa_node: int): - output = TestRun.executor.run_expect_success(f"lscpu --all --parse").stdout - parse_output = re.findall(r'(\d+),(\d+),(?:\d+),(\d+),,', output, re.I) - - return [element[0] for element in parse_output if int(element[2]) == numa_node] - - -def create_user(username, additional_params=None): - command = "useradd " - if additional_params: - command += "".join([f"-{p} " for p in additional_params]) - command += username - return TestRun.executor.run_expect_success(command) - - -def check_if_user_exists(username): - return TestRun.executor.run(f"id {username}").exit_code == 0 - - -def get_block_device_names_list(exclude_list: List[int] = None) -> List[str]: - cmd = "lsblk -lo NAME" - if exclude_list is not None: - cmd += f" -e {','.join(str(type_id) for type_id in exclude_list)}" - devices = TestRun.executor.run_expect_success(cmd).stdout - devices_list = devices.splitlines() - devices_list.sort() - return devices_list From 6970a4310790fc56023661a5c7c1bddca714ceba Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Wed, 11 Dec 2024 09:03:49 +0100 Subject: [PATCH 18/23] Disk finder refactor Signed-off-by: Katarzyna Treder --- core/test_run_utils.py | 2 +- storage_devices/disk.py | 38 ++++++++++++++++++++-- storage_devices/error_device.py | 2 +- storage_devices/lvm.py | 2 +- storage_devices/raid.py | 2 +- {test_utils => test_tools}/disk_finder.py | 39 +++-------------------- 6 files changed, 44 insertions(+), 41 deletions(-) rename {test_utils => test_tools}/disk_finder.py (82%) diff --git a/core/test_run_utils.py b/core/test_run_utils.py index 51e6435..ef4f0b4 100644 --- a/core/test_run_utils.py +++ b/core/test_run_utils.py @@ -19,7 +19,7 @@ from core.pair_testing import generate_pair_testing_testcases, register_testcase from core.plugins import PluginManager from log.base_log import BaseLogResult from storage_devices.disk import Disk -from test_utils import disk_finder +from test_tools import disk_finder from test_utils.dut import Dut TestRun = core.test_run.TestRun diff --git a/storage_devices/disk.py b/storage_devices/disk.py index 59b60ff..ec9e8aa 100644 --- a/storage_devices/disk.py +++ b/storage_devices/disk.py @@ -13,9 +13,9 @@ from enum import IntEnum from core.test_run import TestRun from storage_devices.device import Device from test_tools import disk_utils, fs_utils, nvme_cli -from test_utils import disk_finder from test_tools.common.wait import wait from connection.utils.output import Output +from test_tools.disk_finder import get_block_devices_list, resolve_to_by_id_link from types.size import Unit @@ -159,7 +159,7 @@ class Disk(Device): def is_detected(self): if self.serial_number: - serial_numbers = disk_finder.get_all_serial_numbers() + serial_numbers = Disk.get_all_serial_numbers() return self.serial_number in serial_numbers elif self.path: output = fs_utils.ls_item(f"{self.path}") @@ -214,6 +214,40 @@ class Disk(Device): for disk_type in cls.types_registry: disk_type.plug_all() + @staticmethod + def get_all_serial_numbers(): + serial_numbers = {} + block_devices = get_block_devices_list() + for dev in block_devices: + serial = Disk.get_disk_serial_number(dev) + try: + path = resolve_to_by_id_link(dev) + except Exception: + continue + if serial: + serial_numbers[serial] = path + else: + TestRun.LOGGER.warning(f"Device {path} ({dev}) does not have a serial number.") + serial_numbers[path] = path + return serial_numbers + + @staticmethod + def get_disk_serial_number(dev_path): + commands = [ + f"(udevadm info --query=all --name={dev_path} | grep 'SCSI.*_SERIAL' || " + f"udevadm info --query=all --name={dev_path} | grep 'ID_SERIAL_SHORT') | " + "awk -F '=' '{print $NF}'", + f"sg_inq {dev_path} 2> /dev/null | grep '[Ss]erial number:' | " + "awk '{print $NF}'", + f"udevadm info --query=all --name={dev_path} | grep 'ID_SERIAL' | " + "awk -F '=' '{print $NF}'" + ] + for command in commands: + serial = TestRun.executor.run(command).stdout + if serial: + return serial.split('\n')[0] + return None + @static_init class NvmeDisk(Disk): diff --git a/storage_devices/error_device.py b/storage_devices/error_device.py index 0708603..91d37c7 100644 --- a/storage_devices/error_device.py +++ b/storage_devices/error_device.py @@ -7,7 +7,7 @@ from core.test_run import TestRun from storage_devices.device import Device from test_tools.device_mapper import DmTable, DeviceMapper -from test_utils.disk_finder import resolve_to_by_id_link +from test_tools.disk_finder import resolve_to_by_id_link class ErrorDevice(Device): diff --git a/storage_devices/lvm.py b/storage_devices/lvm.py index 2eb8e5b..ab80379 100644 --- a/storage_devices/lvm.py +++ b/storage_devices/lvm.py @@ -11,7 +11,7 @@ from core.test_run import TestRun from storage_devices.device import Device from storage_devices.disk import Disk from test_tools.fs_utils import readlink -from test_utils.disk_finder import resolve_to_by_id_link, get_system_disks +from test_tools.disk_finder import resolve_to_by_id_link, get_system_disks from test_utils.filesystem.symlink import Symlink from types.size import Size diff --git a/storage_devices/raid.py b/storage_devices/raid.py index 1ac022b..ab68d55 100644 --- a/storage_devices/raid.py +++ b/storage_devices/raid.py @@ -10,7 +10,7 @@ from storage_devices.device import Device from storage_devices.disk import Disk from test_tools.fs_utils import readlink from test_tools.mdadm import Mdadm -from test_utils.disk_finder import resolve_to_by_id_link +from test_tools.disk_finder import resolve_to_by_id_link from types.size import Size, Unit diff --git a/test_utils/disk_finder.py b/test_tools/disk_finder.py similarity index 82% rename from test_utils/disk_finder.py rename to test_tools/disk_finder.py index 86cdfaf..768d2e2 100644 --- a/test_utils/disk_finder.py +++ b/test_tools/disk_finder.py @@ -1,5 +1,6 @@ # # Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # import os @@ -103,40 +104,6 @@ def discover_ssd_devices(block_devices, devices_res): block_devices.remove(dev) -def get_disk_serial_number(dev_path): - commands = [ - f"(udevadm info --query=all --name={dev_path} | grep 'SCSI.*_SERIAL' || " - f"udevadm info --query=all --name={dev_path} | grep 'ID_SERIAL_SHORT') | " - "awk -F '=' '{print $NF}'", - f"sg_inq {dev_path} 2> /dev/null | grep '[Ss]erial number:' | " - "awk '{print $NF}'", - f"udevadm info --query=all --name={dev_path} | grep 'ID_SERIAL' | " - "awk -F '=' '{print $NF}'" - ] - for command in commands: - serial = TestRun.executor.run(command).stdout - if serial: - return serial.split('\n')[0] - return None - - -def get_all_serial_numbers(): - serial_numbers = {} - block_devices = get_block_devices_list() - for dev in block_devices: - serial = get_disk_serial_number(dev) - try: - path = resolve_to_by_id_link(dev) - except Exception: - continue - if serial: - serial_numbers[serial] = path - else: - TestRun.LOGGER.warning(f"Device {path} ({dev}) does not have a serial number.") - serial_numbers[path] = path - return serial_numbers - - def get_system_disks(): system_device = TestRun.executor.run_expect_success('mount | grep " / "').stdout.split()[0] readlink_output = readlink(system_device) @@ -176,7 +143,9 @@ def __get_slaves(device_name: str): def resolve_to_by_id_link(path): by_id_paths = TestRun.executor.run_expect_success("ls /dev/disk/by-id -1").stdout.splitlines() - dev_full_paths = [posixpath.join("/dev/disk/by-id", by_id_path) for by_id_path in by_id_paths] + dev_full_paths = [ + posixpath.join("/dev/disk/by-id", by_id_path) for by_id_path in by_id_paths + ] for full_path in dev_full_paths: # handle exception for broken links From c218f64c81ca76f2c8e46eb5c5cffe6aabe54c50 Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Wed, 11 Dec 2024 13:55:06 +0100 Subject: [PATCH 19/23] Types fix - rename to type_def Signed-off-by: Katarzyna Treder --- storage_devices/device.py | 2 +- storage_devices/disk.py | 2 +- storage_devices/lvm.py | 2 +- storage_devices/partition.py | 2 +- storage_devices/raid.py | 2 +- storage_devices/ramdisk.py | 2 +- test_tools/blktrace.py | 2 +- test_tools/dd.py | 2 +- test_tools/ddrescue.py | 2 +- test_tools/device_mapper.py | 2 +- test_tools/disk_utils.py | 2 +- test_tools/fio/fio_param.py | 2 +- test_tools/fio/fio_result.py | 4 ++-- test_tools/fs_utils.py | 2 +- test_tools/iostat.py | 4 ++-- test_tools/mdadm.py | 2 +- test_tools/memory.py | 2 +- test_utils/filesystem/file.py | 2 +- {types => type_def}/__init__.py | 0 {types => type_def}/size.py | 0 {types => type_def}/time.py | 0 21 files changed, 20 insertions(+), 20 deletions(-) rename {types => type_def}/__init__.py (100%) rename {types => type_def}/size.py (100%) rename {types => type_def}/time.py (100%) diff --git a/storage_devices/device.py b/storage_devices/device.py index 5ce9bce..d8512c1 100644 --- a/storage_devices/device.py +++ b/storage_devices/device.py @@ -9,7 +9,7 @@ from core.test_run import TestRun from test_tools import disk_utils, fs_utils from test_tools.disk_utils import get_device_filesystem_type, get_sysfs_path from test_utils.io_stats import IoStats -from types.size import Size, Unit +from type_def.size import Size, Unit class Device: diff --git a/storage_devices/disk.py b/storage_devices/disk.py index ec9e8aa..fb981e5 100644 --- a/storage_devices/disk.py +++ b/storage_devices/disk.py @@ -16,7 +16,7 @@ from test_tools import disk_utils, fs_utils, nvme_cli from test_tools.common.wait import wait from connection.utils.output import Output from test_tools.disk_finder import get_block_devices_list, resolve_to_by_id_link -from types.size import Unit +from type_def.size import Unit class DiskType(IntEnum): diff --git a/storage_devices/lvm.py b/storage_devices/lvm.py index ab80379..12215e7 100644 --- a/storage_devices/lvm.py +++ b/storage_devices/lvm.py @@ -13,7 +13,7 @@ from storage_devices.disk import Disk from test_tools.fs_utils import readlink from test_tools.disk_finder import resolve_to_by_id_link, get_system_disks from test_utils.filesystem.symlink import Symlink -from types.size import Size +from type_def.size import Size lvm_config_path = "/etc/lvm/lvm.conf" filter_prototype_regex = r"^\sfilter\s=\s\[" diff --git a/storage_devices/partition.py b/storage_devices/partition.py index 966e057..375e4d5 100644 --- a/storage_devices/partition.py +++ b/storage_devices/partition.py @@ -5,7 +5,7 @@ from storage_devices.device import Device from test_tools import disk_utils -from types.size import Size +from type_def.size import Size class Partition(Device): diff --git a/storage_devices/raid.py b/storage_devices/raid.py index ab68d55..2499055 100644 --- a/storage_devices/raid.py +++ b/storage_devices/raid.py @@ -11,7 +11,7 @@ from storage_devices.disk import Disk from test_tools.fs_utils import readlink from test_tools.mdadm import Mdadm from test_tools.disk_finder import resolve_to_by_id_link -from types.size import Size, Unit +from type_def.size import Size, Unit def get_devices_paths_string(devices: [Device]): diff --git a/storage_devices/ramdisk.py b/storage_devices/ramdisk.py index 0baef2f..8a1c4e1 100644 --- a/storage_devices/ramdisk.py +++ b/storage_devices/ramdisk.py @@ -11,7 +11,7 @@ from test_tools import disk_utils from test_tools.fs_utils import ls, parse_ls_output from test_utils.filesystem.symlink import Symlink from test_tools.os_tools import reload_kernel_module, unload_kernel_module, is_kernel_module_loaded -from types.size import Size, Unit +from type_def.size import Size, Unit class RamDisk(Device): diff --git a/test_tools/blktrace.py b/test_tools/blktrace.py index 8776acb..0c928a4 100644 --- a/test_tools/blktrace.py +++ b/test_tools/blktrace.py @@ -15,7 +15,7 @@ from core.test_run import TestRun from storage_devices.device import Device from test_utils.filesystem.directory import Directory from test_tools.os_tools import is_mounted, drop_caches, DropCachesMode -from types.size import Size, Unit +from type_def.size import Size, Unit DEBUGFS_MOUNT_POINT = "/sys/kernel/debug" PREFIX = "trace_" diff --git a/test_tools/dd.py b/test_tools/dd.py index e798281..0428b6f 100644 --- a/test_tools/dd.py +++ b/test_tools/dd.py @@ -4,7 +4,7 @@ # import test_tools.common.linux_command as linux_comm -import types.size as size +import type_def.size as size from core.test_run import TestRun diff --git a/test_tools/ddrescue.py b/test_tools/ddrescue.py index c7da56e..3a6dbb7 100644 --- a/test_tools/ddrescue.py +++ b/test_tools/ddrescue.py @@ -4,7 +4,7 @@ # import test_tools.common.linux_command as linux_comm -import types.size as size +import type_def.size as size from core.test_run import TestRun diff --git a/test_tools/device_mapper.py b/test_tools/device_mapper.py index 920022f..9f78216 100644 --- a/test_tools/device_mapper.py +++ b/test_tools/device_mapper.py @@ -9,7 +9,7 @@ from enum import Enum from core.test_run import TestRun from storage_devices.device import Device from test_tools.common.linux_command import LinuxCommand -from types.size import Size, Unit +from type_def.size import Size, Unit class DmTarget(Enum): diff --git a/test_tools/disk_utils.py b/test_tools/disk_utils.py index 2bc66eb..50ba340 100644 --- a/test_tools/disk_utils.py +++ b/test_tools/disk_utils.py @@ -15,7 +15,7 @@ from test_tools import fs_utils from test_tools.dd import Dd from test_tools.fs_utils import readlink, parse_ls_output, ls from connection.utils.output import CmdException -from types.size import Size, Unit +from type_def.size import Size, Unit SECTOR_SIZE = 512 diff --git a/test_tools/fio/fio_param.py b/test_tools/fio/fio_param.py index 8a14489..b0c71e0 100644 --- a/test_tools/fio/fio_param.py +++ b/test_tools/fio/fio_param.py @@ -14,7 +14,7 @@ from core.test_run import TestRun from storage_devices.device import Device from test_tools.fio.fio_result import FioResult from test_tools.common.linux_command import LinuxCommand -from types.size import Size +from type_def.size import Size class CpusAllowedPolicy(Enum): diff --git a/test_tools/fio/fio_result.py b/test_tools/fio/fio_result.py index e2912f7..5163609 100644 --- a/test_tools/fio/fio_result.py +++ b/test_tools/fio/fio_result.py @@ -4,8 +4,8 @@ # -from types.size import Size, Unit, UnitPerSecond -from types.time import Time +from type_def.size import Size, Unit, UnitPerSecond +from type_def.time import Time class FioResult: diff --git a/test_tools/fs_utils.py b/test_tools/fs_utils.py index 44c3eb2..b16f32c 100644 --- a/test_tools/fs_utils.py +++ b/test_tools/fs_utils.py @@ -15,7 +15,7 @@ from aenum import IntFlag, Enum from core.test_run import TestRun from test_tools.dd import Dd -from types.size import Size, Unit +from type_def.size import Size, Unit class Permissions(IntFlag): diff --git a/test_tools/iostat.py b/test_tools/iostat.py index d355bbc..094327f 100644 --- a/test_tools/iostat.py +++ b/test_tools/iostat.py @@ -5,8 +5,8 @@ from core.test_run import TestRun from storage_devices.device import Device -from types.size import Size, Unit, UnitPerSecond -from types.time import Time +from type_def.size import Size, Unit, UnitPerSecond +from type_def.time import Time import csv diff --git a/test_tools/mdadm.py b/test_tools/mdadm.py index 410ab29..564a428 100644 --- a/test_tools/mdadm.py +++ b/test_tools/mdadm.py @@ -7,7 +7,7 @@ import re from core.test_run import TestRun -from types.size import Unit +from type_def.size import Unit from test_tools.udev import Udev diff --git a/test_tools/memory.py b/test_tools/memory.py index 8bf782f..2f6c166 100644 --- a/test_tools/memory.py +++ b/test_tools/memory.py @@ -6,7 +6,7 @@ from test_tools.dd import Dd from test_tools.fs_utils import check_if_directory_exists, create_directory from test_tools.os_tools import OvercommitMemoryMode, drop_caches, DropCachesMode, \ MEMORY_MOUNT_POINT, is_mounted -from types.size import Size, Unit +from type_def.size import Size, Unit def disable_memory_affecting_functions(): diff --git a/test_utils/filesystem/file.py b/test_utils/filesystem/file.py index efa40c1..1df8e8c 100644 --- a/test_utils/filesystem/file.py +++ b/test_utils/filesystem/file.py @@ -8,7 +8,7 @@ from datetime import timedelta from test_tools import fs_utils from test_tools.dd import Dd from test_utils.filesystem.fs_item import FsItem -from types.size import Size +from type_def.size import Size class File(FsItem): diff --git a/types/__init__.py b/type_def/__init__.py similarity index 100% rename from types/__init__.py rename to type_def/__init__.py diff --git a/types/size.py b/type_def/size.py similarity index 100% rename from types/size.py rename to type_def/size.py diff --git a/types/time.py b/type_def/time.py similarity index 100% rename from types/time.py rename to type_def/time.py From 6dd9c9ca8ce001bb913a5a535f1ade4229e4fb75 Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Wed, 11 Dec 2024 17:54:57 +0100 Subject: [PATCH 20/23] Iostat refactor Signed-off-by: Katarzyna Treder --- test_tools/iostat.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test_tools/iostat.py b/test_tools/iostat.py index 094327f..2a07dcb 100644 --- a/test_tools/iostat.py +++ b/test_tools/iostat.py @@ -4,7 +4,6 @@ # from core.test_run import TestRun -from storage_devices.device import Device from type_def.size import Size, Unit, UnitPerSecond from type_def.time import Time import csv @@ -84,7 +83,7 @@ class IOstatExtended: @classmethod def get_iostat_list( cls, - devices_list: [Device], + devices_list: [str], since_boot: bool = True, interval: int = 1, ): @@ -138,7 +137,7 @@ class IOstatBasic: @classmethod def get_iostat_list( cls, - devices_list: [Device], + devices_list: [str], since_boot: bool = True, interval: int = 1, ): @@ -150,8 +149,8 @@ class IOstatBasic: def _get_iostat_list( - class_type: type, - devices_list: [Device], + class_type: type(IOstatExtended) | type(IOstatBasic), + devices_list: [str], since_boot: bool, interval: int, ): @@ -163,7 +162,7 @@ def _get_iostat_list( if not since_boot: iostat_cmd += f"-y {interval} 1 " - iostat_cmd += " ".join([name.get_device_id() for name in devices_list]) + iostat_cmd += " ".join(devices_list) sed_cmd = "sed -n '/^$/d;s/\s\+/,/g;/^Device/,$p'" From fd869a0afc69fee37528086458e557a6192a8bfe Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Wed, 11 Dec 2024 18:56:23 +0100 Subject: [PATCH 21/23] Refactor disk tools and fs tools Signed-off-by: Katarzyna Treder --- connection/local_executor.py | 2 +- internal_plugins/vdbench/__init__.py | 2 +- log/logger.py | 2 +- storage_devices/device.py | 36 +++++------ storage_devices/disk.py | 31 +++++----- storage_devices/lvm.py | 2 +- storage_devices/nullblk.py | 2 +- storage_devices/partition.py | 4 +- storage_devices/raid.py | 2 +- storage_devices/ramdisk.py | 6 +- test_tools/blktrace.py | 3 +- test_tools/disk_finder.py | 14 ++--- test_tools/{disk_utils.py => disk_tools.py} | 67 +++------------------ test_tools/fio/fio.py | 6 +- test_tools/{fs_utils.py => fs_tools.py} | 65 +++++++++++++++++++- test_tools/fstab.py | 6 +- test_tools/memory.py | 4 +- test_tools/os_tools.py | 11 +--- test_tools/peach_fuzzer/peach_fuzzer.py | 10 +-- test_utils/drbd.py | 2 - test_utils/emergency_escape.py | 2 +- test_utils/filesystem/directory.py | 14 ++--- test_utils/filesystem/file.py | 32 +++++----- test_utils/filesystem/fs_item.py | 28 ++++----- test_utils/filesystem/symlink.py | 2 +- 25 files changed, 179 insertions(+), 176 deletions(-) rename test_tools/{disk_utils.py => disk_tools.py} (84%) rename test_tools/{fs_utils.py => fs_tools.py} (85%) diff --git a/connection/local_executor.py b/connection/local_executor.py index b8ba46d..874988c 100644 --- a/connection/local_executor.py +++ b/connection/local_executor.py @@ -9,7 +9,7 @@ from datetime import timedelta from connection.base_executor import BaseExecutor from core.test_run import TestRun -from test_tools.fs_utils import copy +from test_tools.fs_tools import copy from connection.utils.output import Output, CmdException diff --git a/internal_plugins/vdbench/__init__.py b/internal_plugins/vdbench/__init__.py index ecf8d1a..536db28 100644 --- a/internal_plugins/vdbench/__init__.py +++ b/internal_plugins/vdbench/__init__.py @@ -9,7 +9,7 @@ import posixpath from datetime import timedelta from core.test_run import TestRun -from test_tools import fs_utils +from test_tools import fs_tools class Vdbench: diff --git a/log/logger.py b/log/logger.py index 5d7cf18..bb8a988 100644 --- a/log/logger.py +++ b/log/logger.py @@ -188,7 +188,7 @@ class Log(HtmlLogManager, metaclass=Singleton): def get_additional_logs(self): from core.test_run import TestRun - from test_tools.fs_utils import check_if_file_exists + from test_tools.fs_tools import check_if_file_exists messages_log = "/var/log/messages" if not check_if_file_exists(messages_log): messages_log = "/var/log/syslog" diff --git a/storage_devices/device.py b/storage_devices/device.py index d8512c1..7bfb5c3 100644 --- a/storage_devices/device.py +++ b/storage_devices/device.py @@ -5,27 +5,29 @@ # import posixpath +import test_tools.fs_tools from core.test_run import TestRun -from test_tools import disk_utils, fs_utils -from test_tools.disk_utils import get_device_filesystem_type, get_sysfs_path +from test_tools import disk_tools, fs_tools +from test_tools.disk_tools import get_sysfs_path +from test_tools.fs_tools import get_device_filesystem_type from test_utils.io_stats import IoStats from type_def.size import Size, Unit class Device: def __init__(self, path): - disk_utils.validate_dev_path(path) + disk_tools.validate_dev_path(path) self.path = path - self.size = Size(disk_utils.get_size(self.get_device_id()), Unit.Byte) + self.size = Size(disk_tools.get_size(self.get_device_id()), Unit.Byte) self.filesystem = get_device_filesystem_type(self.get_device_id()) self.mount_point = None - def create_filesystem(self, fs_type: disk_utils.Filesystem, force=True, blocksize=None): - disk_utils.create_filesystem(self, fs_type, force, blocksize) + def create_filesystem(self, fs_type: test_tools.fs_tools.Filesystem, force=True, blocksize=None): + test_tools.fs_tools.create_filesystem(self, fs_type, force, blocksize) self.filesystem = fs_type def wipe_filesystem(self, force=True): - disk_utils.wipe_filesystem(self, force) + test_tools.fs_tools.wipe_filesystem(self, force) self.filesystem = None def is_mounted(self): @@ -34,13 +36,13 @@ class Device: return False else: mount_point_line = output.stdout.split('\n')[1] - device_path = fs_utils.readlink(self.path) + device_path = fs_tools.readlink(self.path) self.mount_point = mount_point_line[0:mount_point_line.find(device_path)].strip() return True def mount(self, mount_point, options: [str] = None): if not self.is_mounted(): - if disk_utils.mount(self, mount_point, options): + if disk_tools.mount(self, mount_point, options): self.mount_point = mount_point else: raise Exception(f"Device is already mounted! Actual mount point: {self.mount_point}") @@ -48,7 +50,7 @@ class Device: def unmount(self): if not self.is_mounted(): TestRun.LOGGER.info("Device is not mounted.") - elif disk_utils.unmount(self): + elif disk_tools.unmount(self): self.mount_point = None def get_device_link(self, directory: str): @@ -56,27 +58,27 @@ class Device: return next(i for i in items if i.full_path.startswith(directory)) def get_device_id(self): - return fs_utils.readlink(self.path).split('/')[-1] + return fs_tools.readlink(self.path).split('/')[-1] def get_all_device_links(self, directory: str): - from test_tools import fs_utils - output = fs_utils.ls(f"$(find -L {directory} -samefile {self.path})") - return fs_utils.parse_ls_output(output, self.path) + from test_tools import fs_tools + output = fs_tools.ls(f"$(find -L {directory} -samefile {self.path})") + return fs_tools.parse_ls_output(output, self.path) def get_io_stats(self): return IoStats.get_io_stats(self.get_device_id()) def get_sysfs_property(self, property_name): - path = posixpath.join(disk_utils.get_sysfs_path(self.get_device_id()), + path = posixpath.join(disk_tools.get_sysfs_path(self.get_device_id()), "queue", property_name) return TestRun.executor.run_expect_success(f"cat {path}").stdout def set_sysfs_property(self, property_name, value): TestRun.LOGGER.info( f"Setting {property_name} for device {self.get_device_id()} to {value}.") - path = posixpath.join(disk_utils.get_sysfs_path(self.get_device_id()), "queue", + path = posixpath.join(disk_tools.get_sysfs_path(self.get_device_id()), "queue", property_name) - fs_utils.write_file(path, str(value)) + fs_tools.write_file(path, str(value)) def set_max_io_size(self, new_max_io_size: Size): self.set_sysfs_property("max_sectors_kb", diff --git a/storage_devices/disk.py b/storage_devices/disk.py index fb981e5..0ac2b29 100644 --- a/storage_devices/disk.py +++ b/storage_devices/disk.py @@ -10,9 +10,10 @@ import re from datetime import timedelta from enum import IntEnum +import test_tools.fs_tools from core.test_run import TestRun from storage_devices.device import Device -from test_tools import disk_utils, fs_utils, nvme_cli +from test_tools import disk_tools, fs_tools, nvme_cli from test_tools.common.wait import wait from connection.utils.output import Output from test_tools.disk_finder import get_block_devices_list, resolve_to_by_id_link @@ -137,24 +138,24 @@ class Disk(Device): ) return recognized_types[0] - def create_partitions(self, sizes: [], partition_table_type=disk_utils.PartitionTable.gpt): - disk_utils.create_partitions(self, sizes, partition_table_type) + def create_partitions(self, sizes: [], partition_table_type=disk_tools.PartitionTable.gpt): + disk_tools.create_partitions(self, sizes, partition_table_type) def remove_partition(self, part): part_number = int(part.path.split("part")[1]) - disk_utils.remove_parition(self, part_number) + disk_tools.remove_parition(self, part_number) self.partitions.remove(part) def umount_all_partitions(self): TestRun.LOGGER.info(f"Unmounting all partitions from: {self.path}") - cmd = f"umount -l {fs_utils.readlink(self.path)}*?" + cmd = f"umount -l {fs_tools.readlink(self.path)}*?" TestRun.executor.run(cmd) def remove_partitions(self): for part in self.partitions: - if part.is_mounted(): + if test_tools.fs_tools.is_mounted(part.path): part.unmount() - if disk_utils.remove_partitions(self): + if disk_tools.remove_partitions(self): self.partitions.clear() def is_detected(self): @@ -162,8 +163,8 @@ class Disk(Device): serial_numbers = Disk.get_all_serial_numbers() return self.serial_number in serial_numbers elif self.path: - output = fs_utils.ls_item(f"{self.path}") - return fs_utils.parse_ls_output(output)[0] is not None + output = fs_tools.ls_item(f"{self.path}") + return fs_tools.parse_ls_output(output)[0] is not None raise Exception("Couldn't check if device is detected by the system") def wait_for_plug_status(self, should_be_visible): @@ -289,8 +290,8 @@ class NvmeDisk(Disk): base = f"/sys/block/{device_id}/device" for suffix in ["/remove", "/device/remove"]: try: - output = fs_utils.ls_item(base + suffix) - fs_utils.parse_ls_output(output)[0] + output = fs_tools.ls_item(base + suffix) + fs_tools.parse_ls_output(output)[0] except TypeError: continue return base + suffix @@ -345,8 +346,8 @@ class SataDisk(Disk): @staticmethod def get_sysfs_addr(device_id): ls_command = f"$(find -H /sys/devices/ -name {device_id} -type d)" - output = fs_utils.ls_item(f"{ls_command}") - sysfs_addr = fs_utils.parse_ls_output(output)[0] + output = fs_tools.ls_item(f"{ls_command}") + sysfs_addr = fs_tools.parse_ls_output(output)[0] if not sysfs_addr: raise Exception(f"Failed to find sysfs address: ls -l {ls_command}") return sysfs_addr.full_path @@ -412,8 +413,8 @@ class VirtioDisk(Disk): @staticmethod def get_sysfs_addr(device_id: str) -> str: ls_command = f"$(find -H /sys/devices/ -name {device_id} -type d)" - output = fs_utils.ls_item(f"{ls_command}") - sysfs_addr = fs_utils.parse_ls_output(output)[0] + output = fs_tools.ls_item(f"{ls_command}") + sysfs_addr = fs_tools.parse_ls_output(output)[0] if not sysfs_addr: raise Exception(f"Failed to find sysfs address: ls -l {ls_command}") diff --git a/storage_devices/lvm.py b/storage_devices/lvm.py index 12215e7..2e94fdd 100644 --- a/storage_devices/lvm.py +++ b/storage_devices/lvm.py @@ -10,7 +10,7 @@ from typing import Union from core.test_run import TestRun from storage_devices.device import Device from storage_devices.disk import Disk -from test_tools.fs_utils import readlink +from test_tools.fs_tools import readlink from test_tools.disk_finder import resolve_to_by_id_link, get_system_disks from test_utils.filesystem.symlink import Symlink from type_def.size import Size diff --git a/storage_devices/nullblk.py b/storage_devices/nullblk.py index 559425e..fd43cc2 100644 --- a/storage_devices/nullblk.py +++ b/storage_devices/nullblk.py @@ -5,7 +5,7 @@ from core.test_run import TestRun from storage_devices.device import Device -from test_tools.fs_utils import ls, parse_ls_output +from test_tools.fs_tools import ls, parse_ls_output from test_tools.os_tools import ( unload_kernel_module, is_kernel_module_loaded, diff --git a/storage_devices/partition.py b/storage_devices/partition.py index 375e4d5..f39b556 100644 --- a/storage_devices/partition.py +++ b/storage_devices/partition.py @@ -4,13 +4,13 @@ # from storage_devices.device import Device -from test_tools import disk_utils +from test_tools import disk_tools from type_def.size import Size class Partition(Device): def __init__(self, parent_dev, type, number, begin: Size, end: Size): - Device.__init__(self, disk_utils.get_partition_path(parent_dev.path, number)) + Device.__init__(self, disk_tools.get_partition_path(parent_dev.path, number)) self.number = number self.parent_device = parent_dev self.type = type diff --git a/storage_devices/raid.py b/storage_devices/raid.py index 2499055..3d1c9c1 100644 --- a/storage_devices/raid.py +++ b/storage_devices/raid.py @@ -8,7 +8,7 @@ from enum import IntEnum, Enum from core.test_run import TestRun from storage_devices.device import Device from storage_devices.disk import Disk -from test_tools.fs_utils import readlink +from test_tools.fs_tools import readlink from test_tools.mdadm import Mdadm from test_tools.disk_finder import resolve_to_by_id_link from type_def.size import Size, Unit diff --git a/storage_devices/ramdisk.py b/storage_devices/ramdisk.py index 8a1c4e1..fc04ba0 100644 --- a/storage_devices/ramdisk.py +++ b/storage_devices/ramdisk.py @@ -7,8 +7,8 @@ import posixpath from core.test_run import TestRun from storage_devices.device import Device -from test_tools import disk_utils -from test_tools.fs_utils import ls, parse_ls_output +from test_tools import disk_tools +from test_tools.fs_tools import ls, parse_ls_output from test_utils.filesystem.symlink import Symlink from test_tools.os_tools import reload_kernel_module, unload_kernel_module, is_kernel_module_loaded from type_def.size import Size, Unit @@ -68,7 +68,7 @@ class RamDisk(Device): ram_disks = cls._list_devices() return ( len(ram_disks) >= disk_count - and Size(disk_utils.get_size(ram_disks[0].name), Unit.Byte).align_down(Unit.MiB.value) + and Size(disk_tools.get_size(ram_disks[0].name), Unit.Byte).align_down(Unit.MiB.value) == disk_size.align_down(Unit.MiB.value) ) diff --git a/test_tools/blktrace.py b/test_tools/blktrace.py index 0c928a4..045742e 100644 --- a/test_tools/blktrace.py +++ b/test_tools/blktrace.py @@ -14,7 +14,8 @@ from datetime import timedelta from core.test_run import TestRun from storage_devices.device import Device from test_utils.filesystem.directory import Directory -from test_tools.os_tools import is_mounted, drop_caches, DropCachesMode +from test_tools.os_tools import drop_caches, DropCachesMode +from test_tools.fs_tools import is_mounted from type_def.size import Size, Unit DEBUGFS_MOUNT_POINT = "/sys/kernel/debug" diff --git a/test_tools/disk_finder.py b/test_tools/disk_finder.py index 768d2e2..1cbbff2 100644 --- a/test_tools/disk_finder.py +++ b/test_tools/disk_finder.py @@ -7,9 +7,9 @@ import os import posixpath from core.test_run import TestRun -from test_tools import disk_utils -from test_tools.disk_utils import get_sysfs_path -from test_tools.fs_utils import check_if_file_exists, readlink +from test_tools import disk_tools +from test_tools.disk_tools import get_sysfs_path +from test_tools.fs_tools import check_if_file_exists, readlink from connection.utils.output import CmdException @@ -50,7 +50,7 @@ def discover_hdd_devices(block_devices, devices_res): for dev in block_devices: if TestRun.executor.run_expect_success(f"cat /sys/block/{dev}/removable").stdout == "1": continue # skip removable drives - block_size = disk_utils.get_block_size(dev) + block_size = disk_tools.get_block_size(dev) if int(block_size) == 4096: disk_type = 'hdd4k' else: @@ -62,7 +62,7 @@ def discover_hdd_devices(block_devices, devices_res): f"sg_inq /dev/{dev} | grep -i 'serial number'" ).stdout.split(': ')[1].strip(), "blocksize": block_size, - "size": disk_utils.get_size(dev)}) + "size": disk_tools.get_size(dev)}) block_devices.clear() @@ -99,8 +99,8 @@ def discover_ssd_devices(block_devices, devices_res): "type": disk_type, "path": resolve_to_by_id_link(device_path), "serial": serial_number, - "blocksize": disk_utils.get_block_size(dev), - "size": disk_utils.get_size(dev)}) + "blocksize": disk_tools.get_block_size(dev), + "size": disk_tools.get_size(dev)}) block_devices.remove(dev) diff --git a/test_tools/disk_utils.py b/test_tools/disk_tools.py similarity index 84% rename from test_tools/disk_utils.py rename to test_tools/disk_tools.py index 50ba340..ca07049 100644 --- a/test_tools/disk_utils.py +++ b/test_tools/disk_tools.py @@ -10,22 +10,17 @@ import time from enum import Enum from typing import List +import test_tools.fs_tools from core.test_run import TestRun -from test_tools import fs_utils from test_tools.dd import Dd -from test_tools.fs_utils import readlink, parse_ls_output, ls -from connection.utils.output import CmdException +from test_tools.fs_tools import readlink, parse_ls_output, ls, check_if_directory_exists, \ + create_directory, wipe_filesystem +from test_tools.udev import Udev from type_def.size import Size, Unit SECTOR_SIZE = 512 -class Filesystem(Enum): - xfs = 0 - ext3 = 1 - ext4 = 2 - - class PartitionTable(Enum): msdos = 0 gpt = 1 @@ -43,21 +38,6 @@ class PartitionType(Enum): unknown = 8 -def create_filesystem(device, filesystem: Filesystem, force=True, blocksize=None): - TestRun.LOGGER.info( - f"Creating filesystem ({filesystem.name}) on device: {device.path}") - force_param = ' -f ' if filesystem == Filesystem.xfs else ' -F ' - force_param = force_param if force else '' - block_size_param = f' -b size={blocksize}' if filesystem == Filesystem.xfs \ - else f' -b {blocksize}' - block_size_param = block_size_param if blocksize else '' - cmd = f'mkfs.{filesystem.name} {force_param} {device.path} {block_size_param}' - cmd = re.sub(' +', ' ', cmd) - TestRun.executor.run_expect_success(cmd) - TestRun.LOGGER.info( - f"Successfully created filesystem on device: {device.path}") - - def create_partition_table(device, partition_table_type: PartitionTable = PartitionTable.gpt): TestRun.LOGGER.info( f"Creating partition table ({partition_table_type.name}) for device: {device.path}") @@ -267,8 +247,7 @@ def get_first_partition_offset(device, aligned: bool): def remove_partitions(device): - from test_tools.udev import Udev - if device.is_mounted(): + if test_tools.fs_tools.is_mounted(device.path): device.unmount() for partition in device.partitions: @@ -276,7 +255,7 @@ def remove_partitions(device): TestRun.LOGGER.info(f"Removing partitions from device: {device.path} " f"({device.get_device_id()}).") - device.wipe_filesystem() + wipe_filesystem(device) Udev.trigger() Udev.settle() output = TestRun.executor.run(f"ls {device.path}* -1") @@ -287,8 +266,8 @@ def remove_partitions(device): def mount(device, mount_point, options: [str] = None): - if not fs_utils.check_if_directory_exists(mount_point): - fs_utils.create_directory(mount_point, True) + if not check_if_directory_exists(mount_point): + create_directory(mount_point, True) TestRun.LOGGER.info(f"Mounting device {device.path} ({device.get_device_id()}) " f"to {mount_point}.") cmd = f"mount {device.path} {mount_point}" @@ -330,15 +309,6 @@ def unit_to_string(unit): return unit_string.get(unit, "Invalid unit.") -def wipe_filesystem(device, force=True): - TestRun.LOGGER.info(f"Erasing the device: {device.path}") - force_param = ' -f' if force else '' - cmd = f'wipefs -a{force_param} {device.path}' - TestRun.executor.run_expect_success(cmd) - TestRun.LOGGER.info( - f"Successfully wiped device: {device.path}") - - def check_if_device_supports_trim(device): if device.get_device_id().startswith("nvme"): return True @@ -351,27 +321,6 @@ def check_if_device_supports_trim(device): return int(command_output.stdout) > 0 -def get_device_filesystem_type(device_id): - cmd = f'lsblk -l -o NAME,FSTYPE | sort | uniq | grep "{device_id} "' - try: - stdout = TestRun.executor.run_expect_success(cmd).stdout - except CmdException: - # unusual devices might not be listed in output (i.e. RAID containers) - if TestRun.executor.run(f"test -b /dev/{device_id}").exit_code != 0: - raise - else: - return None - split_stdout = stdout.strip().split() - if len(split_stdout) <= 1: - return None - else: - try: - return Filesystem[split_stdout[1]] - except KeyError: - TestRun.LOGGER.warning(f"Unrecognized filesystem: {split_stdout[1]}") - return None - - def _is_by_id_path(path: str): """check if given path already is proper by-id path""" dev_by_id_dir = "/dev/disk/by-id" diff --git a/test_tools/fio/fio.py b/test_tools/fio/fio.py index 416f422..8d40599 100644 --- a/test_tools/fio/fio.py +++ b/test_tools/fio/fio.py @@ -9,10 +9,10 @@ import uuid from packaging.version import Version import test_tools.fio.fio_param -import test_tools.fs_utils +import test_tools.fs_tools import test_tools.wget from core.test_run import TestRun -from test_tools import fs_utils +from test_tools import fs_tools from connection.utils.output import CmdException @@ -51,7 +51,7 @@ class Fio: def install(self): fio_url = f"http://brick.kernel.dk/snaps/fio-{self.min_fio_version}.tar.bz2" fio_package = test_tools.wget.download_file(fio_url) - fs_utils.uncompress_archive(fio_package) + fs_tools.uncompress_archive(fio_package) TestRun.executor.run_expect_success( f"cd {fio_package.parent_dir}/fio-{self.min_fio_version}" f" && ./configure && make -j && make install" diff --git a/test_tools/fs_utils.py b/test_tools/fs_tools.py similarity index 85% rename from test_tools/fs_utils.py rename to test_tools/fs_tools.py index b16f32c..87e1425 100644 --- a/test_tools/fs_utils.py +++ b/test_tools/fs_tools.py @@ -7,17 +7,24 @@ import base64 import math +import re import textwrap from collections import namedtuple from datetime import datetime, timedelta +from enum import Enum, IntFlag -from aenum import IntFlag, Enum - +from connection.utils.output import CmdException from core.test_run import TestRun from test_tools.dd import Dd from type_def.size import Size, Unit +class Filesystem(Enum): + xfs = 0 + ext3 = 1 + ext4 = 2 + + class Permissions(IntFlag): r = 4 w = 2 @@ -50,7 +57,7 @@ class PermissionSign(Enum): set = '=' -class FilesPermissions(): +class FilesPermissions: perms_exceptions = {} def __init__(self, files_list: list): @@ -393,3 +400,55 @@ def create_random_test_file(target_file_path: str, dd.run() file.refresh_item() return file + + +def create_filesystem(device, filesystem: Filesystem, force=True, blocksize=None): + TestRun.LOGGER.info( + f"Creating filesystem ({filesystem.name}) on device: {device.path}") + force_param = ' -f ' if filesystem == Filesystem.xfs else ' -F ' + force_param = force_param if force else '' + block_size_param = f' -b size={blocksize}' if filesystem == Filesystem.xfs \ + else f' -b {blocksize}' + block_size_param = block_size_param if blocksize else '' + cmd = f'mkfs.{filesystem.name} {force_param} {device.path} {block_size_param}' + cmd = re.sub(' +', ' ', cmd) + TestRun.executor.run_expect_success(cmd) + TestRun.LOGGER.info( + f"Successfully created filesystem on device: {device.path}") + + +def wipe_filesystem(device, force=True): + TestRun.LOGGER.info(f"Erasing the device: {device.path}") + force_param = ' -f' if force else '' + cmd = f'wipefs -a{force_param} {device.path}' + TestRun.executor.run_expect_success(cmd) + TestRun.LOGGER.info( + f"Successfully wiped device: {device.path}") + + +def get_device_filesystem_type(device_id): + cmd = f'lsblk -l -o NAME,FSTYPE | sort | uniq | grep "{device_id} "' + try: + stdout = TestRun.executor.run_expect_success(cmd).stdout + except CmdException: + # unusual devices might not be listed in output (i.e. RAID containers) + if TestRun.executor.run(f"test -b /dev/{device_id}").exit_code != 0: + raise + else: + return None + split_stdout = stdout.strip().split() + if len(split_stdout) <= 1: + return None + else: + try: + return Filesystem[split_stdout[1]] + except KeyError: + TestRun.LOGGER.warning(f"Unrecognized filesystem: {split_stdout[1]}") + return None + + +def is_mounted(path: str): + if path is None or path.isspace(): + raise Exception("Checked path cannot be empty") + command = f"mount | grep --fixed-strings '{path.rstrip('/')} '" + return TestRun.executor.run(command).exit_code == 0 diff --git a/test_tools/fstab.py b/test_tools/fstab.py index 1b0b669..3df6b88 100644 --- a/test_tools/fstab.py +++ b/test_tools/fstab.py @@ -4,11 +4,11 @@ # SPDX-License-Identifier: BSD-3-Clause # -from test_tools import fs_utils, systemctl +from test_tools import fs_tools, systemctl def add_mountpoint(device, mount_point, fs_type, mount_now=True): - fs_utils.append_line("/etc/fstab", + fs_tools.append_line("/etc/fstab", f"{device.path} {mount_point} {fs_type.name} defaults 0 0") systemctl.reload_daemon() if mount_now: @@ -16,5 +16,5 @@ def add_mountpoint(device, mount_point, fs_type, mount_now=True): def remove_mountpoint(device): - fs_utils.remove_lines("/etc/fstab", device.path) + fs_tools.remove_lines("/etc/fstab", device.path) systemctl.reload_daemon() diff --git a/test_tools/memory.py b/test_tools/memory.py index 2f6c166..eb856b9 100644 --- a/test_tools/memory.py +++ b/test_tools/memory.py @@ -3,9 +3,9 @@ import math from connection.utils.output import CmdException from core.test_run import TestRun from test_tools.dd import Dd -from test_tools.fs_utils import check_if_directory_exists, create_directory +from test_tools.fs_tools import check_if_directory_exists, create_directory, is_mounted from test_tools.os_tools import OvercommitMemoryMode, drop_caches, DropCachesMode, \ - MEMORY_MOUNT_POINT, is_mounted + MEMORY_MOUNT_POINT from type_def.size import Size, Unit diff --git a/test_tools/os_tools.py b/test_tools/os_tools.py index f9427be..27314ee 100644 --- a/test_tools/os_tools.py +++ b/test_tools/os_tools.py @@ -14,8 +14,8 @@ from packaging import version from core.test_run import TestRun from storage_devices.device import Device -from test_tools.disk_utils import get_sysfs_path -from test_tools.fs_utils import check_if_file_exists +from test_tools.disk_tools import get_sysfs_path +from test_tools.fs_tools import check_if_file_exists, is_mounted from test_utils.filesystem.file import File from connection.utils.retry import Retry @@ -110,13 +110,6 @@ def get_kernel_module_parameter(module_name, parameter): return File(param_file_path).read() -def is_mounted(path: str): - if path is None or path.isspace(): - raise Exception("Checked path cannot be empty") - command = f"mount | grep --fixed-strings '{path.rstrip('/')} '" - return TestRun.executor.run(command).exit_code == 0 - - def mount_debugfs(): if not is_mounted(DEBUGFS_MOUNT_POINT): TestRun.executor.run_expect_success(f"mount -t debugfs none {DEBUGFS_MOUNT_POINT}") diff --git a/test_tools/peach_fuzzer/peach_fuzzer.py b/test_tools/peach_fuzzer/peach_fuzzer.py index 0fabfd2..9e54d8f 100644 --- a/test_tools/peach_fuzzer/peach_fuzzer.py +++ b/test_tools/peach_fuzzer/peach_fuzzer.py @@ -14,8 +14,8 @@ from collections import namedtuple import test_tools.wget from core.test_run import TestRun -from test_tools import fs_utils -from test_tools.fs_utils import create_directory, check_if_file_exists, write_file +from test_tools import fs_tools +from test_tools.fs_tools import create_directory, check_if_file_exists, write_file class PeachFuzzer: @@ -75,7 +75,7 @@ class PeachFuzzer: cls._install() if not cls._is_xml_config_prepared(): TestRun.block("No Peach Fuzzer XML config needed to generate fuzzed values was found!") - fs_utils.remove(cls.fuzzy_output_file, force=True, ignore_errors=True) + fs_tools.remove(cls.fuzzy_output_file, force=True, ignore_errors=True) TestRun.LOGGER.info(f"Generate {count} unique fuzzed values") cmd = f"cd {cls.base_dir}; {cls.peach_dir}/peach --range 0,{count - 1} " \ f"--seed {random.randrange(2 ** 32)} {cls.xml_config_file} > " \ @@ -172,7 +172,7 @@ class PeachFuzzer: """ if not cls._is_mono_installed(): TestRun.block("Mono is not installed, can't continue with Peach Fuzzer!") - if fs_utils.check_if_directory_exists(posixpath.join(cls.base_dir, cls.peach_dir)): + if fs_tools.check_if_directory_exists(posixpath.join(cls.base_dir, cls.peach_dir)): return "Peach" in TestRun.executor.run( f"cd {cls.base_dir} && {cls.peach_dir}/peach --version").stdout.strip() else: @@ -197,7 +197,7 @@ class PeachFuzzer: """ Check if Peach Fuzzer XML config is present on the DUT """ - if fs_utils.check_if_file_exists(cls.xml_config_file): + if fs_tools.check_if_file_exists(cls.xml_config_file): return True else: return False diff --git a/test_utils/drbd.py b/test_utils/drbd.py index bec4497..c1b1ddb 100644 --- a/test_utils/drbd.py +++ b/test_utils/drbd.py @@ -3,8 +3,6 @@ # SPDX-License-Identifier: BSD-3-Clause-Clear # -import os - from test_utils.filesystem.file import File diff --git a/test_utils/emergency_escape.py b/test_utils/emergency_escape.py index 8378ab0..1f0ce74 100644 --- a/test_utils/emergency_escape.py +++ b/test_utils/emergency_escape.py @@ -8,7 +8,7 @@ from string import Template from pathlib import Path from test_tools.systemctl import enable_service, reload_daemon, systemd_service_directory -from test_tools.fs_utils import ( +from test_tools.fs_tools import ( create_file, write_file, remove, diff --git a/test_utils/filesystem/directory.py b/test_utils/filesystem/directory.py index 7e46332..fca1ad2 100644 --- a/test_utils/filesystem/directory.py +++ b/test_utils/filesystem/directory.py @@ -3,8 +3,8 @@ # SPDX-License-Identifier: BSD-3-Clause # from core.test_run import TestRun -from test_tools import fs_utils -from test_tools.fs_utils import check_if_directory_exists +from test_tools import fs_tools +from test_tools.fs_tools import check_if_directory_exists from test_utils.filesystem.fs_item import FsItem @@ -13,14 +13,14 @@ class Directory(FsItem): FsItem.__init__(self, full_path) def ls(self): - output = fs_utils.ls(f"{self.full_path}") - return fs_utils.parse_ls_output(output, self.full_path) + output = fs_tools.ls(f"{self.full_path}") + return fs_tools.parse_ls_output(output, self.full_path) @staticmethod def create_directory(path: str, parents: bool = False): - fs_utils.create_directory(path, parents) - output = fs_utils.ls_item(path) - return fs_utils.parse_ls_output(output)[0] + fs_tools.create_directory(path, parents) + output = fs_tools.ls_item(path) + return fs_tools.parse_ls_output(output)[0] @staticmethod def create_temp_directory(parent_dir_path: str = "/tmp"): diff --git a/test_utils/filesystem/file.py b/test_utils/filesystem/file.py index 1df8e8c..3ee3f27 100644 --- a/test_utils/filesystem/file.py +++ b/test_utils/filesystem/file.py @@ -5,7 +5,7 @@ # from datetime import timedelta -from test_tools import fs_utils +from test_tools import fs_tools from test_tools.dd import Dd from test_utils.filesystem.fs_item import FsItem from type_def.size import Size @@ -16,22 +16,22 @@ class File(FsItem): FsItem.__init__(self, full_path) def compare(self, other_file, timeout: timedelta = timedelta(minutes=30)): - return fs_utils.compare(str(self), str(other_file), timeout) + return fs_tools.compare(str(self), str(other_file), timeout) def diff(self, other_file, timeout: timedelta = timedelta(minutes=30)): - return fs_utils.diff(str(self), str(other_file), timeout) + return fs_tools.diff(str(self), str(other_file), timeout) def md5sum(self, binary=True, timeout: timedelta = timedelta(minutes=30)): - return fs_utils.md5sum(str(self), binary, timeout) + return fs_tools.md5sum(str(self), binary, timeout) def crc32sum(self, timeout: timedelta = timedelta(minutes=30)): - return fs_utils.crc32sum(str(self), timeout) + return fs_tools.crc32sum(str(self), timeout) def read(self): - return fs_utils.read_file(str(self)) + return fs_tools.read_file(str(self)) def write(self, content, overwrite: bool = True): - fs_utils.write_file(str(self), content, overwrite) + fs_tools.write_file(str(self), content, overwrite) self.refresh_item() def get_properties(self): @@ -39,9 +39,9 @@ class File(FsItem): @staticmethod def create_file(path: str): - fs_utils.create_file(path) - output = fs_utils.ls_item(path) - return fs_utils.parse_ls_output(output)[0] + fs_tools.create_file(path) + output = fs_tools.ls_item(path) + return fs_tools.parse_ls_output(output)[0] def padding(self, size: Size): dd = Dd().input("/dev/zero").output(self).count(1).block_size(size) @@ -49,7 +49,7 @@ class File(FsItem): self.refresh_item() def remove(self, force: bool = False, ignore_errors: bool = False): - fs_utils.remove(str(self), force=force, ignore_errors=ignore_errors) + fs_tools.remove(str(self), force=force, ignore_errors=ignore_errors) def copy(self, destination, @@ -57,18 +57,18 @@ class File(FsItem): recursive: bool = False, dereference: bool = False, timeout: timedelta = timedelta(minutes=30)): - fs_utils.copy(str(self), destination, force, recursive, dereference, timeout) - if fs_utils.check_if_directory_exists(destination): + fs_tools.copy(str(self), destination, force, recursive, dereference, timeout) + if fs_tools.check_if_directory_exists(destination): path = f"{destination}{'/' if destination[-1] != '/' else ''}{self.name}" else: path = destination - output = fs_utils.ls_item(path) - return fs_utils.parse_ls_output(output)[0] + output = fs_tools.ls_item(path) + return fs_tools.parse_ls_output(output)[0] class FileProperties: def __init__(self, file): - file = fs_utils.parse_ls_output(fs_utils.ls_item(file.full_path))[0] + file = fs_tools.parse_ls_output(fs_tools.ls_item(file.full_path))[0] self.full_path = file.full_path self.parent_dir = FsItem.get_parent_dir(self.full_path) self.name = FsItem.get_name(self.full_path) diff --git a/test_utils/filesystem/fs_item.py b/test_utils/filesystem/fs_item.py index c060ae3..f9ae2a3 100644 --- a/test_utils/filesystem/fs_item.py +++ b/test_utils/filesystem/fs_item.py @@ -5,7 +5,7 @@ import posixpath -from test_tools import fs_utils +from test_tools import fs_tools class FsItem: @@ -38,19 +38,19 @@ class FsItem: return self.full_path def chmod_numerical(self, permissions: int, recursive: bool = False): - fs_utils.chmod_numerical(self.full_path, permissions, recursive) + fs_tools.chmod_numerical(self.full_path, permissions, recursive) self.refresh_item() def chmod(self, - permissions: fs_utils.Permissions, - users: fs_utils.PermissionsUsers, - sign: fs_utils.PermissionSign = fs_utils.PermissionSign.set, + permissions: fs_tools.Permissions, + users: fs_tools.PermissionsUsers, + sign: fs_tools.PermissionSign = fs_tools.PermissionSign.set, recursive: bool = False): - fs_utils.chmod(self.full_path, permissions, users, sign=sign, recursive=recursive) + fs_tools.chmod(self.full_path, permissions, users, sign=sign, recursive=recursive) self.refresh_item() def chown(self, owner, group, recursive: bool = False): - fs_utils.chown(self.full_path, owner, group, recursive) + fs_tools.chown(self.full_path, owner, group, recursive) self.refresh_item() def copy(self, @@ -58,20 +58,20 @@ class FsItem: force: bool = False, recursive: bool = False, dereference: bool = False): - target_dir_exists = fs_utils.check_if_directory_exists(destination) - fs_utils.copy(str(self), destination, force, recursive, dereference) + target_dir_exists = fs_tools.check_if_directory_exists(destination) + fs_tools.copy(str(self), destination, force, recursive, dereference) if target_dir_exists: path = f"{destination}{'/' if destination[-1] != '/' else ''}{self.name}" else: path = destination - output = fs_utils.ls_item(f"{path}") - return fs_utils.parse_ls_output(output)[0] + output = fs_tools.ls_item(f"{path}") + return fs_tools.parse_ls_output(output)[0] def move(self, destination, force: bool = False): - target_dir_exists = fs_utils.check_if_directory_exists(destination) - fs_utils.move(str(self), destination, force) + target_dir_exists = fs_tools.check_if_directory_exists(destination) + fs_tools.move(str(self), destination, force) if target_dir_exists: self.full_path = f"{destination}{'/' if destination[-1] != '/' else ''}{self.name}" else: @@ -80,7 +80,7 @@ class FsItem: return self def refresh_item(self): - updated_file = fs_utils.parse_ls_output(fs_utils.ls_item(self.full_path))[0] + updated_file = fs_tools.parse_ls_output(fs_tools.ls_item(self.full_path))[0] # keep order the same as in __init__() self.parent_dir = updated_file.parent_dir self.name = updated_file.name diff --git a/test_utils/filesystem/symlink.py b/test_utils/filesystem/symlink.py index 5d2e45e..c1c1d05 100644 --- a/test_utils/filesystem/symlink.py +++ b/test_utils/filesystem/symlink.py @@ -5,7 +5,7 @@ # from core.test_run import TestRun -from test_tools.fs_utils import ( +from test_tools.fs_tools import ( readlink, create_directory, check_if_symlink_exists, From 5bd6a656c54c856249e738dc82e00cb7db87af18 Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Fri, 13 Dec 2024 15:18:42 +0100 Subject: [PATCH 22/23] Fix imports Signed-off-by: Katarzyna Treder --- connection/ssh_executor.py | 7 +- connection/utils/__init__.py | 4 + connection/utils/asynchronous.py | 5 +- connection/utils/output.py | 1 + connection/utils/retry.py | 3 +- internal_plugins/vdbench/__init__.py | 98 ------------------------- scripts/__init__.py | 4 + storage_devices/device.py | 34 ++++----- storage_devices/disk.py | 31 ++++---- storage_devices/partition.py | 4 +- storage_devices/raid.py | 1 + storage_devices/ramdisk.py | 4 +- test_tools/common/__init__.py | 4 + test_tools/common/linux_command.py | 1 + test_tools/dd.py | 7 +- test_tools/ddrescue.py | 15 ++-- test_tools/device_mapper.py | 9 +-- test_tools/disk_finder.py | 13 ++-- test_tools/disk_tools.py | 7 +- test_tools/fio/fio.py | 24 +++--- test_tools/fio/fio_patterns.py | 2 +- test_tools/fio/fio_result.py | 6 -- test_tools/fs_tools.py | 7 +- test_tools/fstab.py | 15 ++-- test_tools/git.py | 1 + test_tools/iostat.py | 4 +- test_tools/linux_packaging.py | 1 - test_tools/memory.py | 6 ++ test_tools/nvme_cli.py | 2 + test_tools/os_tools.py | 6 +- test_tools/peach_fuzzer/peach_fuzzer.py | 14 ++-- test_utils/common/__init__.py | 4 + test_utils/common/singleton.py | 1 + test_utils/filesystem/directory.py | 12 +-- test_utils/filesystem/file.py | 21 +++--- test_utils/filesystem/fs_item.py | 19 +++-- type_def/__init__.py | 4 + 37 files changed, 170 insertions(+), 231 deletions(-) delete mode 100644 internal_plugins/vdbench/__init__.py diff --git a/connection/ssh_executor.py b/connection/ssh_executor.py index 173fef5..4f4bfa8 100644 --- a/connection/ssh_executor.py +++ b/connection/ssh_executor.py @@ -3,13 +3,14 @@ # Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # + import os import re +import paramiko import socket import subprocess -from datetime import timedelta, datetime -import paramiko +from datetime import timedelta, datetime from connection.base_executor import BaseExecutor from core.test_run import TestRun, Blocked @@ -46,7 +47,7 @@ class SshExecutor(BaseExecutor): hostname = target["hostname"] key_filename = target.get("identityfile", None) user = target.get("user", user) - port = target.get("port", port) + port = int(target.get("port", port)) if target.get("proxyjump", None) is not None: proxy = config.lookup(target["proxyjump"]) jump = paramiko.SSHClient() diff --git a/connection/utils/__init__.py b/connection/utils/__init__.py index e69de29..18aaf34 100644 --- a/connection/utils/__init__.py +++ b/connection/utils/__init__.py @@ -0,0 +1,4 @@ +# +# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +# diff --git a/connection/utils/asynchronous.py b/connection/utils/asynchronous.py index 9be0159..924e537 100644 --- a/connection/utils/asynchronous.py +++ b/connection/utils/asynchronous.py @@ -1,9 +1,10 @@ # # Copyright(c) 2020-2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # -import concurrent +from concurrent.futures import ThreadPoolExecutor def start_async_func(func, *args): @@ -14,5 +15,5 @@ def start_async_func(func, *args): - done() method returns True when task ended (have a result or ended with an exception) otherwise returns False """ - executor = concurrent.futures.ThreadPoolExecutor() + executor = ThreadPoolExecutor() return executor.submit(func, *args) diff --git a/connection/utils/output.py b/connection/utils/output.py index 73e8f94..d3ae9ae 100644 --- a/connection/utils/output.py +++ b/connection/utils/output.py @@ -1,5 +1,6 @@ # # Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # diff --git a/connection/utils/retry.py b/connection/utils/retry.py index 10ca573..b3bd0f8 100644 --- a/connection/utils/retry.py +++ b/connection/utils/retry.py @@ -1,5 +1,6 @@ # # Copyright(c) 2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # @@ -33,7 +34,7 @@ class Retry: try: result = func() return True - except: + except Exception: return False cls.run_while_false(wrapped_func, retries=retries, timeout=timeout) diff --git a/internal_plugins/vdbench/__init__.py b/internal_plugins/vdbench/__init__.py deleted file mode 100644 index 536db28..0000000 --- a/internal_plugins/vdbench/__init__.py +++ /dev/null @@ -1,98 +0,0 @@ -# -# Copyright(c) 2020-2021 Intel Corporation -# Copyright(c) 2023-2024 Huawei Technologies Co., Ltd. -# SPDX-License-Identifier: BSD-3-Clause -# - -import time -import posixpath - -from datetime import timedelta -from core.test_run import TestRun -from test_tools import fs_tools - - -class Vdbench: - def __init__(self, params, config): - print("VDBench plugin initialization") - self.run_time = timedelta(seconds=60) - - try: - self.working_dir = config["working_dir"] - self.reinstall = config["reinstall"] - self.source_dir = config["source_dir"] - except Exception: - raise Exception("Missing fields in config! ('working_dir', 'source_dir' and " - "'reinstall' required)") - - self.result_dir = posixpath.join(self.working_dir, 'result.tod') - - def pre_setup(self): - pass - - def post_setup(self): - print("VDBench plugin post setup") - if not self.reinstall and fs_utils.check_if_directory_exists(self.working_dir): - return - - if fs_utils.check_if_directory_exists(self.working_dir): - fs_utils.remove(self.working_dir, True, True) - - fs_utils.create_directory(self.working_dir) - TestRun.LOGGER.info("Copying vdbench to working dir.") - fs_utils.copy( - source=self.source_dir, destination=self.working_dir, force=True, recursive=True - ) - - def teardown(self): - pass - - def create_config(self, config, run_time: timedelta): - self.run_time = run_time - if config[-1] != ",": - config += "," - config += f"elapsed={int(run_time.total_seconds())}" - TestRun.LOGGER.info(f"Vdbench config:\n{config}") - fs_utils.write_file(posixpath.join(self.working_dir, "param.ini"), config) - - def run(self): - cmd = f"{posixpath.join(self.working_dir, 'vdbench')} " \ - f"-f {posixpath.join(self.working_dir, 'param.ini')} " \ - f"-vr -o {self.result_dir}" - full_cmd = f"screen -dmS vdbench {cmd}" - TestRun.executor.run(full_cmd) - start_time = time.time() - - timeout = self.run_time * 1.5 - - while True: - if not TestRun.executor.run(f"ps aux | grep '{cmd}' | grep -v grep").exit_code == 0: - return self.analyze_log() - - if time.time() - start_time > timeout.total_seconds(): - TestRun.LOGGER.error("Vdbench timeout.") - return False - time.sleep(1) - - def analyze_log(self): - output = TestRun.executor.run( - f"ls -1td {self.result_dir[0:len(self.result_dir) - 3]}* | head -1") - log_path = posixpath.join(output.stdout if output.exit_code == 0 else self.result_dir, - "logfile.html") - - log_file = fs_utils.read_file(log_path) - - if "Vdbench execution completed successfully" in log_file: - TestRun.LOGGER.info("Vdbench execution completed successfully.") - return True - - if "Data Validation error" in log_file or "data_errors=1" in log_file: - TestRun.LOGGER.error("Data corruption occurred!") - elif "Heartbeat monitor:" in log_file: - TestRun.LOGGER.error("Vdbench: heartbeat.") - else: - TestRun.LOGGER.error("Vdbench unknown result.") - return False - - -plugin_class = Vdbench diff --git a/scripts/__init__.py b/scripts/__init__.py index e69de29..18aaf34 100644 --- a/scripts/__init__.py +++ b/scripts/__init__.py @@ -0,0 +1,4 @@ +# +# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +# diff --git a/storage_devices/device.py b/storage_devices/device.py index 7bfb5c3..a56b0bb 100644 --- a/storage_devices/device.py +++ b/storage_devices/device.py @@ -3,31 +3,32 @@ # Copyright(c) 2023-2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # + import posixpath -import test_tools.fs_tools from core.test_run import TestRun -from test_tools import disk_tools, fs_tools -from test_tools.disk_tools import get_sysfs_path -from test_tools.fs_tools import get_device_filesystem_type +from test_tools import disk_tools +from test_tools.disk_tools import get_sysfs_path, validate_dev_path, get_size +from test_tools.fs_tools import (get_device_filesystem_type, Filesystem, wipefs, + readlink, write_file, mkfs, ls, parse_ls_output) from test_utils.io_stats import IoStats from type_def.size import Size, Unit class Device: def __init__(self, path): - disk_tools.validate_dev_path(path) + validate_dev_path(path) self.path = path - self.size = Size(disk_tools.get_size(self.get_device_id()), Unit.Byte) + self.size = Size(get_size(self.get_device_id()), Unit.Byte) self.filesystem = get_device_filesystem_type(self.get_device_id()) self.mount_point = None - def create_filesystem(self, fs_type: test_tools.fs_tools.Filesystem, force=True, blocksize=None): - test_tools.fs_tools.create_filesystem(self, fs_type, force, blocksize) + def create_filesystem(self, fs_type: Filesystem, force=True, blocksize=None): + mkfs(self, fs_type, force, blocksize) self.filesystem = fs_type def wipe_filesystem(self, force=True): - test_tools.fs_tools.wipe_filesystem(self, force) + wipefs(self, force) self.filesystem = None def is_mounted(self): @@ -36,7 +37,7 @@ class Device: return False else: mount_point_line = output.stdout.split('\n')[1] - device_path = fs_tools.readlink(self.path) + device_path = readlink(self.path) self.mount_point = mount_point_line[0:mount_point_line.find(device_path)].strip() return True @@ -58,27 +59,26 @@ class Device: return next(i for i in items if i.full_path.startswith(directory)) def get_device_id(self): - return fs_tools.readlink(self.path).split('/')[-1] + return readlink(self.path).split('/')[-1] def get_all_device_links(self, directory: str): - from test_tools import fs_tools - output = fs_tools.ls(f"$(find -L {directory} -samefile {self.path})") - return fs_tools.parse_ls_output(output, self.path) + output = ls(f"$(find -L {directory} -samefile {self.path})") + return parse_ls_output(output, self.path) def get_io_stats(self): return IoStats.get_io_stats(self.get_device_id()) def get_sysfs_property(self, property_name): - path = posixpath.join(disk_tools.get_sysfs_path(self.get_device_id()), + path = posixpath.join(get_sysfs_path(self.get_device_id()), "queue", property_name) return TestRun.executor.run_expect_success(f"cat {path}").stdout def set_sysfs_property(self, property_name, value): TestRun.LOGGER.info( f"Setting {property_name} for device {self.get_device_id()} to {value}.") - path = posixpath.join(disk_tools.get_sysfs_path(self.get_device_id()), "queue", + path = posixpath.join(get_sysfs_path(self.get_device_id()), "queue", property_name) - fs_tools.write_file(path, str(value)) + write_file(path, str(value)) def set_max_io_size(self, new_max_io_size: Size): self.set_sysfs_property("max_sectors_kb", diff --git a/storage_devices/disk.py b/storage_devices/disk.py index 0ac2b29..8f3a179 100644 --- a/storage_devices/disk.py +++ b/storage_devices/disk.py @@ -10,13 +10,14 @@ import re from datetime import timedelta from enum import IntEnum -import test_tools.fs_tools from core.test_run import TestRun -from storage_devices.device import Device -from test_tools import disk_tools, fs_tools, nvme_cli -from test_tools.common.wait import wait from connection.utils.output import Output +from storage_devices.device import Device +from test_tools import disk_tools, nvme_cli +from test_tools.common.wait import wait from test_tools.disk_finder import get_block_devices_list, resolve_to_by_id_link +from test_tools.disk_tools import PartitionTable +from test_tools.fs_tools import readlink, is_mounted, ls_item, parse_ls_output from type_def.size import Unit @@ -138,7 +139,7 @@ class Disk(Device): ) return recognized_types[0] - def create_partitions(self, sizes: [], partition_table_type=disk_tools.PartitionTable.gpt): + def create_partitions(self, sizes: [], partition_table_type=PartitionTable.gpt): disk_tools.create_partitions(self, sizes, partition_table_type) def remove_partition(self, part): @@ -148,12 +149,12 @@ class Disk(Device): def umount_all_partitions(self): TestRun.LOGGER.info(f"Unmounting all partitions from: {self.path}") - cmd = f"umount -l {fs_tools.readlink(self.path)}*?" + cmd = f"umount -l {readlink(self.path)}*?" TestRun.executor.run(cmd) def remove_partitions(self): for part in self.partitions: - if test_tools.fs_tools.is_mounted(part.path): + if is_mounted(part.path): part.unmount() if disk_tools.remove_partitions(self): self.partitions.clear() @@ -163,8 +164,8 @@ class Disk(Device): serial_numbers = Disk.get_all_serial_numbers() return self.serial_number in serial_numbers elif self.path: - output = fs_tools.ls_item(f"{self.path}") - return fs_tools.parse_ls_output(output)[0] is not None + output = ls_item(f"{self.path}") + return parse_ls_output(output)[0] is not None raise Exception("Couldn't check if device is detected by the system") def wait_for_plug_status(self, should_be_visible): @@ -290,8 +291,8 @@ class NvmeDisk(Disk): base = f"/sys/block/{device_id}/device" for suffix in ["/remove", "/device/remove"]: try: - output = fs_tools.ls_item(base + suffix) - fs_tools.parse_ls_output(output)[0] + output = ls_item(base + suffix) + parse_ls_output(output)[0] except TypeError: continue return base + suffix @@ -346,8 +347,8 @@ class SataDisk(Disk): @staticmethod def get_sysfs_addr(device_id): ls_command = f"$(find -H /sys/devices/ -name {device_id} -type d)" - output = fs_tools.ls_item(f"{ls_command}") - sysfs_addr = fs_tools.parse_ls_output(output)[0] + output = ls_item(ls_command) + sysfs_addr = parse_ls_output(output)[0] if not sysfs_addr: raise Exception(f"Failed to find sysfs address: ls -l {ls_command}") return sysfs_addr.full_path @@ -413,8 +414,8 @@ class VirtioDisk(Disk): @staticmethod def get_sysfs_addr(device_id: str) -> str: ls_command = f"$(find -H /sys/devices/ -name {device_id} -type d)" - output = fs_tools.ls_item(f"{ls_command}") - sysfs_addr = fs_tools.parse_ls_output(output)[0] + output = ls_item(ls_command) + sysfs_addr = parse_ls_output(output)[0] if not sysfs_addr: raise Exception(f"Failed to find sysfs address: ls -l {ls_command}") diff --git a/storage_devices/partition.py b/storage_devices/partition.py index f39b556..fd83ace 100644 --- a/storage_devices/partition.py +++ b/storage_devices/partition.py @@ -4,13 +4,13 @@ # from storage_devices.device import Device -from test_tools import disk_tools +from test_tools.disk_tools import get_partition_path from type_def.size import Size class Partition(Device): def __init__(self, parent_dev, type, number, begin: Size, end: Size): - Device.__init__(self, disk_tools.get_partition_path(parent_dev.path, number)) + Device.__init__(self, get_partition_path(parent_dev.path, number)) self.number = number self.parent_device = parent_dev self.type = type diff --git a/storage_devices/raid.py b/storage_devices/raid.py index 3d1c9c1..f9d98d9 100644 --- a/storage_devices/raid.py +++ b/storage_devices/raid.py @@ -2,6 +2,7 @@ # Copyright(c) 2020-2021 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # + import threading from enum import IntEnum, Enum diff --git a/storage_devices/ramdisk.py b/storage_devices/ramdisk.py index fc04ba0..1ff9b09 100644 --- a/storage_devices/ramdisk.py +++ b/storage_devices/ramdisk.py @@ -7,7 +7,7 @@ import posixpath from core.test_run import TestRun from storage_devices.device import Device -from test_tools import disk_tools +from test_tools.disk_tools import get_size from test_tools.fs_tools import ls, parse_ls_output from test_utils.filesystem.symlink import Symlink from test_tools.os_tools import reload_kernel_module, unload_kernel_module, is_kernel_module_loaded @@ -68,7 +68,7 @@ class RamDisk(Device): ram_disks = cls._list_devices() return ( len(ram_disks) >= disk_count - and Size(disk_tools.get_size(ram_disks[0].name), Unit.Byte).align_down(Unit.MiB.value) + and Size(get_size(ram_disks[0].name), Unit.Byte).align_down(Unit.MiB.value) == disk_size.align_down(Unit.MiB.value) ) diff --git a/test_tools/common/__init__.py b/test_tools/common/__init__.py index e69de29..18aaf34 100644 --- a/test_tools/common/__init__.py +++ b/test_tools/common/__init__.py @@ -0,0 +1,4 @@ +# +# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +# diff --git a/test_tools/common/linux_command.py b/test_tools/common/linux_command.py index b6d887b..e8c3318 100644 --- a/test_tools/common/linux_command.py +++ b/test_tools/common/linux_command.py @@ -1,5 +1,6 @@ # # Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # diff --git a/test_tools/dd.py b/test_tools/dd.py index 0428b6f..b40e8b1 100644 --- a/test_tools/dd.py +++ b/test_tools/dd.py @@ -1,16 +1,17 @@ # # Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # -import test_tools.common.linux_command as linux_comm import type_def.size as size from core.test_run import TestRun +from test_tools.common.linux_command import LinuxCommand -class Dd(linux_comm.LinuxCommand): +class Dd(LinuxCommand): def __init__(self): - linux_comm.LinuxCommand.__init__(self, TestRun.executor, 'dd') + LinuxCommand.__init__(self, TestRun.executor, 'dd') def block_size(self, value: size.Size): return self.set_param('bs', int(value.get_value())) diff --git a/test_tools/ddrescue.py b/test_tools/ddrescue.py index 3a6dbb7..c319665 100644 --- a/test_tools/ddrescue.py +++ b/test_tools/ddrescue.py @@ -1,16 +1,17 @@ # # Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # -import test_tools.common.linux_command as linux_comm -import type_def.size as size from core.test_run import TestRun +from test_tools.common.linux_command import LinuxCommand +from type_def.size import Size -class Ddrescue(linux_comm.LinuxCommand): +class Ddrescue(LinuxCommand): def __init__(self): - linux_comm.LinuxCommand.__init__(self, TestRun.executor, 'ddrescue') + LinuxCommand.__init__(self, TestRun.executor, 'ddrescue') self.source_path = None self.destination_path = None self.param_name_prefix = "--" @@ -35,13 +36,13 @@ class Ddrescue(linux_comm.LinuxCommand): def force(self): return self.set_flags("force") - def block_size(self, value: size.Size): + def block_size(self, value: Size): return self.set_param('sector-size', int(value.get_value())) - def size(self, value: size.Size): + def size(self, value: Size): return self.set_param('size', int(value.get_value())) def __str__(self): - command = linux_comm.LinuxCommand.__str__(self) + command = LinuxCommand.__str__(self) command += f" {self.source_path} {self.destination_path}" return command diff --git a/test_tools/device_mapper.py b/test_tools/device_mapper.py index 9f78216..a804b5d 100644 --- a/test_tools/device_mapper.py +++ b/test_tools/device_mapper.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019-2022 Intel Corporation +# Copyright(c) 2019-2021 Intel Corporation # Copyright(c) 2023-2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # @@ -30,11 +30,6 @@ class DmTarget(Enum): return self.name.lower() -class DmTable: - class TableEntry: - pass - - class DmTable: class TableEntry: def __init__(self, offset: int, length: int, target: DmTarget, *params): @@ -131,7 +126,7 @@ class DmTable: return self - def add_entry(self, entry: DmTable.TableEntry): + def add_entry(self, entry: TableEntry): self.table.append(entry) return self diff --git a/test_tools/disk_finder.py b/test_tools/disk_finder.py index 1cbbff2..16bc330 100644 --- a/test_tools/disk_finder.py +++ b/test_tools/disk_finder.py @@ -7,8 +7,7 @@ import os import posixpath from core.test_run import TestRun -from test_tools import disk_tools -from test_tools.disk_tools import get_sysfs_path +from test_tools.disk_tools import get_sysfs_path, get_block_size, get_size from test_tools.fs_tools import check_if_file_exists, readlink from connection.utils.output import CmdException @@ -50,7 +49,7 @@ def discover_hdd_devices(block_devices, devices_res): for dev in block_devices: if TestRun.executor.run_expect_success(f"cat /sys/block/{dev}/removable").stdout == "1": continue # skip removable drives - block_size = disk_tools.get_block_size(dev) + block_size = get_block_size(dev) if int(block_size) == 4096: disk_type = 'hdd4k' else: @@ -62,7 +61,7 @@ def discover_hdd_devices(block_devices, devices_res): f"sg_inq /dev/{dev} | grep -i 'serial number'" ).stdout.split(': ')[1].strip(), "blocksize": block_size, - "size": disk_tools.get_size(dev)}) + "size": get_size(dev)}) block_devices.clear() @@ -99,8 +98,8 @@ def discover_ssd_devices(block_devices, devices_res): "type": disk_type, "path": resolve_to_by_id_link(device_path), "serial": serial_number, - "blocksize": disk_tools.get_block_size(dev), - "size": disk_tools.get_size(dev)}) + "blocksize": get_block_size(dev), + "size": get_size(dev)}) block_devices.remove(dev) @@ -125,7 +124,7 @@ def get_system_disks(): def __get_slaves(device_name: str): try: device_names = TestRun.executor.run_expect_success( - f"ls {os.path.join(get_sysfs_path(device_name), "slaves")}").stdout.splitlines() + f"ls {os.path.join(get_sysfs_path(device_name), 'slaves')}").stdout.splitlines() except CmdException as e: if "No such file or directory" not in e.output.stderr: raise diff --git a/test_tools/disk_tools.py b/test_tools/disk_tools.py index ca07049..c2790d6 100644 --- a/test_tools/disk_tools.py +++ b/test_tools/disk_tools.py @@ -10,11 +10,10 @@ import time from enum import Enum from typing import List -import test_tools.fs_tools from core.test_run import TestRun from test_tools.dd import Dd from test_tools.fs_tools import readlink, parse_ls_output, ls, check_if_directory_exists, \ - create_directory, wipe_filesystem + create_directory, wipefs, is_mounted from test_tools.udev import Udev from type_def.size import Size, Unit @@ -247,7 +246,7 @@ def get_first_partition_offset(device, aligned: bool): def remove_partitions(device): - if test_tools.fs_tools.is_mounted(device.path): + if is_mounted(device.path): device.unmount() for partition in device.partitions: @@ -255,7 +254,7 @@ def remove_partitions(device): TestRun.LOGGER.info(f"Removing partitions from device: {device.path} " f"({device.get_device_id()}).") - wipe_filesystem(device) + wipefs(device) Udev.trigger() Udev.settle() output = TestRun.executor.run(f"ls {device.path}* -1") diff --git a/test_tools/fio/fio.py b/test_tools/fio/fio.py index 8d40599..56bab56 100644 --- a/test_tools/fio/fio.py +++ b/test_tools/fio/fio.py @@ -8,12 +8,12 @@ import datetime import uuid from packaging.version import Version -import test_tools.fio.fio_param -import test_tools.fs_tools -import test_tools.wget + from core.test_run import TestRun -from test_tools import fs_tools from connection.utils.output import CmdException +from test_tools import wget +from test_tools.fio.fio_param import FioParam, FioParamCmd, FioOutput, FioParamConfig +from test_tools.fs_tools import uncompress_archive class Fio: @@ -22,12 +22,12 @@ class Fio: self.default_run_time = datetime.timedelta(hours=1) self.jobs = [] self.executor = executor_obj if executor_obj is not None else TestRun.executor - self.base_cmd_parameters: test_tools.fio.fio_param.FioParam = None - self.global_cmd_parameters: test_tools.fio.fio_param.FioParam = None + self.base_cmd_parameters: FioParam = None + self.global_cmd_parameters: FioParam = None - def create_command(self, output_type=test_tools.fio.fio_param.FioOutput.json): - self.base_cmd_parameters = test_tools.fio.fio_param.FioParamCmd(self, self.executor) - self.global_cmd_parameters = test_tools.fio.fio_param.FioParamConfig(self, self.executor) + def create_command(self, output_type=FioOutput.json): + self.base_cmd_parameters = FioParamCmd(self, self.executor) + self.global_cmd_parameters = FioParamConfig(self, self.executor) self.fio_file = \ f'fio_run_{datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_{uuid.uuid4().hex}' @@ -50,8 +50,8 @@ class Fio: def install(self): fio_url = f"http://brick.kernel.dk/snaps/fio-{self.min_fio_version}.tar.bz2" - fio_package = test_tools.wget.download_file(fio_url) - fs_tools.uncompress_archive(fio_package) + fio_package = wget.download_file(fio_url) + uncompress_archive(fio_package) TestRun.executor.run_expect_success( f"cd {fio_package.parent_dir}/fio-{self.min_fio_version}" f" && ./configure && make -j && make install" @@ -106,7 +106,7 @@ class Fio: command = f"echo '{self.execution_cmd_parameters()}' |" \ f" {str(self.base_cmd_parameters)} -" else: - fio_parameters = test_tools.fio.fio_param.FioParamCmd(self, self.executor) + fio_parameters = FioParamCmd(self, self.executor) fio_parameters.command_env_var.update(self.base_cmd_parameters.command_env_var) fio_parameters.command_param.update(self.base_cmd_parameters.command_param) fio_parameters.command_param.update(self.global_cmd_parameters.command_param) diff --git a/test_tools/fio/fio_patterns.py b/test_tools/fio/fio_patterns.py index 22ca70b..0c3f405 100644 --- a/test_tools/fio/fio_patterns.py +++ b/test_tools/fio/fio_patterns.py @@ -4,7 +4,7 @@ # import secrets -from aenum import Enum +from enum import Enum class Pattern(Enum): diff --git a/test_tools/fio/fio_result.py b/test_tools/fio/fio_result.py index 5163609..59eba2c 100644 --- a/test_tools/fio/fio_result.py +++ b/test_tools/fio/fio_result.py @@ -103,9 +103,6 @@ class FioResult: def write_runtime(self): return Time(microseconds=self.job.write.runtime) - def write_completion_latency_average(self): - return Time(nanoseconds=self.job.write.lat_ns.mean) - def write_completion_latency_min(self): return Time(nanoseconds=self.job.write.lat_ns.min) @@ -139,9 +136,6 @@ class FioResult: def trim_runtime(self): return Time(microseconds=self.job.trim.runtime) - def trim_completion_latency_average(self): - return Time(nanoseconds=self.job.trim.lat_ns.mean) - def trim_completion_latency_min(self): return Time(nanoseconds=self.job.trim.lat_ns.min) diff --git a/test_tools/fs_tools.py b/test_tools/fs_tools.py index 87e1425..7724da4 100644 --- a/test_tools/fs_tools.py +++ b/test_tools/fs_tools.py @@ -11,7 +11,8 @@ import re import textwrap from collections import namedtuple from datetime import datetime, timedelta -from enum import Enum, IntFlag +from enum import Enum +from aenum import IntFlag # IntFlag from enum is not able to correctly parse string like "x|y|z" from connection.utils.output import CmdException from core.test_run import TestRun @@ -402,7 +403,7 @@ def create_random_test_file(target_file_path: str, return file -def create_filesystem(device, filesystem: Filesystem, force=True, blocksize=None): +def mkfs(device, filesystem: Filesystem, force=True, blocksize=None): TestRun.LOGGER.info( f"Creating filesystem ({filesystem.name}) on device: {device.path}") force_param = ' -f ' if filesystem == Filesystem.xfs else ' -F ' @@ -417,7 +418,7 @@ def create_filesystem(device, filesystem: Filesystem, force=True, blocksize=None f"Successfully created filesystem on device: {device.path}") -def wipe_filesystem(device, force=True): +def wipefs(device, force=True): TestRun.LOGGER.info(f"Erasing the device: {device.path}") force_param = ' -f' if force else '' cmd = f'wipefs -a{force_param} {device.path}' diff --git a/test_tools/fstab.py b/test_tools/fstab.py index 3df6b88..6bcf504 100644 --- a/test_tools/fstab.py +++ b/test_tools/fstab.py @@ -4,17 +4,18 @@ # SPDX-License-Identifier: BSD-3-Clause # -from test_tools import fs_tools, systemctl +from test_tools.fs_tools import append_line, remove_lines +from test_tools.systemctl import reload_daemon, restart_service def add_mountpoint(device, mount_point, fs_type, mount_now=True): - fs_tools.append_line("/etc/fstab", - f"{device.path} {mount_point} {fs_type.name} defaults 0 0") - systemctl.reload_daemon() + append_line("/etc/fstab", + f"{device.path} {mount_point} {fs_type.name} defaults 0 0") + reload_daemon() if mount_now: - systemctl.restart_service("local-fs.target") + restart_service("local-fs.target") def remove_mountpoint(device): - fs_tools.remove_lines("/etc/fstab", device.path) - systemctl.reload_daemon() + remove_lines("/etc/fstab", device.path) + reload_daemon() diff --git a/test_tools/git.py b/test_tools/git.py index 74c161d..456c08a 100644 --- a/test_tools/git.py +++ b/test_tools/git.py @@ -3,6 +3,7 @@ # Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # + import itertools import os import posixpath diff --git a/test_tools/iostat.py b/test_tools/iostat.py index 2a07dcb..673bf17 100644 --- a/test_tools/iostat.py +++ b/test_tools/iostat.py @@ -3,10 +3,12 @@ # SPDX-License-Identifier: BSD-3-Clause # +import csv + from core.test_run import TestRun from type_def.size import Size, Unit, UnitPerSecond from type_def.time import Time -import csv + class IOstatExtended: diff --git a/test_tools/linux_packaging.py b/test_tools/linux_packaging.py index 8f1940e..b5a8173 100644 --- a/test_tools/linux_packaging.py +++ b/test_tools/linux_packaging.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: BSD-3-Clause # - import os import re diff --git a/test_tools/memory.py b/test_tools/memory.py index eb856b9..145518c 100644 --- a/test_tools/memory.py +++ b/test_tools/memory.py @@ -1,3 +1,9 @@ +# +# Copyright(c) 2019-2022 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +# + import math from connection.utils.output import CmdException diff --git a/test_tools/nvme_cli.py b/test_tools/nvme_cli.py index 5844587..f49f42b 100644 --- a/test_tools/nvme_cli.py +++ b/test_tools/nvme_cli.py @@ -1,7 +1,9 @@ # # Copyright(c) 2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # + import json from core.test_run import TestRun diff --git a/test_tools/os_tools.py b/test_tools/os_tools.py index 27314ee..4a80be9 100644 --- a/test_tools/os_tools.py +++ b/test_tools/os_tools.py @@ -16,7 +16,6 @@ from core.test_run import TestRun from storage_devices.device import Device from test_tools.disk_tools import get_sysfs_path from test_tools.fs_tools import check_if_file_exists, is_mounted -from test_utils.filesystem.file import File from connection.utils.retry import Retry DEBUGFS_MOUNT_POINT = "/sys/kernel/debug" @@ -86,7 +85,8 @@ def get_kernel_version(): def is_kernel_module_loaded(module_name): - output = TestRun.executor.run(f"lsmod | grep ^{module_name}$") + command = f"lsmod | grep -E '^{module_name}\\b'" + output = TestRun.executor.run(command) return output.exit_code == 0 @@ -107,7 +107,7 @@ def get_kernel_module_parameter(module_name, parameter): param_file_path = f"/sys/module/{module_name}/parameters/{parameter}" if not check_if_file_exists(param_file_path): raise FileNotFoundError(f"File {param_file_path} does not exist!") - return File(param_file_path).read() + return TestRun.executor.run(f"cat {param_file_path}").stdout def mount_debugfs(): diff --git a/test_tools/peach_fuzzer/peach_fuzzer.py b/test_tools/peach_fuzzer/peach_fuzzer.py index 9e54d8f..c3aec23 100644 --- a/test_tools/peach_fuzzer/peach_fuzzer.py +++ b/test_tools/peach_fuzzer/peach_fuzzer.py @@ -12,10 +12,10 @@ import tempfile import lxml.etree as etree from collections import namedtuple -import test_tools.wget +from test_tools import wget from core.test_run import TestRun -from test_tools import fs_tools -from test_tools.fs_tools import create_directory, check_if_file_exists, write_file +from test_tools.fs_tools import create_directory, check_if_file_exists, write_file, remove, \ + check_if_directory_exists class PeachFuzzer: @@ -75,7 +75,7 @@ class PeachFuzzer: cls._install() if not cls._is_xml_config_prepared(): TestRun.block("No Peach Fuzzer XML config needed to generate fuzzed values was found!") - fs_tools.remove(cls.fuzzy_output_file, force=True, ignore_errors=True) + remove(cls.fuzzy_output_file, force=True, ignore_errors=True) TestRun.LOGGER.info(f"Generate {count} unique fuzzed values") cmd = f"cd {cls.base_dir}; {cls.peach_dir}/peach --range 0,{count - 1} " \ f"--seed {random.randrange(2 ** 32)} {cls.xml_config_file} > " \ @@ -155,7 +155,7 @@ class PeachFuzzer: Install Peach Fuzzer on the DUT """ create_directory(cls.base_dir, True) - peach_archive = test_tools.wget.download_file( + peach_archive = wget.download_file( cls.peach_fuzzer_3_0_url, destination_dir=cls.base_dir ) TestRun.executor.run_expect_success( @@ -172,7 +172,7 @@ class PeachFuzzer: """ if not cls._is_mono_installed(): TestRun.block("Mono is not installed, can't continue with Peach Fuzzer!") - if fs_tools.check_if_directory_exists(posixpath.join(cls.base_dir, cls.peach_dir)): + if check_if_directory_exists(posixpath.join(cls.base_dir, cls.peach_dir)): return "Peach" in TestRun.executor.run( f"cd {cls.base_dir} && {cls.peach_dir}/peach --version").stdout.strip() else: @@ -197,7 +197,7 @@ class PeachFuzzer: """ Check if Peach Fuzzer XML config is present on the DUT """ - if fs_tools.check_if_file_exists(cls.xml_config_file): + if check_if_file_exists(cls.xml_config_file): return True else: return False diff --git a/test_utils/common/__init__.py b/test_utils/common/__init__.py index e69de29..18aaf34 100644 --- a/test_utils/common/__init__.py +++ b/test_utils/common/__init__.py @@ -0,0 +1,4 @@ +# +# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +# diff --git a/test_utils/common/singleton.py b/test_utils/common/singleton.py index a484929..56a3015 100644 --- a/test_utils/common/singleton.py +++ b/test_utils/common/singleton.py @@ -1,5 +1,6 @@ # # Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # diff --git a/test_utils/filesystem/directory.py b/test_utils/filesystem/directory.py index fca1ad2..2e7a26a 100644 --- a/test_utils/filesystem/directory.py +++ b/test_utils/filesystem/directory.py @@ -1,10 +1,12 @@ # # Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # + from core.test_run import TestRun from test_tools import fs_tools -from test_tools.fs_tools import check_if_directory_exists +from test_tools.fs_tools import check_if_directory_exists, parse_ls_output, ls_item, ls from test_utils.filesystem.fs_item import FsItem @@ -13,14 +15,14 @@ class Directory(FsItem): FsItem.__init__(self, full_path) def ls(self): - output = fs_tools.ls(f"{self.full_path}") - return fs_tools.parse_ls_output(output, self.full_path) + output = ls(self.full_path) + return parse_ls_output(output, self.full_path) @staticmethod def create_directory(path: str, parents: bool = False): fs_tools.create_directory(path, parents) - output = fs_tools.ls_item(path) - return fs_tools.parse_ls_output(output)[0] + output = ls_item(path) + return parse_ls_output(output)[0] @staticmethod def create_temp_directory(parent_dir_path: str = "/tmp"): diff --git a/test_utils/filesystem/file.py b/test_utils/filesystem/file.py index 3ee3f27..986a25a 100644 --- a/test_utils/filesystem/file.py +++ b/test_utils/filesystem/file.py @@ -3,10 +3,13 @@ # Copyright(c) 2023-2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # + from datetime import timedelta from test_tools import fs_tools from test_tools.dd import Dd +from test_tools.fs_tools import read_file, write_file, ls_item, parse_ls_output, remove, \ + check_if_directory_exists from test_utils.filesystem.fs_item import FsItem from type_def.size import Size @@ -28,10 +31,10 @@ class File(FsItem): return fs_tools.crc32sum(str(self), timeout) def read(self): - return fs_tools.read_file(str(self)) + return read_file(str(self)) def write(self, content, overwrite: bool = True): - fs_tools.write_file(str(self), content, overwrite) + write_file(str(self), content, overwrite) self.refresh_item() def get_properties(self): @@ -40,8 +43,8 @@ class File(FsItem): @staticmethod def create_file(path: str): fs_tools.create_file(path) - output = fs_tools.ls_item(path) - return fs_tools.parse_ls_output(output)[0] + output = ls_item(path) + return parse_ls_output(output)[0] def padding(self, size: Size): dd = Dd().input("/dev/zero").output(self).count(1).block_size(size) @@ -49,7 +52,7 @@ class File(FsItem): self.refresh_item() def remove(self, force: bool = False, ignore_errors: bool = False): - fs_tools.remove(str(self), force=force, ignore_errors=ignore_errors) + remove(str(self), force=force, ignore_errors=ignore_errors) def copy(self, destination, @@ -58,17 +61,17 @@ class File(FsItem): dereference: bool = False, timeout: timedelta = timedelta(minutes=30)): fs_tools.copy(str(self), destination, force, recursive, dereference, timeout) - if fs_tools.check_if_directory_exists(destination): + if check_if_directory_exists(destination): path = f"{destination}{'/' if destination[-1] != '/' else ''}{self.name}" else: path = destination - output = fs_tools.ls_item(path) - return fs_tools.parse_ls_output(output)[0] + output = ls_item(path) + return parse_ls_output(output)[0] class FileProperties: def __init__(self, file): - file = fs_tools.parse_ls_output(fs_tools.ls_item(file.full_path))[0] + file = parse_ls_output(ls_item(file.full_path))[0] self.full_path = file.full_path self.parent_dir = FsItem.get_parent_dir(self.full_path) self.name = FsItem.get_name(self.full_path) diff --git a/test_utils/filesystem/fs_item.py b/test_utils/filesystem/fs_item.py index f9ae2a3..3d1b1bc 100644 --- a/test_utils/filesystem/fs_item.py +++ b/test_utils/filesystem/fs_item.py @@ -1,11 +1,14 @@ # # Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # import posixpath from test_tools import fs_tools +from test_tools.fs_tools import Permissions, PermissionsUsers, PermissionSign, \ + check_if_directory_exists, ls_item, parse_ls_output class FsItem: @@ -42,9 +45,9 @@ class FsItem: self.refresh_item() def chmod(self, - permissions: fs_tools.Permissions, - users: fs_tools.PermissionsUsers, - sign: fs_tools.PermissionSign = fs_tools.PermissionSign.set, + permissions: Permissions, + users: PermissionsUsers, + sign: PermissionSign = PermissionSign.set, recursive: bool = False): fs_tools.chmod(self.full_path, permissions, users, sign=sign, recursive=recursive) self.refresh_item() @@ -58,19 +61,19 @@ class FsItem: force: bool = False, recursive: bool = False, dereference: bool = False): - target_dir_exists = fs_tools.check_if_directory_exists(destination) + target_dir_exists = check_if_directory_exists(destination) fs_tools.copy(str(self), destination, force, recursive, dereference) if target_dir_exists: path = f"{destination}{'/' if destination[-1] != '/' else ''}{self.name}" else: path = destination - output = fs_tools.ls_item(f"{path}") - return fs_tools.parse_ls_output(output)[0] + output = ls_item(f"{path}") + return parse_ls_output(output)[0] def move(self, destination, force: bool = False): - target_dir_exists = fs_tools.check_if_directory_exists(destination) + target_dir_exists = check_if_directory_exists(destination) fs_tools.move(str(self), destination, force) if target_dir_exists: self.full_path = f"{destination}{'/' if destination[-1] != '/' else ''}{self.name}" @@ -80,7 +83,7 @@ class FsItem: return self def refresh_item(self): - updated_file = fs_tools.parse_ls_output(fs_tools.ls_item(self.full_path))[0] + updated_file = parse_ls_output(ls_item(self.full_path))[0] # keep order the same as in __init__() self.parent_dir = updated_file.parent_dir self.name = updated_file.name diff --git a/type_def/__init__.py b/type_def/__init__.py index e69de29..18aaf34 100644 --- a/type_def/__init__.py +++ b/type_def/__init__.py @@ -0,0 +1,4 @@ +# +# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +# From f7f2914e41ce73d13dafe6cd749aa6e2d33b4315 Mon Sep 17 00:00:00 2001 From: Katarzyna Treder Date: Tue, 31 Dec 2024 11:30:12 +0100 Subject: [PATCH 23/23] Add trim verification fio param This param will be needed to replace some vdbench configs after removing vdbench plugin Signed-off-by: Katarzyna Treder --- test_tools/fio/fio_param.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test_tools/fio/fio_param.py b/test_tools/fio/fio_param.py index b0c71e0..31240ca 100644 --- a/test_tools/fio/fio_param.py +++ b/test_tools/fio/fio_param.py @@ -316,6 +316,9 @@ class FioParam(LinuxCommand): def verify_only(self, value: bool = True): return self.set_flags('verify_only') if value else self.remove_param('verify_only') + def trim_verify_zero(self, value: bool = True): + return self.set_param('trim_verify_zero', int(value)) + def write_hint(self, value: str): return self.set_param('write_hint', value)