diff --git a/connection/ssh_executor.py b/connection/ssh_executor.py index 173fef5..4f4bfa8 100644 --- a/connection/ssh_executor.py +++ b/connection/ssh_executor.py @@ -3,13 +3,14 @@ # Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # + import os import re +import paramiko import socket import subprocess -from datetime import timedelta, datetime -import paramiko +from datetime import timedelta, datetime from connection.base_executor import BaseExecutor from core.test_run import TestRun, Blocked @@ -46,7 +47,7 @@ class SshExecutor(BaseExecutor): hostname = target["hostname"] key_filename = target.get("identityfile", None) user = target.get("user", user) - port = target.get("port", port) + port = int(target.get("port", port)) if target.get("proxyjump", None) is not None: proxy = config.lookup(target["proxyjump"]) jump = paramiko.SSHClient() diff --git a/connection/utils/__init__.py b/connection/utils/__init__.py index e69de29..18aaf34 100644 --- a/connection/utils/__init__.py +++ b/connection/utils/__init__.py @@ -0,0 +1,4 @@ +# +# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +# diff --git a/connection/utils/asynchronous.py b/connection/utils/asynchronous.py index 9be0159..924e537 100644 --- a/connection/utils/asynchronous.py +++ b/connection/utils/asynchronous.py @@ -1,9 +1,10 @@ # # Copyright(c) 2020-2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # -import concurrent +from concurrent.futures import ThreadPoolExecutor def start_async_func(func, *args): @@ -14,5 +15,5 @@ def start_async_func(func, *args): - done() method returns True when task ended (have a result or ended with an exception) otherwise returns False """ - executor = concurrent.futures.ThreadPoolExecutor() + executor = ThreadPoolExecutor() return executor.submit(func, *args) diff --git a/connection/utils/output.py b/connection/utils/output.py index 73e8f94..d3ae9ae 100644 --- a/connection/utils/output.py +++ b/connection/utils/output.py @@ -1,5 +1,6 @@ # # Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # diff --git a/connection/utils/retry.py b/connection/utils/retry.py index 10ca573..b3bd0f8 100644 --- a/connection/utils/retry.py +++ b/connection/utils/retry.py @@ -1,5 +1,6 @@ # # Copyright(c) 2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # @@ -33,7 +34,7 @@ class Retry: try: result = func() return True - except: + except Exception: return False cls.run_while_false(wrapped_func, retries=retries, timeout=timeout) diff --git a/internal_plugins/vdbench/__init__.py b/internal_plugins/vdbench/__init__.py deleted file mode 100644 index 536db28..0000000 --- a/internal_plugins/vdbench/__init__.py +++ /dev/null @@ -1,98 +0,0 @@ -# -# Copyright(c) 2020-2021 Intel Corporation -# Copyright(c) 2023-2024 Huawei Technologies Co., Ltd. -# SPDX-License-Identifier: BSD-3-Clause -# - -import time -import posixpath - -from datetime import timedelta -from core.test_run import TestRun -from test_tools import fs_tools - - -class Vdbench: - def __init__(self, params, config): - print("VDBench plugin initialization") - self.run_time = timedelta(seconds=60) - - try: - self.working_dir = config["working_dir"] - self.reinstall = config["reinstall"] - self.source_dir = config["source_dir"] - except Exception: - raise Exception("Missing fields in config! ('working_dir', 'source_dir' and " - "'reinstall' required)") - - self.result_dir = posixpath.join(self.working_dir, 'result.tod') - - def pre_setup(self): - pass - - def post_setup(self): - print("VDBench plugin post setup") - if not self.reinstall and fs_utils.check_if_directory_exists(self.working_dir): - return - - if fs_utils.check_if_directory_exists(self.working_dir): - fs_utils.remove(self.working_dir, True, True) - - fs_utils.create_directory(self.working_dir) - TestRun.LOGGER.info("Copying vdbench to working dir.") - fs_utils.copy( - source=self.source_dir, destination=self.working_dir, force=True, recursive=True - ) - - def teardown(self): - pass - - def create_config(self, config, run_time: timedelta): - self.run_time = run_time - if config[-1] != ",": - config += "," - config += f"elapsed={int(run_time.total_seconds())}" - TestRun.LOGGER.info(f"Vdbench config:\n{config}") - fs_utils.write_file(posixpath.join(self.working_dir, "param.ini"), config) - - def run(self): - cmd = f"{posixpath.join(self.working_dir, 'vdbench')} " \ - f"-f {posixpath.join(self.working_dir, 'param.ini')} " \ - f"-vr -o {self.result_dir}" - full_cmd = f"screen -dmS vdbench {cmd}" - TestRun.executor.run(full_cmd) - start_time = time.time() - - timeout = self.run_time * 1.5 - - while True: - if not TestRun.executor.run(f"ps aux | grep '{cmd}' | grep -v grep").exit_code == 0: - return self.analyze_log() - - if time.time() - start_time > timeout.total_seconds(): - TestRun.LOGGER.error("Vdbench timeout.") - return False - time.sleep(1) - - def analyze_log(self): - output = TestRun.executor.run( - f"ls -1td {self.result_dir[0:len(self.result_dir) - 3]}* | head -1") - log_path = posixpath.join(output.stdout if output.exit_code == 0 else self.result_dir, - "logfile.html") - - log_file = fs_utils.read_file(log_path) - - if "Vdbench execution completed successfully" in log_file: - TestRun.LOGGER.info("Vdbench execution completed successfully.") - return True - - if "Data Validation error" in log_file or "data_errors=1" in log_file: - TestRun.LOGGER.error("Data corruption occurred!") - elif "Heartbeat monitor:" in log_file: - TestRun.LOGGER.error("Vdbench: heartbeat.") - else: - TestRun.LOGGER.error("Vdbench unknown result.") - return False - - -plugin_class = Vdbench diff --git a/scripts/__init__.py b/scripts/__init__.py index e69de29..18aaf34 100644 --- a/scripts/__init__.py +++ b/scripts/__init__.py @@ -0,0 +1,4 @@ +# +# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +# diff --git a/storage_devices/device.py b/storage_devices/device.py index 7bfb5c3..a56b0bb 100644 --- a/storage_devices/device.py +++ b/storage_devices/device.py @@ -3,31 +3,32 @@ # Copyright(c) 2023-2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # + import posixpath -import test_tools.fs_tools from core.test_run import TestRun -from test_tools import disk_tools, fs_tools -from test_tools.disk_tools import get_sysfs_path -from test_tools.fs_tools import get_device_filesystem_type +from test_tools import disk_tools +from test_tools.disk_tools import get_sysfs_path, validate_dev_path, get_size +from test_tools.fs_tools import (get_device_filesystem_type, Filesystem, wipefs, + readlink, write_file, mkfs, ls, parse_ls_output) from test_utils.io_stats import IoStats from type_def.size import Size, Unit class Device: def __init__(self, path): - disk_tools.validate_dev_path(path) + validate_dev_path(path) self.path = path - self.size = Size(disk_tools.get_size(self.get_device_id()), Unit.Byte) + self.size = Size(get_size(self.get_device_id()), Unit.Byte) self.filesystem = get_device_filesystem_type(self.get_device_id()) self.mount_point = None - def create_filesystem(self, fs_type: test_tools.fs_tools.Filesystem, force=True, blocksize=None): - test_tools.fs_tools.create_filesystem(self, fs_type, force, blocksize) + def create_filesystem(self, fs_type: Filesystem, force=True, blocksize=None): + mkfs(self, fs_type, force, blocksize) self.filesystem = fs_type def wipe_filesystem(self, force=True): - test_tools.fs_tools.wipe_filesystem(self, force) + wipefs(self, force) self.filesystem = None def is_mounted(self): @@ -36,7 +37,7 @@ class Device: return False else: mount_point_line = output.stdout.split('\n')[1] - device_path = fs_tools.readlink(self.path) + device_path = readlink(self.path) self.mount_point = mount_point_line[0:mount_point_line.find(device_path)].strip() return True @@ -58,27 +59,26 @@ class Device: return next(i for i in items if i.full_path.startswith(directory)) def get_device_id(self): - return fs_tools.readlink(self.path).split('/')[-1] + return readlink(self.path).split('/')[-1] def get_all_device_links(self, directory: str): - from test_tools import fs_tools - output = fs_tools.ls(f"$(find -L {directory} -samefile {self.path})") - return fs_tools.parse_ls_output(output, self.path) + output = ls(f"$(find -L {directory} -samefile {self.path})") + return parse_ls_output(output, self.path) def get_io_stats(self): return IoStats.get_io_stats(self.get_device_id()) def get_sysfs_property(self, property_name): - path = posixpath.join(disk_tools.get_sysfs_path(self.get_device_id()), + path = posixpath.join(get_sysfs_path(self.get_device_id()), "queue", property_name) return TestRun.executor.run_expect_success(f"cat {path}").stdout def set_sysfs_property(self, property_name, value): TestRun.LOGGER.info( f"Setting {property_name} for device {self.get_device_id()} to {value}.") - path = posixpath.join(disk_tools.get_sysfs_path(self.get_device_id()), "queue", + path = posixpath.join(get_sysfs_path(self.get_device_id()), "queue", property_name) - fs_tools.write_file(path, str(value)) + write_file(path, str(value)) def set_max_io_size(self, new_max_io_size: Size): self.set_sysfs_property("max_sectors_kb", diff --git a/storage_devices/disk.py b/storage_devices/disk.py index 0ac2b29..8f3a179 100644 --- a/storage_devices/disk.py +++ b/storage_devices/disk.py @@ -10,13 +10,14 @@ import re from datetime import timedelta from enum import IntEnum -import test_tools.fs_tools from core.test_run import TestRun -from storage_devices.device import Device -from test_tools import disk_tools, fs_tools, nvme_cli -from test_tools.common.wait import wait from connection.utils.output import Output +from storage_devices.device import Device +from test_tools import disk_tools, nvme_cli +from test_tools.common.wait import wait from test_tools.disk_finder import get_block_devices_list, resolve_to_by_id_link +from test_tools.disk_tools import PartitionTable +from test_tools.fs_tools import readlink, is_mounted, ls_item, parse_ls_output from type_def.size import Unit @@ -138,7 +139,7 @@ class Disk(Device): ) return recognized_types[0] - def create_partitions(self, sizes: [], partition_table_type=disk_tools.PartitionTable.gpt): + def create_partitions(self, sizes: [], partition_table_type=PartitionTable.gpt): disk_tools.create_partitions(self, sizes, partition_table_type) def remove_partition(self, part): @@ -148,12 +149,12 @@ class Disk(Device): def umount_all_partitions(self): TestRun.LOGGER.info(f"Unmounting all partitions from: {self.path}") - cmd = f"umount -l {fs_tools.readlink(self.path)}*?" + cmd = f"umount -l {readlink(self.path)}*?" TestRun.executor.run(cmd) def remove_partitions(self): for part in self.partitions: - if test_tools.fs_tools.is_mounted(part.path): + if is_mounted(part.path): part.unmount() if disk_tools.remove_partitions(self): self.partitions.clear() @@ -163,8 +164,8 @@ class Disk(Device): serial_numbers = Disk.get_all_serial_numbers() return self.serial_number in serial_numbers elif self.path: - output = fs_tools.ls_item(f"{self.path}") - return fs_tools.parse_ls_output(output)[0] is not None + output = ls_item(f"{self.path}") + return parse_ls_output(output)[0] is not None raise Exception("Couldn't check if device is detected by the system") def wait_for_plug_status(self, should_be_visible): @@ -290,8 +291,8 @@ class NvmeDisk(Disk): base = f"/sys/block/{device_id}/device" for suffix in ["/remove", "/device/remove"]: try: - output = fs_tools.ls_item(base + suffix) - fs_tools.parse_ls_output(output)[0] + output = ls_item(base + suffix) + parse_ls_output(output)[0] except TypeError: continue return base + suffix @@ -346,8 +347,8 @@ class SataDisk(Disk): @staticmethod def get_sysfs_addr(device_id): ls_command = f"$(find -H /sys/devices/ -name {device_id} -type d)" - output = fs_tools.ls_item(f"{ls_command}") - sysfs_addr = fs_tools.parse_ls_output(output)[0] + output = ls_item(ls_command) + sysfs_addr = parse_ls_output(output)[0] if not sysfs_addr: raise Exception(f"Failed to find sysfs address: ls -l {ls_command}") return sysfs_addr.full_path @@ -413,8 +414,8 @@ class VirtioDisk(Disk): @staticmethod def get_sysfs_addr(device_id: str) -> str: ls_command = f"$(find -H /sys/devices/ -name {device_id} -type d)" - output = fs_tools.ls_item(f"{ls_command}") - sysfs_addr = fs_tools.parse_ls_output(output)[0] + output = ls_item(ls_command) + sysfs_addr = parse_ls_output(output)[0] if not sysfs_addr: raise Exception(f"Failed to find sysfs address: ls -l {ls_command}") diff --git a/storage_devices/partition.py b/storage_devices/partition.py index f39b556..fd83ace 100644 --- a/storage_devices/partition.py +++ b/storage_devices/partition.py @@ -4,13 +4,13 @@ # from storage_devices.device import Device -from test_tools import disk_tools +from test_tools.disk_tools import get_partition_path from type_def.size import Size class Partition(Device): def __init__(self, parent_dev, type, number, begin: Size, end: Size): - Device.__init__(self, disk_tools.get_partition_path(parent_dev.path, number)) + Device.__init__(self, get_partition_path(parent_dev.path, number)) self.number = number self.parent_device = parent_dev self.type = type diff --git a/storage_devices/raid.py b/storage_devices/raid.py index 3d1c9c1..f9d98d9 100644 --- a/storage_devices/raid.py +++ b/storage_devices/raid.py @@ -2,6 +2,7 @@ # Copyright(c) 2020-2021 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # + import threading from enum import IntEnum, Enum diff --git a/storage_devices/ramdisk.py b/storage_devices/ramdisk.py index fc04ba0..1ff9b09 100644 --- a/storage_devices/ramdisk.py +++ b/storage_devices/ramdisk.py @@ -7,7 +7,7 @@ import posixpath from core.test_run import TestRun from storage_devices.device import Device -from test_tools import disk_tools +from test_tools.disk_tools import get_size from test_tools.fs_tools import ls, parse_ls_output from test_utils.filesystem.symlink import Symlink from test_tools.os_tools import reload_kernel_module, unload_kernel_module, is_kernel_module_loaded @@ -68,7 +68,7 @@ class RamDisk(Device): ram_disks = cls._list_devices() return ( len(ram_disks) >= disk_count - and Size(disk_tools.get_size(ram_disks[0].name), Unit.Byte).align_down(Unit.MiB.value) + and Size(get_size(ram_disks[0].name), Unit.Byte).align_down(Unit.MiB.value) == disk_size.align_down(Unit.MiB.value) ) diff --git a/test_tools/common/__init__.py b/test_tools/common/__init__.py index e69de29..18aaf34 100644 --- a/test_tools/common/__init__.py +++ b/test_tools/common/__init__.py @@ -0,0 +1,4 @@ +# +# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +# diff --git a/test_tools/common/linux_command.py b/test_tools/common/linux_command.py index b6d887b..e8c3318 100644 --- a/test_tools/common/linux_command.py +++ b/test_tools/common/linux_command.py @@ -1,5 +1,6 @@ # # Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # diff --git a/test_tools/dd.py b/test_tools/dd.py index 0428b6f..b40e8b1 100644 --- a/test_tools/dd.py +++ b/test_tools/dd.py @@ -1,16 +1,17 @@ # # Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # -import test_tools.common.linux_command as linux_comm import type_def.size as size from core.test_run import TestRun +from test_tools.common.linux_command import LinuxCommand -class Dd(linux_comm.LinuxCommand): +class Dd(LinuxCommand): def __init__(self): - linux_comm.LinuxCommand.__init__(self, TestRun.executor, 'dd') + LinuxCommand.__init__(self, TestRun.executor, 'dd') def block_size(self, value: size.Size): return self.set_param('bs', int(value.get_value())) diff --git a/test_tools/ddrescue.py b/test_tools/ddrescue.py index 3a6dbb7..c319665 100644 --- a/test_tools/ddrescue.py +++ b/test_tools/ddrescue.py @@ -1,16 +1,17 @@ # # Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # -import test_tools.common.linux_command as linux_comm -import type_def.size as size from core.test_run import TestRun +from test_tools.common.linux_command import LinuxCommand +from type_def.size import Size -class Ddrescue(linux_comm.LinuxCommand): +class Ddrescue(LinuxCommand): def __init__(self): - linux_comm.LinuxCommand.__init__(self, TestRun.executor, 'ddrescue') + LinuxCommand.__init__(self, TestRun.executor, 'ddrescue') self.source_path = None self.destination_path = None self.param_name_prefix = "--" @@ -35,13 +36,13 @@ class Ddrescue(linux_comm.LinuxCommand): def force(self): return self.set_flags("force") - def block_size(self, value: size.Size): + def block_size(self, value: Size): return self.set_param('sector-size', int(value.get_value())) - def size(self, value: size.Size): + def size(self, value: Size): return self.set_param('size', int(value.get_value())) def __str__(self): - command = linux_comm.LinuxCommand.__str__(self) + command = LinuxCommand.__str__(self) command += f" {self.source_path} {self.destination_path}" return command diff --git a/test_tools/device_mapper.py b/test_tools/device_mapper.py index 9f78216..a804b5d 100644 --- a/test_tools/device_mapper.py +++ b/test_tools/device_mapper.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019-2022 Intel Corporation +# Copyright(c) 2019-2021 Intel Corporation # Copyright(c) 2023-2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # @@ -30,11 +30,6 @@ class DmTarget(Enum): return self.name.lower() -class DmTable: - class TableEntry: - pass - - class DmTable: class TableEntry: def __init__(self, offset: int, length: int, target: DmTarget, *params): @@ -131,7 +126,7 @@ class DmTable: return self - def add_entry(self, entry: DmTable.TableEntry): + def add_entry(self, entry: TableEntry): self.table.append(entry) return self diff --git a/test_tools/disk_finder.py b/test_tools/disk_finder.py index 1cbbff2..16bc330 100644 --- a/test_tools/disk_finder.py +++ b/test_tools/disk_finder.py @@ -7,8 +7,7 @@ import os import posixpath from core.test_run import TestRun -from test_tools import disk_tools -from test_tools.disk_tools import get_sysfs_path +from test_tools.disk_tools import get_sysfs_path, get_block_size, get_size from test_tools.fs_tools import check_if_file_exists, readlink from connection.utils.output import CmdException @@ -50,7 +49,7 @@ def discover_hdd_devices(block_devices, devices_res): for dev in block_devices: if TestRun.executor.run_expect_success(f"cat /sys/block/{dev}/removable").stdout == "1": continue # skip removable drives - block_size = disk_tools.get_block_size(dev) + block_size = get_block_size(dev) if int(block_size) == 4096: disk_type = 'hdd4k' else: @@ -62,7 +61,7 @@ def discover_hdd_devices(block_devices, devices_res): f"sg_inq /dev/{dev} | grep -i 'serial number'" ).stdout.split(': ')[1].strip(), "blocksize": block_size, - "size": disk_tools.get_size(dev)}) + "size": get_size(dev)}) block_devices.clear() @@ -99,8 +98,8 @@ def discover_ssd_devices(block_devices, devices_res): "type": disk_type, "path": resolve_to_by_id_link(device_path), "serial": serial_number, - "blocksize": disk_tools.get_block_size(dev), - "size": disk_tools.get_size(dev)}) + "blocksize": get_block_size(dev), + "size": get_size(dev)}) block_devices.remove(dev) @@ -125,7 +124,7 @@ def get_system_disks(): def __get_slaves(device_name: str): try: device_names = TestRun.executor.run_expect_success( - f"ls {os.path.join(get_sysfs_path(device_name), "slaves")}").stdout.splitlines() + f"ls {os.path.join(get_sysfs_path(device_name), 'slaves')}").stdout.splitlines() except CmdException as e: if "No such file or directory" not in e.output.stderr: raise diff --git a/test_tools/disk_tools.py b/test_tools/disk_tools.py index ca07049..c2790d6 100644 --- a/test_tools/disk_tools.py +++ b/test_tools/disk_tools.py @@ -10,11 +10,10 @@ import time from enum import Enum from typing import List -import test_tools.fs_tools from core.test_run import TestRun from test_tools.dd import Dd from test_tools.fs_tools import readlink, parse_ls_output, ls, check_if_directory_exists, \ - create_directory, wipe_filesystem + create_directory, wipefs, is_mounted from test_tools.udev import Udev from type_def.size import Size, Unit @@ -247,7 +246,7 @@ def get_first_partition_offset(device, aligned: bool): def remove_partitions(device): - if test_tools.fs_tools.is_mounted(device.path): + if is_mounted(device.path): device.unmount() for partition in device.partitions: @@ -255,7 +254,7 @@ def remove_partitions(device): TestRun.LOGGER.info(f"Removing partitions from device: {device.path} " f"({device.get_device_id()}).") - wipe_filesystem(device) + wipefs(device) Udev.trigger() Udev.settle() output = TestRun.executor.run(f"ls {device.path}* -1") diff --git a/test_tools/fio/fio.py b/test_tools/fio/fio.py index 8d40599..56bab56 100644 --- a/test_tools/fio/fio.py +++ b/test_tools/fio/fio.py @@ -8,12 +8,12 @@ import datetime import uuid from packaging.version import Version -import test_tools.fio.fio_param -import test_tools.fs_tools -import test_tools.wget + from core.test_run import TestRun -from test_tools import fs_tools from connection.utils.output import CmdException +from test_tools import wget +from test_tools.fio.fio_param import FioParam, FioParamCmd, FioOutput, FioParamConfig +from test_tools.fs_tools import uncompress_archive class Fio: @@ -22,12 +22,12 @@ class Fio: self.default_run_time = datetime.timedelta(hours=1) self.jobs = [] self.executor = executor_obj if executor_obj is not None else TestRun.executor - self.base_cmd_parameters: test_tools.fio.fio_param.FioParam = None - self.global_cmd_parameters: test_tools.fio.fio_param.FioParam = None + self.base_cmd_parameters: FioParam = None + self.global_cmd_parameters: FioParam = None - def create_command(self, output_type=test_tools.fio.fio_param.FioOutput.json): - self.base_cmd_parameters = test_tools.fio.fio_param.FioParamCmd(self, self.executor) - self.global_cmd_parameters = test_tools.fio.fio_param.FioParamConfig(self, self.executor) + def create_command(self, output_type=FioOutput.json): + self.base_cmd_parameters = FioParamCmd(self, self.executor) + self.global_cmd_parameters = FioParamConfig(self, self.executor) self.fio_file = \ f'fio_run_{datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_{uuid.uuid4().hex}' @@ -50,8 +50,8 @@ class Fio: def install(self): fio_url = f"http://brick.kernel.dk/snaps/fio-{self.min_fio_version}.tar.bz2" - fio_package = test_tools.wget.download_file(fio_url) - fs_tools.uncompress_archive(fio_package) + fio_package = wget.download_file(fio_url) + uncompress_archive(fio_package) TestRun.executor.run_expect_success( f"cd {fio_package.parent_dir}/fio-{self.min_fio_version}" f" && ./configure && make -j && make install" @@ -106,7 +106,7 @@ class Fio: command = f"echo '{self.execution_cmd_parameters()}' |" \ f" {str(self.base_cmd_parameters)} -" else: - fio_parameters = test_tools.fio.fio_param.FioParamCmd(self, self.executor) + fio_parameters = FioParamCmd(self, self.executor) fio_parameters.command_env_var.update(self.base_cmd_parameters.command_env_var) fio_parameters.command_param.update(self.base_cmd_parameters.command_param) fio_parameters.command_param.update(self.global_cmd_parameters.command_param) diff --git a/test_tools/fio/fio_patterns.py b/test_tools/fio/fio_patterns.py index 22ca70b..0c3f405 100644 --- a/test_tools/fio/fio_patterns.py +++ b/test_tools/fio/fio_patterns.py @@ -4,7 +4,7 @@ # import secrets -from aenum import Enum +from enum import Enum class Pattern(Enum): diff --git a/test_tools/fio/fio_result.py b/test_tools/fio/fio_result.py index 5163609..59eba2c 100644 --- a/test_tools/fio/fio_result.py +++ b/test_tools/fio/fio_result.py @@ -103,9 +103,6 @@ class FioResult: def write_runtime(self): return Time(microseconds=self.job.write.runtime) - def write_completion_latency_average(self): - return Time(nanoseconds=self.job.write.lat_ns.mean) - def write_completion_latency_min(self): return Time(nanoseconds=self.job.write.lat_ns.min) @@ -139,9 +136,6 @@ class FioResult: def trim_runtime(self): return Time(microseconds=self.job.trim.runtime) - def trim_completion_latency_average(self): - return Time(nanoseconds=self.job.trim.lat_ns.mean) - def trim_completion_latency_min(self): return Time(nanoseconds=self.job.trim.lat_ns.min) diff --git a/test_tools/fs_tools.py b/test_tools/fs_tools.py index 87e1425..7724da4 100644 --- a/test_tools/fs_tools.py +++ b/test_tools/fs_tools.py @@ -11,7 +11,8 @@ import re import textwrap from collections import namedtuple from datetime import datetime, timedelta -from enum import Enum, IntFlag +from enum import Enum +from aenum import IntFlag # IntFlag from enum is not able to correctly parse string like "x|y|z" from connection.utils.output import CmdException from core.test_run import TestRun @@ -402,7 +403,7 @@ def create_random_test_file(target_file_path: str, return file -def create_filesystem(device, filesystem: Filesystem, force=True, blocksize=None): +def mkfs(device, filesystem: Filesystem, force=True, blocksize=None): TestRun.LOGGER.info( f"Creating filesystem ({filesystem.name}) on device: {device.path}") force_param = ' -f ' if filesystem == Filesystem.xfs else ' -F ' @@ -417,7 +418,7 @@ def create_filesystem(device, filesystem: Filesystem, force=True, blocksize=None f"Successfully created filesystem on device: {device.path}") -def wipe_filesystem(device, force=True): +def wipefs(device, force=True): TestRun.LOGGER.info(f"Erasing the device: {device.path}") force_param = ' -f' if force else '' cmd = f'wipefs -a{force_param} {device.path}' diff --git a/test_tools/fstab.py b/test_tools/fstab.py index 3df6b88..6bcf504 100644 --- a/test_tools/fstab.py +++ b/test_tools/fstab.py @@ -4,17 +4,18 @@ # SPDX-License-Identifier: BSD-3-Clause # -from test_tools import fs_tools, systemctl +from test_tools.fs_tools import append_line, remove_lines +from test_tools.systemctl import reload_daemon, restart_service def add_mountpoint(device, mount_point, fs_type, mount_now=True): - fs_tools.append_line("/etc/fstab", - f"{device.path} {mount_point} {fs_type.name} defaults 0 0") - systemctl.reload_daemon() + append_line("/etc/fstab", + f"{device.path} {mount_point} {fs_type.name} defaults 0 0") + reload_daemon() if mount_now: - systemctl.restart_service("local-fs.target") + restart_service("local-fs.target") def remove_mountpoint(device): - fs_tools.remove_lines("/etc/fstab", device.path) - systemctl.reload_daemon() + remove_lines("/etc/fstab", device.path) + reload_daemon() diff --git a/test_tools/git.py b/test_tools/git.py index 74c161d..456c08a 100644 --- a/test_tools/git.py +++ b/test_tools/git.py @@ -3,6 +3,7 @@ # Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # + import itertools import os import posixpath diff --git a/test_tools/iostat.py b/test_tools/iostat.py index 2a07dcb..673bf17 100644 --- a/test_tools/iostat.py +++ b/test_tools/iostat.py @@ -3,10 +3,12 @@ # SPDX-License-Identifier: BSD-3-Clause # +import csv + from core.test_run import TestRun from type_def.size import Size, Unit, UnitPerSecond from type_def.time import Time -import csv + class IOstatExtended: diff --git a/test_tools/linux_packaging.py b/test_tools/linux_packaging.py index 8f1940e..b5a8173 100644 --- a/test_tools/linux_packaging.py +++ b/test_tools/linux_packaging.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: BSD-3-Clause # - import os import re diff --git a/test_tools/memory.py b/test_tools/memory.py index eb856b9..145518c 100644 --- a/test_tools/memory.py +++ b/test_tools/memory.py @@ -1,3 +1,9 @@ +# +# Copyright(c) 2019-2022 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +# + import math from connection.utils.output import CmdException diff --git a/test_tools/nvme_cli.py b/test_tools/nvme_cli.py index 5844587..f49f42b 100644 --- a/test_tools/nvme_cli.py +++ b/test_tools/nvme_cli.py @@ -1,7 +1,9 @@ # # Copyright(c) 2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # + import json from core.test_run import TestRun diff --git a/test_tools/os_tools.py b/test_tools/os_tools.py index 27314ee..4a80be9 100644 --- a/test_tools/os_tools.py +++ b/test_tools/os_tools.py @@ -16,7 +16,6 @@ from core.test_run import TestRun from storage_devices.device import Device from test_tools.disk_tools import get_sysfs_path from test_tools.fs_tools import check_if_file_exists, is_mounted -from test_utils.filesystem.file import File from connection.utils.retry import Retry DEBUGFS_MOUNT_POINT = "/sys/kernel/debug" @@ -86,7 +85,8 @@ def get_kernel_version(): def is_kernel_module_loaded(module_name): - output = TestRun.executor.run(f"lsmod | grep ^{module_name}$") + command = f"lsmod | grep -E '^{module_name}\\b'" + output = TestRun.executor.run(command) return output.exit_code == 0 @@ -107,7 +107,7 @@ def get_kernel_module_parameter(module_name, parameter): param_file_path = f"/sys/module/{module_name}/parameters/{parameter}" if not check_if_file_exists(param_file_path): raise FileNotFoundError(f"File {param_file_path} does not exist!") - return File(param_file_path).read() + return TestRun.executor.run(f"cat {param_file_path}").stdout def mount_debugfs(): diff --git a/test_tools/peach_fuzzer/peach_fuzzer.py b/test_tools/peach_fuzzer/peach_fuzzer.py index 9e54d8f..c3aec23 100644 --- a/test_tools/peach_fuzzer/peach_fuzzer.py +++ b/test_tools/peach_fuzzer/peach_fuzzer.py @@ -12,10 +12,10 @@ import tempfile import lxml.etree as etree from collections import namedtuple -import test_tools.wget +from test_tools import wget from core.test_run import TestRun -from test_tools import fs_tools -from test_tools.fs_tools import create_directory, check_if_file_exists, write_file +from test_tools.fs_tools import create_directory, check_if_file_exists, write_file, remove, \ + check_if_directory_exists class PeachFuzzer: @@ -75,7 +75,7 @@ class PeachFuzzer: cls._install() if not cls._is_xml_config_prepared(): TestRun.block("No Peach Fuzzer XML config needed to generate fuzzed values was found!") - fs_tools.remove(cls.fuzzy_output_file, force=True, ignore_errors=True) + remove(cls.fuzzy_output_file, force=True, ignore_errors=True) TestRun.LOGGER.info(f"Generate {count} unique fuzzed values") cmd = f"cd {cls.base_dir}; {cls.peach_dir}/peach --range 0,{count - 1} " \ f"--seed {random.randrange(2 ** 32)} {cls.xml_config_file} > " \ @@ -155,7 +155,7 @@ class PeachFuzzer: Install Peach Fuzzer on the DUT """ create_directory(cls.base_dir, True) - peach_archive = test_tools.wget.download_file( + peach_archive = wget.download_file( cls.peach_fuzzer_3_0_url, destination_dir=cls.base_dir ) TestRun.executor.run_expect_success( @@ -172,7 +172,7 @@ class PeachFuzzer: """ if not cls._is_mono_installed(): TestRun.block("Mono is not installed, can't continue with Peach Fuzzer!") - if fs_tools.check_if_directory_exists(posixpath.join(cls.base_dir, cls.peach_dir)): + if check_if_directory_exists(posixpath.join(cls.base_dir, cls.peach_dir)): return "Peach" in TestRun.executor.run( f"cd {cls.base_dir} && {cls.peach_dir}/peach --version").stdout.strip() else: @@ -197,7 +197,7 @@ class PeachFuzzer: """ Check if Peach Fuzzer XML config is present on the DUT """ - if fs_tools.check_if_file_exists(cls.xml_config_file): + if check_if_file_exists(cls.xml_config_file): return True else: return False diff --git a/test_utils/common/__init__.py b/test_utils/common/__init__.py index e69de29..18aaf34 100644 --- a/test_utils/common/__init__.py +++ b/test_utils/common/__init__.py @@ -0,0 +1,4 @@ +# +# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +# diff --git a/test_utils/common/singleton.py b/test_utils/common/singleton.py index a484929..56a3015 100644 --- a/test_utils/common/singleton.py +++ b/test_utils/common/singleton.py @@ -1,5 +1,6 @@ # # Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # diff --git a/test_utils/filesystem/directory.py b/test_utils/filesystem/directory.py index fca1ad2..2e7a26a 100644 --- a/test_utils/filesystem/directory.py +++ b/test_utils/filesystem/directory.py @@ -1,10 +1,12 @@ # # Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # + from core.test_run import TestRun from test_tools import fs_tools -from test_tools.fs_tools import check_if_directory_exists +from test_tools.fs_tools import check_if_directory_exists, parse_ls_output, ls_item, ls from test_utils.filesystem.fs_item import FsItem @@ -13,14 +15,14 @@ class Directory(FsItem): FsItem.__init__(self, full_path) def ls(self): - output = fs_tools.ls(f"{self.full_path}") - return fs_tools.parse_ls_output(output, self.full_path) + output = ls(self.full_path) + return parse_ls_output(output, self.full_path) @staticmethod def create_directory(path: str, parents: bool = False): fs_tools.create_directory(path, parents) - output = fs_tools.ls_item(path) - return fs_tools.parse_ls_output(output)[0] + output = ls_item(path) + return parse_ls_output(output)[0] @staticmethod def create_temp_directory(parent_dir_path: str = "/tmp"): diff --git a/test_utils/filesystem/file.py b/test_utils/filesystem/file.py index 3ee3f27..986a25a 100644 --- a/test_utils/filesystem/file.py +++ b/test_utils/filesystem/file.py @@ -3,10 +3,13 @@ # Copyright(c) 2023-2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # + from datetime import timedelta from test_tools import fs_tools from test_tools.dd import Dd +from test_tools.fs_tools import read_file, write_file, ls_item, parse_ls_output, remove, \ + check_if_directory_exists from test_utils.filesystem.fs_item import FsItem from type_def.size import Size @@ -28,10 +31,10 @@ class File(FsItem): return fs_tools.crc32sum(str(self), timeout) def read(self): - return fs_tools.read_file(str(self)) + return read_file(str(self)) def write(self, content, overwrite: bool = True): - fs_tools.write_file(str(self), content, overwrite) + write_file(str(self), content, overwrite) self.refresh_item() def get_properties(self): @@ -40,8 +43,8 @@ class File(FsItem): @staticmethod def create_file(path: str): fs_tools.create_file(path) - output = fs_tools.ls_item(path) - return fs_tools.parse_ls_output(output)[0] + output = ls_item(path) + return parse_ls_output(output)[0] def padding(self, size: Size): dd = Dd().input("/dev/zero").output(self).count(1).block_size(size) @@ -49,7 +52,7 @@ class File(FsItem): self.refresh_item() def remove(self, force: bool = False, ignore_errors: bool = False): - fs_tools.remove(str(self), force=force, ignore_errors=ignore_errors) + remove(str(self), force=force, ignore_errors=ignore_errors) def copy(self, destination, @@ -58,17 +61,17 @@ class File(FsItem): dereference: bool = False, timeout: timedelta = timedelta(minutes=30)): fs_tools.copy(str(self), destination, force, recursive, dereference, timeout) - if fs_tools.check_if_directory_exists(destination): + if check_if_directory_exists(destination): path = f"{destination}{'/' if destination[-1] != '/' else ''}{self.name}" else: path = destination - output = fs_tools.ls_item(path) - return fs_tools.parse_ls_output(output)[0] + output = ls_item(path) + return parse_ls_output(output)[0] class FileProperties: def __init__(self, file): - file = fs_tools.parse_ls_output(fs_tools.ls_item(file.full_path))[0] + file = parse_ls_output(ls_item(file.full_path))[0] self.full_path = file.full_path self.parent_dir = FsItem.get_parent_dir(self.full_path) self.name = FsItem.get_name(self.full_path) diff --git a/test_utils/filesystem/fs_item.py b/test_utils/filesystem/fs_item.py index f9ae2a3..3d1b1bc 100644 --- a/test_utils/filesystem/fs_item.py +++ b/test_utils/filesystem/fs_item.py @@ -1,11 +1,14 @@ # # Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2024 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # import posixpath from test_tools import fs_tools +from test_tools.fs_tools import Permissions, PermissionsUsers, PermissionSign, \ + check_if_directory_exists, ls_item, parse_ls_output class FsItem: @@ -42,9 +45,9 @@ class FsItem: self.refresh_item() def chmod(self, - permissions: fs_tools.Permissions, - users: fs_tools.PermissionsUsers, - sign: fs_tools.PermissionSign = fs_tools.PermissionSign.set, + permissions: Permissions, + users: PermissionsUsers, + sign: PermissionSign = PermissionSign.set, recursive: bool = False): fs_tools.chmod(self.full_path, permissions, users, sign=sign, recursive=recursive) self.refresh_item() @@ -58,19 +61,19 @@ class FsItem: force: bool = False, recursive: bool = False, dereference: bool = False): - target_dir_exists = fs_tools.check_if_directory_exists(destination) + target_dir_exists = check_if_directory_exists(destination) fs_tools.copy(str(self), destination, force, recursive, dereference) if target_dir_exists: path = f"{destination}{'/' if destination[-1] != '/' else ''}{self.name}" else: path = destination - output = fs_tools.ls_item(f"{path}") - return fs_tools.parse_ls_output(output)[0] + output = ls_item(f"{path}") + return parse_ls_output(output)[0] def move(self, destination, force: bool = False): - target_dir_exists = fs_tools.check_if_directory_exists(destination) + target_dir_exists = check_if_directory_exists(destination) fs_tools.move(str(self), destination, force) if target_dir_exists: self.full_path = f"{destination}{'/' if destination[-1] != '/' else ''}{self.name}" @@ -80,7 +83,7 @@ class FsItem: return self def refresh_item(self): - updated_file = fs_tools.parse_ls_output(fs_tools.ls_item(self.full_path))[0] + updated_file = parse_ls_output(ls_item(self.full_path))[0] # keep order the same as in __init__() self.parent_dir = updated_file.parent_dir self.name = updated_file.name diff --git a/type_def/__init__.py b/type_def/__init__.py index e69de29..18aaf34 100644 --- a/type_def/__init__.py +++ b/type_def/__init__.py @@ -0,0 +1,4 @@ +# +# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +#