From f7539b46a1c89723765e74086f72f9180d56b52c Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Thu, 2 Jan 2025 03:05:35 +0100 Subject: [PATCH 01/20] conftest: add create/destroy temporary directory in conftest Signed-off-by: Kamil Gierszewski --- test/functional/tests/conftest.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/test/functional/tests/conftest.py b/test/functional/tests/conftest.py index 2e42d24..439032d 100644 --- a/test/functional/tests/conftest.py +++ b/test/functional/tests/conftest.py @@ -1,6 +1,6 @@ # # Copyright(c) 2019-2022 Intel Corporation -# Copyright(c) 2023-2024 Huawei Technologies Co., Ltd. +# Copyright(c) 2023-2025 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # @@ -28,7 +28,7 @@ from test_tools.udev import Udev from test_tools.disk_tools import PartitionTable, create_partition_table from test_tools.device_mapper import DeviceMapper from test_tools.mdadm import Mdadm -from test_tools.fs_tools import remove +from test_tools.fs_tools import remove, check_if_directory_exists, create_directory from test_tools import initramfs, git from log.logger import create_log, Log from test_utils.common.singleton import Singleton @@ -36,6 +36,9 @@ from storage_devices.lvm import Lvm, LvmConfiguration from storage_devices.disk import Disk +TEST_RUN_DATA_PATH = "/tmp/open_cas_test_data" + + def pytest_addoption(parser): TestRun.addoption(parser) parser.addoption("--dut-config", action="append", type=str) @@ -132,6 +135,7 @@ def pytest_runtest_setup(item): TestRun.LOGGER.info(f"DUT info: {TestRun.dut}") TestRun.dut.plugin_manager = TestRun.plugin_manager TestRun.dut.executor = TestRun.executor + TestRun.TEST_RUN_DATA_PATH = TEST_RUN_DATA_PATH TestRun.duts.append(TestRun.dut) base_prepare(item) @@ -184,6 +188,16 @@ def base_prepare(item): Udev.settle() RamDisk.remove_all() + + if check_if_directory_exists(path=TEST_RUN_DATA_PATH): + remove( + path=posixpath.join(TEST_RUN_DATA_PATH, "*"), + force=True, + recursive=True, + ) + else: + create_directory(path=TEST_RUN_DATA_PATH) + for disk in TestRun.disks.values(): disk_serial = Disk.get_disk_serial_number(disk.path) if disk.serial_number and disk.serial_number != disk_serial: @@ -250,6 +264,14 @@ def pytest_runtest_teardown(): DeviceMapper.remove_all() RamDisk.remove_all() + + if check_if_directory_exists(path=TEST_RUN_DATA_PATH): + remove( + path=posixpath.join(TEST_RUN_DATA_PATH, "*"), + force=True, + recursive=True, + ) + except Exception as ex: TestRun.LOGGER.warning( f"Exception occurred during platform cleanup.\n" From 76336c3ef50a19bd7930047eca23c9f98fc433bb Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Thu, 2 Jan 2025 03:07:58 +0100 Subject: [PATCH 02/20] conftest: add cleanup after drbd test Signed-off-by: Kamil Gierszewski --- test/functional/tests/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/functional/tests/conftest.py b/test/functional/tests/conftest.py index 439032d..86fdea8 100644 --- a/test/functional/tests/conftest.py +++ b/test/functional/tests/conftest.py @@ -326,6 +326,8 @@ def __drbd_cleanup(): if installer.check_if_installed(): casadm.stop_all_caches() + remove("/etc/drbd.d/*.res", force=True, ignore_errors=True) + class Opencas(metaclass=Singleton): def __init__(self, repo_dir, working_dir): From e1f842652730f376e27985784880dadfe580f401 Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Fri, 10 Jan 2025 12:40:49 +0100 Subject: [PATCH 03/20] conftest: fix typo Signed-off-by: Kamil Gierszewski --- test/functional/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/tests/conftest.py b/test/functional/tests/conftest.py index 86fdea8..b8dd7c4 100644 --- a/test/functional/tests/conftest.py +++ b/test/functional/tests/conftest.py @@ -321,7 +321,7 @@ def __drbd_cleanup(): from storage_devices.drbd import Drbd Drbd.down_all() - # If drbd instance had been configured on top of the CAS, the previos attempt to stop + # If drbd instance had been configured on top of the CAS, the previous attempt to stop # failed. As drbd has been stopped try to stop CAS one more time. if installer.check_if_installed(): casadm.stop_all_caches() From bef461ccc2a195228ecf15c582c08afeeb7bfef0 Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Thu, 2 Jan 2025 03:08:25 +0100 Subject: [PATCH 04/20] conftest: add TestRun cache/core list Signed-off-by: Kamil Gierszewski --- test/functional/tests/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/functional/tests/conftest.py b/test/functional/tests/conftest.py index b8dd7c4..100f25d 100644 --- a/test/functional/tests/conftest.py +++ b/test/functional/tests/conftest.py @@ -136,6 +136,8 @@ def pytest_runtest_setup(item): TestRun.dut.plugin_manager = TestRun.plugin_manager TestRun.dut.executor = TestRun.executor TestRun.TEST_RUN_DATA_PATH = TEST_RUN_DATA_PATH + TestRun.dut.cache_list = [] + TestRun.dut.core_list = [] TestRun.duts.append(TestRun.dut) base_prepare(item) From 7adc3568893d62d99d0663dd026237e14dec4960 Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Fri, 10 Jan 2025 13:11:10 +0100 Subject: [PATCH 05/20] conftest: move import to the top of file Signed-off-by: Kamil Gierszewski --- test/functional/tests/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/functional/tests/conftest.py b/test/functional/tests/conftest.py index 100f25d..4460d6c 100644 --- a/test/functional/tests/conftest.py +++ b/test/functional/tests/conftest.py @@ -34,6 +34,7 @@ from log.logger import create_log, Log from test_utils.common.singleton import Singleton from storage_devices.lvm import Lvm, LvmConfiguration from storage_devices.disk import Disk +from storage_devices.drbd import Drbd TEST_RUN_DATA_PATH = "/tmp/open_cas_test_data" @@ -320,8 +321,6 @@ def unmount_cas_devices(): def __drbd_cleanup(): - from storage_devices.drbd import Drbd - Drbd.down_all() # If drbd instance had been configured on top of the CAS, the previous attempt to stop # failed. As drbd has been stopped try to stop CAS one more time. From f0753339dd4a011a0c461cf1fcebb2eaf47d8aab Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Thu, 2 Jan 2025 01:59:48 +0100 Subject: [PATCH 06/20] test-api: change default file path Signed-off-by: Kamil Gierszewski --- test/functional/api/cas/ioclass_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/functional/api/cas/ioclass_config.py b/test/functional/api/cas/ioclass_config.py index 93a231e..ebf3d6a 100644 --- a/test/functional/api/cas/ioclass_config.py +++ b/test/functional/api/cas/ioclass_config.py @@ -1,6 +1,6 @@ # # Copyright(c) 2019-2022 Intel Corporation -# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# Copyright(c) 2024-2025 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # @@ -17,7 +17,7 @@ from core.test_run import TestRun from test_tools.fs_tools import write_file from test_tools.os_tools import get_kernel_version -default_config_file_path = "/tmp/opencas_ioclass.conf" +default_config_file_path = TestRun.TEST_RUN_DATA_PATH + "/opencas_ioclass.conf" MAX_IO_CLASS_ID = 32 MAX_IO_CLASS_PRIORITY = 255 From 9fb333a73f6277248edc7848a9f133efdea674a5 Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Thu, 2 Jan 2025 02:03:39 +0100 Subject: [PATCH 07/20] test-api: minor refactors Signed-off-by: Kamil Gierszewski --- test/functional/api/cas/cache.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/functional/api/cas/cache.py b/test/functional/api/cas/cache.py index b114da7..51997db 100644 --- a/test/functional/api/cas/cache.py +++ b/test/functional/api/cas/cache.py @@ -1,6 +1,6 @@ # # Copyright(c) 2019-2021 Intel Corporation -# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# Copyright(c) 2024-2025 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # @@ -195,7 +195,7 @@ class Cache: return casadm.set_param_promotion_nhit( self.cache_id, threshold=promotion_params_nhit.threshold, - trigger=promotion_params_nhit.trigger + trigger=promotion_params_nhit.trigger, ) def get_cache_config(self) -> CacheConfig: @@ -208,7 +208,7 @@ class Cache: def standby_detach(self, shortcut: bool = False) -> Output: return casadm.standby_detach_cache(cache_id=self.cache_id, shortcut=shortcut) - def standby_activate(self, device, shortcut: bool = False) -> Output: + def standby_activate(self, device: Device, shortcut: bool = False) -> Output: return casadm.standby_activate_cache( cache_id=self.cache_id, cache_dev=device, shortcut=shortcut ) From 0f645ac10b8ff92bd960f28d9010360941392150 Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Thu, 2 Jan 2025 02:14:04 +0100 Subject: [PATCH 08/20] test-api: Change Cache init to force use of the cache_id instead of cache_device Signed-off-by: Kamil Gierszewski --- test/functional/api/cas/cache.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/test/functional/api/cas/cache.py b/test/functional/api/cas/cache.py index 51997db..9b54085 100644 --- a/test/functional/api/cas/cache.py +++ b/test/functional/api/cas/cache.py @@ -13,24 +13,26 @@ from test_tools.os_tools import sync class Cache: - def __init__(self, device: Device, cache_id: int = None) -> None: - self.cache_device = device - self.cache_id = cache_id if cache_id else self.__get_cache_id() - self.__cache_line_size = None - - def __get_cache_id(self) -> int: - device_path = self.__get_cache_device_path() + def __init__( + self, cache_id: int, device: Device = None, cache_line_size: CacheLineSize = None + ) -> None: + self.cache_id = cache_id + self.cache_device = device if device else self.__get_cache_device() + self.__cache_line_size = cache_line_size + def __get_cache_device(self) -> Device | None: caches_dict = get_cas_devices_dict()["caches"] + cache = next( + iter([cache for cache in caches_dict.values() if cache["id"] == self.cache_id]) + ) - for cache in caches_dict.values(): - if cache["device_path"] == device_path: - return int(cache["id"]) + if not cache: + return None - raise Exception(f"There is no cache started on {device_path}") + if cache["device_path"] is "-": + return None - def __get_cache_device_path(self) -> str: - return self.cache_device.path if self.cache_device is not None else "-" + return Device(path=cache["device_path"]) def get_core_devices(self) -> list: return get_cores(self.cache_id) From db3dc068f85e427190cd2f3c149a79184a76c3fc Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Thu, 2 Jan 2025 02:17:02 +0100 Subject: [PATCH 09/20] test-api: refactor casadm to use TestRun cache/core list Signed-off-by: Kamil Gierszewski --- test/functional/api/cas/casadm.py | 98 +++++++++++++++++++++++++++---- 1 file changed, 87 insertions(+), 11 deletions(-) diff --git a/test/functional/api/cas/casadm.py b/test/functional/api/cas/casadm.py index ef033a1..1b61444 100644 --- a/test/functional/api/cas/casadm.py +++ b/test/functional/api/cas/casadm.py @@ -1,6 +1,6 @@ # # Copyright(c) 2019-2022 Intel Corporation -# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# Copyright(c) 2024-2025 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # @@ -48,6 +48,7 @@ def start_cache( ) _cache_id = str(cache_id) if cache_id is not None else None _cache_mode = cache_mode.name.lower() if cache_mode else None + output = TestRun.executor.run( start_cmd( cache_dev=cache_dev.path, @@ -59,33 +60,67 @@ def start_cache( shortcut=shortcut, ) ) + if output.exit_code != 0: raise CmdException("Failed to start cache.", output) - return Cache(cache_dev) + + if not _cache_id: + from api.cas.casadm_parser import get_caches + + cache_list = get_caches() + # compare path of old and new caches, returning the only one created now. + # This will be needed in case cache_id not present in cli command + + new_cache = next(cache for cache in cache_list if cache.cache_device.path == cache_dev.path) + _cache_id = new_cache.cache_id + + cache = Cache(cache_id=int(_cache_id), device=cache_dev, cache_line_size=_cache_line_size) + TestRun.dut.cache_list.append(cache) + return cache def load_cache(device: Device, shortcut: bool = False) -> Cache: + from api.cas.casadm_parser import get_caches + + caches_before_load = get_caches() output = TestRun.executor.run(load_cmd(cache_dev=device.path, shortcut=shortcut)) + if output.exit_code != 0: raise CmdException("Failed to load cache.", output) - return Cache(device) + + caches_after_load = get_caches() + new_cache = next(cache for cache in caches_after_load if cache not in caches_before_load) + cache = Cache(cache_id=new_cache.cache_id, device=new_cache.cache_device) + TestRun.dut.cache_list.append(cache) + return cache -def attach_cache(cache_id: int, device: Device, force: bool, shortcut: bool = False) -> Output: +def attach_cache( + cache_id: int, device: Device, force: bool = False, shortcut: bool = False +) -> Output: output = TestRun.executor.run( attach_cache_cmd( cache_dev=device.path, cache_id=str(cache_id), force=force, shortcut=shortcut ) ) + if output.exit_code != 0: raise CmdException("Failed to attach cache.", output) + + attached_cache = next(cache for cache in TestRun.dut.cache_list if cache.cache_id == cache_id) + attached_cache.cache_device = device + return output def detach_cache(cache_id: int, shortcut: bool = False) -> Output: output = TestRun.executor.run(detach_cache_cmd(cache_id=str(cache_id), shortcut=shortcut)) + if output.exit_code != 0: raise CmdException("Failed to detach cache.", output) + + detached_cache = next(cache for cache in TestRun.dut.cache_list if cache.cache_id == cache_id) + detached_cache.cache_device = None return output @@ -93,8 +128,16 @@ def stop_cache(cache_id: int, no_data_flush: bool = False, shortcut: bool = Fals output = TestRun.executor.run( stop_cmd(cache_id=str(cache_id), no_data_flush=no_data_flush, shortcut=shortcut) ) + if output.exit_code != 0: raise CmdException("Failed to stop cache.", output) + + TestRun.dut.cache_list = [ + cache for cache in TestRun.dut.cache_list if cache.cache_id != cache_id + ] + + TestRun.dut.core_list = [core for core in TestRun.dut.core_list if core.cache_id != cache_id] + return output @@ -192,7 +235,7 @@ def set_param_promotion(cache_id: int, policy: PromotionPolicy, shortcut: bool = def set_param_promotion_nhit( - cache_id: int, threshold: int = None, trigger: int = None, shortcut: bool = False + cache_id: int, threshold: int = None, trigger: int = None, shortcut: bool = False ) -> Output: _threshold = str(threshold) if threshold is not None else None _trigger = str(trigger) if trigger is not None else None @@ -267,7 +310,7 @@ def get_param_cleaning_acp( def get_param_promotion( - cache_id: int, output_format: OutputFormat = None, shortcut: bool = False + cache_id: int, output_format: OutputFormat = None, shortcut: bool = False ) -> Output: _output_format = output_format.name if output_format else None output = TestRun.executor.run( @@ -281,7 +324,7 @@ def get_param_promotion( def get_param_promotion_nhit( - cache_id: int, output_format: OutputFormat = None, shortcut: bool = False + cache_id: int, output_format: OutputFormat = None, shortcut: bool = False ) -> Output: _output_format = output_format.name if output_format else None output = TestRun.executor.run( @@ -325,7 +368,11 @@ def add_core(cache: Cache, core_dev: Device, core_id: int = None, shortcut: bool ) if output.exit_code != 0: raise CmdException("Failed to add core.", output) - return Core(core_dev.path, cache.cache_id) + + core = Core(core_dev.path, cache.cache_id) + TestRun.dut.core_list.append(core) + + return core def remove_core(cache_id: int, core_id: int, force: bool = False, shortcut: bool = False) -> Output: @@ -336,6 +383,12 @@ def remove_core(cache_id: int, core_id: int, force: bool = False, shortcut: bool ) if output.exit_code != 0: raise CmdException("Failed to remove core.", output) + + TestRun.dut.core_list = [ + core + for core in TestRun.dut.core_list + if core.cache_id != cache_id or core.core_id != core_id + ] return output @@ -485,22 +538,41 @@ def standby_init( shortcut=shortcut, ) ) + if output.exit_code != 0: raise CmdException("Failed to init standby cache.", output) - return Cache(cache_dev) + return Cache(cache_id=cache_id, device=cache_dev) def standby_load(cache_dev: Device, shortcut: bool = False) -> Cache: + from api.cas.casadm_parser import get_caches + + caches_before_load = get_caches() output = TestRun.executor.run(standby_load_cmd(cache_dev=cache_dev.path, shortcut=shortcut)) + if output.exit_code != 0: - raise CmdException("Failed to load standby cache.", output) - return Cache(cache_dev) + raise CmdException("Failed to load cache.", output) + caches_after_load = get_caches() + # compare ids of old and new caches, returning the only one created now + new_cache = next( + cache + for cache in caches_after_load + if cache.cache_id not in [cache.cache_id for cache in caches_before_load] + ) + cache = Cache(cache_id=new_cache.cache_id, device=new_cache.cache_device) + TestRun.dut.cache_list.append(cache) + + return cache def standby_detach_cache(cache_id: int, shortcut: bool = False) -> Output: output = TestRun.executor.run(standby_detach_cmd(cache_id=str(cache_id), shortcut=shortcut)) if output.exit_code != 0: raise CmdException("Failed to detach standby cache.", output) + + detached_cache = next(cache for cache in TestRun.dut.cache_list if cache.cache_id == cache_id) + detached_cache.cache_device = None + return output @@ -510,6 +582,10 @@ def standby_activate_cache(cache_dev: Device, cache_id: int, shortcut: bool = Fa ) if output.exit_code != 0: raise CmdException("Failed to activate standby cache.", output) + + activated_cache = next(cache for cache in TestRun.dut.cache_list if cache.cache_id == cache_id) + activated_cache.cache_device = cache_dev + return output From 8cd3f4a6318d5537324bd42b19f5e23d826ba212 Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Thu, 2 Jan 2025 02:19:22 +0100 Subject: [PATCH 10/20] test-api: add Byte unit Signed-off-by: Kamil Gierszewski --- test/functional/api/cas/statistics.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/functional/api/cas/statistics.py b/test/functional/api/cas/statistics.py index e103bc5..7ab1d1e 100644 --- a/test/functional/api/cas/statistics.py +++ b/test/functional/api/cas/statistics.py @@ -1,14 +1,14 @@ # # Copyright(c) 2019-2021 Intel Corporation -# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# Copyright(c) 2024-2025 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # import csv + from datetime import timedelta from enum import Enum from typing import List - from api.cas import casadm from api.cas.casadm_params import StatsFilter from type_def.size import Size, Unit @@ -22,6 +22,7 @@ class UnitType(Enum): kibibyte = "[KiB]" gibibyte = "[GiB]" seconds = "[s]" + byte = "[B]" def __str__(self): return self.value From 0e24e5268607b9c5cbe1ed9f95c095f59a9ea405 Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Thu, 2 Jan 2025 03:47:03 +0100 Subject: [PATCH 11/20] test-api: update parser Signed-off-by: Kamil Gierszewski --- test/functional/api/cas/casadm_parser.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/functional/api/cas/casadm_parser.py b/test/functional/api/cas/casadm_parser.py index da3090a..1c222a7 100644 --- a/test/functional/api/cas/casadm_parser.py +++ b/test/functional/api/cas/casadm_parser.py @@ -59,7 +59,7 @@ def get_cores(cache_id: int) -> list: cores_dict = get_cas_devices_dict()["cores"].values() def is_active(core): - return CoreStatus[core["status"].lower()] == CoreStatus.active + return core["status"] == CoreStatus.active return [ Core(core["device_path"], core["cache_id"]) @@ -74,7 +74,7 @@ def get_inactive_cores(cache_id: int) -> list: cores_dict = get_cas_devices_dict()["cores"].values() def is_inactive(core): - return CoreStatus[core["status"].lower()] == CoreStatus.inactive + return core["status"] == CoreStatus.inactive return [ Core(core["device_path"], core["cache_id"]) @@ -89,7 +89,7 @@ def get_detached_cores(cache_id: int) -> list: cores_dict = get_cas_devices_dict()["cores"].values() def is_detached(core): - return CoreStatus[core["status"].lower()] == CoreStatus.detached + return core["status"] == CoreStatus.detached return [ Core(core["device_path"], core["cache_id"]) @@ -110,15 +110,17 @@ def get_cas_devices_dict() -> dict: params = [ ("id", cache_id), ("device_path", device["disk"]), - ("status", device["status"]), + ("status", CacheStatus(device["status"].lower())), ] devices["caches"][cache_id] = dict([(key, value) for key, value in params]) elif device["type"] == "core": params = [ ("cache_id", cache_id), + ("core_id", (int(device["id"]) if device["id"] != "-" else device["id"])), ("device_path", device["disk"]), - ("status", device["status"]), + ("status", CoreStatus(device["status"].lower())), + ("exp_obj", device["device"]), ] if core_pool: params.append(("core_pool", device)) From 3606472e60e77b9b3482223203a5dcb766c13fab Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Thu, 2 Jan 2025 02:30:37 +0100 Subject: [PATCH 12/20] test-api: refactor to fix circular dependencies Signed-off-by: Kamil Gierszewski --- test/functional/api/cas/casadm_parser.py | 9 ++++---- test/functional/api/cas/core.py | 28 +++++++++++++----------- test/functional/api/cas/core_config.py | 16 ++++++++++++++ 3 files changed, 36 insertions(+), 17 deletions(-) create mode 100644 test/functional/api/cas/core_config.py diff --git a/test/functional/api/cas/casadm_parser.py b/test/functional/api/cas/casadm_parser.py index 1c222a7..a8c8e73 100644 --- a/test/functional/api/cas/casadm_parser.py +++ b/test/functional/api/cas/casadm_parser.py @@ -1,6 +1,6 @@ # # Copyright(c) 2019-2022 Intel Corporation -# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# Copyright(c) 2024-2025 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # @@ -14,6 +14,7 @@ from typing import List from api.cas import casadm from api.cas.cache_config import * from api.cas.casadm_params import * +from api.cas.core_config import CoreStatus from api.cas.ioclass_config import IoClass from api.cas.version import CasVersion from core.test_run_utils import TestRun @@ -54,7 +55,7 @@ def get_caches() -> list: def get_cores(cache_id: int) -> list: - from api.cas.core import Core, CoreStatus + from api.cas.core import Core cores_dict = get_cas_devices_dict()["cores"].values() @@ -69,7 +70,7 @@ def get_cores(cache_id: int) -> list: def get_inactive_cores(cache_id: int) -> list: - from api.cas.core import Core, CoreStatus + from api.cas.core import Core cores_dict = get_cas_devices_dict()["cores"].values() @@ -84,7 +85,7 @@ def get_inactive_cores(cache_id: int) -> list: def get_detached_cores(cache_id: int) -> list: - from api.cas.core import Core, CoreStatus + from api.cas.core import Core cores_dict = get_cas_devices_dict()["cores"].values() diff --git a/test/functional/api/cas/core.py b/test/functional/api/cas/core.py index 15ae71b..eb34782 100644 --- a/test/functional/api/cas/core.py +++ b/test/functional/api/cas/core.py @@ -1,17 +1,17 @@ # # Copyright(c) 2019-2021 Intel Corporation -# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# Copyright(c) 2024-2025 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # from datetime import timedelta from typing import List -from enum import Enum from api.cas import casadm from api.cas.cache_config import SeqCutOffParameters, SeqCutOffPolicy from api.cas.casadm_params import StatsFilter -from api.cas.casadm_parser import get_seq_cut_off_parameters, get_core_info_for_cache_by_path +from api.cas.casadm_parser import get_seq_cut_off_parameters, get_cas_devices_dict +from api.cas.core_config import CoreStatus from api.cas.statistics import CoreStats, CoreIoClassStats from core.test_run_utils import TestRun from storage_devices.device import Device @@ -21,13 +21,6 @@ from test_tools.common.wait import wait from type_def.size import Unit, Size -class CoreStatus(Enum): - empty = 0 - active = 1 - inactive = 2 - detached = 3 - - SEQ_CUTOFF_THRESHOLD_MAX = Size(4194181, Unit.KibiByte) SEQ_CUT_OFF_THRESHOLD_DEFAULT = Size(1, Unit.MebiByte) @@ -46,9 +39,18 @@ class Core(Device): self.partitions = [] self.block_size = None - def __get_core_info(self): - return get_core_info_for_cache_by_path(core_disk_path=self.core_device.path, - target_cache_id=self.cache_id) + def __get_core_info(self) -> dict | None: + core_dicts = get_cas_devices_dict()["cores"].values() + return next( + iter( + [ + core + for core in core_dicts + if core["cache_id"] == self.cache_id + and core["device_path"] == self.core_device.path + ] + ) + ) def create_filesystem(self, fs_type: Filesystem, force=True, blocksize=None): super().create_filesystem(fs_type, force, blocksize) diff --git a/test/functional/api/cas/core_config.py b/test/functional/api/cas/core_config.py new file mode 100644 index 0000000..d4d7693 --- /dev/null +++ b/test/functional/api/cas/core_config.py @@ -0,0 +1,16 @@ +# +# Copyright(c) 2024-2025 Huawei Technologies Co., Ltd. +# SPDX-License-Identifier: BSD-3-Clause +# + +from enum import Enum + + +class CoreStatus(Enum): + empty = "empty" + active = "active" + inactive = "inactive" + detached = "detached" + + def __str__(self): + return self.value From 1b5234573207c6e7d6d444e8518ff2e6cb638063 Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Thu, 2 Jan 2025 02:31:55 +0100 Subject: [PATCH 13/20] test-api: fix core pool init Signed-off-by: Kamil Gierszewski --- test/functional/api/cas/core.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/test/functional/api/cas/core.py b/test/functional/api/cas/core.py index eb34782..ad54219 100644 --- a/test/functional/api/cas/core.py +++ b/test/functional/api/cas/core.py @@ -41,16 +41,21 @@ class Core(Device): def __get_core_info(self) -> dict | None: core_dicts = get_cas_devices_dict()["cores"].values() - return next( - iter( - [ - core - for core in core_dicts - if core["cache_id"] == self.cache_id - and core["device_path"] == self.core_device.path - ] - ) - ) + # for core + core_device = [ + core + for core in core_dicts + if core["cache_id"] == self.cache_id and core["device_path"] == self.core_device.path + ] + if core_device: + return core_device[0] + + # for core pool + core_pool_dicts = get_cas_devices_dict()["core_pool"].values() + core_pool_device = [ + core for core in core_pool_dicts if core["device_path"] == self.core_device.path + ] + return core_pool_device[0] def create_filesystem(self, fs_type: Filesystem, force=True, blocksize=None): super().create_filesystem(fs_type, force, blocksize) From 537c9656b8e59cd8c1611ac3a0ac2e17fe54e638 Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Thu, 2 Jan 2025 02:32:46 +0100 Subject: [PATCH 14/20] test-api: rename stat filter Signed-off-by: Kamil Gierszewski --- test/functional/api/cas/casadm_params.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/functional/api/cas/casadm_params.py b/test/functional/api/cas/casadm_params.py index 6f8eadb..c67855f 100644 --- a/test/functional/api/cas/casadm_params.py +++ b/test/functional/api/cas/casadm_params.py @@ -1,6 +1,6 @@ # # Copyright(c) 2019-2021 Intel Corporation -# Copyright(c) 2024 Huawei Technologies Co., Ltd. +# Copyright(c) 2024-2025 Huawei Technologies Co., Ltd. # SPDX-License-Identifier: BSD-3-Clause # @@ -26,7 +26,7 @@ class OutputFormat(Enum): class StatsFilter(Enum): all = "all" - conf = "configuration" + conf = "config" usage = "usage" req = "request" blk = "block" From c82d52bb472eac9794a1e53ca88f6c80cd67c6c0 Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Thu, 2 Jan 2025 02:36:36 +0100 Subject: [PATCH 15/20] test-api: add methods to statistics Signed-off-by: Kamil Gierszewski --- test/functional/api/cas/statistics.py | 119 +++++++++++++++++++++++++- 1 file changed, 115 insertions(+), 4 deletions(-) diff --git a/test/functional/api/cas/statistics.py b/test/functional/api/cas/statistics.py index 7ab1d1e..82eae31 100644 --- a/test/functional/api/cas/statistics.py +++ b/test/functional/api/cas/statistics.py @@ -69,6 +69,16 @@ class CacheStats: getattr(other, stats_item) for stats_item in other.__dict__ ] + def __iter__(self): + return iter([getattr(self, stats_item) for stats_item in self.__dict__]) + + def __sub__(self, other): + self.usage_stats -= other.usage_stats + self.request_stats -= other.request_stats + self.block_stats -= other.block_stats + self.error_stats -= other.error_stats + return self + class CoreStats: def __init__( @@ -93,6 +103,13 @@ class CoreStats: case StatsFilter.err: self.error_stats = ErrorStats(stats_dict, percentage_val) + def __sub__(self, other): + self.usage_stats -= other.usage_stats + self.request_stats -= other.request_stats + self.block_stats -= other.block_stats + self.error_stats -= other.error_stats + return self + def __str__(self): # stats_list contains all Class.__str__ methods initialized in CacheStats stats_list = [str(getattr(self, stats_item)) for stats_item in self.__dict__] @@ -104,6 +121,9 @@ class CoreStats: getattr(other, stats_item) for stats_item in other.__dict__ ] + def __iter__(self): + return iter([getattr(self, stats_item) for stats_item in self.__dict__]) + class CoreIoClassStats: def __init__( @@ -140,6 +160,14 @@ class CoreIoClassStats: stats_list = [str(getattr(self, stats_item)) for stats_item in self.__dict__] return "\n".join(stats_list) + def __iter__(self): + return iter([getattr(self, stats_item) for stats_item in self.__dict__]) + + def __sub__(self, other): + self.usage_stats -= other.usage_stats + self.request_stats -= other.request_stats + self.block_stats -= other.block_stats + class CacheIoClassStats(CoreIoClassStats): def __init__( @@ -220,6 +248,9 @@ class CacheConfigStats: and self.status == other.status ) + def __iter__(self): + return iter([getattr(self, stats_item) for stats_item in self.__dict__]) + class CoreConfigStats: def __init__(self, stats_dict): @@ -263,6 +294,9 @@ class CoreConfigStats: and self.seq_cutoff_policy == other.seq_cutoff_policy ) + def __iter__(self): + return iter([getattr(self, stats_item) for stats_item in self.__dict__]) + class IoClassConfigStats: def __init__(self, stats_dict): @@ -290,6 +324,9 @@ class IoClassConfigStats: and self.max_size == other.max_size ) + def __iter__(self): + return iter([getattr(self, stats_item) for stats_item in self.__dict__]) + class UsageStats: def __init__(self, stats_dict, percentage_val): @@ -336,6 +373,16 @@ class UsageStats: def __ne__(self, other): return not self == other + def __iter__(self): + return iter([getattr(self, stats_item) for stats_item in self.__dict__]) + + def __sub__(self, other): + self.occupancy -= other.occupancy + self.free -= other.free + self.clean -= other.clean + self.dirty -= other.dirty + return self + class IoClassUsageStats: def __init__(self, stats_dict, percentage_val): @@ -367,6 +414,15 @@ class IoClassUsageStats: def __ne__(self, other): return not self == other + def __iter__(self): + return iter([getattr(self, stats_item) for stats_item in self.__dict__]) + + def __sub__(self, other): + self.occupancy -= other.occupancy + self.clean -= other.clean + self.dirty -= other.dirty + return self + class RequestStats: def __init__(self, stats_dict, percentage_val): @@ -413,6 +469,18 @@ class RequestStats: and self.requests_total == other.requests_total ) + def __iter__(self): + return iter([getattr(self, stats_item) for stats_item in self.__dict__]) + + def __sub__(self, other): + self.read -= other.read + self.write -= other.write + self.pass_through_reads -= other.pass_through_reads + self.pass_through_writes -= other.pass_through_writes + self.requests_serviced -= other.requests_serviced + self.requests_total -= other.requests_total + return self + class RequestStatsChunk: def __init__(self, stats_dict, percentage_val: bool, operation: OperationType): @@ -444,6 +512,16 @@ class RequestStatsChunk: and self.total == other.total ) + def __iter__(self): + return iter([getattr(self, stats_item) for stats_item in self.__dict__]) + + def __sub__(self, other): + self.hits -= other.hits + self.part_misses -= other.part_misses + self.full_misses -= other.full_misses + self.total -= other.total + return self + class BlockStats: def __init__(self, stats_dict, percentage_val): @@ -474,6 +552,15 @@ class BlockStats: self.core == other.core and self.cache == other.cache and self.exp_obj == other.exp_obj ) + def __iter__(self): + return iter([getattr(self, stats_item) for stats_item in self.__dict__]) + + def __sub__(self, other): + self.core -= other.core + self.cache -= other.cache + self.exp_obj -= other.exp_obj + return self + class ErrorStats: def __init__(self, stats_dict, percentage_val): @@ -503,6 +590,15 @@ class ErrorStats: and self.total_errors == other.total_errors ) + def __iter__(self): + return iter([getattr(self, stats_item) for stats_item in self.__dict__]) + + def __sub__(self, other): + self.cache -= other.cache + self.core -= other.core + self.total_errors -= other.total_errors + return self + class BasicStatsChunk: def __init__(self, stats_dict: dict, percentage_val: bool, device: str): @@ -521,6 +617,15 @@ class BasicStatsChunk: self.reads == other.reads and self.writes == other.writes and self.total == other.total ) + def __iter__(self): + return iter([getattr(self, stats_item) for stats_item in self.__dict__]) + + def __sub__(self, other): + self.reads -= other.reads + self.writes -= other.writes + self.total -= other.total + return self + class BasicStatsChunkError: def __init__(self, stats_dict: dict, percentage_val: bool, device: str): @@ -539,6 +644,15 @@ class BasicStatsChunkError: self.reads == other.reads and self.writes == other.writes and self.total == other.total ) + def __iter__(self): + return iter([getattr(self, stats_item) for stats_item in self.__dict__]) + + def __sub__(self, other): + self.reads -= other.reads + self.writes -= other.writes + self.total -= other.total + return self + def get_stat_value(stat_dict: dict, key: str): idx = key.index("[") @@ -584,10 +698,7 @@ def _get_section_filters(filter: List[StatsFilter], io_class_stats: bool = False def get_stats_dict( - filter: List[StatsFilter], - cache_id: int, - core_id: int = None, - io_class_id: int = None + filter: List[StatsFilter], cache_id: int, core_id: int = None, io_class_id: int = None ): csv_stats = casadm.print_statistics( cache_id=cache_id, From d3f4d80612e33fa92c0925a7b71aed2d8e00db18 Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Thu, 2 Jan 2025 02:37:37 +0100 Subject: [PATCH 16/20] test-api: add attach/detach methods to Cache Signed-off-by: Kamil Gierszewski --- test/functional/api/cas/cache.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/functional/api/cas/cache.py b/test/functional/api/cas/cache.py index 9b54085..b2fbeb8 100644 --- a/test/functional/api/cas/cache.py +++ b/test/functional/api/cas/cache.py @@ -215,5 +215,13 @@ class Cache: cache_id=self.cache_id, cache_dev=device, shortcut=shortcut ) + def attach(self, device: Device, force: bool = False) -> Output: + cmd_output = casadm.attach_cache(cache_id=self.cache_id, device=device, force=force) + return cmd_output + + def detach(self) -> Output: + cmd_output = casadm.detach_cache(cache_id=self.cache_id) + return cmd_output + def has_volatile_metadata(self) -> bool: return self.get_metadata_size_on_disk() == Size.zero() From 0ab7c2ca369f1d3b137e94b616c448589cdccf59 Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Thu, 2 Jan 2025 02:38:16 +0100 Subject: [PATCH 17/20] test-api: fix get status method Signed-off-by: Kamil Gierszewski --- test/functional/api/cas/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/functional/api/cas/core.py b/test/functional/api/cas/core.py index ad54219..3bf1fe1 100644 --- a/test/functional/api/cas/core.py +++ b/test/functional/api/cas/core.py @@ -85,8 +85,8 @@ class Core(Device): percentage_val=percentage_val, ) - def get_status(self): - return CoreStatus[self.__get_core_info()["status"].lower()] + def get_status(self) -> CoreStatus: + return self.__get_core_info()["status"] def get_seq_cut_off_parameters(self): return get_seq_cut_off_parameters(self.cache_id, self.core_id) From 8ad0193a84ec6803d4cdbc6734cbb17d3991078b Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Thu, 2 Jan 2025 02:39:30 +0100 Subject: [PATCH 18/20] test-api: refactor cache imports Signed-off-by: Kamil Gierszewski --- test/functional/api/cas/cache.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/test/functional/api/cas/cache.py b/test/functional/api/cas/cache.py index b2fbeb8..5c23cc4 100644 --- a/test/functional/api/cas/cache.py +++ b/test/functional/api/cas/cache.py @@ -4,12 +4,33 @@ # SPDX-License-Identifier: BSD-3-Clause # -from api.cas.casadm_parser import * +from datetime import timedelta +from typing import List + +from api.cas import casadm +from api.cas.cache_config import ( + CacheLineSize, + CleaningPolicy, + CacheStatus, + CacheMode, + FlushParametersAlru, + FlushParametersAcp, + SeqCutOffParameters, + SeqCutOffPolicy, + PromotionPolicy, + PromotionParametersNhit, + CacheConfig, +) +from api.cas.casadm_params import StatsFilter +from api.cas.casadm_parser import (get_cas_devices_dict, get_cores, get_flush_parameters_alru, + get_flush_parameters_acp, get_io_class_list) from api.cas.core import Core from api.cas.dmesg import get_metadata_size_on_device from api.cas.statistics import CacheStats, CacheIoClassStats from connection.utils.output import Output +from storage_devices.device import Device from test_tools.os_tools import sync +from type_def.size import Size class Cache: From 7b52a2fc001ba3f79d90b9bd2b66a02639920322 Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Thu, 2 Jan 2025 02:44:42 +0100 Subject: [PATCH 19/20] test-api: add attach cli msg Signed-off-by: Kamil Gierszewski --- test/functional/api/cas/cli_messages.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/functional/api/cas/cli_messages.py b/test/functional/api/cas/cli_messages.py index 78da17b..eb7e0de 100644 --- a/test/functional/api/cas/cli_messages.py +++ b/test/functional/api/cas/cli_messages.py @@ -1,6 +1,6 @@ # # Copyright(c) 2019-2022 Intel Corporation -# Copyright(c) 2024 Huawei Technologies +# Copyright(c) 2024-2025 Huawei Technologies # SPDX-License-Identifier: BSD-3-Clause # @@ -233,6 +233,12 @@ malformed_io_class_header = [ unexpected_cls_option = [r"Option '--cache-line-size \(-x\)' is not allowed"] +attach_not_enough_memory = [ + r"Not enough free RAM\.\nYou need at least \d+.\d+GB to attach a device to cache " + r"with cache line size equal \d+kB.\n" + r"Try with greater cache line size\." +] + def check_stderr_msg(output: Output, expected_messages, negate=False): return __check_string_msg(output.stderr, expected_messages, negate) From bf7c72ccbae44723644f11c964a4cd11fc66a434 Mon Sep 17 00:00:00 2001 From: Kamil Gierszewski Date: Thu, 2 Jan 2025 02:54:19 +0100 Subject: [PATCH 20/20] test-api: add a check for each stat parsing Signed-off-by: Kamil Gierszewski --- test/functional/api/cas/statistics.py | 168 ++++++++++++++------------ 1 file changed, 93 insertions(+), 75 deletions(-) diff --git a/test/functional/api/cas/statistics.py b/test/functional/api/cas/statistics.py index 82eae31..3041a3e 100644 --- a/test/functional/api/cas/statistics.py +++ b/test/functional/api/cas/statistics.py @@ -11,6 +11,7 @@ from enum import Enum from typing import List from api.cas import casadm from api.cas.casadm_params import StatsFilter +from connection.utils.output import CmdException from type_def.size import Size, Unit @@ -58,6 +59,9 @@ class CacheStats: case StatsFilter.err: self.error_stats = ErrorStats(stats_dict, percentage_val) + if stats_dict: + raise CmdException(f"Unknown stat(s) left after parsing output cmd\n{stats_dict}") + def __str__(self): # stats_list contains all Class.__str__ methods initialized in CacheStats stats_list = [str(getattr(self, stats_item)) for stats_item in self.__dict__] @@ -72,13 +76,6 @@ class CacheStats: def __iter__(self): return iter([getattr(self, stats_item) for stats_item in self.__dict__]) - def __sub__(self, other): - self.usage_stats -= other.usage_stats - self.request_stats -= other.request_stats - self.block_stats -= other.block_stats - self.error_stats -= other.error_stats - return self - class CoreStats: def __init__( @@ -103,12 +100,8 @@ class CoreStats: case StatsFilter.err: self.error_stats = ErrorStats(stats_dict, percentage_val) - def __sub__(self, other): - self.usage_stats -= other.usage_stats - self.request_stats -= other.request_stats - self.block_stats -= other.block_stats - self.error_stats -= other.error_stats - return self + if stats_dict: + raise CmdException(f"Unknown stat(s) left after parsing output cmd\n{stats_dict}") def __str__(self): # stats_list contains all Class.__str__ methods initialized in CacheStats @@ -149,6 +142,9 @@ class CoreIoClassStats: case StatsFilter.blk: self.block_stats = BlockStats(stats_dict, percentage_val) + if stats_dict: + raise CmdException(f"Unknown stat(s) left after parsing output cmd\n{stats_dict}") + def __eq__(self, other): # check if all initialized variable in self(CacheStats) match other(CacheStats) return [getattr(self, stats_item) for stats_item in self.__dict__] == [ @@ -163,11 +159,6 @@ class CoreIoClassStats: def __iter__(self): return iter([getattr(self, stats_item) for stats_item in self.__dict__]) - def __sub__(self, other): - self.usage_stats -= other.usage_stats - self.request_stats -= other.request_stats - self.block_stats -= other.block_stats - class CacheIoClassStats(CoreIoClassStats): def __init__( @@ -211,6 +202,22 @@ class CacheConfigStats: self.dirty_for = parse_value(value=stats_dict["Dirty for [s]"], unit_type=UnitType.seconds) self.status = stats_dict["Status"] + del stats_dict["Cache Id"] + del stats_dict["Cache Size [4KiB Blocks]"] + del stats_dict["Cache Size [GiB]"] + del stats_dict["Cache Device"] + del stats_dict["Exported Object"] + del stats_dict["Core Devices"] + del stats_dict["Inactive Core Devices"] + del stats_dict["Write Policy"] + del stats_dict["Cleaning Policy"] + del stats_dict["Promotion Policy"] + del stats_dict["Cache line size [KiB]"] + del stats_dict[footprint_key] + del stats_dict["Dirty for [s]"] + del stats_dict["Dirty for"] + del stats_dict["Status"] + def __str__(self): return ( f"Config stats:\n" @@ -254,7 +261,7 @@ class CacheConfigStats: class CoreConfigStats: def __init__(self, stats_dict): - self.core_id = stats_dict["Core Id"] + self.core_id = int(stats_dict["Core Id"]) self.core_dev = stats_dict["Core Device"] self.exp_obj = stats_dict["Exported Object"] self.core_size = parse_value( @@ -267,6 +274,17 @@ class CoreConfigStats: ) self.seq_cutoff_policy = stats_dict["Seq cutoff policy"] + del stats_dict["Core Id"] + del stats_dict["Core Device"] + del stats_dict["Exported Object"] + del stats_dict["Core Size [4KiB Blocks]"] + del stats_dict["Core Size [GiB]"] + del stats_dict["Dirty for [s]"] + del stats_dict["Dirty for"] + del stats_dict["Status"] + del stats_dict["Seq cutoff threshold [KiB]"] + del stats_dict["Seq cutoff policy"] + def __str__(self): return ( f"Config stats:\n" @@ -305,6 +323,11 @@ class IoClassConfigStats: self.eviction_priority = stats_dict["Eviction priority"] self.max_size = stats_dict["Max size"] + del stats_dict["IO class ID"] + del stats_dict["IO class name"] + del stats_dict["Eviction priority"] + del stats_dict["Max size"] + def __str__(self): return ( f"Config stats:\n" @@ -348,6 +371,18 @@ class UsageStats: value=stats_dict[f"Inactive Dirty {unit}"], unit_type=unit ) + for unit in [UnitType.percentage, UnitType.block_4k]: + del stats_dict[f"Occupancy {unit}"] + del stats_dict[f"Free {unit}"] + del stats_dict[f"Clean {unit}"] + del stats_dict[f"Dirty {unit}"] + if f"Inactive Dirty {unit}" in stats_dict: + del stats_dict[f"Inactive Occupancy {unit}"] + if f"Inactive Clean {unit}" in stats_dict: + del stats_dict[f"Inactive Clean {unit}"] + if f"Inactive Dirty {unit}" in stats_dict: + del stats_dict[f"Inactive Dirty {unit}"] + def __str__(self): return ( f"Usage stats:\n" @@ -376,13 +411,6 @@ class UsageStats: def __iter__(self): return iter([getattr(self, stats_item) for stats_item in self.__dict__]) - def __sub__(self, other): - self.occupancy -= other.occupancy - self.free -= other.free - self.clean -= other.clean - self.dirty -= other.dirty - return self - class IoClassUsageStats: def __init__(self, stats_dict, percentage_val): @@ -391,6 +419,11 @@ class IoClassUsageStats: self.clean = parse_value(value=stats_dict[f"Clean {unit}"], unit_type=unit) self.dirty = parse_value(value=stats_dict[f"Dirty {unit}"], unit_type=unit) + for unit in [UnitType.percentage, UnitType.block_4k]: + del stats_dict[f"Occupancy {unit}"] + del stats_dict[f"Clean {unit}"] + del stats_dict[f"Dirty {unit}"] + def __str__(self): return ( f"Usage stats:\n" @@ -417,21 +450,19 @@ class IoClassUsageStats: def __iter__(self): return iter([getattr(self, stats_item) for stats_item in self.__dict__]) - def __sub__(self, other): - self.occupancy -= other.occupancy - self.clean -= other.clean - self.dirty -= other.dirty - return self - class RequestStats: def __init__(self, stats_dict, percentage_val): unit = UnitType.percentage if percentage_val else UnitType.requests self.read = RequestStatsChunk( - stats_dict=stats_dict, percentage_val=percentage_val, operation=OperationType.read + stats_dict=stats_dict, + percentage_val=percentage_val, + operation=OperationType.read, ) self.write = RequestStatsChunk( - stats_dict=stats_dict, percentage_val=percentage_val, operation=OperationType.write + stats_dict=stats_dict, + percentage_val=percentage_val, + operation=OperationType.write, ) self.pass_through_reads = parse_value( value=stats_dict[f"Pass-Through reads {unit}"], unit_type=unit @@ -446,6 +477,17 @@ class RequestStats: value=stats_dict[f"Total requests {unit}"], unit_type=unit ) + for unit in [UnitType.percentage, UnitType.requests]: + for operation in [OperationType.read, OperationType.write]: + del stats_dict[f"{operation} hits {unit}"] + del stats_dict[f"{operation} partial misses {unit}"] + del stats_dict[f"{operation} full misses {unit}"] + del stats_dict[f"{operation} total {unit}"] + del stats_dict[f"Pass-Through reads {unit}"] + del stats_dict[f"Pass-Through writes {unit}"] + del stats_dict[f"Serviced requests {unit}"] + del stats_dict[f"Total requests {unit}"] + def __str__(self): return ( f"Request stats:\n" @@ -472,15 +514,6 @@ class RequestStats: def __iter__(self): return iter([getattr(self, stats_item) for stats_item in self.__dict__]) - def __sub__(self, other): - self.read -= other.read - self.write -= other.write - self.pass_through_reads -= other.pass_through_reads - self.pass_through_writes -= other.pass_through_writes - self.requests_serviced -= other.requests_serviced - self.requests_total -= other.requests_total - return self - class RequestStatsChunk: def __init__(self, stats_dict, percentage_val: bool, operation: OperationType): @@ -515,13 +548,6 @@ class RequestStatsChunk: def __iter__(self): return iter([getattr(self, stats_item) for stats_item in self.__dict__]) - def __sub__(self, other): - self.hits -= other.hits - self.part_misses -= other.part_misses - self.full_misses -= other.full_misses - self.total -= other.total - return self - class BlockStats: def __init__(self, stats_dict, percentage_val): @@ -537,6 +563,12 @@ class BlockStats: device="exported object", ) + for unit in [UnitType.percentage, UnitType.block_4k]: + for device in ["core", "cache", "exported object"]: + del stats_dict[f"Reads from {device} {unit}"] + del stats_dict[f"Writes to {device} {unit}"] + del stats_dict[f"Total to/from {device} {unit}"] + def __str__(self): return ( f"Block stats:\n" @@ -555,12 +587,6 @@ class BlockStats: def __iter__(self): return iter([getattr(self, stats_item) for stats_item in self.__dict__]) - def __sub__(self, other): - self.core -= other.core - self.cache -= other.cache - self.exp_obj -= other.exp_obj - return self - class ErrorStats: def __init__(self, stats_dict, percentage_val): @@ -573,6 +599,13 @@ class ErrorStats: ) self.total_errors = parse_value(value=stats_dict[f"Total errors {unit}"], unit_type=unit) + for unit in [UnitType.percentage, UnitType.requests]: + for device in ["Core", "Cache"]: + del stats_dict[f"{device} read errors {unit}"] + del stats_dict[f"{device} write errors {unit}"] + del stats_dict[f"{device} total errors {unit}"] + del stats_dict[f"Total errors {unit}"] + def __str__(self): return ( f"Error stats:\n" @@ -593,12 +626,6 @@ class ErrorStats: def __iter__(self): return iter([getattr(self, stats_item) for stats_item in self.__dict__]) - def __sub__(self, other): - self.cache -= other.cache - self.core -= other.core - self.total_errors -= other.total_errors - return self - class BasicStatsChunk: def __init__(self, stats_dict: dict, percentage_val: bool, device: str): @@ -620,12 +647,6 @@ class BasicStatsChunk: def __iter__(self): return iter([getattr(self, stats_item) for stats_item in self.__dict__]) - def __sub__(self, other): - self.reads -= other.reads - self.writes -= other.writes - self.total -= other.total - return self - class BasicStatsChunkError: def __init__(self, stats_dict: dict, percentage_val: bool, device: str): @@ -647,12 +668,6 @@ class BasicStatsChunkError: def __iter__(self): return iter([getattr(self, stats_item) for stats_item in self.__dict__]) - def __sub__(self, other): - self.reads -= other.reads - self.writes -= other.writes - self.total -= other.total - return self - def get_stat_value(stat_dict: dict, key: str): idx = key.index("[") @@ -698,7 +713,10 @@ def _get_section_filters(filter: List[StatsFilter], io_class_stats: bool = False def get_stats_dict( - filter: List[StatsFilter], cache_id: int, core_id: int = None, io_class_id: int = None + filter: List[StatsFilter], + cache_id: int, + core_id: int = None, + io_class_id: int = None, ): csv_stats = casadm.print_statistics( cache_id=cache_id,