Merge pull request #1595 from Kamoppl/kamilg/update_api_dec

Few api fixes/improvements
This commit is contained in:
Katarzyna Treder 2025-02-06 07:17:32 +01:00 committed by GitHub
commit 69a4da4b38
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 365 additions and 71 deletions

View File

@ -1,36 +1,59 @@
# #
# Copyright(c) 2019-2021 Intel Corporation # Copyright(c) 2019-2021 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd. # Copyright(c) 2024-2025 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause # SPDX-License-Identifier: BSD-3-Clause
# #
from api.cas.casadm_parser import * from datetime import timedelta
from typing import List
from api.cas import casadm
from api.cas.cache_config import (
CacheLineSize,
CleaningPolicy,
CacheStatus,
CacheMode,
FlushParametersAlru,
FlushParametersAcp,
SeqCutOffParameters,
SeqCutOffPolicy,
PromotionPolicy,
PromotionParametersNhit,
CacheConfig,
)
from api.cas.casadm_params import StatsFilter
from api.cas.casadm_parser import (get_cas_devices_dict, get_cores, get_flush_parameters_alru,
get_flush_parameters_acp, get_io_class_list)
from api.cas.core import Core from api.cas.core import Core
from api.cas.dmesg import get_metadata_size_on_device from api.cas.dmesg import get_metadata_size_on_device
from api.cas.statistics import CacheStats, CacheIoClassStats from api.cas.statistics import CacheStats, CacheIoClassStats
from connection.utils.output import Output from connection.utils.output import Output
from storage_devices.device import Device
from test_tools.os_tools import sync from test_tools.os_tools import sync
from type_def.size import Size
class Cache: class Cache:
def __init__(self, device: Device, cache_id: int = None) -> None: def __init__(
self.cache_device = device self, cache_id: int, device: Device = None, cache_line_size: CacheLineSize = None
self.cache_id = cache_id if cache_id else self.__get_cache_id() ) -> None:
self.__cache_line_size = None self.cache_id = cache_id
self.cache_device = device if device else self.__get_cache_device()
def __get_cache_id(self) -> int: self.__cache_line_size = cache_line_size
device_path = self.__get_cache_device_path()
def __get_cache_device(self) -> Device | None:
caches_dict = get_cas_devices_dict()["caches"] caches_dict = get_cas_devices_dict()["caches"]
cache = next(
iter([cache for cache in caches_dict.values() if cache["id"] == self.cache_id])
)
for cache in caches_dict.values(): if not cache:
if cache["device_path"] == device_path: return None
return int(cache["id"])
raise Exception(f"There is no cache started on {device_path}") if cache["device_path"] is "-":
return None
def __get_cache_device_path(self) -> str: return Device(path=cache["device_path"])
return self.cache_device.path if self.cache_device is not None else "-"
def get_core_devices(self) -> list: def get_core_devices(self) -> list:
return get_cores(self.cache_id) return get_cores(self.cache_id)
@ -195,7 +218,7 @@ class Cache:
return casadm.set_param_promotion_nhit( return casadm.set_param_promotion_nhit(
self.cache_id, self.cache_id,
threshold=promotion_params_nhit.threshold, threshold=promotion_params_nhit.threshold,
trigger=promotion_params_nhit.trigger trigger=promotion_params_nhit.trigger,
) )
def get_cache_config(self) -> CacheConfig: def get_cache_config(self) -> CacheConfig:
@ -208,10 +231,18 @@ class Cache:
def standby_detach(self, shortcut: bool = False) -> Output: def standby_detach(self, shortcut: bool = False) -> Output:
return casadm.standby_detach_cache(cache_id=self.cache_id, shortcut=shortcut) return casadm.standby_detach_cache(cache_id=self.cache_id, shortcut=shortcut)
def standby_activate(self, device, shortcut: bool = False) -> Output: def standby_activate(self, device: Device, shortcut: bool = False) -> Output:
return casadm.standby_activate_cache( return casadm.standby_activate_cache(
cache_id=self.cache_id, cache_dev=device, shortcut=shortcut cache_id=self.cache_id, cache_dev=device, shortcut=shortcut
) )
def attach(self, device: Device, force: bool = False) -> Output:
cmd_output = casadm.attach_cache(cache_id=self.cache_id, device=device, force=force)
return cmd_output
def detach(self) -> Output:
cmd_output = casadm.detach_cache(cache_id=self.cache_id)
return cmd_output
def has_volatile_metadata(self) -> bool: def has_volatile_metadata(self) -> bool:
return self.get_metadata_size_on_disk() == Size.zero() return self.get_metadata_size_on_disk() == Size.zero()

View File

@ -1,6 +1,6 @@
# #
# Copyright(c) 2019-2022 Intel Corporation # Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd. # Copyright(c) 2024-2025 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause # SPDX-License-Identifier: BSD-3-Clause
# #
@ -48,6 +48,7 @@ def start_cache(
) )
_cache_id = str(cache_id) if cache_id is not None else None _cache_id = str(cache_id) if cache_id is not None else None
_cache_mode = cache_mode.name.lower() if cache_mode else None _cache_mode = cache_mode.name.lower() if cache_mode else None
output = TestRun.executor.run( output = TestRun.executor.run(
start_cmd( start_cmd(
cache_dev=cache_dev.path, cache_dev=cache_dev.path,
@ -59,33 +60,67 @@ def start_cache(
shortcut=shortcut, shortcut=shortcut,
) )
) )
if output.exit_code != 0: if output.exit_code != 0:
raise CmdException("Failed to start cache.", output) raise CmdException("Failed to start cache.", output)
return Cache(cache_dev)
if not _cache_id:
from api.cas.casadm_parser import get_caches
cache_list = get_caches()
# compare path of old and new caches, returning the only one created now.
# This will be needed in case cache_id not present in cli command
new_cache = next(cache for cache in cache_list if cache.cache_device.path == cache_dev.path)
_cache_id = new_cache.cache_id
cache = Cache(cache_id=int(_cache_id), device=cache_dev, cache_line_size=_cache_line_size)
TestRun.dut.cache_list.append(cache)
return cache
def load_cache(device: Device, shortcut: bool = False) -> Cache: def load_cache(device: Device, shortcut: bool = False) -> Cache:
from api.cas.casadm_parser import get_caches
caches_before_load = get_caches()
output = TestRun.executor.run(load_cmd(cache_dev=device.path, shortcut=shortcut)) output = TestRun.executor.run(load_cmd(cache_dev=device.path, shortcut=shortcut))
if output.exit_code != 0: if output.exit_code != 0:
raise CmdException("Failed to load cache.", output) raise CmdException("Failed to load cache.", output)
return Cache(device)
caches_after_load = get_caches()
new_cache = next(cache for cache in caches_after_load if cache not in caches_before_load)
cache = Cache(cache_id=new_cache.cache_id, device=new_cache.cache_device)
TestRun.dut.cache_list.append(cache)
return cache
def attach_cache(cache_id: int, device: Device, force: bool, shortcut: bool = False) -> Output: def attach_cache(
cache_id: int, device: Device, force: bool = False, shortcut: bool = False
) -> Output:
output = TestRun.executor.run( output = TestRun.executor.run(
attach_cache_cmd( attach_cache_cmd(
cache_dev=device.path, cache_id=str(cache_id), force=force, shortcut=shortcut cache_dev=device.path, cache_id=str(cache_id), force=force, shortcut=shortcut
) )
) )
if output.exit_code != 0: if output.exit_code != 0:
raise CmdException("Failed to attach cache.", output) raise CmdException("Failed to attach cache.", output)
attached_cache = next(cache for cache in TestRun.dut.cache_list if cache.cache_id == cache_id)
attached_cache.cache_device = device
return output return output
def detach_cache(cache_id: int, shortcut: bool = False) -> Output: def detach_cache(cache_id: int, shortcut: bool = False) -> Output:
output = TestRun.executor.run(detach_cache_cmd(cache_id=str(cache_id), shortcut=shortcut)) output = TestRun.executor.run(detach_cache_cmd(cache_id=str(cache_id), shortcut=shortcut))
if output.exit_code != 0: if output.exit_code != 0:
raise CmdException("Failed to detach cache.", output) raise CmdException("Failed to detach cache.", output)
detached_cache = next(cache for cache in TestRun.dut.cache_list if cache.cache_id == cache_id)
detached_cache.cache_device = None
return output return output
@ -93,8 +128,16 @@ def stop_cache(cache_id: int, no_data_flush: bool = False, shortcut: bool = Fals
output = TestRun.executor.run( output = TestRun.executor.run(
stop_cmd(cache_id=str(cache_id), no_data_flush=no_data_flush, shortcut=shortcut) stop_cmd(cache_id=str(cache_id), no_data_flush=no_data_flush, shortcut=shortcut)
) )
if output.exit_code != 0: if output.exit_code != 0:
raise CmdException("Failed to stop cache.", output) raise CmdException("Failed to stop cache.", output)
TestRun.dut.cache_list = [
cache for cache in TestRun.dut.cache_list if cache.cache_id != cache_id
]
TestRun.dut.core_list = [core for core in TestRun.dut.core_list if core.cache_id != cache_id]
return output return output
@ -192,7 +235,7 @@ def set_param_promotion(cache_id: int, policy: PromotionPolicy, shortcut: bool =
def set_param_promotion_nhit( def set_param_promotion_nhit(
cache_id: int, threshold: int = None, trigger: int = None, shortcut: bool = False cache_id: int, threshold: int = None, trigger: int = None, shortcut: bool = False
) -> Output: ) -> Output:
_threshold = str(threshold) if threshold is not None else None _threshold = str(threshold) if threshold is not None else None
_trigger = str(trigger) if trigger is not None else None _trigger = str(trigger) if trigger is not None else None
@ -267,7 +310,7 @@ def get_param_cleaning_acp(
def get_param_promotion( def get_param_promotion(
cache_id: int, output_format: OutputFormat = None, shortcut: bool = False cache_id: int, output_format: OutputFormat = None, shortcut: bool = False
) -> Output: ) -> Output:
_output_format = output_format.name if output_format else None _output_format = output_format.name if output_format else None
output = TestRun.executor.run( output = TestRun.executor.run(
@ -281,7 +324,7 @@ def get_param_promotion(
def get_param_promotion_nhit( def get_param_promotion_nhit(
cache_id: int, output_format: OutputFormat = None, shortcut: bool = False cache_id: int, output_format: OutputFormat = None, shortcut: bool = False
) -> Output: ) -> Output:
_output_format = output_format.name if output_format else None _output_format = output_format.name if output_format else None
output = TestRun.executor.run( output = TestRun.executor.run(
@ -325,7 +368,11 @@ def add_core(cache: Cache, core_dev: Device, core_id: int = None, shortcut: bool
) )
if output.exit_code != 0: if output.exit_code != 0:
raise CmdException("Failed to add core.", output) raise CmdException("Failed to add core.", output)
return Core(core_dev.path, cache.cache_id)
core = Core(core_dev.path, cache.cache_id)
TestRun.dut.core_list.append(core)
return core
def remove_core(cache_id: int, core_id: int, force: bool = False, shortcut: bool = False) -> Output: def remove_core(cache_id: int, core_id: int, force: bool = False, shortcut: bool = False) -> Output:
@ -336,6 +383,12 @@ def remove_core(cache_id: int, core_id: int, force: bool = False, shortcut: bool
) )
if output.exit_code != 0: if output.exit_code != 0:
raise CmdException("Failed to remove core.", output) raise CmdException("Failed to remove core.", output)
TestRun.dut.core_list = [
core
for core in TestRun.dut.core_list
if core.cache_id != cache_id or core.core_id != core_id
]
return output return output
@ -485,22 +538,41 @@ def standby_init(
shortcut=shortcut, shortcut=shortcut,
) )
) )
if output.exit_code != 0: if output.exit_code != 0:
raise CmdException("Failed to init standby cache.", output) raise CmdException("Failed to init standby cache.", output)
return Cache(cache_dev) return Cache(cache_id=cache_id, device=cache_dev)
def standby_load(cache_dev: Device, shortcut: bool = False) -> Cache: def standby_load(cache_dev: Device, shortcut: bool = False) -> Cache:
from api.cas.casadm_parser import get_caches
caches_before_load = get_caches()
output = TestRun.executor.run(standby_load_cmd(cache_dev=cache_dev.path, shortcut=shortcut)) output = TestRun.executor.run(standby_load_cmd(cache_dev=cache_dev.path, shortcut=shortcut))
if output.exit_code != 0: if output.exit_code != 0:
raise CmdException("Failed to load standby cache.", output) raise CmdException("Failed to load cache.", output)
return Cache(cache_dev) caches_after_load = get_caches()
# compare ids of old and new caches, returning the only one created now
new_cache = next(
cache
for cache in caches_after_load
if cache.cache_id not in [cache.cache_id for cache in caches_before_load]
)
cache = Cache(cache_id=new_cache.cache_id, device=new_cache.cache_device)
TestRun.dut.cache_list.append(cache)
return cache
def standby_detach_cache(cache_id: int, shortcut: bool = False) -> Output: def standby_detach_cache(cache_id: int, shortcut: bool = False) -> Output:
output = TestRun.executor.run(standby_detach_cmd(cache_id=str(cache_id), shortcut=shortcut)) output = TestRun.executor.run(standby_detach_cmd(cache_id=str(cache_id), shortcut=shortcut))
if output.exit_code != 0: if output.exit_code != 0:
raise CmdException("Failed to detach standby cache.", output) raise CmdException("Failed to detach standby cache.", output)
detached_cache = next(cache for cache in TestRun.dut.cache_list if cache.cache_id == cache_id)
detached_cache.cache_device = None
return output return output
@ -510,6 +582,10 @@ def standby_activate_cache(cache_dev: Device, cache_id: int, shortcut: bool = Fa
) )
if output.exit_code != 0: if output.exit_code != 0:
raise CmdException("Failed to activate standby cache.", output) raise CmdException("Failed to activate standby cache.", output)
activated_cache = next(cache for cache in TestRun.dut.cache_list if cache.cache_id == cache_id)
activated_cache.cache_device = cache_dev
return output return output

View File

@ -1,6 +1,6 @@
# #
# Copyright(c) 2019-2021 Intel Corporation # Copyright(c) 2019-2021 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd. # Copyright(c) 2024-2025 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause # SPDX-License-Identifier: BSD-3-Clause
# #
@ -26,7 +26,7 @@ class OutputFormat(Enum):
class StatsFilter(Enum): class StatsFilter(Enum):
all = "all" all = "all"
conf = "configuration" conf = "config"
usage = "usage" usage = "usage"
req = "request" req = "request"
blk = "block" blk = "block"

View File

@ -1,6 +1,6 @@
# #
# Copyright(c) 2019-2022 Intel Corporation # Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd. # Copyright(c) 2024-2025 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause # SPDX-License-Identifier: BSD-3-Clause
# #
@ -14,6 +14,7 @@ from typing import List
from api.cas import casadm from api.cas import casadm
from api.cas.cache_config import * from api.cas.cache_config import *
from api.cas.casadm_params import * from api.cas.casadm_params import *
from api.cas.core_config import CoreStatus
from api.cas.ioclass_config import IoClass from api.cas.ioclass_config import IoClass
from api.cas.version import CasVersion from api.cas.version import CasVersion
from core.test_run_utils import TestRun from core.test_run_utils import TestRun
@ -54,12 +55,12 @@ def get_caches() -> list:
def get_cores(cache_id: int) -> list: def get_cores(cache_id: int) -> list:
from api.cas.core import Core, CoreStatus from api.cas.core import Core
cores_dict = get_cas_devices_dict()["cores"].values() cores_dict = get_cas_devices_dict()["cores"].values()
def is_active(core): def is_active(core):
return CoreStatus[core["status"].lower()] == CoreStatus.active return core["status"] == CoreStatus.active
return [ return [
Core(core["device_path"], core["cache_id"]) Core(core["device_path"], core["cache_id"])
@ -69,12 +70,12 @@ def get_cores(cache_id: int) -> list:
def get_inactive_cores(cache_id: int) -> list: def get_inactive_cores(cache_id: int) -> list:
from api.cas.core import Core, CoreStatus from api.cas.core import Core
cores_dict = get_cas_devices_dict()["cores"].values() cores_dict = get_cas_devices_dict()["cores"].values()
def is_inactive(core): def is_inactive(core):
return CoreStatus[core["status"].lower()] == CoreStatus.inactive return core["status"] == CoreStatus.inactive
return [ return [
Core(core["device_path"], core["cache_id"]) Core(core["device_path"], core["cache_id"])
@ -84,12 +85,12 @@ def get_inactive_cores(cache_id: int) -> list:
def get_detached_cores(cache_id: int) -> list: def get_detached_cores(cache_id: int) -> list:
from api.cas.core import Core, CoreStatus from api.cas.core import Core
cores_dict = get_cas_devices_dict()["cores"].values() cores_dict = get_cas_devices_dict()["cores"].values()
def is_detached(core): def is_detached(core):
return CoreStatus[core["status"].lower()] == CoreStatus.detached return core["status"] == CoreStatus.detached
return [ return [
Core(core["device_path"], core["cache_id"]) Core(core["device_path"], core["cache_id"])
@ -110,15 +111,17 @@ def get_cas_devices_dict() -> dict:
params = [ params = [
("id", cache_id), ("id", cache_id),
("device_path", device["disk"]), ("device_path", device["disk"]),
("status", device["status"]), ("status", CacheStatus(device["status"].lower())),
] ]
devices["caches"][cache_id] = dict([(key, value) for key, value in params]) devices["caches"][cache_id] = dict([(key, value) for key, value in params])
elif device["type"] == "core": elif device["type"] == "core":
params = [ params = [
("cache_id", cache_id), ("cache_id", cache_id),
("core_id", (int(device["id"]) if device["id"] != "-" else device["id"])),
("device_path", device["disk"]), ("device_path", device["disk"]),
("status", device["status"]), ("status", CoreStatus(device["status"].lower())),
("exp_obj", device["device"]),
] ]
if core_pool: if core_pool:
params.append(("core_pool", device)) params.append(("core_pool", device))

View File

@ -1,6 +1,6 @@
# #
# Copyright(c) 2019-2022 Intel Corporation # Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies # Copyright(c) 2024-2025 Huawei Technologies
# SPDX-License-Identifier: BSD-3-Clause # SPDX-License-Identifier: BSD-3-Clause
# #
@ -233,6 +233,12 @@ malformed_io_class_header = [
unexpected_cls_option = [r"Option '--cache-line-size \(-x\)' is not allowed"] unexpected_cls_option = [r"Option '--cache-line-size \(-x\)' is not allowed"]
attach_not_enough_memory = [
r"Not enough free RAM\.\nYou need at least \d+.\d+GB to attach a device to cache "
r"with cache line size equal \d+kB.\n"
r"Try with greater cache line size\."
]
def check_stderr_msg(output: Output, expected_messages, negate=False): def check_stderr_msg(output: Output, expected_messages, negate=False):
return __check_string_msg(output.stderr, expected_messages, negate) return __check_string_msg(output.stderr, expected_messages, negate)

View File

@ -1,17 +1,17 @@
# #
# Copyright(c) 2019-2021 Intel Corporation # Copyright(c) 2019-2021 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd. # Copyright(c) 2024-2025 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause # SPDX-License-Identifier: BSD-3-Clause
# #
from datetime import timedelta from datetime import timedelta
from typing import List from typing import List
from enum import Enum
from api.cas import casadm from api.cas import casadm
from api.cas.cache_config import SeqCutOffParameters, SeqCutOffPolicy from api.cas.cache_config import SeqCutOffParameters, SeqCutOffPolicy
from api.cas.casadm_params import StatsFilter from api.cas.casadm_params import StatsFilter
from api.cas.casadm_parser import get_seq_cut_off_parameters, get_core_info_for_cache_by_path from api.cas.casadm_parser import get_seq_cut_off_parameters, get_cas_devices_dict
from api.cas.core_config import CoreStatus
from api.cas.statistics import CoreStats, CoreIoClassStats from api.cas.statistics import CoreStats, CoreIoClassStats
from core.test_run_utils import TestRun from core.test_run_utils import TestRun
from storage_devices.device import Device from storage_devices.device import Device
@ -21,13 +21,6 @@ from test_tools.common.wait import wait
from type_def.size import Unit, Size from type_def.size import Unit, Size
class CoreStatus(Enum):
empty = 0
active = 1
inactive = 2
detached = 3
SEQ_CUTOFF_THRESHOLD_MAX = Size(4194181, Unit.KibiByte) SEQ_CUTOFF_THRESHOLD_MAX = Size(4194181, Unit.KibiByte)
SEQ_CUT_OFF_THRESHOLD_DEFAULT = Size(1, Unit.MebiByte) SEQ_CUT_OFF_THRESHOLD_DEFAULT = Size(1, Unit.MebiByte)
@ -46,9 +39,23 @@ class Core(Device):
self.partitions = [] self.partitions = []
self.block_size = None self.block_size = None
def __get_core_info(self): def __get_core_info(self) -> dict | None:
return get_core_info_for_cache_by_path(core_disk_path=self.core_device.path, core_dicts = get_cas_devices_dict()["cores"].values()
target_cache_id=self.cache_id) # for core
core_device = [
core
for core in core_dicts
if core["cache_id"] == self.cache_id and core["device_path"] == self.core_device.path
]
if core_device:
return core_device[0]
# for core pool
core_pool_dicts = get_cas_devices_dict()["core_pool"].values()
core_pool_device = [
core for core in core_pool_dicts if core["device_path"] == self.core_device.path
]
return core_pool_device[0]
def create_filesystem(self, fs_type: Filesystem, force=True, blocksize=None): def create_filesystem(self, fs_type: Filesystem, force=True, blocksize=None):
super().create_filesystem(fs_type, force, blocksize) super().create_filesystem(fs_type, force, blocksize)
@ -78,8 +85,8 @@ class Core(Device):
percentage_val=percentage_val, percentage_val=percentage_val,
) )
def get_status(self): def get_status(self) -> CoreStatus:
return CoreStatus[self.__get_core_info()["status"].lower()] return self.__get_core_info()["status"]
def get_seq_cut_off_parameters(self): def get_seq_cut_off_parameters(self):
return get_seq_cut_off_parameters(self.cache_id, self.core_id) return get_seq_cut_off_parameters(self.cache_id, self.core_id)

View File

@ -0,0 +1,16 @@
#
# Copyright(c) 2024-2025 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
from enum import Enum
class CoreStatus(Enum):
empty = "empty"
active = "active"
inactive = "inactive"
detached = "detached"
def __str__(self):
return self.value

View File

@ -1,6 +1,6 @@
# #
# Copyright(c) 2019-2022 Intel Corporation # Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd. # Copyright(c) 2024-2025 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause # SPDX-License-Identifier: BSD-3-Clause
# #
@ -17,7 +17,7 @@ from core.test_run import TestRun
from test_tools.fs_tools import write_file from test_tools.fs_tools import write_file
from test_tools.os_tools import get_kernel_version from test_tools.os_tools import get_kernel_version
default_config_file_path = "/tmp/opencas_ioclass.conf" default_config_file_path = TestRun.TEST_RUN_DATA_PATH + "/opencas_ioclass.conf"
MAX_IO_CLASS_ID = 32 MAX_IO_CLASS_ID = 32
MAX_IO_CLASS_PRIORITY = 255 MAX_IO_CLASS_PRIORITY = 255

View File

@ -1,16 +1,17 @@
# #
# Copyright(c) 2019-2021 Intel Corporation # Copyright(c) 2019-2021 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd. # Copyright(c) 2024-2025 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause # SPDX-License-Identifier: BSD-3-Clause
# #
import csv import csv
from datetime import timedelta from datetime import timedelta
from enum import Enum from enum import Enum
from typing import List from typing import List
from api.cas import casadm from api.cas import casadm
from api.cas.casadm_params import StatsFilter from api.cas.casadm_params import StatsFilter
from connection.utils.output import CmdException
from type_def.size import Size, Unit from type_def.size import Size, Unit
@ -22,6 +23,7 @@ class UnitType(Enum):
kibibyte = "[KiB]" kibibyte = "[KiB]"
gibibyte = "[GiB]" gibibyte = "[GiB]"
seconds = "[s]" seconds = "[s]"
byte = "[B]"
def __str__(self): def __str__(self):
return self.value return self.value
@ -57,6 +59,9 @@ class CacheStats:
case StatsFilter.err: case StatsFilter.err:
self.error_stats = ErrorStats(stats_dict, percentage_val) self.error_stats = ErrorStats(stats_dict, percentage_val)
if stats_dict:
raise CmdException(f"Unknown stat(s) left after parsing output cmd\n{stats_dict}")
def __str__(self): def __str__(self):
# stats_list contains all Class.__str__ methods initialized in CacheStats # stats_list contains all Class.__str__ methods initialized in CacheStats
stats_list = [str(getattr(self, stats_item)) for stats_item in self.__dict__] stats_list = [str(getattr(self, stats_item)) for stats_item in self.__dict__]
@ -68,6 +73,9 @@ class CacheStats:
getattr(other, stats_item) for stats_item in other.__dict__ getattr(other, stats_item) for stats_item in other.__dict__
] ]
def __iter__(self):
return iter([getattr(self, stats_item) for stats_item in self.__dict__])
class CoreStats: class CoreStats:
def __init__( def __init__(
@ -92,6 +100,9 @@ class CoreStats:
case StatsFilter.err: case StatsFilter.err:
self.error_stats = ErrorStats(stats_dict, percentage_val) self.error_stats = ErrorStats(stats_dict, percentage_val)
if stats_dict:
raise CmdException(f"Unknown stat(s) left after parsing output cmd\n{stats_dict}")
def __str__(self): def __str__(self):
# stats_list contains all Class.__str__ methods initialized in CacheStats # stats_list contains all Class.__str__ methods initialized in CacheStats
stats_list = [str(getattr(self, stats_item)) for stats_item in self.__dict__] stats_list = [str(getattr(self, stats_item)) for stats_item in self.__dict__]
@ -103,6 +114,9 @@ class CoreStats:
getattr(other, stats_item) for stats_item in other.__dict__ getattr(other, stats_item) for stats_item in other.__dict__
] ]
def __iter__(self):
return iter([getattr(self, stats_item) for stats_item in self.__dict__])
class CoreIoClassStats: class CoreIoClassStats:
def __init__( def __init__(
@ -128,6 +142,9 @@ class CoreIoClassStats:
case StatsFilter.blk: case StatsFilter.blk:
self.block_stats = BlockStats(stats_dict, percentage_val) self.block_stats = BlockStats(stats_dict, percentage_val)
if stats_dict:
raise CmdException(f"Unknown stat(s) left after parsing output cmd\n{stats_dict}")
def __eq__(self, other): def __eq__(self, other):
# check if all initialized variable in self(CacheStats) match other(CacheStats) # check if all initialized variable in self(CacheStats) match other(CacheStats)
return [getattr(self, stats_item) for stats_item in self.__dict__] == [ return [getattr(self, stats_item) for stats_item in self.__dict__] == [
@ -139,6 +156,9 @@ class CoreIoClassStats:
stats_list = [str(getattr(self, stats_item)) for stats_item in self.__dict__] stats_list = [str(getattr(self, stats_item)) for stats_item in self.__dict__]
return "\n".join(stats_list) return "\n".join(stats_list)
def __iter__(self):
return iter([getattr(self, stats_item) for stats_item in self.__dict__])
class CacheIoClassStats(CoreIoClassStats): class CacheIoClassStats(CoreIoClassStats):
def __init__( def __init__(
@ -182,6 +202,22 @@ class CacheConfigStats:
self.dirty_for = parse_value(value=stats_dict["Dirty for [s]"], unit_type=UnitType.seconds) self.dirty_for = parse_value(value=stats_dict["Dirty for [s]"], unit_type=UnitType.seconds)
self.status = stats_dict["Status"] self.status = stats_dict["Status"]
del stats_dict["Cache Id"]
del stats_dict["Cache Size [4KiB Blocks]"]
del stats_dict["Cache Size [GiB]"]
del stats_dict["Cache Device"]
del stats_dict["Exported Object"]
del stats_dict["Core Devices"]
del stats_dict["Inactive Core Devices"]
del stats_dict["Write Policy"]
del stats_dict["Cleaning Policy"]
del stats_dict["Promotion Policy"]
del stats_dict["Cache line size [KiB]"]
del stats_dict[footprint_key]
del stats_dict["Dirty for [s]"]
del stats_dict["Dirty for"]
del stats_dict["Status"]
def __str__(self): def __str__(self):
return ( return (
f"Config stats:\n" f"Config stats:\n"
@ -219,10 +255,13 @@ class CacheConfigStats:
and self.status == other.status and self.status == other.status
) )
def __iter__(self):
return iter([getattr(self, stats_item) for stats_item in self.__dict__])
class CoreConfigStats: class CoreConfigStats:
def __init__(self, stats_dict): def __init__(self, stats_dict):
self.core_id = stats_dict["Core Id"] self.core_id = int(stats_dict["Core Id"])
self.core_dev = stats_dict["Core Device"] self.core_dev = stats_dict["Core Device"]
self.exp_obj = stats_dict["Exported Object"] self.exp_obj = stats_dict["Exported Object"]
self.core_size = parse_value( self.core_size = parse_value(
@ -235,6 +274,17 @@ class CoreConfigStats:
) )
self.seq_cutoff_policy = stats_dict["Seq cutoff policy"] self.seq_cutoff_policy = stats_dict["Seq cutoff policy"]
del stats_dict["Core Id"]
del stats_dict["Core Device"]
del stats_dict["Exported Object"]
del stats_dict["Core Size [4KiB Blocks]"]
del stats_dict["Core Size [GiB]"]
del stats_dict["Dirty for [s]"]
del stats_dict["Dirty for"]
del stats_dict["Status"]
del stats_dict["Seq cutoff threshold [KiB]"]
del stats_dict["Seq cutoff policy"]
def __str__(self): def __str__(self):
return ( return (
f"Config stats:\n" f"Config stats:\n"
@ -262,6 +312,9 @@ class CoreConfigStats:
and self.seq_cutoff_policy == other.seq_cutoff_policy and self.seq_cutoff_policy == other.seq_cutoff_policy
) )
def __iter__(self):
return iter([getattr(self, stats_item) for stats_item in self.__dict__])
class IoClassConfigStats: class IoClassConfigStats:
def __init__(self, stats_dict): def __init__(self, stats_dict):
@ -270,6 +323,11 @@ class IoClassConfigStats:
self.eviction_priority = stats_dict["Eviction priority"] self.eviction_priority = stats_dict["Eviction priority"]
self.max_size = stats_dict["Max size"] self.max_size = stats_dict["Max size"]
del stats_dict["IO class ID"]
del stats_dict["IO class name"]
del stats_dict["Eviction priority"]
del stats_dict["Max size"]
def __str__(self): def __str__(self):
return ( return (
f"Config stats:\n" f"Config stats:\n"
@ -289,6 +347,9 @@ class IoClassConfigStats:
and self.max_size == other.max_size and self.max_size == other.max_size
) )
def __iter__(self):
return iter([getattr(self, stats_item) for stats_item in self.__dict__])
class UsageStats: class UsageStats:
def __init__(self, stats_dict, percentage_val): def __init__(self, stats_dict, percentage_val):
@ -310,6 +371,18 @@ class UsageStats:
value=stats_dict[f"Inactive Dirty {unit}"], unit_type=unit value=stats_dict[f"Inactive Dirty {unit}"], unit_type=unit
) )
for unit in [UnitType.percentage, UnitType.block_4k]:
del stats_dict[f"Occupancy {unit}"]
del stats_dict[f"Free {unit}"]
del stats_dict[f"Clean {unit}"]
del stats_dict[f"Dirty {unit}"]
if f"Inactive Dirty {unit}" in stats_dict:
del stats_dict[f"Inactive Occupancy {unit}"]
if f"Inactive Clean {unit}" in stats_dict:
del stats_dict[f"Inactive Clean {unit}"]
if f"Inactive Dirty {unit}" in stats_dict:
del stats_dict[f"Inactive Dirty {unit}"]
def __str__(self): def __str__(self):
return ( return (
f"Usage stats:\n" f"Usage stats:\n"
@ -335,6 +408,9 @@ class UsageStats:
def __ne__(self, other): def __ne__(self, other):
return not self == other return not self == other
def __iter__(self):
return iter([getattr(self, stats_item) for stats_item in self.__dict__])
class IoClassUsageStats: class IoClassUsageStats:
def __init__(self, stats_dict, percentage_val): def __init__(self, stats_dict, percentage_val):
@ -343,6 +419,11 @@ class IoClassUsageStats:
self.clean = parse_value(value=stats_dict[f"Clean {unit}"], unit_type=unit) self.clean = parse_value(value=stats_dict[f"Clean {unit}"], unit_type=unit)
self.dirty = parse_value(value=stats_dict[f"Dirty {unit}"], unit_type=unit) self.dirty = parse_value(value=stats_dict[f"Dirty {unit}"], unit_type=unit)
for unit in [UnitType.percentage, UnitType.block_4k]:
del stats_dict[f"Occupancy {unit}"]
del stats_dict[f"Clean {unit}"]
del stats_dict[f"Dirty {unit}"]
def __str__(self): def __str__(self):
return ( return (
f"Usage stats:\n" f"Usage stats:\n"
@ -366,15 +447,22 @@ class IoClassUsageStats:
def __ne__(self, other): def __ne__(self, other):
return not self == other return not self == other
def __iter__(self):
return iter([getattr(self, stats_item) for stats_item in self.__dict__])
class RequestStats: class RequestStats:
def __init__(self, stats_dict, percentage_val): def __init__(self, stats_dict, percentage_val):
unit = UnitType.percentage if percentage_val else UnitType.requests unit = UnitType.percentage if percentage_val else UnitType.requests
self.read = RequestStatsChunk( self.read = RequestStatsChunk(
stats_dict=stats_dict, percentage_val=percentage_val, operation=OperationType.read stats_dict=stats_dict,
percentage_val=percentage_val,
operation=OperationType.read,
) )
self.write = RequestStatsChunk( self.write = RequestStatsChunk(
stats_dict=stats_dict, percentage_val=percentage_val, operation=OperationType.write stats_dict=stats_dict,
percentage_val=percentage_val,
operation=OperationType.write,
) )
self.pass_through_reads = parse_value( self.pass_through_reads = parse_value(
value=stats_dict[f"Pass-Through reads {unit}"], unit_type=unit value=stats_dict[f"Pass-Through reads {unit}"], unit_type=unit
@ -389,6 +477,17 @@ class RequestStats:
value=stats_dict[f"Total requests {unit}"], unit_type=unit value=stats_dict[f"Total requests {unit}"], unit_type=unit
) )
for unit in [UnitType.percentage, UnitType.requests]:
for operation in [OperationType.read, OperationType.write]:
del stats_dict[f"{operation} hits {unit}"]
del stats_dict[f"{operation} partial misses {unit}"]
del stats_dict[f"{operation} full misses {unit}"]
del stats_dict[f"{operation} total {unit}"]
del stats_dict[f"Pass-Through reads {unit}"]
del stats_dict[f"Pass-Through writes {unit}"]
del stats_dict[f"Serviced requests {unit}"]
del stats_dict[f"Total requests {unit}"]
def __str__(self): def __str__(self):
return ( return (
f"Request stats:\n" f"Request stats:\n"
@ -412,6 +511,9 @@ class RequestStats:
and self.requests_total == other.requests_total and self.requests_total == other.requests_total
) )
def __iter__(self):
return iter([getattr(self, stats_item) for stats_item in self.__dict__])
class RequestStatsChunk: class RequestStatsChunk:
def __init__(self, stats_dict, percentage_val: bool, operation: OperationType): def __init__(self, stats_dict, percentage_val: bool, operation: OperationType):
@ -443,6 +545,9 @@ class RequestStatsChunk:
and self.total == other.total and self.total == other.total
) )
def __iter__(self):
return iter([getattr(self, stats_item) for stats_item in self.__dict__])
class BlockStats: class BlockStats:
def __init__(self, stats_dict, percentage_val): def __init__(self, stats_dict, percentage_val):
@ -458,6 +563,12 @@ class BlockStats:
device="exported object", device="exported object",
) )
for unit in [UnitType.percentage, UnitType.block_4k]:
for device in ["core", "cache", "exported object"]:
del stats_dict[f"Reads from {device} {unit}"]
del stats_dict[f"Writes to {device} {unit}"]
del stats_dict[f"Total to/from {device} {unit}"]
def __str__(self): def __str__(self):
return ( return (
f"Block stats:\n" f"Block stats:\n"
@ -473,6 +584,9 @@ class BlockStats:
self.core == other.core and self.cache == other.cache and self.exp_obj == other.exp_obj self.core == other.core and self.cache == other.cache and self.exp_obj == other.exp_obj
) )
def __iter__(self):
return iter([getattr(self, stats_item) for stats_item in self.__dict__])
class ErrorStats: class ErrorStats:
def __init__(self, stats_dict, percentage_val): def __init__(self, stats_dict, percentage_val):
@ -485,6 +599,13 @@ class ErrorStats:
) )
self.total_errors = parse_value(value=stats_dict[f"Total errors {unit}"], unit_type=unit) self.total_errors = parse_value(value=stats_dict[f"Total errors {unit}"], unit_type=unit)
for unit in [UnitType.percentage, UnitType.requests]:
for device in ["Core", "Cache"]:
del stats_dict[f"{device} read errors {unit}"]
del stats_dict[f"{device} write errors {unit}"]
del stats_dict[f"{device} total errors {unit}"]
del stats_dict[f"Total errors {unit}"]
def __str__(self): def __str__(self):
return ( return (
f"Error stats:\n" f"Error stats:\n"
@ -502,6 +623,9 @@ class ErrorStats:
and self.total_errors == other.total_errors and self.total_errors == other.total_errors
) )
def __iter__(self):
return iter([getattr(self, stats_item) for stats_item in self.__dict__])
class BasicStatsChunk: class BasicStatsChunk:
def __init__(self, stats_dict: dict, percentage_val: bool, device: str): def __init__(self, stats_dict: dict, percentage_val: bool, device: str):
@ -520,6 +644,9 @@ class BasicStatsChunk:
self.reads == other.reads and self.writes == other.writes and self.total == other.total self.reads == other.reads and self.writes == other.writes and self.total == other.total
) )
def __iter__(self):
return iter([getattr(self, stats_item) for stats_item in self.__dict__])
class BasicStatsChunkError: class BasicStatsChunkError:
def __init__(self, stats_dict: dict, percentage_val: bool, device: str): def __init__(self, stats_dict: dict, percentage_val: bool, device: str):
@ -538,6 +665,9 @@ class BasicStatsChunkError:
self.reads == other.reads and self.writes == other.writes and self.total == other.total self.reads == other.reads and self.writes == other.writes and self.total == other.total
) )
def __iter__(self):
return iter([getattr(self, stats_item) for stats_item in self.__dict__])
def get_stat_value(stat_dict: dict, key: str): def get_stat_value(stat_dict: dict, key: str):
idx = key.index("[") idx = key.index("[")
@ -583,10 +713,10 @@ def _get_section_filters(filter: List[StatsFilter], io_class_stats: bool = False
def get_stats_dict( def get_stats_dict(
filter: List[StatsFilter], filter: List[StatsFilter],
cache_id: int, cache_id: int,
core_id: int = None, core_id: int = None,
io_class_id: int = None io_class_id: int = None,
): ):
csv_stats = casadm.print_statistics( csv_stats = casadm.print_statistics(
cache_id=cache_id, cache_id=cache_id,

View File

@ -1,6 +1,6 @@
# #
# Copyright(c) 2019-2022 Intel Corporation # Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2023-2024 Huawei Technologies Co., Ltd. # Copyright(c) 2023-2025 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause # SPDX-License-Identifier: BSD-3-Clause
# #
@ -28,12 +28,16 @@ from test_tools.udev import Udev
from test_tools.disk_tools import PartitionTable, create_partition_table from test_tools.disk_tools import PartitionTable, create_partition_table
from test_tools.device_mapper import DeviceMapper from test_tools.device_mapper import DeviceMapper
from test_tools.mdadm import Mdadm from test_tools.mdadm import Mdadm
from test_tools.fs_tools import remove from test_tools.fs_tools import remove, check_if_directory_exists, create_directory
from test_tools import initramfs, git from test_tools import initramfs, git
from log.logger import create_log, Log from log.logger import create_log, Log
from test_utils.common.singleton import Singleton from test_utils.common.singleton import Singleton
from storage_devices.lvm import Lvm, LvmConfiguration from storage_devices.lvm import Lvm, LvmConfiguration
from storage_devices.disk import Disk from storage_devices.disk import Disk
from storage_devices.drbd import Drbd
TEST_RUN_DATA_PATH = "/tmp/open_cas_test_data"
def pytest_addoption(parser): def pytest_addoption(parser):
@ -132,6 +136,9 @@ def pytest_runtest_setup(item):
TestRun.LOGGER.info(f"DUT info: {TestRun.dut}") TestRun.LOGGER.info(f"DUT info: {TestRun.dut}")
TestRun.dut.plugin_manager = TestRun.plugin_manager TestRun.dut.plugin_manager = TestRun.plugin_manager
TestRun.dut.executor = TestRun.executor TestRun.dut.executor = TestRun.executor
TestRun.TEST_RUN_DATA_PATH = TEST_RUN_DATA_PATH
TestRun.dut.cache_list = []
TestRun.dut.core_list = []
TestRun.duts.append(TestRun.dut) TestRun.duts.append(TestRun.dut)
base_prepare(item) base_prepare(item)
@ -184,6 +191,16 @@ def base_prepare(item):
Udev.settle() Udev.settle()
RamDisk.remove_all() RamDisk.remove_all()
if check_if_directory_exists(path=TEST_RUN_DATA_PATH):
remove(
path=posixpath.join(TEST_RUN_DATA_PATH, "*"),
force=True,
recursive=True,
)
else:
create_directory(path=TEST_RUN_DATA_PATH)
for disk in TestRun.disks.values(): for disk in TestRun.disks.values():
disk_serial = Disk.get_disk_serial_number(disk.path) disk_serial = Disk.get_disk_serial_number(disk.path)
if disk.serial_number and disk.serial_number != disk_serial: if disk.serial_number and disk.serial_number != disk_serial:
@ -250,6 +267,14 @@ def pytest_runtest_teardown():
DeviceMapper.remove_all() DeviceMapper.remove_all()
RamDisk.remove_all() RamDisk.remove_all()
if check_if_directory_exists(path=TEST_RUN_DATA_PATH):
remove(
path=posixpath.join(TEST_RUN_DATA_PATH, "*"),
force=True,
recursive=True,
)
except Exception as ex: except Exception as ex:
TestRun.LOGGER.warning( TestRun.LOGGER.warning(
f"Exception occurred during platform cleanup.\n" f"Exception occurred during platform cleanup.\n"
@ -296,14 +321,14 @@ def unmount_cas_devices():
def __drbd_cleanup(): def __drbd_cleanup():
from storage_devices.drbd import Drbd
Drbd.down_all() Drbd.down_all()
# If drbd instance had been configured on top of the CAS, the previos attempt to stop # If drbd instance had been configured on top of the CAS, the previous attempt to stop
# failed. As drbd has been stopped try to stop CAS one more time. # failed. As drbd has been stopped try to stop CAS one more time.
if installer.check_if_installed(): if installer.check_if_installed():
casadm.stop_all_caches() casadm.stop_all_caches()
remove("/etc/drbd.d/*.res", force=True, ignore_errors=True)
class Opencas(metaclass=Singleton): class Opencas(metaclass=Singleton):
def __init__(self, repo_dir, working_dir): def __init__(self, repo_dir, working_dir):