github/open-cas-linux
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1599 from katlapinka/kasiat/tf-refactor

Browse Source 
Tests and CAS API fixes after TF refactor
This commit is contained in:
Katarzyna Treder 2024-12-31 12:44:52 +01:00 committed by GitHub
commit 1934e801e7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
173 changed files with 762 additions and 1051 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
test/functional/api/cas/cache.py
View File

@ -8,8 +8,8 @@ from api.cas.casadm_parser import *
from api.cas.core import Core
from api.cas.dmesg import get_metadata_size_on_device
from api.cas.statistics import CacheStats, CacheIoClassStats
from test_utils.os_utils import *
from test_utils.output import Output
from connection.utils.output import Output
from test_tools.os_tools import sync
class Cache:

6
test/functional/api/cas/cache_config.py
View File

@ -6,9 +6,9 @@
from enum import Enum, IntFlag
from test_utils.os_utils import get_kernel_module_parameter
from test_utils.size import Size, Unit
from test_utils.time import Time
from test_tools.os_tools import get_kernel_module_parameter
from type_def.size import Size, Unit
from type_def.time import Time
class CacheLineSize(Enum):

9
test/functional/api/cas/cas_module.py
View File

@ -6,8 +6,7 @@
from enum import Enum
from core.test_run import TestRun
from test_utils import os_utils
from test_utils.os_utils import ModuleRemoveMethod
from test_tools.os_tools import unload_kernel_module, load_kernel_module
class CasModule(Enum):
@ -15,12 +14,12 @@ class CasModule(Enum):
def reload_all_cas_modules():
os_utils.unload_kernel_module(CasModule.cache.value, ModuleRemoveMethod.modprobe)
os_utils.load_kernel_module(CasModule.cache.value)
unload_kernel_module(CasModule.cache.value)
load_kernel_module(CasModule.cache.value)
def unload_all_cas_modules():
os_utils.unload_kernel_module(CasModule.cache.value, os_utils.ModuleRemoveMethod.rmmod)
unload_kernel_module(CasModule.cache.value)
def is_cas_management_dev_present():

2
test/functional/api/cas/cas_packaging.py
View File

@ -9,7 +9,7 @@ import os
import re
from core.test_run import TestRun
from test_tools.fs_utils import check_if_directory_exists, find_all_files
from test_tools.fs_tools import check_if_directory_exists, find_all_files
from test_tools.linux_packaging import DebSet, RpmSet

4
test/functional/api/cas/cas_service.py
View File

@ -9,13 +9,13 @@ from datetime import timedelta
from string import Template
from textwrap import dedent
from test_tools.fs_utils import (
from test_tools.fs_tools import (
check_if_directory_exists,
create_directory,
write_file,
remove,
)
from test_utils.systemd import reload_daemon
from test_tools.systemctl import reload_daemon
opencas_drop_in_directory = Path("/etc/systemd/system/open-cas.service.d/")
test_drop_in_file = Path("10-modified-timeout.conf")

6
test/functional/api/cas/casadm.py
View File

@ -20,9 +20,9 @@ from api.cas.cli import *
from api.cas.core import Core
from core.test_run import TestRun
from storage_devices.device import Device
from test_utils.os_utils import reload_kernel_module
from test_utils.output import CmdException, Output
from test_utils.size import Size, Unit
from test_tools.os_tools import reload_kernel_module
from connection.utils.output import CmdException, Output
from type_def.size import Size, Unit
# casadm commands

2
test/functional/api/cas/casadm_parser.py
View File

@ -18,7 +18,7 @@ from api.cas.ioclass_config import IoClass
from api.cas.version import CasVersion
from core.test_run_utils import TestRun
from storage_devices.device import Device
from test_utils.output import CmdException
from connection.utils.output import CmdException
class Stats(dict):

2
test/functional/api/cas/cli_messages.py
View File

@ -7,7 +7,7 @@
import re
from core.test_run import TestRun
from test_utils.output import Output
from connection.utils.output import Output
load_inactive_core_missing = [
r"WARNING: Can not resolve path to core \d+ from cache \d+\. By-id path will be shown for that "

11
test/functional/api/cas/core.py
View File

@ -15,9 +15,10 @@ from api.cas.casadm_parser import get_seq_cut_off_parameters, get_core_info_for_
from api.cas.statistics import CoreStats, CoreIoClassStats
from core.test_run_utils import TestRun
from storage_devices.device import Device
from test_tools import fs_utils, disk_utils
from test_utils.os_utils import wait, sync
from test_utils.size import Unit, Size
from test_tools.fs_tools import Filesystem, ls_item
from test_tools.os_tools import sync
from test_tools.common.wait import wait
from type_def.size import Unit, Size
class CoreStatus(Enum):
@ -49,7 +50,7 @@ class Core(Device):
return get_core_info_for_cache_by_path(core_disk_path=self.core_device.path,
target_cache_id=self.cache_id)
def create_filesystem(self, fs_type: disk_utils.Filesystem, force=True, blocksize=None):
def create_filesystem(self, fs_type: Filesystem, force=True, blocksize=None):
super().create_filesystem(fs_type, force, blocksize)
self.core_device.filesystem = self.filesystem
@ -138,7 +139,7 @@ class Core(Device):
def check_if_is_present_in_os(self, should_be_visible=True):
device_in_system_message = "CAS device exists in OS."
device_not_in_system_message = "CAS device does not exist in OS."
item = fs_utils.ls_item(f"{self.path}")
item = ls_item(self.path)
if item is not None:
if should_be_visible:
TestRun.LOGGER.info(device_in_system_message)

4
test/functional/api/cas/dmesg.py
View File

@ -6,8 +6,8 @@
import re
from test_utils.dmesg import get_dmesg
from test_utils.size import Size, Unit
from test_tools.dmesg import get_dmesg
from type_def.size import Size, Unit
def get_metadata_size_on_device(cache_id: int) -> Size:

9
test/functional/api/cas/init_config.py
View File

@ -7,8 +7,7 @@
from api.cas import casadm_parser
from api.cas.cache_config import CacheMode
from storage_devices.device import Device
from test_tools import fs_utils
from test_tools.fs_tools import remove, write_file
opencas_conf_path = "/etc/opencas/opencas.conf"
@ -34,7 +33,7 @@ class InitConfig:
@staticmethod
def remove_config_file():
fs_utils.remove(opencas_conf_path, force=False)
remove(opencas_conf_path, force=False)
def save_config_file(self):
config_lines = []
@ -47,7 +46,7 @@ class InitConfig:
config_lines.append(CoreConfigLine.header)
for c in self.core_config_lines:
config_lines.append(str(c))
fs_utils.write_file(opencas_conf_path, "\n".join(config_lines), False)
write_file(opencas_conf_path, "\n".join(config_lines), False)
@classmethod
def create_init_config_from_running_configuration(
@ -69,7 +68,7 @@ class InitConfig:
@classmethod
def create_default_init_config(cls):
cas_version = casadm_parser.get_casadm_version()
fs_utils.write_file(opencas_conf_path, f"version={cas_version.base}")
write_file(opencas_conf_path, f"version={cas_version.base}")
class CacheConfigLine:

7
test/functional/api/cas/installer.py
View File

@ -9,8 +9,9 @@ import os
from core.test_run import TestRun
from api.cas import cas_module
from api.cas.version import get_installed_cas_version
from test_utils import os_utils, git
from test_utils.output import CmdException
from test_tools import git
from connection.utils.output import CmdException
from test_tools.os_tools import is_kernel_module_loaded
def rsync_opencas_sources():
@ -98,7 +99,7 @@ def reinstall_opencas(version: str = ""):
def check_if_installed(version: str = ""):
TestRun.LOGGER.info("Check if Open CAS Linux is installed")
output = TestRun.executor.run("which casadm")
modules_loaded = os_utils.is_kernel_module_loaded(cas_module.CasModule.cache.value)
modules_loaded = is_kernel_module_loaded(cas_module.CasModule.cache.value)
if output.exit_code != 0 or not modules_loaded:
TestRun.LOGGER.info("CAS is not installed")

14
test/functional/api/cas/ioclass_config.py
View File

@ -14,9 +14,8 @@ from datetime import timedelta
from packaging import version
from core.test_run import TestRun
from test_tools import fs_utils
from test_utils import os_utils
from test_utils.generator import random_string
from test_tools.fs_tools import write_file
from test_tools.os_tools import get_kernel_version
default_config_file_path = "/tmp/opencas_ioclass.conf"
@ -109,7 +108,7 @@ class IoClass:
ioclass_config_path: str = default_config_file_path,
):
TestRun.LOGGER.info(f"Creating config file {ioclass_config_path}")
fs_utils.write_file(
write_file(
ioclass_config_path, IoClass.list_to_csv(ioclass_list, add_default_rule)
)
@ -167,7 +166,7 @@ class IoClass:
"file_offset",
"request_size",
]
if os_utils.get_kernel_version() >= version.Version("4.13"):
if get_kernel_version() >= version.Version("4.13"):
rules.append("wlth")
rule = random.choice(rules)
@ -178,13 +177,14 @@ class IoClass:
def add_random_params(rule: str):
if rule == "directory":
allowed_chars = string.ascii_letters + string.digits + "/"
rule += f":/{random_string(random.randint(1, 40), allowed_chars)}"
rule += f":/{''.join(random.choices(allowed_chars, k=random.randint(1, 40)))}"
elif rule in ["file_size", "lba", "pid", "file_offset", "request_size", "wlth"]:
rule += f":{Operator(random.randrange(len(Operator))).name}:{random.randrange(1000000)}"
elif rule == "io_class":
rule += f":{random.randrange(MAX_IO_CLASS_PRIORITY + 1)}"
elif rule in ["extension", "process_name", "file_name_prefix"]:
rule += f":{random_string(random.randint(1, 10))}"
allowed_chars = string.ascii_letters + string.digits
rule += f":{''.join(random.choices(allowed_chars, k=random.randint(1, 10)))}"
if random.randrange(2):
rule += "&done"
return rule

2
test/functional/api/cas/progress_bar.py
View File

@ -10,7 +10,7 @@ from datetime import timedelta
import paramiko
from core.test_run import TestRun
from test_utils.os_utils import wait
from test_tools.common.wait import wait
def check_progress_bar(command: str, progress_bar_expected: bool = True):

2
test/functional/api/cas/statistics.py
View File

@ -11,7 +11,7 @@ from typing import List
from api.cas import casadm
from api.cas.casadm_params import StatsFilter
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
class UnitType(Enum):

6
test/functional/api/cas/version.py
View File

@ -6,9 +6,9 @@
import re
from test_utils import git
from test_tools import git
from core.test_run import TestRun
from test_utils.output import CmdException
from connection.utils.output import CmdException
class CasVersion:
@ -43,7 +43,7 @@ class CasVersion:
def get_available_cas_versions():
release_tags = git.get_release_tags()
release_tags = git.get_tags()
versions = [CasVersion.from_git_tag(tag) for tag in release_tags]

2
test/functional/test-framework

@ -1 +1 @@
Subproject commit 072c72b08c9d5657c131b5cd325175a19a5417df
Subproject commit f7f2914e41ce73d13dafe6cd749aa6e2d33b4315

9
test/functional/tests/512b/test_different_io_sizes_support.py
View File

@ -12,9 +12,8 @@ from core.test_run import TestRun
from api.cas import casadm
from storage_devices.disk import DiskType, DiskTypeSet
from api.cas.cache_config import CacheMode
from test_tools import fs_utils
from test_tools.disk_utils import Filesystem
from test_utils.size import Size, Unit
from test_tools.fs_tools import Filesystem, remove, create_directory
from type_def.size import Size, Unit
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine
@ -52,8 +51,8 @@ def test_support_different_io_size(cache_mode):
cache.load_io_class(opencas_ioclass_conf_path)
with TestRun.step("Create a filesystem on the core device and mount it"):
fs_utils.remove(path=mountpoint, force=True, recursive=True, ignore_errors=True)
fs_utils.create_directory(path=mountpoint)
remove(path=mountpoint, force=True, recursive=True, ignore_errors=True)
create_directory(path=mountpoint)
core.create_filesystem(Filesystem.xfs)
core.mount(mountpoint)

9
test/functional/tests/basic/test_basic.py
View File

@ -17,12 +17,11 @@ from api.cas.cli_messages import (
)
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools import fs_utils
from test_tools.dd import Dd
from test_tools.disk_utils import Filesystem
from test_tools.fs_tools import Filesystem, read_file
from test_utils.filesystem.file import File
from test_utils.output import CmdException
from test_utils.size import Size, Unit
from connection.utils.output import CmdException
from type_def.size import Size, Unit
version_file_path = r"/var/lib/opencas/cas_version"
mountpoint = "/mnt"
@ -45,7 +44,7 @@ def test_cas_version():
cmd_cas_versions = [version.split(",")[1] for version in cmd_version.split("\n")[1:]]
with TestRun.step(f"Read cas version from {version_file_path} location"):
file_read = fs_utils.read_file(version_file_path).split("\n")
file_read = read_file(version_file_path).split("\n")
file_cas_version = next(
(line.split("=")[1] for line in file_read if "CAS_VERSION=" in line)
)

2
test/functional/tests/basic/test_start_cas.py
View File

@ -9,7 +9,7 @@ import pytest
from api.cas import casadm
from core.test_run import TestRun
from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
@pytest.mark.CI

4
test/functional/tests/cache_ops/test_cleaning_policy_operation.py
View File

@ -17,8 +17,8 @@ from api.cas.cache_config import (
)
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from core.test_run import TestRun
from test_utils.size import Size, Unit
from test_utils.os_utils import Udev
from type_def.size import Size, Unit
from test_tools.udev import Udev
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine

4
test/functional/tests/cache_ops/test_concurrent_flushes.py
View File

@ -15,8 +15,8 @@ from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import IoEngine, ReadWrite
from test_utils.output import CmdException
from test_utils.size import Size, Unit
from connection.utils.output import CmdException
from type_def.size import Size, Unit
@pytest.mark.parametrize("cache_mode", CacheMode.with_traits(CacheModeTrait.LazyWrites))

7
test/functional/tests/cache_ops/test_core_remove.py
View File

@ -5,15 +5,14 @@
#
import random
import pytest
from api.cas import casadm
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeLowerThan, DiskTypeSet
from test_tools.disk_utils import Filesystem
from test_utils.output import CmdException
from test_utils.size import Size, Unit
from test_tools.fs_tools import Filesystem
from connection.utils.output import CmdException
from type_def.size import Size, Unit
mount_point = "/mnt/cas"
cores_amount = 3

5
test/functional/tests/cache_ops/test_dynamic_cache_mode_switching.py
View File

@ -15,8 +15,9 @@ from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine, VerifyMethod
from test_utils.os_utils import Udev, sync
from test_utils.size import Size, Unit
from test_tools.os_tools import sync
from test_tools.udev import Udev
from type_def.size import Size, Unit
io_size = Size(10000, Unit.Blocks4096)

2
test/functional/tests/cache_ops/test_multilevel_cache.py
View File

@ -9,7 +9,7 @@ import pytest
from api.cas import casadm, cli, cli_messages
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))

8
test/functional/tests/cache_ops/test_multistream_seq_cutoff.py
View File

@ -21,12 +21,12 @@ from api.cas.casadm_params import StatsFilter
from core.test_run_utils import TestRun
from storage_devices.disk import DiskTypeSet, DiskTypeLowerThan, DiskType
from test_tools.dd import Dd
from test_tools.disk_utils import Filesystem
from test_tools.fs_tools import Filesystem
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import IoEngine, ReadWrite
from test_utils.os_utils import Udev
from test_utils.output import CmdException
from test_utils.size import Size, Unit
from test_tools.udev import Udev
from connection.utils.output import CmdException
from type_def.size import Size, Unit
random_thresholds = random.sample(range(1028, 1024**2, 4), 3)
random_stream_numbers = random.sample(range(2, 128), 3)

5
test/functional/tests/cache_ops/test_seq_cutoff.py
View File

@ -15,8 +15,9 @@ from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine, CpusAllowedPolicy
from test_utils.os_utils import Udev, sync, get_dut_cpu_physical_cores
from test_utils.size import Size, Unit
from test_tools.os_tools import sync, get_dut_cpu_physical_cores
from test_tools.udev import Udev
from type_def.size import Size, Unit
class VerifyType(Enum):

33
test/functional/tests/ci/test_ci_read_write.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
@ -9,8 +10,8 @@ from api.cas import casadm
from api.cas.cache_config import CacheMode
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_utils.os_utils import Udev
from test_utils.size import Unit, Size
from test_tools.udev import Udev
from type_def.size import Unit, Size
from test_tools.dd import Dd
from test_tools.iostat import IOstatBasic
@ -62,7 +63,7 @@ def test_ci_read(cache_mode):
dd.run()
with TestRun.step("Collect iostat"):
iostat = IOstatBasic.get_iostat_list([cache_device.parent_device])
iostat = IOstatBasic.get_iostat_list([cache_device.parent_device.get_device_id()])
read_cache_1 = iostat[0].total_reads
with TestRun.step("Generate cache hits using reads"):
@ -77,7 +78,7 @@ def test_ci_read(cache_mode):
dd.run()
with TestRun.step("Collect iostat"):
iostat = IOstatBasic.get_iostat_list([cache_device.parent_device])
iostat = IOstatBasic.get_iostat_list([cache_device.parent_device.get_device_id()])
read_cache_2 = iostat[0].total_reads
with TestRun.step("Stop cache"):
@ -117,10 +118,10 @@ def test_ci_write_around_write():
casadm.add_core(cache, core_device)
with TestRun.step("Collect iostat before I/O"):
iostat_core = IOstatBasic.get_iostat_list([core_device.parent_device])
iostat_core = IOstatBasic.get_iostat_list([core_device.parent_device.get_device_id()])
write_core_0 = iostat_core[0].total_writes
iostat_cache = IOstatBasic.get_iostat_list([cache_device.parent_device])
iostat_cache = IOstatBasic.get_iostat_list([cache_device.parent_device.get_device_id()])
write_cache_0 = iostat_cache[0].total_writes
with TestRun.step("Submit writes to exported object"):
@ -136,11 +137,11 @@ def test_ci_write_around_write():
dd.run()
with TestRun.step("Collect iostat"):
iostat_core = IOstatBasic.get_iostat_list([core_device.parent_device])
iostat_core = IOstatBasic.get_iostat_list([core_device.parent_device.get_device_id()])
write_core_1 = iostat_core[0].total_writes
read_core_1 = iostat_core[0].total_reads
iostat_cache = IOstatBasic.get_iostat_list([cache_device.parent_device])
iostat_cache = IOstatBasic.get_iostat_list([cache_device.parent_device.get_device_id()])
write_cache_1 = iostat_cache[0].total_writes
read_cache_1 = iostat_cache[0].total_reads
@ -156,10 +157,10 @@ def test_ci_write_around_write():
dd.run()
with TestRun.step("Collect iostat"):
iostat_core = IOstatBasic.get_iostat_list([core_device.parent_device])
iostat_core = IOstatBasic.get_iostat_list([core_device.parent_device.get_device_id()])
read_core_2 = iostat_core[0].total_reads
iostat_cache = IOstatBasic.get_iostat_list([cache_device.parent_device])
iostat_cache = IOstatBasic.get_iostat_list([cache_device.parent_device.get_device_id()])
read_cache_2 = iostat_cache[0].total_reads
with TestRun.step("Stop cache"):
@ -221,10 +222,10 @@ def test_ci_write_through_write():
casadm.add_core(cache, core_device)
with TestRun.step("Collect iostat before I/O"):
iostat_core = IOstatBasic.get_iostat_list([core_device.parent_device])
iostat_core = IOstatBasic.get_iostat_list([core_device.parent_device.get_device_id()])
write_core_0 = iostat_core[0].total_writes
iostat_cache = IOstatBasic.get_iostat_list([cache_device.parent_device])
iostat_cache = IOstatBasic.get_iostat_list([cache_device.parent_device.get_device_id()])
write_cache_0 = iostat_cache[0].total_writes
with TestRun.step("Insert data into the cache using writes"):
@ -241,11 +242,11 @@ def test_ci_write_through_write():
dd.run()
with TestRun.step("Collect iostat"):
iostat_core = IOstatBasic.get_iostat_list([core_device.parent_device])
iostat_core = IOstatBasic.get_iostat_list([core_device.parent_device.get_device_id()])
write_core_1 = iostat_core[0].total_writes
read_core_1 = iostat_core[0].total_reads
iostat_cache = IOstatBasic.get_iostat_list([cache_device.parent_device])
iostat_cache = IOstatBasic.get_iostat_list([cache_device.parent_device.get_device_id()])
write_cache_1 = iostat_cache[0].total_writes
read_cache_1 = iostat_cache[0].total_reads
@ -262,10 +263,10 @@ def test_ci_write_through_write():
dd.run()
with TestRun.step("Collect iostat"):
iostat_core = IOstatBasic.get_iostat_list([core_device.parent_device])
iostat_core = IOstatBasic.get_iostat_list([core_device.parent_device.get_device_id()])
read_core_2 = iostat_core[0].total_reads
iostat_cache = IOstatBasic.get_iostat_list([cache_device.parent_device])
iostat_cache = IOstatBasic.get_iostat_list([cache_device.parent_device.get_device_id()])
read_cache_2 = iostat_cache[0].total_reads
with TestRun.step("Stop cache"):

8
test/functional/tests/cli/test_cas_memory_usage.py
View File

@ -8,15 +8,13 @@ import pytest
from api.cas.cas_module import CasModule
from core.test_run import TestRun
from test_utils.size import Unit
from test_utils.os_utils import (allocate_memory,
disable_memory_affecting_functions,
drop_caches,
get_mem_free,
from type_def.size import Unit
from test_tools.os_tools import (drop_caches,
is_kernel_module_loaded,
load_kernel_module,
unload_kernel_module,
)
from test_tools.memory import disable_memory_affecting_functions, get_mem_free, allocate_memory
@pytest.mark.os_dependent

4
test/functional/tests/cli/test_cleaning_policy.py
View File

@ -8,14 +8,14 @@ import pytest
import time
from core.test_run_utils import TestRun
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine
from api.cas import casadm
from api.cas.cache_config import CacheMode, CleaningPolicy
from test_utils.os_utils import Udev
from test_tools.udev import Udev
@pytest.mark.CI

7
test/functional/tests/cli/test_cli_script.py
View File

@ -1,16 +1,17 @@
#
# Copyright(c) 2020-2021 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import pytest
from api.cas import casadm, casadm_parser
from api.cas import casadm
from core.test_run import TestRun
from test_utils.os_utils import sync
from test_tools.os_tools import sync
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_utils.size import Unit, Size
from type_def.size import Unit, Size
from test_tools.dd import Dd

8
test/functional/tests/cli/test_cli_standby.py
View File

@ -13,11 +13,11 @@ from core.test_run import TestRun
from storage_devices.device import Device
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools.dd import Dd
from test_tools.disk_utils import Filesystem
from test_tools.fs_tools import Filesystem
from test_utils.filesystem.file import File
from test_utils.os_utils import sync
from test_utils.output import CmdException
from test_utils.size import Size, Unit
from test_tools.os_tools import sync
from connection.utils.output import CmdException
from type_def.size import Size, Unit
from api.cas.cli_messages import (
check_stderr_msg,
missing_param,

3
test/functional/tests/cli/test_cli_start_stop.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2019-2021 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
@ -10,7 +11,7 @@ from api.cas import casadm, casadm_parser, cli_messages
from api.cas.cli import start_cmd
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_utils.size import Unit, Size
from type_def.size import Unit, Size
CACHE_ID_RANGE = (1, 16384)
CORE_ID_RANGE = (0, 4095)

5
test/functional/tests/cli/test_manual_casadm_flush.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
@ -8,14 +9,14 @@ import time
from core.test_run_utils import TestRun
from storage_devices.device import Device
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine
from api.cas import casadm
from api.cas.cache_config import CacheMode, CleaningPolicy
from test_utils.os_utils import Udev
from test_tools.udev import Udev
@pytest.mark.CI

3
test/functional/tests/cli/test_seq_cutoff_settings.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
@ -14,7 +15,7 @@ from api.cas.casadm import set_param_cutoff_cmd
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))

3
test/functional/tests/cli/test_set_get_params.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2020-2021 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
@ -19,7 +20,7 @@ from api.cas.cache_config import (
)
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
# There should be at least 2 cache instances and 2 cores per cache

8
test/functional/tests/cli/test_zero_metadata_command.py
View File

@ -12,12 +12,12 @@ from api.cas import casadm, cli_messages, cli
from api.cas.cache_config import CacheMode, CleaningPolicy
from core.test_run import TestRun
from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan
from test_tools.disk_utils import get_device_filesystem_type, Filesystem
from test_tools.fs_tools import Filesystem, get_device_filesystem_type
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import IoEngine, ReadWrite
from test_utils.disk_finder import get_system_disks
from test_utils.output import CmdException
from test_utils.size import Size, Unit
from test_tools.disk_finder import get_system_disks
from connection.utils.output import CmdException
from type_def.size import Size, Unit
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))

35
test/functional/tests/common/lvm_common_utils.py
View File

@ -1,35 +0,0 @@
#
# Copyright(c) 2021 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause
#
import datetime
from storage_devices.lvm import get_block_devices_list
from api.cas.init_config import InitConfig
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import IoEngine, ReadWrite, VerifyMethod
from test_utils.size import Size, Unit
def run_fio_on_lvm(volumes: []):
fio_run = (Fio().create_command()
.read_write(ReadWrite.randrw)
.io_engine(IoEngine.sync)
.io_depth(1)
.time_based()
.run_time(datetime.timedelta(seconds=180))
.do_verify()
.verify(VerifyMethod.md5)
.block_size(Size(1, Unit.Blocks4096)))
for lvm in volumes:
fio_run.add_job().target(lvm).size(lvm.size)
fio_run.run()
def get_test_configuration():
config = InitConfig.create_init_config_from_running_configuration()
devices = get_block_devices_list()
return config, devices

16
test/functional/tests/conftest.py
View File

@ -20,20 +20,20 @@ from core.test_run import Blocked
from core.test_run_utils import TestRun
from api.cas import installer
from api.cas import casadm
from test_utils import git
from api.cas.cas_service import opencas_drop_in_directory
from storage_devices.raid import Raid
from storage_devices.ramdisk import RamDisk
from test_utils.os_utils import Udev, kill_all_io
from test_utils.disk_finder import get_disk_serial_number
from test_tools.disk_utils import PartitionTable, create_partition_table
from test_tools.os_tools import kill_all_io
from test_tools.udev import Udev
from test_tools.disk_tools import PartitionTable, create_partition_table
from test_tools.device_mapper import DeviceMapper
from test_tools.mdadm import Mdadm
from test_tools.fs_utils import remove
from test_tools import initramfs
from test_tools.fs_tools import remove
from test_tools import initramfs, git
from log.logger import create_log, Log
from test_utils.singleton import Singleton
from test_utils.common.singleton import Singleton
from storage_devices.lvm import Lvm, LvmConfiguration
from storage_devices.disk import Disk
def pytest_addoption(parser):
@ -185,7 +185,7 @@ def base_prepare(item):
RamDisk.remove_all()
for disk in TestRun.disks.values():
disk_serial = get_disk_serial_number(disk.path)
disk_serial = Disk.get_disk_serial_number(disk.path)
if disk.serial_number and disk.serial_number != disk_serial:
raise Exception(
f"Serial for {disk.path} doesn't match the one from the config."

3
test/functional/tests/data_integrity/test_data_integrity_12h.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2019-2021 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
@ -14,7 +15,7 @@ from core.test_run import TestRun
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine, VerifyMethod
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_utils.size import Unit, Size
from type_def.size import Unit, Size
start_size = int(Size(512, Unit.Byte))

21
test/functional/tests/data_integrity/test_data_integrity_5d.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2019-2021 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
@ -12,14 +13,12 @@ import pytest
from api.cas import casadm
from api.cas.cache_config import CacheMode
from core.test_run import TestRun
from test_tools import fs_utils
from test_tools.disk_utils import Filesystem
from test_tools.fs_tools import Filesystem, create_directory, check_if_directory_exists, md5sum
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine, VerifyMethod
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_utils.filesystem.file import File
from test_utils.os_utils import sync
from test_utils.size import Unit, Size
from test_tools.os_tools import sync
from type_def.size import Unit, Size
start_size = Size(512, Unit.Byte).get_value()
@ -72,8 +71,8 @@ def test_data_integrity_5d_dss(filesystems):
with TestRun.step("Create filesystems and mount cores"):
for i, core in enumerate(cores):
mount_point = core.path.replace('/dev/', '/mnt/')
if not fs_utils.check_if_directory_exists(mount_point):
fs_utils.create_directory(mount_point)
if not check_if_directory_exists(mount_point):
create_directory(mount_point)
TestRun.LOGGER.info(f"Create filesystem {filesystems[i].name} on {core.path}")
core.create_filesystem(filesystems[i])
TestRun.LOGGER.info(f"Mount filesystem {filesystems[i].name} on {core.path} to "
@ -106,14 +105,14 @@ def test_data_integrity_5d_dss(filesystems):
core.unmount()
with TestRun.step("Calculate md5 for each core"):
core_md5s = [File(core.full_path).md5sum() for core in cores]
core_md5s = [md5sum(core.path) for core in cores]
with TestRun.step("Stop caches"):
for cache in caches:
cache.stop()
with TestRun.step("Calculate md5 for each core"):
dev_md5s = [File(dev.full_path).md5sum() for dev in core_devices]
dev_md5s = [md5sum(dev.full_path) for dev in core_devices]
with TestRun.step("Compare md5 sums for cores and core devices"):
for core_md5, dev_md5, mode, fs in zip(core_md5s, dev_md5s, cache_modes, filesystems):
@ -171,14 +170,14 @@ def test_data_integrity_5d():
fio_run.run()
with TestRun.step("Calculate md5 for each core"):
core_md5s = [File(core.full_path).md5sum() for core in cores]
core_md5s = [md5sum(core.path) for core in cores]
with TestRun.step("Stop caches"):
for cache in caches:
cache.stop()
with TestRun.step("Calculate md5 for each core"):
dev_md5s = [File(dev.full_path).md5sum() for dev in core_devices]
dev_md5s = [md5sum(dev.full_path) for dev in core_devices]
with TestRun.step("Compare md5 sums for cores and core devices"):
for core_md5, dev_md5, mode in zip(core_md5s, dev_md5s, cache_modes):

6
test/functional/tests/data_integrity/test_data_integrity_unplug.py
View File

@ -19,10 +19,10 @@ from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from storage_devices.ramdisk import RamDisk
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite
from test_utils.asynchronous import start_async_func
from connection.utils.asynchronous import start_async_func
from test_utils.filesystem.directory import Directory
from test_utils.output import CmdException
from test_utils.size import Unit, Size
from connection.utils.output import CmdException
from type_def.size import Unit, Size
ram_disk, tmp_dir, fio_seed = None, None, None
num_jobs = 8

20
test/functional/tests/example/example_test.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
@ -13,14 +14,13 @@ from storage_devices.ramdisk import RamDisk
from test_utils.drbd import Resource, Node
from storage_devices.drbd import Drbd
from test_tools.drbdadm import Drbdadm
from test_tools import fs_utils
from test_tools.disk_utils import Filesystem
from test_tools.fs_utils import copy, check_if_file_exists
from test_tools.fs_tools import copy, Filesystem, replace_in_lines, remove, Permissions, \
PermissionsUsers
from test_utils.filesystem.directory import Directory
from test_utils.filesystem.file import File
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
from test_utils.emergency_escape import EmergencyEscape
from test_utils.fstab import add_mountpoint
from test_tools.fstab import add_mountpoint
from storage_devices.lvm import Lvm
@ -102,7 +102,7 @@ def test_create_example_files():
content_before_change = file1.read()
TestRun.LOGGER.info(f"File content: {content_before_change}")
with TestRun.step("Replace single line in file"):
fs_utils.replace_in_lines(file1, 'content line', 'replaced line')
replace_in_lines(file1, 'content line', 'replaced line')
with TestRun.step("Read file content and check if it changed"):
content_after_change = file1.read()
if content_before_change == content_after_change:
@ -115,19 +115,19 @@ def test_create_example_files():
with TestRun.step("Change permissions of second file"):
file2.chmod_numerical(123)
with TestRun.step("Remove second file"):
fs_utils.remove(file2.full_path, True)
remove(file2.full_path, True)
with TestRun.step("List contents of home directory"):
dir1 = Directory("~")
dir_content = dir1.ls()
with TestRun.step("Change permissions of file"):
file1.chmod(fs_utils.Permissions['r'] | fs_utils.Permissions['w'],
fs_utils.PermissionsUsers(7))
file1.chmod(Permissions['r'] | Permissions['w'],
PermissionsUsers(7))
with TestRun.step("Log home directory content"):
for item in dir_content:
TestRun.LOGGER.info(f"Item {str(item)} - {type(item).__name__}")
with TestRun.step("Remove file"):
fs_utils.remove(file1.full_path, True)
remove(file1.full_path, True)
@pytest.mark.require_disk("cache1", DiskTypeSet([DiskType.optane, DiskType.nand]))

5
test/functional/tests/failover_standby/test_cache_activation_perf.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
from datetime import timedelta, datetime
@ -10,8 +11,8 @@ from core.test_run import TestRun
from test_tools.dd import Dd
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine
from test_utils.os_utils import Udev
from test_utils.size import Size, Unit
from test_tools.udev import Udev
from type_def.size import Size, Unit
from storage_devices.disk import DiskType, DiskTypeSet
from storage_devices.device import Device
from api.cas import casadm, dmesg

14
test/functional/tests/failover_standby/test_failover_advanced.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
@ -22,14 +23,12 @@ from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet
from storage_devices.drbd import Drbd
from test_tools.dd import Dd
from test_tools.disk_utils import Filesystem
from test_utils.size import Size, Unit
from test_utils.filesystem.file import File
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite
from test_tools.fs_utils import readlink, create_directory
from test_tools.fs_tools import create_directory, Filesystem
from test_utils.drbd import Resource, Node
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
from test_failover_multihost import check_drbd_installed
@ -1096,13 +1095,6 @@ def test_failover_io_long(cls, cleaning_policy, num_iterations):
TestRun.executor.wait_for_connection()
def check_drbd_installed(duts):
for dut in duts:
with TestRun.use_dut(dut):
if not Drbd.is_installed():
TestRun.fail(f"DRBD is not installed on DUT {dut.ip}")
def prepare_devices(duts):
for dut in duts:
with TestRun.use_dut(dut):

17
test/functional/tests/failover_standby/test_failover_multihost.py
View File

@ -4,7 +4,6 @@
# SPDX-License-Identifier: BSD-3-Clause
#
from time import sleep
import pytest
from api.cas import casadm
@ -22,15 +21,13 @@ from storage_devices.disk import DiskType, DiskTypeSet
from storage_devices.drbd import Drbd
from storage_devices.raid import Raid, RaidConfiguration, MetadataVariant, Level
from test_tools.dd import Dd
from test_tools.drbdadm import Drbdadm
from test_tools.disk_utils import Filesystem
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite
from test_tools.fs_utils import readlink
from test_tools.fs_tools import readlink, Filesystem, create_directory
from test_utils.drbd import Resource, Node
from test_utils.os_utils import sync, Udev
from test_utils.size import Size, Unit
from test_tools import fs_utils
from test_tools.os_tools import sync
from test_tools.udev import Udev
from type_def.size import Size, Unit
cache_id = 5
@ -147,7 +144,7 @@ def test_functional_activate_twice_round_trip(filesystem):
primary_node.cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
if filesystem:
TestRun.executor.run(f"rm -rf {mountpoint}")
fs_utils.create_directory(path=mountpoint)
create_directory(path=mountpoint)
core.create_filesystem(filesystem)
core.mount(mountpoint)
@ -318,7 +315,7 @@ def test_functional_activate_twice_new_host(filesystem):
primary_node.cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
if filesystem:
TestRun.executor.run(f"rm -rf {mountpoint}")
fs_utils.create_directory(path=mountpoint)
create_directory(path=mountpoint)
core.create_filesystem(filesystem)
core.mount(mountpoint)
@ -494,7 +491,7 @@ def failover_sequence(standby_node, drbd_resource, filesystem, core):
if filesystem:
with TestRun.use_dut(standby_node), TestRun.step(f"Mount core"):
TestRun.executor.run(f"rm -rf {mountpoint}")
fs_utils.create_directory(path=mountpoint)
create_directory(path=mountpoint)
core.mount(mountpoint)

5
test/functional/tests/fault_injection/test_another_device_with_same_id.py
View File

@ -1,16 +1,15 @@
#
# Copyright(c) 2019-2021 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import time
import pytest
from api.cas import cli, casadm
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from core.test_run import TestRun
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
@pytest.mark.require_disk("cache_1", DiskTypeSet([DiskType.optane, DiskType.nand]))

8
test/functional/tests/fault_injection/test_cache_insert_error.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
@ -16,11 +17,12 @@ from api.cas.cache_config import (
from core.test_run import TestRun
from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan
from test_tools.device_mapper import ErrorDevice, DmTable
from test_tools.device_mapper import DmTable
from storage_devices.error_device import ErrorDevice
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine, ErrorFilter, VerifyMethod
from test_utils.os_utils import Udev
from test_utils.size import Size, Unit
from test_tools.udev import Udev
from type_def.size import Size, Unit
start_size = Size(512, Unit.Byte)
stop_size = Size(128, Unit.KibiByte)

5
test/functional/tests/fault_injection/test_error_device_as_cache.py
View File

@ -9,7 +9,8 @@ from time import sleep
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine, ErrorFilter
from test_tools.device_mapper import ErrorDevice, DmTable
from test_tools.device_mapper import DmTable
from storage_devices.error_device import ErrorDevice
from core.test_run import TestRun
from api.cas import casadm
from api.cas.cache_config import (
@ -21,7 +22,7 @@ from api.cas.cache_config import (
)
from storage_devices.disk import DiskTypeSet, DiskType
from test_utils.io_stats import IoStats
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
@pytest.mark.parametrizex("cache_line_size", CacheLineSize)

31
test/functional/tests/fault_injection/test_fault_injection_interrupts.py
View File

@ -11,12 +11,11 @@ from api.cas.cache_config import CacheMode, CleaningPolicy, CacheModeTrait
from api.cas.casadm_parser import wait_for_flushing
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools import fs_utils
from test_tools.dd import Dd
from test_tools.disk_utils import Filesystem
from test_utils import os_utils
from test_utils.os_utils import Udev, DropCachesMode
from test_utils.size import Size, Unit
from test_tools.fs_tools import Filesystem, create_random_test_file
from test_tools.os_tools import DropCachesMode, sync, drop_caches
from test_tools.udev import Udev
from type_def.size import Size, Unit
from tests.lazy_writes.recovery.recovery_tests_methods import compare_files
mount_point = "/mnt/cas"
@ -65,8 +64,8 @@ def test_interrupt_core_flush(cache_mode, filesystem):
test_file_md5sum_before = test_file.md5sum()
with TestRun.step("Get number of dirty data on exported object before interruption."):
os_utils.sync()
os_utils.drop_caches(DropCachesMode.ALL)
sync()
drop_caches(DropCachesMode.ALL)
core_dirty_blocks_before = core.get_dirty_blocks()
with TestRun.step("Start flushing core device."):
@ -149,8 +148,8 @@ def test_interrupt_cache_flush(cache_mode, filesystem):
test_file_md5sum_before = test_file.md5sum()
with TestRun.step("Get number of dirty data on exported object before interruption."):
os_utils.sync()
os_utils.drop_caches(DropCachesMode.ALL)
sync()
drop_caches(DropCachesMode.ALL)
cache_dirty_blocks_before = cache.get_dirty_blocks()
with TestRun.step("Start flushing cache."):
@ -238,8 +237,8 @@ def test_interrupt_core_remove(cache_mode, filesystem):
with TestRun.step(
"Get number of dirty data on exported object before core removal interruption"
):
os_utils.sync()
os_utils.drop_caches(DropCachesMode.ALL)
sync()
drop_caches(DropCachesMode.ALL)
cache_dirty_blocks_before = cache.get_dirty_blocks()
with TestRun.step("Unmount core"):
@ -342,7 +341,7 @@ def test_interrupt_cache_mode_switch_parametrized(cache_mode, stop_percentage):
core = cache.add_core(core_part)
with TestRun.step(f"Create test file in mount point of exported object"):
test_file = fs_utils.create_random_test_file(test_file_path, test_file_size)
test_file = create_random_test_file(test_file_path, test_file_size)
with TestRun.step("Calculate md5sum of test file"):
test_file_md5_before = test_file.md5sum()
@ -358,8 +357,8 @@ def test_interrupt_cache_mode_switch_parametrized(cache_mode, stop_percentage):
dd.run()
with TestRun.step("Get number of dirty data on exported object before interruption"):
os_utils.sync()
os_utils.drop_caches(DropCachesMode.ALL)
sync()
drop_caches(DropCachesMode.ALL)
cache_dirty_blocks_before = cache.get_dirty_blocks()
with TestRun.step("Start switching cache mode"):
@ -450,8 +449,8 @@ def test_interrupt_cache_stop(cache_mode, filesystem):
test_file = create_test_file()
with TestRun.step("Get number of dirty data on exported object before interruption."):
os_utils.sync()
os_utils.drop_caches(DropCachesMode.ALL)
sync()
drop_caches(DropCachesMode.ALL)
cache_dirty_blocks_before = cache.get_dirty_blocks()
with TestRun.step("Unmount core."):

6
test/functional/tests/fault_injection/test_fault_injection_many_to_one.py
View File

@ -12,8 +12,8 @@ from api.cas.core import Core
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from core.test_run import TestRun
from test_tools.dd import Dd
from test_utils.os_utils import Udev, sync
from test_utils.size import Size, Unit
from test_tools.udev import Udev
from type_def.size import Size, Unit
block_size = Size(1, Unit.Blocks4096)
@ -289,7 +289,7 @@ def test_one_core_fail_dirty():
with TestRun.step("Check if core device is really out of cache."):
output = str(casadm.list_caches().stdout.splitlines())
if core_part1.path in output:
TestRun.exception("The first core device should be unplugged!")
TestRun.LOGGER.exception("The first core device should be unplugged!")
with TestRun.step("Verify that I/O to the remaining cores does not insert to cache"):
dd_builder(cache_mode, core2, Size(100, Unit.MebiByte)).run()

13
test/functional/tests/fault_injection/test_fault_injection_opencas_load.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2020-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies
# SPDX-License-Identifier: BSD-3-Clause
#
@ -7,14 +8,12 @@ import pytest
from api.cas import casadm, casadm_parser, cli, cli_messages
from api.cas.cache_config import CacheMode, CleaningPolicy, CacheModeTrait
from test_tools.fs_tools import create_random_test_file
from test_tools.udev import Udev
from tests.lazy_writes.recovery.recovery_tests_methods import copy_file, compare_files
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools import fs_utils
from test_tools.dd import Dd
from test_tools.disk_utils import Filesystem
from test_utils import os_utils
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
mount_point = "/mnt/cas"
test_file_path = f"{mount_point}/test_file"
@ -48,7 +47,7 @@ def test_stop_no_flush_load_cache(cache_mode):
with TestRun.step(f"Create test file in mount point of exported object and check its md5 sum."):
test_file_size = Size(48, Unit.MebiByte)
test_file = fs_utils.create_random_test_file(test_file_path, test_file_size)
test_file = create_random_test_file(test_file_path, test_file_size)
test_file_md5_before = test_file.md5sum()
copy_file(source=test_file.full_path, target=core.path, size=test_file_size,
direct="oflag")
@ -101,5 +100,5 @@ def prepare():
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(2, Unit.GibiByte)])
core_part = core_dev.partitions[0]
os_utils.Udev.disable()
Udev.disable()
return cache_part, core_part

9
test/functional/tests/fault_injection/test_fault_injection_standby.py
View File

@ -5,22 +5,17 @@
#
import pytest
from collections import namedtuple
import random
from api.cas import casadm
from api.cas import dmesg
from api.cas.cli import casadm_bin
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_utils.size import Size, Unit
from api.cas.cli_messages import check_stderr_msg, missing_param, disallowed_param
from type_def.size import Size, Unit
from api.cas.cache_config import CacheLineSize, CacheMode
from api.cas.cli import standby_activate_cmd, standby_load_cmd
from api.cas.dmesg import get_md_section_size
from api.cas.ioclass_config import IoClass
from test_tools.dd import Dd
from test_utils.os_utils import sync
from test_tools.os_tools import sync
from test_utils.filesystem.file import File

6
test/functional/tests/fault_injection/test_fault_injection_standby_core.py
View File

@ -6,11 +6,11 @@
import pytest
from api.cas import cli, casadm
from api.cas import casadm
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from core.test_run import TestRun
from test_utils.size import Size, Unit
from api.cas.cache_config import CacheLineSize, CacheMode, CacheStatus
from type_def.size import Size, Unit
from api.cas.cache_config import CacheLineSize, CacheStatus
from api.cas.casadm_params import StatsFilter
from api.cas.casadm_parser import get_core_info_for_cache_by_path
from api.cas.core import CoreStatus, Core

16
test/functional/tests/fault_injection/test_fault_injection_with_mounted_core.py
View File

@ -9,11 +9,11 @@ import pytest
from api.cas import casadm, casadm_parser, cli, cli_messages
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools import fs_utils, disk_utils
from test_tools.disk_utils import Filesystem
from test_tools.disk_tools import get_block_size, create_partitions
from test_tools.fs_tools import Filesystem, create_random_test_file, check_if_file_exists
from test_utils.filesystem.file import File
from test_utils.filesystem.symlink import Symlink
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
mount_point, mount_point2 = "/mnt/cas", "/mnt/cas2"
test_file_path = f"{mount_point}/test_file"
@ -46,7 +46,7 @@ def test_load_cache_with_mounted_core():
core.mount(mount_point)
with TestRun.step(f"Create test file in mount point of exported object and check its md5 sum."):
test_file = fs_utils.create_random_test_file(test_file_path)
test_file = create_random_test_file(test_file_path)
test_file_md5_before = test_file.md5sum()
with TestRun.step("Unmount core device."):
@ -110,8 +110,8 @@ def test_stop_cache_with_mounted_partition():
core2 = cache.add_core(core_dev2)
with TestRun.step("Create partitions on one exported object."):
core.block_size = Size(disk_utils.get_block_size(core.get_device_id()))
disk_utils.create_partitions(core, 2 * [Size(4, Unit.GibiByte)])
core.block_size = Size(get_block_size(core.get_device_id()))
create_partitions(core, 2 * [Size(4, Unit.GibiByte)])
fs_part = core.partitions[0]
with TestRun.step("Create xfs filesystems on one exported object partition "
@ -124,7 +124,7 @@ def test_stop_cache_with_mounted_partition():
core2.mount(mount_point2)
with TestRun.step("Ensure /etc/mtab exists."):
if not fs_utils.check_if_file_exists("/etc/mtab"):
if not check_if_file_exists("/etc/mtab"):
Symlink.create_symlink("/proc/self/mounts", "/etc/mtab")
with TestRun.step("Try to remove the core with partitions from cache."):
@ -185,7 +185,7 @@ def test_stop_cache_with_mounted_partition_no_mtab():
core.mount(mount_point)
with TestRun.step("Move /etc/mtab"):
if fs_utils.check_if_file_exists("/etc/mtab"):
if check_if_file_exists("/etc/mtab"):
mtab = File("/etc/mtab")
else:
mtab = Symlink.create_symlink("/proc/self/mounts", "/etc/mtab")

6
test/functional/tests/fault_injection/test_fault_power_reboot.py
View File

@ -1,17 +1,17 @@
#
# Copyright(c) 2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies
# SPDX-License-Identifier: BSD-3-Clause
#
import time
import pytest
from api.cas import casadm, cli, cli_messages
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_utils.output import CmdException
from test_utils.size import Size, Unit
from connection.utils.output import CmdException
from type_def.size import Size, Unit
log_path = "/var/log/opencas.log"
wait_long_time = 180

5
test/functional/tests/fault_injection/test_max_io_greater_in_core.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import pytest
@ -9,8 +10,8 @@ from api.cas.cache_config import CacheMode, CacheLineSize
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet
from test_tools.dd import Dd
from test_utils.os_utils import Udev
from test_utils.size import Size, Unit
from test_tools.udev import Udev
from type_def.size import Size, Unit
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))

8
test/functional/tests/fault_injection/test_primary_device_error.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
@ -7,7 +8,8 @@ import pytest
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine, ErrorFilter, VerifyMethod
from test_tools.device_mapper import ErrorDevice, DmTable
from test_tools.device_mapper import DmTable
from storage_devices.error_device import ErrorDevice
from core.test_run import TestRun
from api.cas import casadm
from api.cas.cache_config import (
@ -18,8 +20,8 @@ from api.cas.cache_config import (
CleaningPolicy,
)
from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan
from test_utils.os_utils import Udev
from test_utils.size import Size, Unit
from test_tools.udev import Udev
from type_def.size import Size, Unit
@pytest.mark.parametrizex("cache_mode", CacheMode.without_traits(CacheModeTrait.LazyWrites))

4
test/functional/tests/fault_injection/test_soft_hot_plug_device.py
View File

@ -16,8 +16,8 @@ from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine, ErrorFilter
from test_utils.output import CmdException
from test_utils.size import Size, Unit
from connection.utils.output import CmdException
from type_def.size import Size, Unit
@pytest.mark.parametrizex("cache_mode", CacheMode)

6
test/functional/tests/fault_injection/test_standby_activate_different_cache_path.py
View File

@ -11,12 +11,12 @@ from api.cas.cli import standby_activate_cmd
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_utils.filesystem.symlink import Symlink
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
from api.cas.cache_config import CacheLineSize
from api.cas.cache import CacheStatus
from test_tools.dd import Dd
from test_tools.fs_utils import check_if_symlink_exists
from test_utils.os_utils import sync
from test_tools.fs_tools import check_if_symlink_exists
from test_tools.os_tools import sync
@pytest.mark.CI

2
test/functional/tests/fault_injection/test_standby_activate_the_same_cache_path.py
View File

@ -10,7 +10,7 @@ from api.cas import casadm
from api.cas.casadm_parser import get_caches
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
from api.cas.cache_config import CacheStatus

7
test/functional/tests/incremental_load/test_core_pool.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
@ -9,9 +10,9 @@ from api.cas import casadm
from api.cas.core import CoreStatus
from core.test_run import TestRun
from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan
from test_tools.disk_utils import Filesystem
from test_utils.output import CmdException
from test_utils.size import Size, Unit
from test_tools.fs_tools import Filesystem
from connection.utils.output import CmdException
from type_def.size import Size, Unit
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))

2
test/functional/tests/incremental_load/test_inactive_cores.py
View File

@ -13,7 +13,7 @@ from api.cas.core import Core, CoreStatus
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools.dd import Dd
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))

11
test/functional/tests/incremental_load/test_incremental_load.py
View File

@ -25,14 +25,15 @@ from api.cas.statistics import CacheStats
from core.test_run import TestRun
from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan
from test_tools.dd import Dd
from test_tools.disk_utils import Filesystem
from test_tools.fs_tools import Filesystem
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import IoEngine, ReadWrite
from test_utils.filesystem.file import File
from test_utils.os_utils import Udev, sync
from test_utils.output import CmdException
from test_utils.size import Size, Unit
from test_utils.time import Time
from test_tools.os_tools import sync
from test_tools.udev import Udev
from connection.utils.output import CmdException
from type_def.size import Size, Unit
from type_def.time import Time
@pytest.mark.CI

2
test/functional/tests/incremental_load/test_udev.py
View File

@ -16,7 +16,7 @@ from api.cas.init_config import InitConfig
from core.test_run import TestRun
from storage_devices.disk import DiskTypeSet, DiskType
from storage_devices.raid import RaidConfiguration, Raid, Level, MetadataVariant
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
@pytest.mark.os_dependent

6
test/functional/tests/initialize/test_clean_reboot.py
View File

@ -12,10 +12,10 @@ from api.cas.cache_config import CacheMode
from core.test_run import TestRun
from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan
from test_tools.dd import Dd
from test_tools.disk_utils import Filesystem
from test_tools.fs_tools import Filesystem
from test_utils.filesystem.file import File
from test_utils.os_utils import drop_caches, DropCachesMode, sync
from test_utils.size import Size, Unit
from test_tools.os_tools import drop_caches, DropCachesMode, sync
from type_def.size import Size, Unit
@pytest.mark.os_dependent

6
test/functional/tests/initialize/test_initialize_load.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
@ -12,10 +13,11 @@ from api.cas.cache_config import (CleaningPolicy,
FlushParametersAlru,
Time,
FlushParametersAcp)
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import *
from test_utils.size import Size, Unit
from test_tools.fio.fio_param import IoEngine, ReadWrite, CpusAllowedPolicy
from type_def.size import Size, Unit
@pytest.mark.parametrizex("cores_amount", [1, 4])

18
test/functional/tests/initialize/test_initialize_runlevel.py
View File

@ -1,22 +1,24 @@
#
# Copyright(c) 2020-2021 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import os
import pytest
import time
from api.cas import casadm, casadm_parser
from api.cas.cache_config import CacheMode
from api.cas.init_config import InitConfig
from core.test_run import TestRun
from storage_devices.disk import DiskTypeLowerThan, DiskTypeSet, DiskType
from test_tools.disk_utils import Filesystem
from test_tools.fs_tools import Filesystem
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine
from test_utils import os_utils
from test_utils.os_utils import Runlevel
from test_utils.size import Size, Unit
from test_tools.os_tools import sync, drop_caches
from test_tools.runlevel import Runlevel, change_runlevel
from type_def.size import Size, Unit
mount_point = "/mnt/test"
@ -36,7 +38,7 @@ def test_init_reboot_runlevels(runlevel, cache_mode):
- Cache should be loaded successfully after reboot.
"""
with TestRun.step(f"Set runlevel to {runlevel.value}."):
os_utils.change_runlevel(runlevel)
change_runlevel(runlevel)
with TestRun.step("Prepare CAS device."):
cache_disk = TestRun.disks['cache']
@ -65,8 +67,8 @@ def test_init_reboot_runlevels(runlevel, cache_mode):
.file_size(Size(30, Unit.GibiByte))
fio.run_in_background()
os_utils.sync()
os_utils.drop_caches()
sync()
drop_caches()
time.sleep(10)
TestRun.executor.run_expect_success("pgrep fio")
@ -86,5 +88,5 @@ def test_init_reboot_runlevels(runlevel, cache_mode):
with TestRun.step("Stop cache and set default runlevel."):
if len(caches) != 0:
casadm.stop_all_caches()
os_utils.change_runlevel(Runlevel.runlevel3)
change_runlevel(Runlevel.runlevel3)
TestRun.executor.reboot()

4
test/functional/tests/initialize/test_initialize_status.py
View File

@ -10,7 +10,7 @@ import pytest
from api.cas import cas_module, casctl
from api.cas.cas_module import CasModule
from core.test_run import TestRun
from test_utils import os_utils
from test_tools.os_tools import load_kernel_module
@pytest.mark.os_dependent
@ -46,5 +46,5 @@ def test_init_status():
TestRun.fail("CAS management device is present in OS when CAS module is not loaded.")
with TestRun.step("Load CAS modules and start CAS service"):
os_utils.load_kernel_module(CasModule.cache.value)
load_kernel_module(CasModule.cache.value)
casctl.start()

5
test/functional/tests/initialize/test_negative_load.py
View File

@ -1,14 +1,15 @@
#
# Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import pytest
from api.cas import casadm, casadm_parser, cli, cli_messages
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))

7
test/functional/tests/initialize/test_recovery.py
View File

@ -14,11 +14,10 @@ from api.cas.cli_messages import check_stderr_msg, stop_cache_errors
from core.test_run import TestRun
from storage_devices.disk import DiskTypeLowerThan, DiskTypeSet, DiskType
from test_tools.dd import Dd
from test_tools.disk_utils import Filesystem, mount
from test_tools.fs_utils import check_if_file_exists
from test_tools.fs_tools import check_if_file_exists, Filesystem
from test_utils.filesystem.file import File
from test_utils.os_utils import sync
from test_utils.size import Size, Unit
from test_tools.os_tools import sync
from type_def.size import Size, Unit
mount_point = "/mnt/cas"

2
test/functional/tests/initialize/test_simulation_startup.py
View File

@ -13,7 +13,7 @@ from api.cas.core import CoreStatus
from api.cas.init_config import InitConfig
from core.test_run import TestRun
from storage_devices.disk import DiskTypeLowerThan, DiskTypeSet, DiskType
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
@pytest.mark.CI

21
test/functional/tests/initialize/test_startup_init_config.py
View File

@ -14,11 +14,12 @@ from api.cas.init_config import InitConfig
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_utils.filesystem.file import File
from test_tools.disk_utils import Filesystem
from test_utils import fstab
from test_tools.fs_tools import Filesystem
from test_tools import fstab
from test_tools.dd import Dd
from test_utils.size import Unit, Size
from test_utils.os_utils import sync, Udev
from type_def.size import Unit, Size
from test_tools.os_tools import sync
from test_tools.udev import Udev
from test_utils.emergency_escape import EmergencyEscape
from api.cas.cas_service import set_cas_service_timeout, clear_cas_service_timeout
@ -221,7 +222,7 @@ def test_cas_startup_lazy():
core_pool_paths = {c["device_path"] for c in core_pool_list}
if core_pool_paths != expected_core_pool_paths:
TestRun.error(
TestRun.LOGGER.error(
f"Expected the following devices in core pool "
f"{expected_core_pool_paths}. Got {core_pool_paths}"
)
@ -230,7 +231,7 @@ def test_cas_startup_lazy():
caches_paths = {c["device_path"] for c in caches_list}
if caches_paths != expected_caches_paths:
TestRun.error(
TestRun.LOGGER.error(
f"Expected the following devices as caches "
f"{expected_caches_paths}. Got {caches_paths}"
)
@ -239,7 +240,7 @@ def test_cas_startup_lazy():
cores_paths = {c["device_path"] for c in cores_list}
if cores_paths != expected_cores_paths:
TestRun.error(
TestRun.LOGGER.error(
f"Expected the following devices as cores "
f"{expected_caches_paths}. Got {cores_paths}"
)
@ -446,14 +447,14 @@ def test_failover_config_startup():
cores_list = get_cas_devices_dict()["cores"].values()
if len(core_pool_list) != 0:
TestRun.error(f"No cores expected in core pool. Got {core_pool_list}")
TestRun.LOGGER.error(f"No cores expected in core pool. Got {core_pool_list}")
else:
TestRun.LOGGER.info("Core pool is ok")
expected_caches_paths = set([active_cache_path, standby_cache_path])
caches_paths = {c["device"] for c in caches_list}
if caches_paths != expected_caches_paths:
TestRun.error(
TestRun.LOGGER.error(
f"Expected the following devices as caches "
f"{expected_caches_paths}. Got {caches_paths}"
)
@ -463,7 +464,7 @@ def test_failover_config_startup():
expected_core_paths = set([active_core_path])
cores_paths = {c["device"] for c in cores_list}
if cores_paths != expected_core_paths:
TestRun.error(
TestRun.LOGGER.error(
f"Expected the following devices as cores "
f"{expected_core_paths}. Got {cores_paths}"
)

11
test/functional/tests/io/test_io_engines.py
View File

@ -1,21 +1,22 @@
#
# Copyright(c) 2020-2021 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
from datetime import timedelta
import pytest
from datetime import timedelta
from api.cas import casadm
from api.cas.cache_config import CacheMode
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools.disk_utils import Filesystem
from test_tools.fs_tools import Filesystem
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import IoEngine, CpusAllowedPolicy, ReadWrite
from test_utils.os_utils import get_dut_cpu_physical_cores
from test_utils.size import Size, Unit
from test_tools.os_tools import get_dut_cpu_physical_cores
from type_def.size import Size, Unit
mount_point = "/mnt/test"
runtime = timedelta(minutes=15)

4
test/functional/tests/io/test_remove_device_during_io.py
View File

@ -1,10 +1,10 @@
#
# Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import time
import pytest
from datetime import timedelta
@ -13,7 +13,7 @@ from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from core.test_run import TestRun
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))

8
test/functional/tests/io/test_write_fetch.py
View File

@ -1,11 +1,13 @@
#
# Copyright(c) 2020-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import os
import os
import pytest
import uuid
from api.cas import casadm
from api.cas.cache_config import CacheMode, CacheLineSize, CacheModeTrait
from core.test_run import TestRun
@ -13,8 +15,8 @@ from storage_devices.disk import DiskTypeSet, DiskTypeLowerThan, DiskType
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import IoEngine, ReadWrite
from test_utils.filesystem.symlink import Symlink
from test_utils.os_utils import Udev
from test_utils.size import Size, Unit
from test_tools.udev import Udev
from type_def.size import Size, Unit
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))

26
test/functional/tests/io/trim/test_trim.py
View File

@ -3,23 +3,23 @@
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import os
import posixpath
import time
import pytest
from test_tools.os_tools import sync, drop_caches
from test_tools.udev import Udev
from api.cas import casadm
from api.cas.cache_config import CacheMode, CacheModeTrait, CleaningPolicy, SeqCutOffPolicy
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet
from test_tools import fs_utils
from test_tools.blktrace import BlkTrace, BlkTraceMask, RwbsKind
from test_tools.disk_utils import Filesystem, check_if_device_supports_trim
from test_tools.disk_tools import check_if_device_supports_trim
from test_tools.fs_tools import Filesystem, create_random_test_file
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine
from test_utils import os_utils
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
from api.cas.dmesg import get_metadata_size_on_device
@ -129,7 +129,7 @@ def test_trim_propagation():
raise Exception("Core device doesn't support discards")
with TestRun.step(f"Disable udev"):
os_utils.Udev.disable()
Udev.disable()
with TestRun.step(f"Prepare cache instance in WB with one core"):
cache = casadm.start_cache(cache_dev, CacheMode.WB, force=True)
@ -161,7 +161,7 @@ def test_trim_propagation():
with TestRun.step("Power cycle"):
power_control = TestRun.plugin_manager.get_plugin("power_control")
power_control.power_cycle()
os_utils.Udev.disable()
Udev.disable()
with TestRun.step("Load cache"):
cache = casadm.start_cache(cache_dev, load=True)
@ -235,7 +235,7 @@ def test_trim_device_discard_support(
core.mount(mount_point, ["discard"])
with TestRun.step("Create random file."):
test_file = fs_utils.create_random_test_file(
test_file = create_random_test_file(
posixpath.join(mount_point, "test_file"), core_dev.size * 0.2
)
occupancy_before = core.get_occupancy()
@ -246,8 +246,8 @@ def test_trim_device_discard_support(
with TestRun.step("Remove file."):
test_file.remove()
os_utils.sync()
os_utils.drop_caches()
sync()
drop_caches()
time.sleep(5)
with TestRun.step("Ensure that discards were detected by blktrace on proper devices."):
@ -315,8 +315,8 @@ def compare_properties(value, expected_value, property_name):
def stop_monitoring_and_check_discards(blktraces, discard_support):
time.sleep(60)
os_utils.sync()
os_utils.drop_caches()
sync()
drop_caches()
time.sleep(5)
discard_flag = RwbsKind.D # Discard

70
test/functional/tests/io/trim/test_trim_stress.py
View File

@ -1,17 +1,20 @@
#
# Copyright(c) 2020-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import os
from datetime import timedelta
import pytest
from datetime import timedelta
from api.cas import casadm
from api.cas.cache_config import CacheMode, CacheLineSize
from core.test_run import TestRun
from storage_devices.disk import DiskTypeSet, DiskType
from test_tools.disk_utils import Filesystem
from test_utils.size import Size, Unit
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine
from type_def.size import Size, Unit
@pytest.mark.os_dependent
@ -19,28 +22,23 @@ from test_utils.size import Size, Unit
@pytest.mark.require_disk("core", DiskTypeSet([DiskType.optane, DiskType.nand, DiskType.sata]))
@pytest.mark.parametrizex("cache_mode", CacheMode)
@pytest.mark.parametrizex("cache_line_size", CacheLineSize)
@pytest.mark.require_plugin("vdbench")
def test_trim_stress(cache_mode, cache_line_size):
"""
title: Trim support on cache devices in different cache modes stress test.
description: |
Stress test validating the ability of CAS to handle trim requests in different modes,
on different filesystem types.
Stress test validating the ability of CAS to handle trim requests in different modes.
pass_criteria:
- No kernel bug.
- Cache should still work correctly.
"""
cores_number = 4
mount_point = "/mnt"
with TestRun.step("Prepare devices."):
cache_disk = TestRun.disks['cache']
cache_disk.create_partitions([Size(1, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_disk = TestRun.disks['core']
core_disk.create_partitions([Size(1, Unit.GibiByte)] * cores_number)
core_disk.create_partitions([Size(1, Unit.GibiByte)] * 3)
core_devices = core_disk.partitions
with TestRun.step("Start cache and add cores."):
@ -49,41 +47,35 @@ def test_trim_stress(cache_mode, cache_line_size):
for d in core_devices:
cores.append(cache.add_core(d))
with TestRun.step("Create filesystem and mount CAS devices."):
directories = []
for counter, core in enumerate(cores):
core.create_filesystem(Filesystem(counter % len(Filesystem)))
mount_dir = os.path.join(mount_point, str(counter + 1))
directories.append(mount_dir)
core.mount(mount_dir, ["discard"])
with TestRun.step("Run I/O workload."):
for _ in TestRun.iteration(range(1, 7)):
run_vdbench(directories)
for _ in TestRun.iteration(range(1, 6)):
run_fio([core.path for core in cores])
with TestRun.step("Stop CAS."):
for c in cores:
c.unmount()
c.remove_core()
cache.stop()
def run_vdbench(directories):
vdbench = TestRun.plugin_manager.get_plugin('vdbench')
config = f"data_errors=1,validate=yes,messagescan=no,create_anchors=yes\n" \
f"fsd=default,depth=4,width=5,files=10,sizes=" \
f"(1k,10,2k,10,4k,10,8k,10,16k,10,32k,10,64k,10,128k,10,256k,10,512k,10)\n"
def run_fio(paths):
block_sizes = [f"{2 ** n}k" for n in range(0, 10)]
for i, dir in enumerate(directories):
config += f"fsd=fsd{i},anchor={dir}\n"
fio = (
Fio().create_command()
.io_engine(IoEngine.libaio)
.io_depth(16)
.bs_split(":".join([f"{bs}/{100 // len(block_sizes)}" for bs in block_sizes]))
.time_based()
.run_time(timedelta(minutes=10))
.trim_verify_zero()
.verify_fatal()
)
config += f"\nfwd=fwd1,fsd=fsd*," \
f"fileio=(random),fileselect=random,threads=32," \
f"xfersizes=(512,10,1k,10,2k,10,4k,10,8k,10,16k,10,32k,10,64k,10," \
f"128k,10,256k,10)\nrd=rd1,fwd=fwd*,fwdrate=max,format=yes," \
f"interval=5,operations=(read,write,open,close,getattr,setattr)"
vdbench.create_config(config, run_time=timedelta(minutes=5))
if vdbench.run():
TestRun.LOGGER.info("VDBench finished with status zero.")
else:
TestRun.LOGGER.error("VDBench finished with non-zero status.")
for path, rw in zip(paths, [ReadWrite.trim, ReadWrite.randtrim, ReadWrite.trimwrite]):
(
fio.add_job(path + rw.name)
.file_name(path)
.read_write(rw)
)
fio.run(fio_timeout=timedelta(minutes=20))

8
test/functional/tests/io_class/io_class_common.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
@ -16,9 +17,10 @@ from core.test_run import TestRun
from test_tools.dd import Dd
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine
from test_utils.os_utils import Udev, sync
from test_utils.os_utils import drop_caches, DropCachesMode
from test_utils.size import Size, Unit
from test_tools.os_tools import sync
from test_tools.udev import Udev
from test_tools.os_tools import drop_caches, DropCachesMode
from type_def.size import Size, Unit
ioclass_config_path = "/etc/opencas/ioclass.conf"
template_config_path = "/etc/opencas/ioclass-config.csv"

10
test/functional/tests/io_class/test_io_class_cli.py
View File

@ -1,9 +1,9 @@
#
# Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import pytest
from api.cas import casadm, ioclass_config
@ -12,8 +12,8 @@ from api.cas.casadm_params import OutputFormat
from api.cas.ioclass_config import IoClass
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools import fs_utils
from test_utils.size import Size, Unit
from test_tools.fs_tools import read_file, remove
from type_def.size import Size, Unit
ioclass_config_path = "/tmp/opencas_ioclass.conf"
@ -61,7 +61,7 @@ def test_io_class_export_configuration():
f"{casadm.list_io_classes_cmd(str(cache.cache_id), OutputFormat.csv.name)}"
f" > {saved_config_path}"
)
csv = fs_utils.read_file(saved_config_path)
csv = read_file(saved_config_path)
if not IoClass.compare_ioclass_lists(IoClass.csv_to_list(csv), random_list):
TestRun.LOGGER.error(
"Exported configuration does not match expected\n"
@ -98,7 +98,7 @@ def test_io_class_export_configuration():
)
with TestRun.LOGGER.step(f"Test cleanup"):
fs_utils.remove(saved_config_path)
remove(saved_config_path)
def prepare(cache_mode: CacheMode = None):

6
test/functional/tests/io_class/test_io_class_core_id.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
@ -10,8 +11,9 @@ from api.cas.cache_config import CacheMode, CleaningPolicy, SeqCutOffPolicy
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools.dd import Dd
from test_utils.os_utils import sync, Udev, drop_caches
from test_utils.size import Unit, Size
from test_tools.os_tools import sync, drop_caches
from test_tools.udev import Udev
from type_def.size import Unit, Size
dd_bs = Size(1, Unit.Blocks4096)
dd_count = 1230

21
test/functional/tests/io_class/test_io_class_directory.py
View File

@ -1,24 +1,25 @@
#
# Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import random
import time
from datetime import datetime
import pytest
from datetime import datetime
from api.cas import ioclass_config, casadm
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools import fs_utils
from test_tools.dd import Dd
from test_tools.disk_utils import Filesystem
from test_tools.fs_tools import Filesystem, create_directory, remove
from test_utils.filesystem.directory import Directory
from test_utils.filesystem.file import File
from test_utils.os_utils import drop_caches, DropCachesMode, sync, Udev
from test_utils.size import Size, Unit
from test_tools.os_tools import drop_caches, DropCachesMode, sync
from test_tools.udev import Udev
from type_def.size import Size, Unit
from tests.io_class.io_class_common import mountpoint, prepare, ioclass_config_path
@ -50,14 +51,14 @@ def test_ioclass_directory_depth(filesystem):
sync()
with TestRun.step(f"Create the base directory: {base_dir_path}."):
fs_utils.create_directory(base_dir_path)
create_directory(base_dir_path)
with TestRun.step(f"Create a nested directory."):
nested_dir_path = base_dir_path
random_depth = random.randint(40, 80)
for i in range(random_depth):
nested_dir_path += f"/dir_{i}"
fs_utils.create_directory(path=nested_dir_path, parents=True)
create_directory(path=nested_dir_path, parents=True)
# Test classification in nested dir by reading a previously unclassified file
with TestRun.step("Create the first file in the nested directory."):
@ -175,7 +176,7 @@ def test_ioclass_directory_file_operations(filesystem):
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
with TestRun.step(
f"Prepare {filesystem.name} filesystem " f"and mounting {core.path} at {mountpoint}."
f"Prepare {filesystem.name} filesystem and mounting {core.path} at {mountpoint}."
):
core.create_filesystem(fs_type=filesystem)
core.mount(mount_point=mountpoint)
@ -397,7 +398,7 @@ def test_ioclass_directory_dir_operations(filesystem):
)
with TestRun.step(f"Remove {classified_dir_path_2}."):
fs_utils.remove(path=classified_dir_path_2, force=True, recursive=True)
remove(path=classified_dir_path_2, force=True, recursive=True)
sync()
drop_caches(DropCachesMode.ALL)

9
test/functional/tests/io_class/test_io_class_eviction_pinned_after_unpin.py
View File

@ -1,19 +1,22 @@
#
# Copyright(c) 2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
from collections import namedtuple
import pytest
from collections import namedtuple
from api.cas.ioclass_config import default_config_file_path
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from core.test_run import TestRun
from api.cas.cache_config import CacheMode, CleaningPolicy, SeqCutOffPolicy
from .io_class_common import ioclass_config, get_io_class_occupancy, run_io_dir
from api.cas import casadm
from test_utils.os_utils import sync, Udev, drop_caches
from test_utils.size import Unit, Size
from test_tools.os_tools import sync, drop_caches
from test_tools.udev import Udev
from type_def.size import Unit, Size
cache_size = Size(100, Unit.MiB)

10
test/functional/tests/io_class/test_io_class_eviction_priority.py
View File

@ -1,16 +1,16 @@
#
# Copyright(c) 2020-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import pytest
from collections import namedtuple
from math import isclose
import pytest
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools import fs_utils
from test_tools.disk_utils import Filesystem
from test_tools.fs_tools import Filesystem, create_directory
from .io_class_common import *
@ -50,7 +50,7 @@ def test_io_class_eviction_priority():
IoclassConfig(4, 1, 1.00, f"{mountpoint}/D"),
]
for io_class in io_classes:
fs_utils.create_directory(io_class.dir_path, parents=True)
create_directory(io_class.dir_path, parents=True)
with TestRun.step("Adding io classes for all dirs"):
for io_class in io_classes:

8
test/functional/tests/io_class/test_io_class_file.py
View File

@ -1,20 +1,20 @@
#
# Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import random
import pytest
from api.cas import ioclass_config, casadm
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools.dd import Dd
from test_tools.disk_utils import Filesystem
from test_tools.fs_tools import Filesystem
from test_utils.filesystem.file import File
from test_utils.os_utils import sync, DropCachesMode, drop_caches
from test_utils.size import Size, Unit
from test_tools.os_tools import sync, DropCachesMode, drop_caches
from type_def.size import Size, Unit
from tests.io_class.io_class_common import mountpoint, prepare, ioclass_config_path

9
test/functional/tests/io_class/test_io_class_lvm_on_cas.py
View File

@ -12,11 +12,10 @@ from api.cas.ioclass_config import IoClass
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from storage_devices.lvm import Lvm, LvmConfiguration
from test_tools import fs_utils
from test_tools.disk_utils import Filesystem
from test_tools.fs_tools import Filesystem, read_file, remove
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
from tests.volumes.common import lvm_filters
mount_point = "/mnt/"
@ -62,7 +61,7 @@ def test_io_class_lvm_on_cas():
lvm.mount(mount_point)
with TestRun.step("Prepare and load IO class config."):
io_classes = IoClass.csv_to_list(fs_utils.read_file("/etc/opencas/ioclass-config.csv"))
io_classes = IoClass.csv_to_list(read_file("/etc/opencas/ioclass-config.csv"))
# remove two firs elements/lines: unclassified and metadata
io_classes.pop(1)
io_classes.pop(0)
@ -123,7 +122,7 @@ def test_io_class_lvm_on_cas():
else:
file_size = Size(1100, Unit.MebiByte)
fs_utils.remove(io_target)
remove(io_target)
with TestRun.step("Remove LVMs."):
TestRun.executor.run(f"umount {mount_point}")

19
test/functional/tests/io_class/test_io_class_occupancy.py
View File

@ -4,21 +4,20 @@
# SPDX-License-Identifier: BSD-3-Clause
#
import pytest
from collections import namedtuple
from math import isclose
import pytest
from api.cas import ioclass_config, casadm
from api.cas.cache_config import CacheMode, CacheLineSize
from api.cas.ioclass_config import IoClass, default_config_file_path
from api.cas.statistics import IoClassUsageStats
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools import fs_utils
from test_tools.disk_utils import Filesystem
from test_utils.os_utils import sync, Udev
from test_utils.size import Unit, Size
from test_tools.fs_tools import Filesystem, create_directory
from test_tools.os_tools import sync
from test_tools.udev import Udev
from type_def.size import Unit, Size
from tests.io_class.io_class_common import (
prepare,
mountpoint,
@ -69,7 +68,7 @@ def test_io_class_occupancy_directory_write(io_size_multiplication, cache_mode):
]
for io_class in io_classes:
fs_utils.create_directory(io_class.dir_path, parents=True)
create_directory(io_class.dir_path, parents=True)
with TestRun.step("Remove old ioclass config"):
ioclass_config.remove_ioclass_config()
@ -199,7 +198,7 @@ def test_io_class_occupancy_directory_read(io_size_multiplication):
]
for io_class in io_classes:
fs_utils.create_directory(io_class.dir_path, parents=True)
create_directory(io_class.dir_path, parents=True)
with TestRun.step(
f"In each directory create file with size of {io_size_multiplication} "
@ -330,7 +329,7 @@ def test_ioclass_occupancy_sum_cache():
]
for io_class in io_classes:
fs_utils.create_directory(io_class.dir_path, parents=True)
create_directory(io_class.dir_path, parents=True)
with TestRun.step("Remove old ioclass config"):
ioclass_config.remove_ioclass_config()

13
test/functional/tests/io_class/test_io_class_occupancy_load.py
View File

@ -1,21 +1,22 @@
#
# Copyright(c) 2020-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import pytest
from collections import namedtuple
from math import isclose
import pytest
from api.cas import ioclass_config, casadm
from api.cas.cache_config import CacheMode, CacheLineSize
from api.cas.ioclass_config import IoClass, default_config_file_path
from storage_devices.device import Device
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools import fs_utils
from test_tools.disk_utils import Filesystem
from test_utils.os_utils import sync, Udev
from test_tools.fs_tools import Filesystem, create_directory
from test_tools.os_tools import sync
from test_tools.udev import Udev
from tests.io_class.io_class_common import (
prepare,
mountpoint,
@ -65,7 +66,7 @@ def test_ioclass_occupancy_load():
]
for io_class in io_classes:
fs_utils.create_directory(io_class.dir_path, parents=True)
create_directory(io_class.dir_path, parents=True)
with TestRun.step("Remove old ioclass config"):
ioclass_config.remove_ioclass_config()

17
test/functional/tests/io_class/test_io_class_occupancy_repart.py
View File

@ -1,22 +1,23 @@
#
# Copyright(c) 2020-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import pytest
from collections import namedtuple
from math import isclose
import pytest
from api.cas import ioclass_config, casadm
from api.cas.cache_config import CacheMode, CacheLineSize
from api.cas.ioclass_config import IoClass, default_config_file_path
from core.test_run_utils import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools import fs_utils
from test_tools.disk_utils import Filesystem
from test_utils.os_utils import sync, Udev
from test_utils.size import Unit
from test_tools.fs_tools import Filesystem, create_directory, move
from test_tools.os_tools import sync
from test_tools.udev import Udev
from type_def.size import Unit
from tests.io_class.io_class_common import (
prepare,
mountpoint,
@ -67,7 +68,7 @@ def test_ioclass_repart(io_class_size_multiplication):
]
for io_class in io_classes:
fs_utils.create_directory(io_class.dir_path, parents=True)
create_directory(io_class.dir_path, parents=True)
with TestRun.step("Remove old io class config"):
ioclass_config.remove_ioclass_config()
@ -123,7 +124,7 @@ def test_ioclass_repart(io_class_size_multiplication):
with TestRun.step("Force repart - move files to created directories and read theirs contents"):
for i, io_class in enumerate(io_classes):
fs_utils.move(source=f"{mountpoint}/{i}", destination=io_class.dir_path)
move(source=f"{mountpoint}/{i}", destination=io_class.dir_path)
run_io_dir_read(f"{io_class.dir_path}/{i}")
with TestRun.step("Check if each ioclass reached it's occupancy limit"):

12
test/functional/tests/io_class/test_io_class_occupancy_resize.py
View File

@ -1,9 +1,11 @@
#
# Copyright(c) 2020-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import pytest
from recordclass import recordclass
from api.cas import ioclass_config, casadm
@ -11,10 +13,10 @@ from api.cas.cache_config import CacheMode, CacheLineSize
from api.cas.ioclass_config import IoClass, default_config_file_path
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools import fs_utils
from test_tools.disk_utils import Filesystem
from test_utils.os_utils import sync, Udev
from test_utils.size import Unit
from test_tools.fs_tools import Filesystem, create_directory
from test_tools.os_tools import sync
from test_tools.udev import Udev
from type_def.size import Unit
from tests.io_class.io_class_common import (
mountpoint,
prepare,
@ -54,7 +56,7 @@ def test_ioclass_resize(cache_line_size, new_occupancy):
IoclassConfig = recordclass("IoclassConfig", "id eviction_prio max_occupancy dir_path")
io_class = IoclassConfig(2, 3, 0.10, f"{mountpoint}/A")
fs_utils.create_directory(io_class.dir_path, parents=True)
create_directory(io_class.dir_path, parents=True)
with TestRun.step("Remove old ioclass config"):
ioclass_config.remove_ioclass_config()

12
test/functional/tests/io_class/test_io_class_pinning_eviction.py
View File

@ -1,19 +1,19 @@
#
# Copyright(c) 2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import pytest
from collections import namedtuple
from math import isclose
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from api.cas.ioclass_config import IoClass, default_config_file_path
from test_tools import fs_utils
from test_tools.disk_utils import Filesystem
from test_tools.fs_tools import Filesystem, create_directory
from core.test_run import TestRun
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
from .io_class_common import (
prepare,
mountpoint,
@ -58,7 +58,7 @@ def test_io_class_pinning_eviction():
]
for io_class in io_classes:
fs_utils.create_directory(io_class.dir_path, parents=True)
create_directory(io_class.dir_path, parents=True)
with TestRun.step("Remove old config"):
ioclass_config.remove_ioclass_config()
@ -152,7 +152,7 @@ def test_pinned_ioclasses_eviction():
]
for io_class in io_classes:
fs_utils.create_directory(io_class.dir_path, parents=True)
create_directory(io_class.dir_path, parents=True)
with TestRun.step("Remove old config"):
ioclass_config.remove_ioclass_config()

3
test/functional/tests/io_class/test_io_class_preserve_config.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
@ -9,7 +10,7 @@ from api.cas import casadm, ioclass_config
from api.cas.ioclass_config import IoClass
from core.test_run_utils import TestRun
from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan
from test_utils.size import Size, Unit
from type_def.size import Size, Unit
from tests.io_class.io_class_common import (
compare_io_classes_list,
generate_and_load_random_io_class_config,

17
test/functional/tests/io_class/test_io_class_prevent_wrong_configuration.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
@ -9,9 +10,9 @@ from api.cas import ioclass_config, cli_messages
from api.cas.ioclass_config import IoClass
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools import fs_utils
from test_utils.output import CmdException
from test_utils.size import Unit, Size
from connection.utils.output import CmdException
from test_tools.fs_tools import read_file, write_file
from type_def.size import Unit, Size
from tests.io_class.io_class_common import prepare, ioclass_config_path
headerless_configuration = "1,unclassified,22,1.00"
@ -86,7 +87,7 @@ def test_io_class_prevent_wrong_configuration():
f"{IoClass.default_header()}\n{loaded_io_classes_str}"
)
config_io_classes = IoClass.csv_to_list(fs_utils.read_file(ioclass_config_path))
config_io_classes = IoClass.csv_to_list(read_file(ioclass_config_path))
if not IoClass.compare_ioclass_lists(config_io_classes, loaded_io_classes):
TestRun.fail("Default IO class configuration not loaded correctly.")
@ -98,7 +99,7 @@ def test_io_class_prevent_wrong_configuration():
f"Preparing headerless configuration file with following content:\n"
f"{headerless_configuration}"
)
fs_utils.write_file(ioclass_config_path, headerless_configuration)
write_file(ioclass_config_path, headerless_configuration)
try_load_malformed_config(
cache,
config_io_classes,
@ -114,7 +115,7 @@ def test_io_class_prevent_wrong_configuration():
TestRun.LOGGER.info(
f"Testing following header with default IO class:\n" f"{config_content}"
)
fs_utils.write_file(ioclass_config_path, config_content)
write_file(ioclass_config_path, config_content)
try_load_malformed_config(cache, config_io_classes, expected_err_msg=err_message)
with TestRun.step(
@ -123,7 +124,7 @@ def test_io_class_prevent_wrong_configuration():
):
config_content = f"{IoClass.default_header()}\n{double_io_class_configuration}"
TestRun.LOGGER.info(f"Testing following configuration file:\n{config_content}")
fs_utils.write_file(ioclass_config_path, config_content)
write_file(ioclass_config_path, config_content)
try_load_malformed_config(
cache,
config_io_classes,
@ -139,7 +140,7 @@ def test_io_class_prevent_wrong_configuration():
TestRun.LOGGER.info(
f"Testing following header with default IO class:\n{config_content}"
)
fs_utils.write_file(ioclass_config_path, config_content)
write_file(ioclass_config_path, config_content)
try_load_malformed_config(cache, config_io_classes, expected_err_msg=err_message)

7
test/functional/tests/io_class/test_io_class_process.py
View File

@ -1,18 +1,19 @@
#
# Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import time
import pytest
from api.cas import ioclass_config, casadm
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools.dd import Dd
from test_utils.os_utils import sync, Udev
from test_utils.size import Size, Unit
from test_tools.os_tools import sync
from test_tools.udev import Udev
from type_def.size import Size, Unit
from tests.io_class.io_class_common import prepare, ioclass_config_path

15
test/functional/tests/io_class/test_io_class_purge.py
View File

@ -4,19 +4,18 @@
# SPDX-License-Identifier: BSD-3-Clause
#
from collections import namedtuple
import pytest
from collections import namedtuple
from api.cas import ioclass_config, casadm
from api.cas.statistics import IoClassUsageStats
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools import fs_utils
from test_tools.dd import Dd
from test_tools.disk_utils import Filesystem
from test_utils.os_utils import drop_caches, DropCachesMode, sync, Udev
from test_utils.size import Unit, Size
from test_tools.fs_tools import Filesystem, create_directory
from test_tools.os_tools import drop_caches, DropCachesMode, sync
from test_tools.udev import Udev
from type_def.size import Unit, Size
from tests.io_class.io_class_common import prepare, mountpoint, ioclass_config_path
@ -56,7 +55,7 @@ def test_ioclass_usage_sum():
]
for io_class in io_classes:
fs_utils.create_directory(io_class.dir_path, parents=True)
create_directory(io_class.dir_path, parents=True)
with TestRun.step("Add io classes for all dirs"):
ioclass_config.remove_ioclass_config()

39
test/functional/tests/io_class/test_io_class_service_support.py
View File

@ -1,25 +1,28 @@
#
# Copyright(c) 2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import os
import pytest
from datetime import timedelta
from api.cas import ioclass_config, casadm_parser
from api.cas.cache_config import CacheMode
from api.cas.casadm_params import StatsFilter
from api.cas.casadm_parser import get_caches, get_cores
from api.cas.init_config import InitConfig
from api.cas.ioclass_config import IoClass
from api.cas.ioclass_config import IoClass, remove_ioclass_config, create_ioclass_config, \
add_ioclass
from core.test_run_utils import TestRun
from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan
from test_tools import fs_utils
from test_tools.disk_utils import Filesystem
from test_tools.fs_tools import Filesystem, copy, read_file
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import IoEngine, ReadWrite
from test_utils import os_utils
from test_utils.os_utils import Runlevel
from test_utils.size import Size, Unit
from test_tools.os_tools import sync, drop_caches
from test_tools.runlevel import Runlevel, change_runlevel
from type_def.size import Size, Unit
from tests.io_class.io_class_common import (
prepare,
mountpoint,
@ -73,21 +76,21 @@ def test_io_class_service_load(runlevel):
InitConfig.create_init_config_from_running_configuration(
cache_extra_flags=f"ioclass_file={ioclass_config_path}"
)
os_utils.sync()
sync()
with TestRun.step(f"Reboot system to runlevel {runlevel}."):
os_utils.change_runlevel(runlevel)
change_runlevel(runlevel)
TestRun.executor.reboot()
with TestRun.step(
"Check if CAS device loads properly - "
"IO class configuration and statistics shall not change"
):
caches = casadm_parser.get_caches()
caches = get_caches()
if len(caches) != 1:
TestRun.fail("Cache did not start at boot time.")
cache = caches[0]
cores = casadm_parser.get_cores(cache.cache_id)
cores = get_cores(cache.cache_id)
if len(cores) != 1:
TestRun.fail(f"Actual number of cores: {len(cores)}\nExpected number of cores: 1")
core = cores[0]
@ -152,22 +155,22 @@ def run_io():
)
fio.run()
os_utils.sync()
os_utils.drop_caches()
sync()
drop_caches()
def prepare_and_load_io_class_config(cache, metadata_not_cached=False):
ioclass_config.remove_ioclass_config()
remove_ioclass_config()
if metadata_not_cached:
ioclass_config.create_ioclass_config(
create_ioclass_config(
add_default_rule=True, ioclass_config_path=ioclass_config_path
)
ioclass_config.add_ioclass(1, "metadata&done", 1, "0.00", ioclass_config_path)
add_ioclass(1, "metadata&done", 1, "0.00", ioclass_config_path)
else:
fs_utils.copy(template_config_path, ioclass_config_path)
copy(template_config_path, ioclass_config_path)
config_io_classes = IoClass.csv_to_list(fs_utils.read_file(ioclass_config_path))
config_io_classes = IoClass.csv_to_list(read_file(ioclass_config_path))
cache.load_io_class(ioclass_config_path)
output_io_classes = cache.list_io_classes()
if not IoClass.compare_ioclass_lists(config_io_classes, output_io_classes):

13
test/functional/tests/io_class/test_io_class_stats_file_size_core.py
View File

@ -13,12 +13,11 @@ from api.cas.cache_config import CacheMode, CleaningPolicy, SeqCutOffPolicy
from api.cas.ioclass_config import IoClass
from core.test_run_utils import TestRun
from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan
from test_tools import fs_utils
from test_tools.disk_utils import Filesystem
from test_tools.fs_tools import Filesystem, remove, read_file
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import IoEngine, ReadWrite
from test_utils.os_utils import sync, drop_caches
from test_utils.size import Size, Unit
from test_tools.os_tools import sync, drop_caches
from type_def.size import Size, Unit
from tests.io_class.io_class_common import template_config_path
@ -83,7 +82,7 @@ def test_io_class_stats_file_size_core_fs(cache_mode: CacheMode, filesystem: Fil
issued_reqs_no = \
result[0].write_requests_number() + result[0].read_requests_number()
check_statistics(cache, core, io_classes, io_class, issued_reqs_no)
fs_utils.remove(f"{core.mount_point}/*", force=True, recursive=True)
remove(f"{core.mount_point}/*", force=True, recursive=True)
size_min = size + Size(512, Unit.Byte)
@ -140,7 +139,7 @@ def test_io_class_stats_file_size_core_direct(cache_mode: CacheMode):
issued_reqs_no = \
result[0].write_requests_number() + result[0].read_requests_number()
check_statistics(cache, core, io_classes, io_class_direct, issued_reqs_no)
fs_utils.remove(f"{core.path}/*", force=True, recursive=True)
remove(f"{core.path}/*", force=True, recursive=True)
size_min = size + Size(512, Unit.Byte)
@ -191,7 +190,7 @@ def check_statistics(cache, core, io_classes, tested_io_class, issued_reqs_no):
def prepare_io_classes(cache):
template_io_classes = IoClass.csv_to_list(fs_utils.read_file(template_config_path))
template_io_classes = IoClass.csv_to_list(read_file(template_config_path))
test_io_classes = []
for io_class in template_io_classes:

6
test/functional/tests/io_class/test_io_class_wlth.py
View File

@ -1,5 +1,6 @@
#
# Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
@ -10,8 +11,9 @@ from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine
from test_utils.os_utils import sync, Udev, drop_caches
from test_utils.size import Unit, Size
from test_tools.os_tools import sync, drop_caches
from test_tools.udev import Udev
from type_def.size import Unit, Size
from tests.io_class.io_class_common import prepare

26
test/functional/tests/io_class/test_io_classification.py
View File

@ -1,26 +1,28 @@
#
# Copyright(c) 2019-2022 Intel Corporation
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
# SPDX-License-Identifier: BSD-3-Clause
#
import random
import time
from itertools import permutations
import pytest
from itertools import permutations
from api.cas import ioclass_config, casadm
from api.cas.ioclass_config import IoClass
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools import fs_utils
from test_tools.dd import Dd
from test_tools.disk_utils import Filesystem
from test_tools.fs_tools import Filesystem, create_directory, remove, \
replace_first_pattern_occurrence
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import ReadWrite, IoEngine
from test_utils.filesystem.file import File
from test_utils.os_utils import sync, Udev
from test_utils.size import Size, Unit
from test_tools.os_tools import sync
from test_tools.udev import Udev
from type_def.size import Size, Unit
from tests.io_class.io_class_common import prepare, ioclass_config_path, mountpoint
@ -371,7 +373,7 @@ def test_ioclass_metadata(filesystem):
with TestRun.step(f"Create directory {test_dir_path}."):
requests_to_metadata_before = requests_to_metadata_after
fs_utils.create_directory(path=test_dir_path)
create_directory(path=test_dir_path)
TestRun.LOGGER.info(f"Moving test files into {test_dir_path}")
for file in files:
@ -386,7 +388,7 @@ def test_ioclass_metadata(filesystem):
TestRun.fail("No requests to metadata while moving files!")
with TestRun.step(f"Remove {test_dir_path}."):
fs_utils.remove(path=test_dir_path, force=True, recursive=True)
remove(path=test_dir_path, force=True, recursive=True)
with TestRun.step("Check requests to metadata."):
requests_to_metadata_after = cache.get_io_class_statistics(
@ -475,7 +477,7 @@ def test_ioclass_id_as_condition():
):
core.create_filesystem(filesystem)
core.mount(mountpoint)
fs_utils.create_directory(base_dir_path)
create_directory(base_dir_path)
sync()
# CAS needs some time to resolve directory to inode
time.sleep(ioclass_config.MAX_CLASSIFICATION_DELAY.seconds)
@ -617,7 +619,7 @@ def test_ioclass_conditions_or():
core.create_filesystem(filesystem)
core.mount(mountpoint)
for i in range(1, 6):
fs_utils.create_directory(f"{mountpoint}/dir{i}")
create_directory(f"{mountpoint}/dir{i}")
sync()
with TestRun.step("Perform IO fulfilling each condition and check if occupancy raises."):
@ -744,7 +746,7 @@ def test_ioclass_effective_ioclass():
):
core.create_filesystem(filesystem)
core.mount(mountpoint)
fs_utils.create_directory(test_dir)
create_directory(test_dir)
sync()
for i, permutation in TestRun.iteration(enumerate(permutations(range(1, 5)), start=1)):
@ -830,7 +832,7 @@ def add_done_to_second_non_exclusive_condition(rules, permutation, cache):
if non_exclusive_conditions == 2:
break
second_class_id += 1
fs_utils.replace_first_pattern_occurrence(ioclass_config_path, rules[idx], f"{rules[idx]}&done")
replace_first_pattern_occurrence(ioclass_config_path, rules[idx], f"{rules[idx]}&done")
sync()
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
return second_class_id

10
test/functional/tests/lazy_writes/cleaning_policy/test_acp.py
View File

@ -6,11 +6,11 @@
import random
import time
import pytest
from collections import namedtuple
from datetime import timedelta
import pytest
from api.cas import casadm
from api.cas.cache_config import (
CacheMode,
@ -24,9 +24,9 @@ from storage_devices.disk import DiskTypeSet, DiskTypeLowerThan, DiskType
from test_tools.blktrace import BlkTrace, BlkTraceMask, ActionKind, RwbsKind
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import IoEngine, ReadWrite
from test_utils.os_utils import kill_all_io
from test_utils.size import Size, Unit
from test_utils.time import Time
from test_tools.os_tools import kill_all_io
from type_def.size import Size, Unit
from type_def.time import Time
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))

Some files were not shown because too many files have changed in this diff Show More