From 5edbf2a5a9a88562bce32a9fbec55bcba12f6237 Mon Sep 17 00:00:00 2001 From: Michal Rakowski Date: Thu, 2 Jan 2020 16:34:40 +0100 Subject: [PATCH 1/4] conftest: execute casadm-related cleanup only if CAS is installed during teardown Signed-off-by: Michal Rakowski --- test/functional/tests/conftest.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/functional/tests/conftest.py b/test/functional/tests/conftest.py index 3390636..d3681b1 100644 --- a/test/functional/tests/conftest.py +++ b/test/functional/tests/conftest.py @@ -94,10 +94,11 @@ def pytest_runtest_teardown(): TestRun.executor.wait_for_connection() Udev.enable() unmount_cas_devices() - casadm.remove_all_detached_cores() - casadm.stop_all_caches() - from api.cas.init_config import InitConfig - InitConfig.create_default_init_config() + if installer.check_if_installed(): + casadm.remove_all_detached_cores() + casadm.stop_all_caches() + from api.cas.init_config import InitConfig + InitConfig.create_default_init_config() DeviceMapper.remove_all() except Exception as ex: TestRun.LOGGER.warning(f"Exception occured during platform cleanup.\n" From d9c0c0e83a855d5b1afc0d70c7d7ba24361d38ea Mon Sep 17 00:00:00 2001 From: Michal Rakowski Date: Fri, 3 Jan 2020 12:17:54 +0100 Subject: [PATCH 2/4] cas_module.py: fixup load/unload methods Signed-off-by: Michal Rakowski --- test/functional/api/cas/cas_module.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/test/functional/api/cas/cas_module.py b/test/functional/api/cas/cas_module.py index 09faa7e..58f0dd6 100644 --- a/test/functional/api/cas/cas_module.py +++ b/test/functional/api/cas/cas_module.py @@ -2,7 +2,9 @@ # Copyright(c) 2019 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause-Clear # + from aenum import Enum + from test_utils import os_utils from test_utils.os_utils import ModuleRemoveMethod @@ -13,5 +15,12 @@ class CasModule(Enum): def reload_all_cas_modules(): - os_utils.unload_kernel_module(CasModule.cache, ModuleRemoveMethod.modprobe) - os_utils.load_kernel_module(CasModule.cache) + os_utils.unload_kernel_module(CasModule.cache.value, ModuleRemoveMethod.modprobe) + os_utils.load_kernel_module(CasModule.cache.value) + + +def unload_all_cas_modules(): + os_utils.unload_kernel_module(CasModule.cache.value, + os_utils.ModuleRemoveMethod.rmmod) + os_utils.unload_kernel_module(CasModule.disk.value, + os_utils.ModuleRemoveMethod.rmmod) From 9acb679ede38b1f6f2433c429c44c5f839651561 Mon Sep 17 00:00:00 2001 From: Michal Rakowski Date: Fri, 3 Jan 2020 12:19:39 +0100 Subject: [PATCH 3/4] installer.py: determine if CAS is installed on modules presence instead of casadm only Signed-off-by: Michal Rakowski --- test/functional/api/cas/installer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/functional/api/cas/installer.py b/test/functional/api/cas/installer.py index 1690898..82547ad 100644 --- a/test/functional/api/cas/installer.py +++ b/test/functional/api/cas/installer.py @@ -8,6 +8,8 @@ import logging from tests import conftest from core.test_run import TestRun +from api.cas import cas_module +from test_utils import os_utils from test_utils.output import CmdException @@ -64,7 +66,9 @@ def reinstall_opencas(): def check_if_installed(): TestRun.LOGGER.info("Check if Open-CAS-Linux is installed") output = TestRun.executor.run("which casadm") - if output.exit_code == 0: + modules_loaded = os_utils.is_kernel_module_loaded(cas_module.CasModule.cache.value) + + if output.exit_code == 0 and modules_loaded: TestRun.LOGGER.info("CAS is installed") return True From 34e7a81635ada5c3ec938f989c44b2e37a2382b9 Mon Sep 17 00:00:00 2001 From: Michal Rakowski Date: Fri, 3 Jan 2020 12:24:38 +0100 Subject: [PATCH 4/4] Add tests using kedr - load/unload modules, start/stop cache Signed-off-by: Michal Rakowski --- test/functional/tests/stress/test_kedr.py | 210 ++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 test/functional/tests/stress/test_kedr.py diff --git a/test/functional/tests/stress/test_kedr.py b/test/functional/tests/stress/test_kedr.py new file mode 100644 index 0000000..728f72d --- /dev/null +++ b/test/functional/tests/stress/test_kedr.py @@ -0,0 +1,210 @@ +# +# Copyright(c) 2020 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +import pytest + +from datetime import timedelta + +from test_tools.kedr import Kedr, KedrProfile +from api.cas import cas_module, installer, casadm +from core.test_run import TestRun +from test_utils import os_utils +from test_utils.size import Size, Unit +from test_tools.fio.fio import Fio +from test_tools.fio.fio_param import ReadWrite, IoEngine +from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan + + +@pytest.fixture(scope="module") +def install_kedr(): + TestRun.LOGGER.info("Checking if kedr is installed") + if not Kedr.is_installed(): + TestRun.LOGGER.info("Installing kedr") + Kedr.install() + + +@pytest.fixture(scope="function") +def unload_modules(): + TestRun.LOGGER.info("Check if CAS is installed") + if installer.check_if_installed(): + TestRun.LOGGER.info("Unloading modules") + cas_module.unload_all_cas_modules() + + TestRun.LOGGER.info("Stop kedr if it is running") + if Kedr.is_loaded(): + Kedr.stop() + + TestRun.LOGGER.info("Mounting debugfs") + os_utils.mount_debugfs() + + +@pytest.mark.parametrize("module", cas_module.CasModule) +def test_kedr_memleak_load_cas_module(module, unload_modules, install_kedr): + """ + title: Loading modules with kedr started with 'memleak' configuration + description: Load and unload modules with kedr started to watch for memory leaks + pass_criteria: + - No memory leaks observed after loading and unloading module + """ + with TestRun.step(f"Starting kedr against {module}"): + Kedr.start(module.value) + + with TestRun.step(f"Loading {module}"): + os_utils.load_kernel_module(module.value) + + with TestRun.step(f"Unloading {module}"): + os_utils.unload_kernel_module(module.value, os_utils.ModuleRemoveMethod.modprobe) + + with TestRun.step(f"Checking for memory leaks for {module}"): + try: + Kedr.check_for_mem_leaks(module.value) + except Exception as e: + TestRun.LOGGER.error(f"{e}") + + with TestRun.step(f"Stopping kedr"): + Kedr.stop() + + +@pytest.mark.parametrize("module", cas_module.CasModule) +def test_kedr_fsim_load_cas_module(module, unload_modules, install_kedr): + """ + title: Loading modules with kedr started with 'fsim' configuration + description: Load and unload modules with kedr started to simulate kmalloc fails + pass_criteria: + - Module fails to load + """ + with TestRun.step(f"Starting kedr against {module}"): + Kedr.start(module.value, KedrProfile.FAULT_SIM) + + with TestRun.step("Setting up fault simulation parameters"): + Kedr.setup_fault_injections() + + with TestRun.step(f"Trying to load {module}"): + out = os_utils.load_kernel_module(module.value) + if out.exit_code == 0 \ + or "Cannot allocate memory" not in out.stderr: + TestRun.LOGGER.error(f"Loading module should fail because of alloc error, instead " + f"modprobe's output is: {out}") + + with TestRun.step(f"Stopping kedr"): + Kedr.stop() + + +@pytest.mark.parametrize("module", cas_module.CasModule) +@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand])) +@pytest.mark.require_disk("core", DiskTypeLowerThan("cache")) +def test_kedr_start_cache(module, unload_modules, install_kedr): + """ + title: Start cache and add core with kedr started against one of CAS modules + description: | + Load CAS modules, start kedr against one of them, start cache and add core, + stop cache and unload modules + pass_criteria: + - No memory leaks observed + """ + with TestRun.step("Preparing cache device"): + cache_device = TestRun.disks['cache'] + cache_device.create_partitions([Size(500, Unit.MebiByte)]) + cache_part = cache_device.partitions[0] + + with TestRun.step("Preparing core device"): + core_device = TestRun.disks['core'] + core_device.create_partitions([Size(1, Unit.GibiByte)]) + core_part = core_device.partitions[0] + + with TestRun.step("Unload CAS modules if needed"): + if os_utils.is_kernel_module_loaded(module.value): + cas_module.unload_all_cas_modules() + + with TestRun.step(f"Starting kedr against {module.value}"): + Kedr.start(module.value) + + with TestRun.step(f"Loading CAS modules"): + os_utils.load_kernel_module(cas_module.CasModule.cache.value) + + with TestRun.step("Starting cache"): + cache = casadm.start_cache(cache_part, force=True) + + with TestRun.step("Adding core"): + cache.add_core(core_dev=core_part) + + with TestRun.step("Stopping cache"): + cache.stop() + + with TestRun.step(f"Unloading CAS modules"): + cas_module.unload_all_cas_modules() + + with TestRun.step(f"Checking for memory leaks for {module}"): + try: + Kedr.check_for_mem_leaks(module.value) + except Exception as e: + TestRun.LOGGER.error(f"{e}") + + with TestRun.step(f"Stopping kedr"): + Kedr.stop() + + +@pytest.mark.parametrize("module", cas_module.CasModule) +@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand])) +@pytest.mark.require_disk("core", DiskTypeLowerThan("cache")) +def test_kedr_basic_io(module, unload_modules, install_kedr): + """ + title: Basic IO test with kedr started with memory leaks profile + description: | + Load CAS modules, start kedr against one of them, start cache and add core, + run simple 4 minute random IO, stop cache and unload modules + pass_criteria: + - No memory leaks observed + """ + with TestRun.step("Preparing cache device"): + cache_device = TestRun.disks['cache'] + cache_device.create_partitions([Size(500, Unit.MebiByte)]) + cache_part = cache_device.partitions[0] + + with TestRun.step("Preparing core device"): + core_device = TestRun.disks['core'] + core_device.create_partitions([Size(1, Unit.GibiByte)]) + core_part = core_device.partitions[0] + + with TestRun.step("Unload CAS modules if needed"): + if os_utils.is_kernel_module_loaded(module.value): + cas_module.unload_all_cas_modules() + + with TestRun.step(f"Starting kedr against {module.value}"): + Kedr.start(module.value) + + with TestRun.step(f"Loading CAS modules"): + os_utils.load_kernel_module(cas_module.CasModule.cache.value) + + with TestRun.step("Starting cache"): + cache = casadm.start_cache(cache_part, force=True) + + with TestRun.step("Adding core"): + core = cache.add_core(core_dev=core_part) + + with TestRun.step(f"Running IO"): + (Fio().create_command() + .io_engine(IoEngine.libaio) + .run_time(timedelta(minutes=4)) + .time_based() + .read_write(ReadWrite.randrw) + .target(f"{core.system_path}") + .direct() + ).run() + + with TestRun.step("Stopping cache"): + cache.stop() + + with TestRun.step(f"Unloading CAS modules"): + cas_module.unload_all_cas_modules() + + with TestRun.step(f"Checking for memory leaks for {module.value}"): + try: + Kedr.check_for_mem_leaks(module.value) + except Exception as e: + TestRun.LOGGER.error(f"{e}") + + with TestRun.step(f"Stopping kedr"): + Kedr.stop()