Merge pull request #265 from micrakow/kedr

Add basic kedr test

test/functional/api/cas/cas_module.py

@@ -2,7 +2,9 @@
 # Copyright(c) 2019 Intel Corporation
 # SPDX-License-Identifier: BSD-3-Clause-Clear
 #
+
 from aenum import Enum
+
 from test_utils import os_utils
 from test_utils.os_utils import ModuleRemoveMethod
 
@@ -13,5 +15,12 @@ class CasModule(Enum):
 
 
 def reload_all_cas_modules():
-    os_utils.unload_kernel_module(CasModule.cache, ModuleRemoveMethod.modprobe)
+    os_utils.unload_kernel_module(CasModule.cache.value, ModuleRemoveMethod.modprobe)
-    os_utils.load_kernel_module(CasModule.cache)
+    os_utils.load_kernel_module(CasModule.cache.value)
+
+
+def unload_all_cas_modules():
+    os_utils.unload_kernel_module(CasModule.cache.value,
+                                  os_utils.ModuleRemoveMethod.rmmod)
+    os_utils.unload_kernel_module(CasModule.disk.value,
+                                  os_utils.ModuleRemoveMethod.rmmod)

test/functional/api/cas/installer.py

@@ -8,6 +8,8 @@ import logging
 
 from tests import conftest
 from core.test_run import TestRun
+from api.cas import cas_module
+from test_utils import os_utils
 from test_utils.output import CmdException
 
 
@@ -64,7 +66,9 @@ def reinstall_opencas():
 def check_if_installed():
     TestRun.LOGGER.info("Check if Open-CAS-Linux is installed")
     output = TestRun.executor.run("which casadm")
-    if output.exit_code == 0:
+    modules_loaded = os_utils.is_kernel_module_loaded(cas_module.CasModule.cache.value)
+
+    if output.exit_code == 0 and modules_loaded:
         TestRun.LOGGER.info("CAS is installed")
 
         return True

test/functional/tests/conftest.py

@@ -94,6 +94,7 @@ def pytest_runtest_teardown():
                     TestRun.executor.wait_for_connection()
                 Udev.enable()
                 unmount_cas_devices()
+                if installer.check_if_installed():
                     casadm.remove_all_detached_cores()
                     casadm.stop_all_caches()
                     from api.cas.init_config import InitConfig

test/functional/tests/stress/test_kedr.py

@@ -0,0 +1,210 @@
+#
+# Copyright(c) 2020 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+import pytest
+
+from datetime import timedelta
+
+from test_tools.kedr import Kedr, KedrProfile
+from api.cas import cas_module, installer, casadm
+from core.test_run import TestRun
+from test_utils import os_utils
+from test_utils.size import Size, Unit
+from test_tools.fio.fio import Fio
+from test_tools.fio.fio_param import ReadWrite, IoEngine
+from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
+
+
+@pytest.fixture(scope="module")
+def install_kedr():
+    TestRun.LOGGER.info("Checking if kedr is installed")
+    if not Kedr.is_installed():
+        TestRun.LOGGER.info("Installing kedr")
+        Kedr.install()
+
+
+@pytest.fixture(scope="function")
+def unload_modules():
+    TestRun.LOGGER.info("Check if CAS is installed")
+    if installer.check_if_installed():
+        TestRun.LOGGER.info("Unloading modules")
+        cas_module.unload_all_cas_modules()
+
+    TestRun.LOGGER.info("Stop kedr if it is running")
+    if Kedr.is_loaded():
+        Kedr.stop()
+
+    TestRun.LOGGER.info("Mounting debugfs")
+    os_utils.mount_debugfs()
+
+
+@pytest.mark.parametrize("module", cas_module.CasModule)
+def test_kedr_memleak_load_cas_module(module, unload_modules, install_kedr):
+    """
+    title: Loading modules with kedr started with 'memleak' configuration
+    description: Load and unload modules with kedr started to watch for memory leaks
+    pass_criteria:
+      - No memory leaks observed after loading and unloading module
+    """
+    with TestRun.step(f"Starting kedr against {module}"):
+        Kedr.start(module.value)
+
+    with TestRun.step(f"Loading {module}"):
+        os_utils.load_kernel_module(module.value)
+
+    with TestRun.step(f"Unloading {module}"):
+        os_utils.unload_kernel_module(module.value, os_utils.ModuleRemoveMethod.modprobe)
+
+    with TestRun.step(f"Checking for memory leaks for {module}"):
+        try:
+            Kedr.check_for_mem_leaks(module.value)
+        except Exception as e:
+            TestRun.LOGGER.error(f"{e}")
+
+    with TestRun.step(f"Stopping kedr"):
+        Kedr.stop()
+
+
+@pytest.mark.parametrize("module", cas_module.CasModule)
+def test_kedr_fsim_load_cas_module(module, unload_modules, install_kedr):
+    """
+    title: Loading modules with kedr started with 'fsim' configuration
+    description: Load and unload modules with kedr started to simulate kmalloc fails
+    pass_criteria:
+      - Module fails to load
+    """
+    with TestRun.step(f"Starting kedr against {module}"):
+        Kedr.start(module.value, KedrProfile.FAULT_SIM)
+
+    with TestRun.step("Setting up fault simulation parameters"):
+        Kedr.setup_fault_injections()
+
+    with TestRun.step(f"Trying to load {module}"):
+        out = os_utils.load_kernel_module(module.value)
+        if out.exit_code == 0 \
+                or "Cannot allocate memory" not in out.stderr:
+            TestRun.LOGGER.error(f"Loading module should fail because of alloc error, instead "
+                                 f"modprobe's output is: {out}")
+
+    with TestRun.step(f"Stopping kedr"):
+        Kedr.stop()
+
+
+@pytest.mark.parametrize("module", cas_module.CasModule)
+@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
+@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
+def test_kedr_start_cache(module, unload_modules, install_kedr):
+    """
+    title: Start cache and add core with kedr started against one of CAS modules
+    description: |
+        Load CAS modules, start kedr against one of them, start cache and add core,
+        stop cache and unload modules
+    pass_criteria:
+      - No memory leaks observed
+    """
+    with TestRun.step("Preparing cache device"):
+        cache_device = TestRun.disks['cache']
+        cache_device.create_partitions([Size(500, Unit.MebiByte)])
+        cache_part = cache_device.partitions[0]
+
+    with TestRun.step("Preparing core device"):
+        core_device = TestRun.disks['core']
+        core_device.create_partitions([Size(1, Unit.GibiByte)])
+        core_part = core_device.partitions[0]
+
+    with TestRun.step("Unload CAS modules if needed"):
+        if os_utils.is_kernel_module_loaded(module.value):
+            cas_module.unload_all_cas_modules()
+
+    with TestRun.step(f"Starting kedr against {module.value}"):
+        Kedr.start(module.value)
+
+    with TestRun.step(f"Loading CAS modules"):
+        os_utils.load_kernel_module(cas_module.CasModule.cache.value)
+
+    with TestRun.step("Starting cache"):
+        cache = casadm.start_cache(cache_part, force=True)
+
+    with TestRun.step("Adding core"):
+        cache.add_core(core_dev=core_part)
+
+    with TestRun.step("Stopping cache"):
+        cache.stop()
+
+    with TestRun.step(f"Unloading CAS modules"):
+        cas_module.unload_all_cas_modules()
+
+    with TestRun.step(f"Checking for memory leaks for {module}"):
+        try:
+            Kedr.check_for_mem_leaks(module.value)
+        except Exception as e:
+            TestRun.LOGGER.error(f"{e}")
+
+    with TestRun.step(f"Stopping kedr"):
+        Kedr.stop()
+
+
+@pytest.mark.parametrize("module", cas_module.CasModule)
+@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
+@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
+def test_kedr_basic_io(module, unload_modules, install_kedr):
+    """
+    title: Basic IO test with kedr started with memory leaks profile
+    description: |
+        Load CAS modules, start kedr against one of them, start cache and add core,
+        run simple 4 minute random IO, stop cache and unload modules
+    pass_criteria:
+      - No memory leaks observed
+    """
+    with TestRun.step("Preparing cache device"):
+        cache_device = TestRun.disks['cache']
+        cache_device.create_partitions([Size(500, Unit.MebiByte)])
+        cache_part = cache_device.partitions[0]
+
+    with TestRun.step("Preparing core device"):
+        core_device = TestRun.disks['core']
+        core_device.create_partitions([Size(1, Unit.GibiByte)])
+        core_part = core_device.partitions[0]
+
+    with TestRun.step("Unload CAS modules if needed"):
+        if os_utils.is_kernel_module_loaded(module.value):
+            cas_module.unload_all_cas_modules()
+
+    with TestRun.step(f"Starting kedr against {module.value}"):
+        Kedr.start(module.value)
+
+    with TestRun.step(f"Loading CAS modules"):
+        os_utils.load_kernel_module(cas_module.CasModule.cache.value)
+
+    with TestRun.step("Starting cache"):
+        cache = casadm.start_cache(cache_part, force=True)
+
+    with TestRun.step("Adding core"):
+        core = cache.add_core(core_dev=core_part)
+
+    with TestRun.step(f"Running IO"):
+        (Fio().create_command()
+              .io_engine(IoEngine.libaio)
+              .run_time(timedelta(minutes=4))
+              .time_based()
+              .read_write(ReadWrite.randrw)
+              .target(f"{core.system_path}")
+              .direct()
+         ).run()
+
+    with TestRun.step("Stopping cache"):
+        cache.stop()
+
+    with TestRun.step(f"Unloading CAS modules"):
+        cas_module.unload_all_cas_modules()
+
+    with TestRun.step(f"Checking for memory leaks for {module.value}"):
+        try:
+            Kedr.check_for_mem_leaks(module.value)
+        except Exception as e:
+            TestRun.LOGGER.error(f"{e}")
+
+    with TestRun.step(f"Stopping kedr"):
+        Kedr.stop()