Merge pull request #197 from Ostrokrzew/last

Add fault injection tests and update stress tests file

test/functional/.gitignore

@@ -1,2 +1,3 @@
 plugins/
 results/
+config/dut_config.yml

test/functional/tests/fault_injection/test_fault_injection_with_mounted_core.py

@@ -0,0 +1,168 @@
+#
+# Copyright(c) 2019 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+import pytest
+
+from api.cas import casadm, casadm_parser, cli
+from api.cas.cache_config import CacheMode
+from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
+from core.test_run import TestRun
+from test_tools import fs_utils
+from test_tools.disk_utils import Filesystem
+from test_utils.filesystem.file import File
+from test_utils.size import Size, Unit
+
+mount_point = "/mnt/cas"
+test_file_path = f"{mount_point}/test_file"
+
+
+@pytest.mark.parametrize("cache_mode", CacheMode)
+@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
+@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
+def test_load_cache_with_mounted_core(cache_mode):
+    """
+        title: Fault injection test for adding mounted core on cache load.
+        description: |
+          Negative test of the ability of CAS to add to cache while its loading
+          core device which is mounted.
+        pass_criteria:
+          - No system crash while loading cache.
+          - Adding mounted core while loading cache fails.
+    """
+    with TestRun.step("Prepare cache and core devices. Start CAS."):
+        cache_dev = TestRun.disks['cache']
+        cache_dev.create_partitions([Size(1, Unit.GibiByte)])
+        cache_dev = cache_dev.partitions[0]
+        core_dev = TestRun.disks['core']
+        core_dev.create_partitions([Size(4, Unit.GibiByte)])
+        core_dev = core_dev.partitions[0]
+        cache = casadm.start_cache(cache_dev, cache_mode, force=True)
+
+    with TestRun.step("Add core device with xfs filesystem and mount it."):
+        core_dev.create_filesystem(Filesystem.xfs)
+        core = cache.add_core(core_dev)
+        core.mount(mount_point)
+
+    with TestRun.step(f"Create test file in /tmp directory."):
+        file = fs_utils.create_test_file('/tmp/test_file')
+
+    with TestRun.step("Copy file to cache's exported object"):
+        copied_file = File.copy(file.full_path, test_file_path, force=True)
+
+    with TestRun.step("Unmount core device."):
+        core.unmount()
+
+    with TestRun.step("Stop cache."):
+        cache.stop()
+        caches_count = len(casadm_parser.get_caches())
+        if caches_count != 0:
+            TestRun.fail(f"Expected caches count: 0; Actual caches count: {caches_count}.")
+
+    with TestRun.step("Mount core device."):
+        core_dev.mount(mount_point)
+
+    with TestRun.step("Load cache."):
+        cache = casadm.load_cache(cache.cache_device)
+        caches_count = len(casadm_parser.get_caches())
+        if caches_count != 1:
+            TestRun.fail(f"Expected caches count: 1 Actual caches count: {caches_count}.")
+        cores_count = len(casadm_parser.get_cores(cache.cache_id))
+        if cores_count != 0:
+            TestRun.fail(f"Expected cores count: 0; Actual cores count: {cores_count}.")
+
+    with TestRun.step("Check properties of test file."):
+        if file.get_properties() != copied_file.get_properties():
+            TestRun.LOGGER.error("File properties before and after copying are different.")
+        core_dev.unmount()
+
+    with TestRun.step("Stop all caches."):
+        casadm.stop_all_caches()
+
+
+@pytest.mark.parametrize("cache_mode", CacheMode)
+@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
+@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
+def test_stop_cache_with_mounted_partition(cache_mode):
+    """
+        title: Fault injection test for removing core and stopping cache with mounted core.
+        description: |
+          Negative test of the ability of CAS to remove core and stop cache while core
+          is still mounted.
+        pass_criteria:
+          - No system crash.
+          - Unable to stop CAS device.
+          - Unable to remove core when partition is mounted.
+    """
+    with TestRun.step("Prepare cache and core devices. Start CAS."):
+        cache_dev = TestRun.disks['cache']
+        cache_dev.create_partitions([Size(1, Unit.GibiByte)])
+        cache_dev = cache_dev.partitions[0]
+        core_dev = TestRun.disks['core']
+        core_dev.create_partitions([Size(4, Unit.GibiByte)])
+        core_dev = core_dev.partitions[0]
+        cache = casadm.start_cache(cache_dev, cache_mode, force=True)
+
+    with TestRun.step("Add core device with xfs filesystem and mount it."):
+        core_dev.create_filesystem(Filesystem.xfs)
+        core = cache.add_core(core_dev)
+        core.mount(mount_point)
+
+    with TestRun.step("Try to remove core from cache."):
+        output = TestRun.executor.run_expect_fail(cli.remove_core_cmd(cache_id=str(cache.cache_id),
+                                                                      core_id=str(core.core_id)))
+        if not output.stderr:
+            TestRun.fail("Removing core succeeded (should fail)!")
+
+    with TestRun.step("Try to stop CAS."):
+        output = TestRun.executor.run_expect_fail(cli.stop_cmd(cache_id=str(cache.cache_id)))
+        if not output.stderr:
+            TestRun.fail("Stopping CAS succeeded (should fail)!")
+
+    with TestRun.step("Unmount core device."):
+        core.unmount()
+
+    with TestRun.step("Stop all caches."):
+        casadm.stop_all_caches()
+
+
+@pytest.mark.parametrize("cache_mode", CacheMode)
+@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
+@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
+def test_add_occupied_core(cache_mode):
+    """
+        title: Fault injection test for adding already used core to a cache.
+        description: |
+          Negative test of the ability to add core to cache
+          while the core is already used by the another cache instance.
+        pass_criteria:
+          - Adding already used core to another cache instance fails.
+    """
+    with TestRun.step("Prepare two caches and one core device."):
+        cache_dev = TestRun.disks['cache']
+        cache_dev.create_partitions([Size(2, Unit.GibiByte), Size(2, Unit.GibiByte)])
+        cache_dev1 = cache_dev.partitions[0]
+        cache_dev2 = cache_dev.partitions[1]
+        core_dev = TestRun.disks['core']
+        core_dev.create_partitions([Size(4, Unit.GibiByte)])
+        core_dev = core_dev.partitions[0]
+
+    with TestRun.step("Start first cache instance"):
+        cache1 = casadm.start_cache(cache_dev1, cache_mode, force=True)
+
+    with TestRun.step("Add core device to first cache instance."):
+        core = cache1.add_core(core_dev)
+
+    with TestRun.step("Start second cache instance"):
+        cache2 = casadm.start_cache(cache_dev2, cache_mode, force=True)
+
+    with TestRun.step("Try adding the same core device to second cache instance."):
+        output = TestRun.executor.run_expect_fail(cli.add_core_cmd(cache_id=str(cache2.cache_id),
+                                                                   core_dev=str(core_dev),
+                                                                   core_id=str(core.core_id)))
+        if not output.stderr:
+            TestRun.fail("Adding same core to other cache succeeded (should fail)!")
+
+    with TestRun.step("Stop all caches."):
+        casadm.stop_all_caches()

test/functional/tests/stress/test_stress_without_io.py

@@ -13,7 +13,6 @@ from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
 from core.test_run import TestRun
 from test_tools import fs_utils
 from test_tools.disk_utils import Filesystem
-from test_utils.filesystem.file import File
 from test_utils.size import Size, Unit
 
 iterations_per_config = 50
@@ -159,9 +158,8 @@ def test_stress_reload_cache(cache_mode):
           - CAS device loads successfully.
           - No data corruption.
     """
-    with TestRun.step("Prepare cache and core. Create test file and count it's checksum."):
+    with TestRun.step("Prepare cache and core. Create test file and count its checksum."):
-        cache, core, md5_before_load, size_before_load, permissions_before_load = \
+        cache, core, file, file_before = prepare_with_file_creation(cache_mode)
-            prepare_with_file_creation(cache_mode)
 
     for _ in TestRun.iteration(range(0, iterations_per_config),
                                f"Stop and load cache {iterations_per_config} times."):
@@ -180,7 +178,12 @@ def test_stress_reload_cache(cache_mode):
                 TestRun.fail(f"Expected cores count: 1; Actual cores count: {cores_count}.")
 
     with TestRun.step("Check md5 of test file."):
-        check_files(core, size_before_load, permissions_before_load, md5_before_load)
+        core.mount(mount_point)
+        file_after = file.get_properties()
+        if file_after != file_before:
+            TestRun.LOGGER.error("File properties before and after are different.")
+        core.unmount()
+
     with TestRun.step("Stop all caches."):
         casadm.stop_all_caches()
 
@@ -197,9 +200,8 @@ def test_stress_add_remove_core(cache_mode):
           - Core is added and removed successfully.
           - No data corruption.
     """
-    with TestRun.step("Prepare cache and core. Create test file and count it's checksum."):
+    with TestRun.step("Prepare cache and core. Create test file and count its checksum."):
-        cache, core, md5_before_load, size_before_load, permissions_before_load = \
+        cache, core, file, file_before = prepare_with_file_creation(cache_mode)
-            prepare_with_file_creation(cache_mode)
 
     for _ in TestRun.iteration(range(0, iterations_per_config),
                                f"Add and remove core {iterations_per_config} times."):
@@ -221,7 +223,12 @@ def test_stress_add_remove_core(cache_mode):
                 TestRun.fail(f"Expected cores count: 1; Actual cores count: {cores_count}.")
 
     with TestRun.step("Check md5 of test file."):
-        check_files(core, size_before_load, permissions_before_load, md5_before_load)
+        core.mount(mount_point)
+        file_after = file.get_properties()
+        if file_after != file_before:
+            TestRun.LOGGER.error("File properties before and after are different.")
+        core.unmount()
+
     with TestRun.step("Stop all caches."):
         casadm.stop_all_caches()
 
@@ -238,9 +245,9 @@ def test_stress_reload_module(cache_mode):
           - CAS modules reloads with no errors.
           - No data corruption.
     """
-    with TestRun.step("Prepare cache and core. Create test file and count it's checksum."):
+    with TestRun.step("Prepare cache and core. Create test file and count its checksum."):
-        cache, core, md5_before_load, size_before_load, permissions_before_load = \
+        cache, core, file, file_before = prepare_with_file_creation(cache_mode)
-            prepare_with_file_creation(cache_mode)
+
     with TestRun.step("Save current cache configuration."):
         cache_config = cache.get_cache_config()
 
@@ -270,47 +277,26 @@ def test_stress_reload_module(cache_mode):
                 TestRun.fail("Cache configuration is different than before reloading modules.")
 
     with TestRun.step("Check md5 of test file."):
-        check_files(core, size_before_load, permissions_before_load, md5_before_load)
+        core.mount(mount_point)
+        file_after = file.get_properties()
+        if file_after != file_before:
+            TestRun.LOGGER.error("File properties before and after are different.")
+        core.unmount()
+
     with TestRun.step("Stop all caches."):
         casadm.stop_all_caches()
 
 
-def check_files(core, size_before, permissions_before, md5_before):
-    TestRun.LOGGER.info("Checking file md5.")
-    core.mount(mount_point)
-    file_after = fs_utils.parse_ls_output(fs_utils.ls(test_file_path))[0]
-    md5_after = file_after.md5sum()
-    if md5_before != md5_after:
-        TestRun.LOGGER.error(f"Md5 before ({md5_before}) and after ({md5_after}) are different.")
-
-    if permissions_before.user == file_after.permissions.user:
-        TestRun.LOGGER.error(f"User permissions before ({permissions_before.user}) "
-                             f"and after ({file_after.permissions.user}) are different.")
-    if permissions_before.group != file_after.permissions.group:
-        TestRun.LOGGER.error(f"Group permissions before ({permissions_before.group}) "
-                             f"and after ({file_after.permissions.group}) are different.")
-    if permissions_before.other != file_after.permissions.other:
-        TestRun.LOGGER.error(f"Other permissions before ({permissions_before.other}) "
-                             f"and after ({file_after.permissions.other}) are different.")
-    if size_before != file_after.size:
-        TestRun.LOGGER.error(f"Size before ({size_before}) and after ({file_after.size}) "
-                             f"are different.")
-    core.unmount()
-
-
 def prepare_with_file_creation(config):
     cache_dev, core_dev = prepare()
     cache = casadm.start_cache(cache_dev, config, force=True)
     core = cache.add_core(core_dev)
     core.create_filesystem(Filesystem.ext3)
     core.mount(mount_point)
-    file = File.create_file(test_file_path)
+    file = fs_utils.create_test_file(test_file_path)
-    file.write("Test content")
+    file_properties = file.get_properties()
-    md5_before_load = file.md5sum()
-    size_before_load = file.size
-    permissions_before_load = file.permissions
     core.unmount()
-    return cache, core, md5_before_load, size_before_load, permissions_before_load
+    return cache, core, file, file_properties
 
 
 def prepare():