diff --git a/test/functional/tests/fault_injection/test_fault_injection_many_to_one.py b/test/functional/tests/fault_injection/test_fault_injection_many_to_one.py new file mode 100644 index 0000000..f0c55a9 --- /dev/null +++ b/test/functional/tests/fault_injection/test_fault_injection_many_to_one.py @@ -0,0 +1,238 @@ +# +# Copyright(c) 2020 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +import pytest + +from api.cas import casadm, casadm_parser +from api.cas.cache_config import CacheMode, SeqCutOffPolicy, CacheModeTrait +from api.cas.core import Core +from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan +from core.test_run import TestRun +from test_tools.dd import Dd +from test_utils.os_utils import Udev +from test_utils.size import Size, Unit + +block_size = Size(1, Unit.Blocks4096) + + +@pytest.mark.parametrize("cache_mode", CacheMode.with_any_trait(CacheModeTrait.InsertRead + | CacheModeTrait.InsertWrite)) +@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand])) +@pytest.mark.require_disk("core", DiskTypeLowerThan("cache")) +def test_one_core_remove(cache_mode): + """ + title: Test if OpenCAS correctly handles removal of one of multiple core devices. + description: | + When one core device is removed from a cache instance all blocks previously occupied + by the data from that core device should be removed. That means that the number of free + cache blocks should increase by the number of removed blocks. + Test is without pass through mode. + pass_criteria: + - No system crash. + - The remaining core is able to use cache. + - Removing core frees cache blocks occupied by this core. + """ + with TestRun.step("Prepare one device for cache and two for core."): + cache_dev = TestRun.disks['cache'] + cache_dev.create_partitions([Size(512, Unit.MebiByte)]) + cache_dev = cache_dev.partitions[0] + core_dev = TestRun.disks['core'] + core_dev.create_partitions([Size(1, Unit.GibiByte)] * 2) + core_part1 = core_dev.partitions[0] + core_part2 = core_dev.partitions[1] + Udev.disable() + + with TestRun.step("Start cache"): + cache = casadm.start_cache(cache_dev, cache_mode, force=True) + caches_count = len(casadm_parser.get_caches()) + if caches_count != 1: + TestRun.fail(f"Expected caches count: 1; Actual caches count: {caches_count}.") + + with TestRun.step("Add both core devices to cache."): + core1 = cache.add_core(core_part1) + core2 = cache.add_core(core_part2) + cores_count = len(casadm_parser.get_cores(cache.cache_id)) + if cores_count != 2: + TestRun.fail(f"Expected cores count: 2; Actual cores count: {cores_count}.") + + with TestRun.step("Fill cache with pages from the first core."): + dd_builder(cache_mode, core1, cache.size).run() + core1_occupied_blocks = core1.get_occupancy() + occupied_blocks_before = cache.get_occupancy() + + with TestRun.step("Remove the first core."): + cache.remove_core(core1.core_id, True) + caches_count = len(casadm_parser.get_caches()) + if caches_count != 1: + TestRun.fail(f"Expected caches count: 1; Actual caches count: {caches_count}.") + cores_count = len(casadm_parser.get_cores(cache.cache_id)) + if cores_count != 1: + TestRun.fail(f"Expected cores count: 1; Actual cores count: {cores_count}.") + + with TestRun.step("Check if occupancy from the first core is removed from cache."): + # Blocks occupied by the first core should be completely released. + if cache.get_occupancy() != occupied_blocks_before - core1_occupied_blocks: + TestRun.LOGGER.error("Blocks previously occupied by the first core " + "aren't released by removing this core.") + + with TestRun.step("Check if the remaining core is able to use cache."): + dd_builder(cache_mode, core2, Size(100, Unit.MebiByte)).run() + if not float(core2.get_occupancy().get_value()) > 0: + TestRun.fail("The remaining core is not able to use cache.") + + with TestRun.step("Stop cache."): + casadm.stop_all_caches() + + +@pytest.mark.parametrize("cache_mode", CacheMode.with_any_trait(CacheModeTrait.InsertRead + | CacheModeTrait.InsertWrite)) +@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand])) +@pytest.mark.require_disk("core", DiskTypeLowerThan("cache")) +def test_one_core_release(cache_mode): + """ + title: Test if OpenCAS dynamically allocates space according to core devices needs. + description: | + When one or more core devices are unused in a single cache instance all blocks + previously occupied should be available to other core devices. + Test is without pass through mode. + pass_criteria: + - No system crash. + - The remaining core is able to use cache. + - OpenCAS frees blocks occupied by unused core and allocates it to the remaining core. + """ + with TestRun.step("Prepare two cache and one core devices."): + cache_dev = TestRun.disks['cache'] + cache_dev.create_partitions([Size(512, Unit.MebiByte)]) + cache_part = cache_dev.partitions[0] + core_dev = TestRun.disks['core'] + core_dev.create_partitions([Size(1, Unit.GibiByte)] * 2) + core_part1 = core_dev.partitions[0] + core_part2 = core_dev.partitions[1] + Udev.disable() + + with TestRun.step("Start cache"): + cache = casadm.start_cache(cache_part, cache_mode, force=True) + caches_count = len(casadm_parser.get_caches()) + if caches_count != 1: + TestRun.fail(f"Expected caches count: 1; Actual caches count: {caches_count}.") + + with TestRun.step("Add both core devices to cache."): + core1 = cache.add_core(core_part1) + core2 = cache.add_core(core_part2) + cores_count = len(casadm_parser.get_cores(cache.cache_id)) + if cores_count != 2: + TestRun.fail(f"Expected cores count: 2; Actual cores count: {cores_count}.") + + with TestRun.step("Change sequential cutoff policy to 'never'."): + cache.set_seq_cutoff_policy(SeqCutOffPolicy.never) + + with TestRun.step("Fill cache with pages from the first core."): + dd_builder(cache_mode, core1, cache.size).run() + core1_occupied_blocks_before = core1.get_occupancy() + + with TestRun.step("Check if the remaining core is able to use cache."): + dd_builder(cache_mode, core2, Size(100, Unit.MebiByte)).run() + core1_occupied_blocks_after = core1.get_occupancy() + + with TestRun.step("Check if occupancy from the first core is removed from cache."): + # The first core's occupancy should be lower than cache's occupancy + # by the value of the remaining core's occupancy because cache + # should reallocate blocks from unused core to used core. + if core1_occupied_blocks_after >= core1_occupied_blocks_before \ + or cache.get_occupancy() <= core1_occupied_blocks_after \ + or not float(core2.get_occupancy().get_value()) > 0: + TestRun.LOGGER.error("Blocks previously occupied by the first core aren't released.") + + with TestRun.step("Stop cache."): + casadm.stop_all_caches() + + +@pytest.mark.parametrize("cache_mode", CacheMode.with_any_trait(CacheModeTrait.InsertRead + | CacheModeTrait.InsertWrite)) +@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand])) +@pytest.mark.require_disk("core1", DiskTypeLowerThan("cache")) +@pytest.mark.require_disk("core2", DiskTypeLowerThan("cache")) +def test_one_core_fail(cache_mode): + """ + title: Test if OpenCAS correctly handles failure of one of multiple core devices. + description: | + When one core device fails in a single cache instance all blocks previously occupied + should be available to other core devices. + Test is without pass through mode. + pass_criteria: + - No system crash. + - Second core is able to use OpenCAS. + """ + with TestRun.step("Prepare one cache and two core devices."): + cache_dev = TestRun.disks['cache'] + cache_dev.create_partitions([Size(1, Unit.GibiByte)] * 2) + cache_part = cache_dev.partitions[0] + core_dev1 = TestRun.disks['core1'] # This device would be unplugged. + core_dev1.create_partitions([Size(2, Unit.GibiByte)]) + core_part1 = core_dev1.partitions[0] + core_dev2 = TestRun.disks['core2'] + core_dev2.create_partitions([Size(2, Unit.GibiByte)]) + core_part2 = core_dev2.partitions[0] + Udev.disable() + + with TestRun.step("Start cache"): + cache = casadm.start_cache(cache_part, cache_mode, force=True) + caches_count = len(casadm_parser.get_caches()) + if caches_count != 1: + TestRun.fail(f"Expected caches count: 1; Actual caches count: {caches_count}.") + + with TestRun.step("Add both core devices to cache."): + core1 = cache.add_core(core_part1) + core2 = cache.add_core(core_part2) + cores_count = len(casadm_parser.get_cores(cache.cache_id)) + if cores_count != 2: + TestRun.fail(f"Expected cores count: 2; Actual cores count: {cores_count}.") + + with TestRun.step("Change sequential cutoff policy."): + cache.set_seq_cutoff_policy(SeqCutOffPolicy.never) + + with TestRun.step("Fill cache with pages from the first core."): + dd_builder(cache_mode, core1, cache.size).run() + cache_occupied_blocks_before = cache.get_occupancy() + + with TestRun.step("Unplug the first core device."): + core_dev1.unplug() + + with TestRun.step("Check if core device is really out of cache."): + output = str(casadm.list_caches().stdout.splitlines()) + if core_part1.system_path in output: + TestRun.fail("The first core device should be unplugged!") + + with TestRun.step("Check if the remaining core is able to use cache."): + dd_builder(cache_mode, core2, Size(100, Unit.MebiByte)).run() + if not float(core2.get_occupancy().get_value()) > 0: + TestRun.LOGGER.error("The remaining core is not able to use cache.") + + with TestRun.step("Check if occupancy from the first core is removed from cache."): + # Cache occupancy cannot be lower than before the first core fails and after that + # should be equal to the sum of occupancy of the first and the remaining core + cache_occupied_blocks_after = cache.get_occupancy() + if cache_occupied_blocks_before > cache_occupied_blocks_after \ + or cache_occupied_blocks_after != core2.get_occupancy() + core1.get_occupancy(): + TestRun.fail("Blocks previously occupied by the first core " + "aren't released after this core failure.") + + with TestRun.step("Stop cache."): + casadm.stop_all_caches() + + with TestRun.step("Plug back the first core."): + core_dev1.plug() + + +def dd_builder(cache_mode: CacheMode, dev: Core, size: Size): + blocks = int(size.value / block_size.value) + dd = (Dd() + .block_size(block_size) + .count(blocks)) + if CacheModeTrait.InsertRead in CacheMode.get_traits(cache_mode): + dd.input(dev.system_path).output("/dev/null") + else: + dd.input("/dev/urandom").output(dev.system_path) + return dd