From 948eccb0b952823808b6b67bbe7736105528b060 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Thu, 21 Apr 2022 12:34:19 +0200 Subject: [PATCH 1/6] test api: add missing `standby_activate`() method Signed-off-by: Michal Mielewczyk --- test/functional/api/cas/cache.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/functional/api/cas/cache.py b/test/functional/api/cas/cache.py index 9729b64..abc13d8 100644 --- a/test/functional/api/cas/cache.py +++ b/test/functional/api/cas/cache.py @@ -180,3 +180,8 @@ class Cache: def standby_detach(self, shortcut: bool = False): return casadm.standby_detach_cache(cache_id=self.cache_id, shortcut=shortcut) + + def standby_activate(self, device, shortcut: bool = False): + return casadm.standby_activate_cache( + cache_id=self.cache_id, cache_dev=device, shortcut=shortcut + ) From ca790e4f35393c425f1d2b34fb1a16b37be9c859 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Thu, 21 Apr 2022 12:35:50 +0200 Subject: [PATCH 2/6] test api: add missing cli messages Signed-off-by: Michal Mielewczyk --- test/functional/api/cas/cli_messages.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/functional/api/cas/cli_messages.py b/test/functional/api/cas/cli_messages.py index 70881a9..5e0246b 100644 --- a/test/functional/api/cas/cli_messages.py +++ b/test/functional/api/cas/cli_messages.py @@ -149,6 +149,18 @@ activate_with_different_cache_id = [ r"Cache id specified by user and loaded from metadata are different" ] +cache_activated_successfully = [ + r"Successfully activated cache instance \d+" +] + +invalid_core_volume_size = [ + r"Core volume size does not match the size stored in cache metadata" +] + +error_activating_cache = [ + r"Error activating cache \d+" +] + def check_stderr_msg(output: Output, expected_messages): return __check_string_msg(output.stderr, expected_messages) From aec57d3c81067d79d722a6dcc114edbeb5ca8275 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Thu, 21 Apr 2022 12:39:10 +0200 Subject: [PATCH 3/6] test api: extract utility for retrieving core info Signed-off-by: Michal Mielewczyk --- test/functional/api/cas/casadm_parser.py | 14 ++++++++++++++ test/functional/api/cas/core.py | 13 ++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/test/functional/api/cas/casadm_parser.py b/test/functional/api/cas/casadm_parser.py index 6dcfc07..b496c4e 100644 --- a/test/functional/api/cas/casadm_parser.py +++ b/test/functional/api/cas/casadm_parser.py @@ -4,6 +4,7 @@ # import csv +import io import json import re from datetime import timedelta @@ -286,3 +287,16 @@ def get_io_class_list(cache_id: int): ioclass = IoClass(int(values[0]), values[1], int(values[2]), values[3]) ret.append(ioclass) return ret + + +def get_core_info_by_path(core_disk_path): + output = casadm.list_caches(OutputFormat.csv, by_id_path=True) + reader = csv.DictReader(io.StringIO(output.stdout)) + for row in reader: + if row['type'] == "core" and row['disk'] == core_disk_path: + return {"core_id": row['id'], + "core_device": row['disk'], + "status": row['status'], + "exp_obj": row['device']} + + return None diff --git a/test/functional/api/cas/core.py b/test/functional/api/cas/core.py index f29d9ef..4482c9f 100644 --- a/test/functional/api/cas/core.py +++ b/test/functional/api/cas/core.py @@ -2,8 +2,6 @@ # Copyright(c) 2019-2021 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # -import csv -import io from datetime import timedelta from typing import List @@ -12,7 +10,7 @@ from aenum import Enum from api.cas import casadm from api.cas.cache_config import SeqCutOffParameters, SeqCutOffPolicy from api.cas.casadm_params import OutputFormat, StatsFilter -from api.cas.casadm_parser import get_statistics, get_seq_cut_off_parameters +from api.cas.casadm_parser import get_statistics, get_seq_cut_off_parameters, get_core_info_by_path from api.cas.statistics import CoreStats, CoreIoClassStats from core.test_run_utils import TestRun from storage_devices.device import Device @@ -47,14 +45,7 @@ class Core(Device): self.block_size = None def __get_core_info(self): - output = casadm.list_caches(OutputFormat.csv, by_id_path=True) - reader = csv.DictReader(io.StringIO(output.stdout)) - for row in reader: - if row['type'] == "core" and row['disk'] == self.core_device.path: - return {"core_id": row['id'], - "core_device": row['disk'], - "status": row['status'], - "exp_obj": row['device']} + return get_core_info_by_path(self.core_device.path) def create_filesystem(self, fs_type: disk_utils.Filesystem, force=True, blocksize=None): super().create_filesystem(fs_type, force, blocksize) From f05443c06bb7c1788e15897a394e4368c82b60ba Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Mon, 25 Apr 2022 10:59:07 +0200 Subject: [PATCH 4/6] tests: more meaningful name for `test_standby.py` Signed-off-by: Michal Mielewczyk --- ...ndby.py => test_fault_injection_standby_core.py} | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) rename test/functional/tests/fault_injection/{test_standby.py => test_fault_injection_standby_core.py} (89%) diff --git a/test/functional/tests/fault_injection/test_standby.py b/test/functional/tests/fault_injection/test_fault_injection_standby_core.py similarity index 89% rename from test/functional/tests/fault_injection/test_standby.py rename to test/functional/tests/fault_injection/test_fault_injection_standby_core.py index 03b29f9..530a506 100644 --- a/test/functional/tests/fault_injection/test_standby.py +++ b/test/functional/tests/fault_injection/test_fault_injection_standby_core.py @@ -10,9 +10,20 @@ from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan from core.test_run import TestRun from test_utils.size import Size, Unit from api.cas.cache_config import CacheLineSize, CacheMode, CacheStatus +from api.cas.casadm_params import StatsFilter +from api.cas.casadm_parser import get_core_info_by_path +from api.cas.core import CoreStatus, Core from test_tools.dd import Dd from api.cas.cli import standby_activate_cmd -from api.cas.cli_messages import check_stderr_msg, activate_with_different_cache_id +from api.cas.cli_messages import ( + check_stderr_msg, + check_stdout_msg, + activate_with_different_cache_id, + load_inactive_core_missing, + cache_activated_successfully, + invalid_core_volume_size, + error_activating_cache, +) @pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand])) From 150c20608c1d8c24d5672df90f985b4dd4b5d7ba Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Mon, 25 Apr 2022 11:00:23 +0200 Subject: [PATCH 5/6] tests: test_activate_incomplete_cache Signed-off-by: Michal Mielewczyk --- .../test_fault_injection_standby_core.py | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/test/functional/tests/fault_injection/test_fault_injection_standby_core.py b/test/functional/tests/fault_injection/test_fault_injection_standby_core.py index 530a506..9cef1cd 100644 --- a/test/functional/tests/fault_injection/test_fault_injection_standby_core.py +++ b/test/functional/tests/fault_injection/test_fault_injection_standby_core.py @@ -94,3 +94,119 @@ def test_activate_neg_cache_id(): "The standby cache instance is in an invalid state " f"Expected {CacheStatus.standby_detached}. Got {status}" ) + + +@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand])) +@pytest.mark.require_disk("core", DiskTypeLowerThan("cache")) +def test_activate_incomplete_cache(): + """ + title: Activating cache with a missing core device. + description: | + Try restoring cache operations from a standby cache when the core device + used before is missing. + pass_criteria: + -The activation succeedes when the required core device is missing + -The cache is transitioned into Incomplete state after a successful activation + -The message about activating into “Incomplete” state is displayed + -The cache instance is switched into “Running” mode when the core device appears + in the system + """ + with TestRun.step("Prepare partitions"): + core_part_size = Size(200, Unit.MebiByte) + cache_disk = TestRun.disks["cache"] + core_disk = TestRun.disks["core"] + cache_disk.create_partitions([Size(200, Unit.MebiByte)]) + core_disk.create_partitions([core_part_size]) + cache_dev = cache_disk.partitions[0] + core_dev = core_disk.partitions[0] + core_dev_path = core_dev.path + + with TestRun.step("Start a regular cache instance with a core"): + cache = casadm.start_cache(cache_dev, force=True) + cache.add_core(core_dev) + + with TestRun.step("Stop the cache device"): + cache.stop() + + with TestRun.step("Remove the partition used as core"): + core_disk.remove_partitions() + core_dev = None + + with TestRun.step("Load standby cache instance"): + cache = casadm.standby_load(cache_dev) + + with TestRun.step("Verify if the cache exported object appeared in the system"): + output = TestRun.executor.run_expect_success(f"ls -la /dev/ | grep cas-cache-1") + if output.stdout[0] != "b": + TestRun.fail("The cache exported object is not a block device") + + with TestRun.step("Detach the standby instance"): + cache.standby_detach() + + with TestRun.step("Activate standby cache and check if a proper incompleteness info appeared"): + output = cache.standby_activate(device=cache_dev) + check_stderr_msg(output, load_inactive_core_missing) + check_stdout_msg(output, cache_activated_successfully) + + with TestRun.step("Verify that the cache is in Incomplete state"): + status = cache.get_status() + if status != CacheStatus.incomplete: + TestRun.LOGGER.error( + "The cache instance is in an invalid state. " + f"Expected {CacheStatus.incomplete}. Got {status}" + ) + + with TestRun.step("Check if the number of cores is valid"): + cache_conf_stats = cache.get_statistics(stat_filter=[StatsFilter.conf]) + core_count = int(cache_conf_stats.config_stats.core_dev) + if core_count != 1: + TestRun.fail(f"Expected one core. Got {core_count}") + + with TestRun.step("Check if the number of inactive cores is valid"): + inactive_core_count = int(cache_conf_stats.config_stats.inactive_core_dev) + if inactive_core_count != 1: + TestRun.fail(f"Expected one inactive core. Got {inactive_core_count}") + + with TestRun.step("Check if core is in an appropriate state"): + core_status = CoreStatus[get_core_info_by_path(core_dev_path)["status"].lower()] + if core_status != CoreStatus.inactive: + TestRun.fail( + "The core is in an invalid state. " + f"Expected {CoreStatus.inactive}. Got {core_status}" + ) + + with TestRun.step("Restore core partition"): + core_disk.create_partitions([core_part_size]) + core_dev = core_disk.partitions[0] + + with TestRun.step("Add core using try-add script command"): + casadm.try_add(core_dev, cache_id=1, core_id=1) + + with TestRun.step("Verify that the cache is in Running state"): + status = cache.get_status() + if status != CacheStatus.running: + TestRun.LOGGER.error( + "The cache instance is in an invalid state. " + f"Expected {CacheStatus.running}. Got {status}" + ) + + with TestRun.step("Verify that the core is in Active state"): + cache_conf_stats = cache.get_statistics(stat_filter=[StatsFilter.conf]) + core_count = int(cache_conf_stats.config_stats.core_dev) + if core_count != 1: + TestRun.fail(f"Expected one core. Got {core_count}") + + with TestRun.step("Check if the number of inactive cores is valid"): + inactive_core_count = int(cache_conf_stats.config_stats.inactive_core_dev) + if inactive_core_count != 0: + TestRun.fail( + f"The test didn't expect inactive cores at this point. " + f"Got {inactive_core_count}" + ) + + core_status = Core(core_device=core_dev.path, cache_id=1).get_status() + if core_status != CoreStatus.active: + TestRun.LOGGER.error( + "The core is in an invalid state. " + f"Expected {CoreStatus.active}. Got {core_status}" + ) From 00b34646445c931364ca21071f49c2f4f66490df Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Thu, 21 Apr 2022 13:55:16 +0200 Subject: [PATCH 6/6] tests: test_activate_neg_core_size Signed-off-by: Michal Mielewczyk --- .../test_fault_injection_standby_core.py | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/test/functional/tests/fault_injection/test_fault_injection_standby_core.py b/test/functional/tests/fault_injection/test_fault_injection_standby_core.py index 9cef1cd..9083e57 100644 --- a/test/functional/tests/fault_injection/test_fault_injection_standby_core.py +++ b/test/functional/tests/fault_injection/test_fault_injection_standby_core.py @@ -210,3 +210,102 @@ def test_activate_incomplete_cache(): "The core is in an invalid state. " f"Expected {CoreStatus.active}. Got {core_status}" ) + + +@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand])) +@pytest.mark.require_disk("core", DiskTypeLowerThan("cache")) +def test_activate_neg_core_size(): + """ + title: Activating cache with a core of a altered size. + description: | + Try restoring cache operations from a standby cache when the core device size has been altered + pass_criteria: + - The cache activation is cancelled when a core device size mismatch occurs + - The cache remains in standby detached state after an unsuccessful activation + - A proper error message is displayed + """ + with TestRun.step("Prepare partitions"): + core_part_size = Size(200, Unit.MebiByte) + cache_disk = TestRun.disks["cache"] + core_disk = TestRun.disks["core"] + cache_disk.create_partitions([Size(200, Unit.MebiByte)]) + core_disk.create_partitions([core_part_size]) + cache_dev = cache_disk.partitions[0] + core_dev = core_disk.partitions[0] + core_dev_path = core_dev.path + + with TestRun.step("Start a regular cache instance with a core"): + cache = casadm.start_cache(cache_dev, force=True) + cache.add_core(core_dev) + + with TestRun.step("Stop the cache device"): + cache.stop() + + with TestRun.step("Resize the partition used as core"): + core_disk.remove_partitions() + core_disk.create_partitions([Size(400, Unit.MebiByte)]) + + with TestRun.step("Load standby cache instance"): + cache = casadm.standby_load(cache_dev) + + with TestRun.step("Verify if the cache exported object appeared in the system"): + output = TestRun.executor.run_expect_success(f"ls -la /dev/ | grep cas-cache-1") + if output.stdout[0] != "b": + TestRun.fail("The cache exported object is not a block device") + + with TestRun.step("Detach the standby instance"): + cache.standby_detach() + + with TestRun.step("Try if activate fails and returns the same error every time"): + activate_cmd = standby_activate_cmd(cache_dev=cache_dev.path, cache_id="1") + for i in range(10): + output = TestRun.executor.run_expect_fail(activate_cmd) + check_stderr_msg(output, error_activating_cache) + check_stderr_msg(output, invalid_core_volume_size) + + with TestRun.step("Verify that the cache is in Standby detached state"): + status = cache.get_status() + if status != CacheStatus.standby_detached: + TestRun.LOGGER.error( + "The cache instance is in an invalid state. " + f"Expected {CacheStatus.standby_detached}. Got {status}" + ) + + with TestRun.step("Restore the original size of core partition"): + core_disk.remove_partitions() + core_disk.create_partitions([core_part_size]) + core_dev = core_disk.partitions[0] + + with TestRun.step("Activate standby cache"): + output = cache.standby_activate(device=cache_dev) + check_stdout_msg(output, cache_activated_successfully) + + with TestRun.step("Verify that the cache is in Running state"): + status = cache.get_status() + if status != CacheStatus.running: + TestRun.LOGGER.error( + "The cache instance is in an invalid state. " + f"Expected {CacheStatus.running}. Got {status}" + ) + + with TestRun.step("Verify that the core is in Active state"): + core_status = Core(core_device=core_dev.path, cache_id=1).get_status() + if core_status != CoreStatus.active: + TestRun.LOGGER.error( + "The core is in an invalid state. " + f"Expected {CoreStatus.active}. Got {core_status}" + ) + + with TestRun.step("Check if the number of active cores is valid"): + cache_conf_stats = cache.get_statistics(stat_filter=[StatsFilter.conf]) + core_count = int(cache_conf_stats.config_stats.core_dev) + if core_count != 1: + TestRun.fail(f"Expected one core. Got {core_count}") + + with TestRun.step("Check if the number of inactive cores is valid"): + inactive_core_count = int(cache_conf_stats.config_stats.inactive_core_dev) + if inactive_core_count != 0: + TestRun.fail( + f"The test didn't expect inactive cores at this point. " + f"Got {inactive_core_count}" + )