From 0b299b7d969bcce4c652e3d209aa9b3fe20b90f5 Mon Sep 17 00:00:00 2001 From: Slawomir Jankowski Date: Mon, 22 Mar 2021 16:38:20 +0100 Subject: [PATCH 1/2] Fix interruption tests Add helper functions and use them in interruption tests. Fix tests descriptions and messages. Update test according to current behaviour of cache mode switching. Signed-off-by: Slawomir Jankowski --- .../test_fault_injection_interrupts.py | 73 +++++++++++++------ 1 file changed, 50 insertions(+), 23 deletions(-) diff --git a/test/functional/tests/fault_injection/test_fault_injection_interrupts.py b/test/functional/tests/fault_injection/test_fault_injection_interrupts.py index 7992f5a..201bea8 100644 --- a/test/functional/tests/fault_injection/test_fault_injection_interrupts.py +++ b/test/functional/tests/fault_injection/test_fault_injection_interrupts.py @@ -2,17 +2,21 @@ # Copyright(c) 2020-2021 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # -from time import sleep + +from datetime import timedelta, datetime import pytest from api.cas import casadm, casadm_parser, cli +from api.cas.cache import Cache from api.cas.cache_config import CacheMode, CleaningPolicy, CacheModeTrait -from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan +from api.cas.core import Core from core.test_run import TestRun +from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan from test_tools.disk_utils import Filesystem from test_utils import os_utils from test_utils.os_utils import Udev, DropCachesMode +from test_utils.output import CmdException from test_utils.size import Size, Unit mount_point = "/mnt/cas" @@ -34,7 +38,7 @@ def test_interrupt_core_flush(cache_mode, filesystem): - No system crash. - Flushing would be stopped after interruption. - Md5sum are correct during all test steps. - - Dirty blocks quantity after interruption is equal or lower. + - Dirty blocks quantity after interruption is equal or lower but non-zero. """ with TestRun.step("Prepare cache and core."): cache_part, core_part = prepare() @@ -67,9 +71,9 @@ def test_interrupt_core_flush(cache_mode, filesystem): with TestRun.step("Start flushing core device."): flush_pid = TestRun.executor.run_in_background( cli.flush_core_cmd(str(cache.cache_id), str(core.core_id))) - sleep(2) with TestRun.step("Interrupt core flushing."): + wait_for_flushing(cache, core) percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id) while percentage < 50: percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id) @@ -113,7 +117,7 @@ def test_interrupt_cache_flush(cache_mode, filesystem): - No system crash. - Flushing would be stopped after interruption. - Md5sum are correct during all test steps. - - Dirty blocks quantity after interruption is equal or lower. + - Dirty blocks quantity after interruption is equal or lower but non-zero. """ with TestRun.step("Prepare cache and core."): cache_part, core_part = prepare() @@ -146,9 +150,9 @@ def test_interrupt_cache_flush(cache_mode, filesystem): with TestRun.step("Start flushing cache."): flush_pid = TestRun.executor.run_in_background( cli.flush_cache_cmd(str(cache.cache_id))) - sleep(2) with TestRun.step("Interrupt cache flushing"): + wait_for_flushing(cache, core) percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id) while percentage < 50: percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id) @@ -193,7 +197,7 @@ def test_interrupt_core_remove(cache_mode, filesystem): - Core would not be removed from cache after interruption. - Flushing would be stopped after interruption. - Md5sum are correct during all test steps. - - Dirty blocks quantity after interruption is equal or lower. + - Dirty blocks quantity after interruption is lower but non-zero. """ with TestRun.step("Prepare cache and core."): cache_dev = TestRun.disks['cache'] @@ -235,9 +239,9 @@ def test_interrupt_core_remove(cache_mode, filesystem): with TestRun.step("Start removing core device."): flush_pid = TestRun.executor.run_in_background( cli.remove_core_cmd(str(cache.cache_id), str(core.core_id))) - sleep(2) with TestRun.step("Interrupt core removing"): + wait_for_flushing(cache, core) percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id) while percentage < 50: percentage = casadm_parser.get_flushing_progress(cache.cache_i, core.core_id) @@ -286,7 +290,7 @@ def test_interrupt_cache_mode_switch_immediately(cache_mode, filesystem): - Cache mode will not be switched after interruption. - Flushing would be stopped after interruption. - Md5sum are correct during all test steps. - - Dirty blocks quantity after interruption is equal or lower. + - Dirty blocks quantity after interruption is lower but non-zero. """ with TestRun.step("Prepare cache and core."): cache_part, core_part = prepare() @@ -326,11 +330,11 @@ def test_interrupt_cache_mode_switch_immediately(cache_mode, filesystem): with TestRun.step("Check number of dirty data on exported object after interruption."): cache_dirty_blocks_after = cache.get_dirty_blocks() if cache_dirty_blocks_after >= cache_dirty_blocks_before: - TestRun.LOGGER.error("Quantity of dirty lines after cache flush interruption " - "should be lower.") + TestRun.LOGGER.error("Quantity of dirty lines after cache mode switching " + "interruption should be lower.") if int(cache_dirty_blocks_after) == 0: - TestRun.LOGGER.error("Quantity of dirty lines after cache flush interruption " - "should not be zero.") + TestRun.LOGGER.error("Quantity of dirty lines after cache mode switching " + "interruption should not be zero.") with TestRun.step("Check cache mode."): if cache.get_cache_mode() != cache_mode: @@ -346,7 +350,7 @@ def test_interrupt_cache_mode_switch_immediately(cache_mode, filesystem): with TestRun.step("Check md5 sum of test file again."): if test_file_md5sum_before != test_file.md5sum(): TestRun.LOGGER.error( - "Md5 sums before and after interrupting core removal are different.") + "Md5 sums before and after interrupting mode switching are different.") with TestRun.step("Unmount core device."): core_part.unmount() @@ -364,9 +368,10 @@ def test_interrupt_cache_mode_switch_delayed(cache_mode, filesystem): interruption with delay. pass_criteria: - No system crash. - - Cache mode cannot be interrupted with delay. + - Cache mode will not be switched after interruption. + - Flushing would be stopped after interruption. - Md5sum are correct during all test steps. - - Dirty blocks quantity after cache mode switching is zero. + - Dirty blocks quantity after interruption is lower but non-zero. """ with TestRun.step("Prepare cache and core."): cache_part, core_part = prepare() @@ -391,12 +396,18 @@ def test_interrupt_cache_mode_switch_delayed(cache_mode, filesystem): with TestRun.step("Check md5 sum of test file."): test_file_md5sum_before = test_file.md5sum() + with TestRun.step("Get number of dirty data on exported object before " + "switching cache mode interruption."): + os_utils.sync() + os_utils.drop_caches(DropCachesMode.ALL) + cache_dirty_blocks_before = cache.get_dirty_blocks() + with TestRun.step("Start switching cache mode."): flush_pid = TestRun.executor.run_in_background(cli.set_cache_mode_cmd( str(CacheMode.DEFAULT.name.lower()), str(cache.cache_id), "yes")) - sleep(2) with TestRun.step("Send interruption signal."): + wait_for_flushing(cache, core) percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id) while percentage < 50: percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id) @@ -405,12 +416,17 @@ def test_interrupt_cache_mode_switch_delayed(cache_mode, filesystem): with TestRun.step( "Get quantity of dirty data on exported object after sending interruption " "signal to cas to stop mode switching."): - if int(cache.get_dirty_blocks()) != 0: - TestRun.LOGGER.error("Quantity of dirty lines should be zero now.") + cache_dirty_blocks_after = cache.get_dirty_blocks() + if cache_dirty_blocks_after >= cache_dirty_blocks_before: + TestRun.LOGGER.error("Quantity of dirty lines after cache mode switching " + "interruption should be lower.") + if int(cache_dirty_blocks_after) == 0: + TestRun.LOGGER.error("Quantity of dirty lines after cache mode switching " + "interruption should not be zero.") with TestRun.step("Check cache mode."): - if cache.get_cache_mode() == cache_mode: - TestRun.LOGGER.error("Cache mode should have changed.") + if cache.get_cache_mode() != cache_mode: + TestRun.LOGGER.error("Cache mode should remain the same.") with TestRun.step("Unmount core and stop cache."): core.unmount() @@ -422,7 +438,7 @@ def test_interrupt_cache_mode_switch_delayed(cache_mode, filesystem): with TestRun.step("Check md5 sum of test file again."): if test_file_md5sum_before != test_file.md5sum(): TestRun.LOGGER.error( - "Md5 sums before and after interrupting core removal are different.") + "Md5 sums before and after interrupting mode switching are different.") with TestRun.step("Unmount core device."): core_part.unmount() @@ -476,9 +492,9 @@ def test_interrupt_cache_stop(cache_mode, filesystem): with TestRun.step("Start stopping cache."): flush_pid = TestRun.executor.run_in_background(cli.stop_cmd(str(cache.cache_id))) - sleep(2) with TestRun.step("Interrupt cache stopping."): + wait_for_flushing(cache, core) percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id) while percentage < 50: percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id) @@ -532,3 +548,14 @@ def create_test_file(): dd.run() test_file.refresh_item() return test_file + + +def wait_for_flushing(cache: Cache, core: Core, timeout: timedelta = timedelta(seconds=30)): + start_time = datetime.now() + while datetime.now() - start_time < timeout: + try: + casadm_parser.get_flushing_progress(cache.cache_id, core.core_id) + return + except CmdException: + continue + raise Exception("Management flush not started!") From 7ab8423b559f9779bead2ec37df241a133f939c5 Mon Sep 17 00:00:00 2001 From: Daniel Madej Date: Fri, 5 Nov 2021 12:26:09 +0100 Subject: [PATCH 2/2] Send actual output to CmdException Signed-off-by: Daniel Madej --- test/functional/api/cas/casadm_parser.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/functional/api/cas/casadm_parser.py b/test/functional/api/cas/casadm_parser.py index df7e403..6dcfc07 100644 --- a/test/functional/api/cas/casadm_parser.py +++ b/test/functional/api/cas/casadm_parser.py @@ -15,8 +15,8 @@ from api.cas.casadm_params import * from api.cas.ioclass_config import IoClass from api.cas.version import CasVersion from storage_devices.device import Device -from test_utils.size import parse_unit from test_utils.output import CmdException +from test_utils.size import parse_unit class Stats(dict): @@ -210,8 +210,9 @@ def get_cas_devices_dict(): def get_flushing_progress(cache_id: int, core_id: int = None): - casadm_output = casadm.list_caches(OutputFormat.csv).stdout.splitlines() - for line in casadm_output: + casadm_output = casadm.list_caches(OutputFormat.csv) + lines = casadm_output.stdout.splitlines() + for line in lines: line_elements = line.split(',') if core_id is not None and line_elements[0] == "core" \ and int(line_elements[1]) == core_id \