github/open-cas-linux
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tests: refactor fault_injection tests

Browse Source 
Signed-off-by: Damian Raczkowski <damian.raczkowski@intel.com>
This commit is contained in:
Damian Raczkowski
2022-08-16 04:32:55 +02:00
parent 5069652fb4
commit f57b01ee31
12 changed files with 339 additions and 458 deletions
Download Patch File Download Diff File Expand all files Collapse all files

355
test/functional/tests/fault_injection/test_fault_injection_interrupts.py
View File

@@ -11,9 +11,12 @@ from api.cas.casadm_parser import wait_for_flushing
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools.disk_utils import Filesystem
from test_tools.dd import Dd
from test_utils import os_utils
from test_utils.os_utils import Udev, DropCachesMode
from test_utils.size import Size, Unit
from tests.lazy_writes.recovery.recovery_tests_methods import compare_files
from test_tools import fs_utils
mount_point = "/mnt/cas"
test_file_path = f"{mount_point}/test_file"
@@ -27,20 +30,21 @@ cache_size = Size(16, Unit.GibiByte)
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
def test_interrupt_core_flush(cache_mode, filesystem):
"""
title: Test if OpenCAS works correctly after core's flushing interruption.
description: |
Negative test of the ability of OpenCAS to handle core flushing interruption.
pass_criteria:
- No system crash.
- Flushing would be stopped after interruption.
- Md5sum are correct during all test steps.
- Dirty blocks quantity after interruption is equal or lower but non-zero.
title: Test if OpenCAS works correctly after core's flushing interruption.
description: |
Negative test of the ability of OpenCAS to handle core flushing interruption.
pass_criteria:
- No system crash.
- Flushing would be stopped after interruption.
- Md5sum are correct during all test steps.
- Dirty blocks quantity after interruption is equal or lower but non-zero.
"""
with TestRun.step("Prepare cache and core."):
cache_part, core_part = prepare()
for _ in TestRun.iteration(range(iterations_per_config),
f"Reload cache configuration {iterations_per_config} times."):
for _ in TestRun.iteration(
range(iterations_per_config), f"Reload cache configuration {iterations_per_config} times."
):
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
@@ -66,7 +70,8 @@ def test_interrupt_core_flush(cache_mode, filesystem):
with TestRun.step("Start flushing core device."):
flush_pid = TestRun.executor.run_in_background(
cli.flush_core_cmd(str(cache.cache_id), str(core.core_id)))
cli.flush_core_cmd(str(cache.cache_id), str(core.core_id))
)
with TestRun.step("Interrupt core flushing."):
wait_for_flushing(cache, core)
@@ -78,11 +83,13 @@ def test_interrupt_core_flush(cache_mode, filesystem):
with TestRun.step("Check number of dirty data on exported object after interruption."):
core_dirty_blocks_after = core.get_dirty_blocks()
if core_dirty_blocks_after >= core_dirty_blocks_before:
TestRun.LOGGER.error("Quantity of dirty lines after core flush interruption "
"should be lower.")
TestRun.LOGGER.error(
"Quantity of dirty lines after core flush interruption " "should be lower."
)
if int(core_dirty_blocks_after) == 0:
TestRun.LOGGER.error("Quantity of dirty lines after core flush interruption "
"should not be zero.")
TestRun.LOGGER.error(
"Quantity of dirty lines after core flush interruption " "should not be zero."
)
with TestRun.step("Unmount core and stop cache."):
core.unmount()
@@ -94,7 +101,8 @@ def test_interrupt_core_flush(cache_mode, filesystem):
with TestRun.step("Check md5 sum of test file again."):
if test_file_md5sum_before != test_file.md5sum():
TestRun.LOGGER.error(
"Md5 sums before and after interrupting core flush are different.")
"Md5 sums before and after interrupting core flush are different."
)
with TestRun.step("Unmount core device."):
core_part.unmount()
@@ -106,20 +114,21 @@ def test_interrupt_core_flush(cache_mode, filesystem):
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
def test_interrupt_cache_flush(cache_mode, filesystem):
"""
title: Test if OpenCAS works correctly after cache's flushing interruption.
description: |
Negative test of the ability of OpenCAS to handle cache flushing interruption.
pass_criteria:
- No system crash.
- Flushing would be stopped after interruption.
- Md5sum are correct during all test steps.
- Dirty blocks quantity after interruption is equal or lower but non-zero.
title: Test if OpenCAS works correctly after cache's flushing interruption.
description: |
Negative test of the ability of OpenCAS to handle cache flushing interruption.
pass_criteria:
- No system crash.
- Flushing would be stopped after interruption.
- Md5sum are correct during all test steps.
- Dirty blocks quantity after interruption is equal or lower but non-zero.
"""
with TestRun.step("Prepare cache and core."):
cache_part, core_part = prepare()
for _ in TestRun.iteration(range(iterations_per_config),
f"Reload cache configuration {iterations_per_config} times."):
for _ in TestRun.iteration(
range(iterations_per_config), f"Reload cache configuration {iterations_per_config} times."
):
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
@@ -144,8 +153,7 @@ def test_interrupt_cache_flush(cache_mode, filesystem):
cache_dirty_blocks_before = cache.get_dirty_blocks()
with TestRun.step("Start flushing cache."):
flush_pid = TestRun.executor.run_in_background(
cli.flush_cache_cmd(str(cache.cache_id)))
flush_pid = TestRun.executor.run_in_background(cli.flush_cache_cmd(str(cache.cache_id)))
with TestRun.step("Interrupt cache flushing"):
wait_for_flushing(cache, core)
@@ -157,11 +165,13 @@ def test_interrupt_cache_flush(cache_mode, filesystem):
with TestRun.step("Check number of dirty data on exported object after interruption."):
cache_dirty_blocks_after = cache.get_dirty_blocks()
if cache_dirty_blocks_after >= cache_dirty_blocks_before:
TestRun.LOGGER.error("Quantity of dirty lines after cache flush interruption "
"should be lower.")
TestRun.LOGGER.error(
"Quantity of dirty lines after cache flush interruption " "should be lower."
)
if int(cache_dirty_blocks_after) == 0:
TestRun.LOGGER.error("Quantity of dirty lines after cache flush interruption "
"should not be zero.")
TestRun.LOGGER.error(
"Quantity of dirty lines after cache flush interruption " "should not be zero."
)
with TestRun.step("Unmount core and stop cache."):
core.unmount()
@@ -173,7 +183,8 @@ def test_interrupt_cache_flush(cache_mode, filesystem):
with TestRun.step("Check md5 sum of test file again."):
if test_file_md5sum_before != test_file.md5sum():
TestRun.LOGGER.error(
"Md5 sums before and after interrupting cache flush are different.")
"Md5 sums before and after interrupting cache flush are different."
)
with TestRun.step("Unmount core device."):
core_part.unmount()
@@ -185,26 +196,27 @@ def test_interrupt_cache_flush(cache_mode, filesystem):
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
def test_interrupt_core_remove(cache_mode, filesystem):
"""
title: Test if OpenCAS works correctly after core's removing interruption.
description: |
Negative test of the ability of OpenCAS to handle core's removing interruption.
pass_criteria:
- No system crash.
- Core would not be removed from cache after interruption.
- Flushing would be stopped after interruption.
- Md5sum are correct during all test steps.
- Dirty blocks quantity after interruption is lower but non-zero.
title: Test if OpenCAS works correctly after core's removing interruption.
description: |
Negative test of the ability of OpenCAS to handle core's removing interruption.
pass_criteria:
- No system crash.
- Core would not be removed from cache after interruption.
- Flushing would be stopped after interruption.
- Md5sum are correct during all test steps.
- Dirty blocks quantity after interruption is lower but non-zero.
"""
with TestRun.step("Prepare cache and core."):
cache_dev = TestRun.disks['cache']
cache_dev = TestRun.disks["cache"]
cache_dev.create_partitions([cache_size])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev = TestRun.disks["core"]
core_dev.create_partitions([cache_size * 2])
core_part = core_dev.partitions[0]
for _ in TestRun.iteration(range(iterations_per_config),
f"Reload cache configuration {iterations_per_config} times."):
for _ in TestRun.iteration(
range(iterations_per_config), f"Reload cache configuration {iterations_per_config} times."
):
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
@@ -224,7 +236,8 @@ def test_interrupt_core_remove(cache_mode, filesystem):
test_file_md5sum_before = test_file.md5sum()
with TestRun.step(
"Get number of dirty data on exported object before core removal interruption."):
"Get number of dirty data on exported object before core removal interruption."
):
os_utils.sync()
os_utils.drop_caches(DropCachesMode.ALL)
cache_dirty_blocks_before = cache.get_dirty_blocks()
@@ -234,7 +247,8 @@ def test_interrupt_core_remove(cache_mode, filesystem):
with TestRun.step("Start removing core device."):
flush_pid = TestRun.executor.run_in_background(
cli.remove_core_cmd(str(cache.cache_id), str(core.core_id)))
cli.remove_core_cmd(str(cache.cache_id), str(core.core_id))
)
with TestRun.step("Interrupt core removing"):
wait_for_flushing(cache, core)
@@ -243,15 +257,23 @@ def test_interrupt_core_remove(cache_mode, filesystem):
percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id)
TestRun.executor.run(f"kill -s SIGINT {flush_pid}")
with TestRun.step("Check md5 sum of test file after interruption."):
cache.set_cache_mode(CacheMode.WO)
test_file_md5sum_interrupt = test_file.md5sum()
cache.set_cache_mode(cache_mode)
with TestRun.step(
"Check number of dirty data on exported object after core removal interruption."):
"Check number of dirty data on exported object after core removal interruption."
):
cache_dirty_blocks_after = cache.get_dirty_blocks()
if cache_dirty_blocks_after >= cache_dirty_blocks_before:
TestRun.LOGGER.error("Quantity of dirty lines after core removal interruption "
"should be lower.")
TestRun.LOGGER.error(
"Quantity of dirty lines after core removal interruption " "should be lower."
)
if int(cache_dirty_blocks_after) == 0:
TestRun.LOGGER.error("Quantity of dirty lines after core removal interruption "
"should not be zero.")
TestRun.LOGGER.error(
"Quantity of dirty lines after core removal interruption " "should not be zero."
)
with TestRun.step("Remove core from cache."):
core.remove_core()
@@ -264,35 +286,40 @@ def test_interrupt_core_remove(cache_mode, filesystem):
with TestRun.step("Check md5 sum of test file again."):
if test_file_md5sum_before != test_file.md5sum():
TestRun.LOGGER.error(
"Md5 sums before and after interrupting core removal are different.")
TestRun.LOGGER.error("Md5 sum before interrupting core removal is different.")
is_sum_diff_after_interrupt = test_file_md5sum_interrupt != test_file.md5sum()
if is_sum_diff_after_interrupt:
TestRun.LOGGER.error("Md5 sum after interrupting core removal is different.")
with TestRun.step("Unmount core device."):
core_part.unmount()
@pytest.mark.parametrizex("filesystem", Filesystem)
@pytest.mark.parametrize("stop_percentage", [0, 50])
@pytest.mark.parametrizex("cache_mode", CacheMode.with_traits(CacheModeTrait.LazyWrites))
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
def test_interrupt_cache_mode_switch_immediately(cache_mode, filesystem):
def test_interrupt_cache_mode_switch_parametrized(cache_mode, stop_percentage):
"""
title: Test if OpenCAS works correctly after cache mode switching immediate interruption.
description: |
Negative test of the ability of OpenCAS to handle cache mode switching
immediate interruption.
pass_criteria:
- No system crash.
- Cache mode will not be switched after interruption.
- Flushing would be stopped after interruption.
- Md5sum are correct during all test steps.
- Dirty blocks quantity after interruption is lower but non-zero.
title: Test if OpenCAS works correctly after cache mode switching
immediate or delayed interruption.
description: |
Negative test of the ability of OpenCAS to handle cache mode switching
immediate or delayed interruption.
pass_criteria:
- No system crash.
- Cache mode will not be switched after interruption.
- Flushing would be stopped after interruption.
- Md5sum are correct during all test steps.
- Dirty blocks quantity after interruption is lower but non-zero.
"""
with TestRun.step("Prepare cache and core."):
cache_part, core_part = prepare()
for _ in TestRun.iteration(range(iterations_per_config),
f"Reload cache configuration {iterations_per_config} times."):
for _ in TestRun.iteration(
range(iterations_per_config), f"Reload cache configuration {iterations_per_config} times."
):
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
@@ -300,16 +327,20 @@ def test_interrupt_cache_mode_switch_immediately(cache_mode, filesystem):
with TestRun.step("Set cleaning policy to NOP."):
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step(f"Add core device with {filesystem} filesystem and mount it."):
core_part.create_filesystem(filesystem)
with TestRun.step(f"Add core device."):
core = cache.add_core(core_part)
core.mount(mount_point)
with TestRun.step(f"Create test file in mount point of exported object."):
test_file = create_test_file()
test_file_size = Size(1024, Unit.MebiByte)
test_file = fs_utils.create_random_test_file(test_file_path, test_file_size)
with TestRun.step("Check md5 sum of test file."):
test_file_md5sum_before = test_file.md5sum()
test_file_md5_before = test_file.md5sum()
with TestRun.step("Export file to CAS"):
Dd().block_size(test_file_size).input(test_file.full_path).output(core.path).oflag(
"direct"
).run()
with TestRun.step("Get number of dirty data on exported object before interruption."):
os_utils.sync()
@@ -317,128 +348,45 @@ def test_interrupt_cache_mode_switch_immediately(cache_mode, filesystem):
cache_dirty_blocks_before = cache.get_dirty_blocks()
with TestRun.step("Start switching cache mode."):
flush_pid = TestRun.executor.run_in_background(cli.set_cache_mode_cmd(
str(CacheMode.DEFAULT.name.lower()), str(cache.cache_id), "yes"))
wait_for_flushing(cache, core)
flush_pid = TestRun.executor.run_in_background(
cli.set_cache_mode_cmd(
str(CacheMode.DEFAULT.name.lower()), str(cache.cache_id), "yes"
)
)
with TestRun.step("Send interruption signal."):
wait_for_flushing(cache, core)
percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id)
while percentage < stop_percentage:
percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id)
TestRun.executor.run(f"kill -s SIGINT {flush_pid}")
with TestRun.step("Check number of dirty data on exported object after interruption."):
cache_dirty_blocks_after = cache.get_dirty_blocks()
if cache_dirty_blocks_after >= cache_dirty_blocks_before:
TestRun.LOGGER.error("Quantity of dirty lines after cache mode switching "
"interruption should be lower.")
TestRun.LOGGER.error(
"Quantity of dirty lines after cache mode switching "
"interruption should be lower."
)
if int(cache_dirty_blocks_after) == 0:
TestRun.LOGGER.error("Quantity of dirty lines after cache mode switching "
"interruption should not be zero.")
TestRun.LOGGER.error(
"Quantity of dirty lines after cache mode switching "
"interruption should not be zero."
)
with TestRun.step("Check cache mode."):
if cache.get_cache_mode() != cache_mode:
TestRun.LOGGER.error("Cache mode should remain the same.")
with TestRun.step("Unmount core and stop cache."):
core.unmount()
cache.stop()
with TestRun.step("Mount core device."):
core_part.mount(mount_point)
with TestRun.step("Check md5 sum of test file again."):
if test_file_md5sum_before != test_file.md5sum():
TestRun.LOGGER.error(
"Md5 sums before and after interrupting mode switching are different.")
with TestRun.step("Unmount core device."):
core_part.unmount()
@pytest.mark.parametrizex("filesystem", Filesystem)
@pytest.mark.parametrizex("cache_mode", CacheMode.with_traits(CacheModeTrait.LazyWrites))
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
def test_interrupt_cache_mode_switch_delayed(cache_mode, filesystem):
"""
title: Test if OpenCAS works correctly after cache mode switching delayed interruption.
description: |
Negative test of the ability of OpenCAS to handle cache mode switching
interruption with delay.
pass_criteria:
- No system crash.
- Cache mode will not be switched after interruption.
- Flushing would be stopped after interruption.
- Md5sum are correct during all test steps.
- Dirty blocks quantity after interruption is lower but non-zero.
"""
with TestRun.step("Prepare cache and core."):
cache_part, core_part = prepare()
for _ in TestRun.iteration(range(iterations_per_config),
f"Reload cache configuration {iterations_per_config} times."):
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Set cleaning policy to NOP."):
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step(f"Add core device with {filesystem} filesystem and mount it."):
core_part.create_filesystem(filesystem)
core = cache.add_core(core_part)
core.mount(mount_point)
with TestRun.step(f"Create test file in mount point of exported object."):
test_file = create_test_file()
with TestRun.step("Check md5 sum of test file."):
test_file_md5sum_before = test_file.md5sum()
with TestRun.step("Get number of dirty data on exported object before "
"switching cache mode interruption."):
os_utils.sync()
os_utils.drop_caches(DropCachesMode.ALL)
cache_dirty_blocks_before = cache.get_dirty_blocks()
with TestRun.step("Start switching cache mode."):
flush_pid = TestRun.executor.run_in_background(cli.set_cache_mode_cmd(
str(CacheMode.DEFAULT.name.lower()), str(cache.cache_id), "yes"))
with TestRun.step("Send interruption signal."):
wait_for_flushing(cache, core)
percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id)
while percentage < 50:
percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id)
TestRun.executor.run(f"kill -s SIGINT {flush_pid}")
with TestRun.step(
"Get quantity of dirty data on exported object after sending interruption "
"signal to cas to stop mode switching."):
cache_dirty_blocks_after = cache.get_dirty_blocks()
if cache_dirty_blocks_after >= cache_dirty_blocks_before:
TestRun.LOGGER.error("Quantity of dirty lines after cache mode switching "
"interruption should be lower.")
if int(cache_dirty_blocks_after) == 0:
TestRun.LOGGER.error("Quantity of dirty lines after cache mode switching "
"interruption should not be zero.")
with TestRun.step("Check cache mode."):
if cache.get_cache_mode() != cache_mode:
TestRun.LOGGER.error("Cache mode should remain the same.")
with TestRun.step("Unmount core and stop cache."):
core.unmount()
cache.stop()
with TestRun.step("Mount core device."):
core_part.mount(mount_point)
with TestRun.step("Check md5 sum of test file again."):
if test_file_md5sum_before != test_file.md5sum():
TestRun.LOGGER.error(
"Md5 sums before and after interrupting mode switching are different.")
with TestRun.step("Unmount core device."):
core_part.unmount()
Dd().block_size(test_file_size).input(core.path).output(test_file.full_path).oflag(
"direct"
).run()
target_file_md5 = test_file.md5sum()
compare_files(test_file_md5_before, target_file_md5)
@pytest.mark.parametrizex("filesystem", Filesystem)
@@ -447,20 +395,21 @@ def test_interrupt_cache_mode_switch_delayed(cache_mode, filesystem):
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
def test_interrupt_cache_stop(cache_mode, filesystem):
"""
title: Test if OpenCAS works correctly after cache stopping interruption.
description: |
Negative test of the ability of OpenCAS to handle cache's stop interruption.
pass_criteria:
- No system crash.
- Flushing would be stopped after interruption.
- Md5sum are correct during all test steps.
- Dirty blocks quantity after interruption is lower but non-zero.
title: Test if OpenCAS works correctly after cache stopping interruption.
description: |
Negative test of the ability of OpenCAS to handle cache's stop interruption.
pass_criteria:
- No system crash.
- Flushing would be stopped after interruption.
- Md5sum are correct during all test steps.
- Dirty blocks quantity after interruption is lower but non-zero.
"""
with TestRun.step("Prepare cache and core."):
cache_part, core_part = prepare()
for _ in TestRun.iteration(range(iterations_per_config),
f"Reload cache configuration {iterations_per_config} times."):
for _ in TestRun.iteration(
range(iterations_per_config), f"Reload cache configuration {iterations_per_config} times."
):
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
@@ -476,9 +425,6 @@ def test_interrupt_cache_stop(cache_mode, filesystem):
with TestRun.step(f"Create test file in mount point of exported object."):
test_file = create_test_file()
with TestRun.step("Check md5 sum of test file."):
test_file_md5sum_before = test_file.md5sum()
with TestRun.step("Get number of dirty data on exported object before interruption."):
os_utils.sync()
os_utils.drop_caches(DropCachesMode.ALL)
@@ -500,11 +446,13 @@ def test_interrupt_cache_stop(cache_mode, filesystem):
with TestRun.step("Check number of dirty data on exported object after interruption."):
cache_dirty_blocks_after = cache.get_dirty_blocks()
if cache_dirty_blocks_after >= cache_dirty_blocks_before:
TestRun.LOGGER.error("Quantity of dirty lines after cache stop interruption "
"should be lower.")
TestRun.LOGGER.error(
"Quantity of dirty lines after cache stop interruption " "should be lower."
)
if int(cache_dirty_blocks_after) == 0:
TestRun.LOGGER.error("Quantity of dirty lines after cache stop interruption "
"should not be zero.")
TestRun.LOGGER.error(
"Quantity of dirty lines after cache stop interruption " "should not be zero."
)
with TestRun.step("Stop cache."):
cache.stop()
@@ -512,20 +460,15 @@ def test_interrupt_cache_stop(cache_mode, filesystem):
with TestRun.step("Mount core device."):
core_part.mount(mount_point)
with TestRun.step("Check md5 sum of test file again."):
if test_file_md5sum_before != test_file.md5sum():
TestRun.LOGGER.error("Md5 sums before and after interrupting"
" cache stop are different.")
with TestRun.step("Unmount core device."):
core_part.unmount()
def prepare():
cache_dev = TestRun.disks['cache']
cache_dev = TestRun.disks["cache"]
cache_dev.create_partitions([cache_size])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev = TestRun.disks["core"]
core_dev.create_partitions([cache_size * 2])
core_part = core_dev.partitions[0]
Udev.disable()
@@ -535,13 +478,11 @@ def prepare():
def create_test_file():
from test_utils.filesystem.file import File
from test_tools.dd import Dd
bs = Size(512, Unit.KibiByte)
cnt = int(cache_size.value / bs.value)
test_file = File.create_file(test_file_path)
dd = Dd().output(test_file_path) \
.input("/dev/zero") \
.block_size(bs) \
.count(cnt)
dd = Dd().output(test_file_path).input("/dev/zero").block_size(bs).count(cnt)
dd.run()
test_file.refresh_item()
return test_file