tests: fix fault injestion interrupt test
Signed-off-by: Kamil Gierszewski <kamil.gierszewski@huawei.com>
This commit is contained in:
parent
9ab60fe679
commit
4ebc00bac8
@ -1,6 +1,6 @@
|
|||||||
#
|
#
|
||||||
# Copyright(c) 2020-2022 Intel Corporation
|
# Copyright(c) 2020-2022 Intel Corporation
|
||||||
# Copyright(c) 2024 Huawei Technologies Co., Ltd.
|
# Copyright(c) 2024 Huawei Technologies
|
||||||
# SPDX-License-Identifier: BSD-3-Clause
|
# SPDX-License-Identifier: BSD-3-Clause
|
||||||
#
|
#
|
||||||
|
|
||||||
@ -11,13 +11,13 @@ from api.cas.cache_config import CacheMode, CleaningPolicy, CacheModeTrait
|
|||||||
from api.cas.casadm_parser import wait_for_flushing
|
from api.cas.casadm_parser import wait_for_flushing
|
||||||
from core.test_run import TestRun
|
from core.test_run import TestRun
|
||||||
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
|
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
|
||||||
from test_tools.disk_utils import Filesystem
|
from test_tools import fs_utils
|
||||||
from test_tools.dd import Dd
|
from test_tools.dd import Dd
|
||||||
|
from test_tools.disk_utils import Filesystem
|
||||||
from test_utils import os_utils
|
from test_utils import os_utils
|
||||||
from test_utils.os_utils import Udev, DropCachesMode
|
from test_utils.os_utils import Udev, DropCachesMode
|
||||||
from test_utils.size import Size, Unit
|
from test_utils.size import Size, Unit
|
||||||
from tests.lazy_writes.recovery.recovery_tests_methods import compare_files
|
from tests.lazy_writes.recovery.recovery_tests_methods import compare_files
|
||||||
from test_tools import fs_utils
|
|
||||||
|
|
||||||
mount_point = "/mnt/cas"
|
mount_point = "/mnt/cas"
|
||||||
test_file_path = f"{mount_point}/test_file"
|
test_file_path = f"{mount_point}/test_file"
|
||||||
@ -197,17 +197,17 @@ def test_interrupt_cache_flush(cache_mode, filesystem):
|
|||||||
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
|
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
|
||||||
def test_interrupt_core_remove(cache_mode, filesystem):
|
def test_interrupt_core_remove(cache_mode, filesystem):
|
||||||
"""
|
"""
|
||||||
title: Test if OpenCAS works correctly after core's removing interruption.
|
title: Core removal interruption.
|
||||||
description: |
|
description: |
|
||||||
Negative test of the ability of OpenCAS to handle core's removing interruption.
|
Test for proper handling of 'core remove' operation interruption.
|
||||||
pass_criteria:
|
pass_criteria:
|
||||||
- No system crash.
|
- No system crash.
|
||||||
- Core would not be removed from cache after interruption.
|
- Core would not be removed from cache after interruption.
|
||||||
- Flushing would be stopped after interruption.
|
- Flushing would be stopped after interruption.
|
||||||
- Md5sum are correct during all test steps.
|
- Checksums are correct during all test steps.
|
||||||
- Dirty blocks quantity after interruption is lower but non-zero.
|
- Dirty blocks quantity after interruption is lower but non-zero.
|
||||||
"""
|
"""
|
||||||
with TestRun.step("Prepare cache and core."):
|
with TestRun.step("Prepare cache and core devices"):
|
||||||
cache_dev = TestRun.disks["cache"]
|
cache_dev = TestRun.disks["cache"]
|
||||||
cache_dev.create_partitions([cache_size])
|
cache_dev.create_partitions([cache_size])
|
||||||
cache_part = cache_dev.partitions[0]
|
cache_part = cache_dev.partitions[0]
|
||||||
@ -216,37 +216,36 @@ def test_interrupt_core_remove(cache_mode, filesystem):
|
|||||||
core_part = core_dev.partitions[0]
|
core_part = core_dev.partitions[0]
|
||||||
|
|
||||||
for _ in TestRun.iteration(
|
for _ in TestRun.iteration(
|
||||||
range(iterations_per_config), f"Reload cache configuration {iterations_per_config} times."
|
range(iterations_per_config), f"Reload cache configuration {iterations_per_config} times"
|
||||||
):
|
):
|
||||||
|
with TestRun.step("Start cache"):
|
||||||
with TestRun.step("Start cache."):
|
|
||||||
cache = casadm.start_cache(cache_part, cache_mode, force=True)
|
cache = casadm.start_cache(cache_part, cache_mode, force=True)
|
||||||
|
|
||||||
with TestRun.step("Set cleaning policy to NOP."):
|
with TestRun.step("Set cleaning policy to NOP"):
|
||||||
cache.set_cleaning_policy(CleaningPolicy.nop)
|
cache.set_cleaning_policy(CleaningPolicy.nop)
|
||||||
|
|
||||||
with TestRun.step(f"Add core device with {filesystem} filesystem and mount it."):
|
with TestRun.step(f"Add core device with {filesystem} filesystem and mount it"):
|
||||||
core_part.create_filesystem(filesystem)
|
core_part.create_filesystem(filesystem)
|
||||||
core = cache.add_core(core_part)
|
core = cache.add_core(core_part)
|
||||||
core.mount(mount_point)
|
core.mount(mount_point)
|
||||||
|
|
||||||
with TestRun.step(f"Create test file in mount point of exported object."):
|
with TestRun.step("Create test file in mount point of exported object"):
|
||||||
test_file = create_test_file()
|
test_file = create_test_file()
|
||||||
|
|
||||||
with TestRun.step("Check md5 sum of test file."):
|
with TestRun.step("Calculate checksum of test file"):
|
||||||
test_file_md5sum_before = test_file.md5sum()
|
test_file_crc32sum_before = test_file.crc32sum()
|
||||||
|
|
||||||
with TestRun.step(
|
with TestRun.step(
|
||||||
"Get number of dirty data on exported object before core removal interruption."
|
"Get number of dirty data on exported object before core removal interruption"
|
||||||
):
|
):
|
||||||
os_utils.sync()
|
os_utils.sync()
|
||||||
os_utils.drop_caches(DropCachesMode.ALL)
|
os_utils.drop_caches(DropCachesMode.ALL)
|
||||||
cache_dirty_blocks_before = cache.get_dirty_blocks()
|
cache_dirty_blocks_before = cache.get_dirty_blocks()
|
||||||
|
|
||||||
with TestRun.step("Unmount core."):
|
with TestRun.step("Unmount core"):
|
||||||
core.unmount()
|
core.unmount()
|
||||||
|
|
||||||
with TestRun.step("Start removing core device."):
|
with TestRun.step("Start removing core"):
|
||||||
flush_pid = TestRun.executor.run_in_background(
|
flush_pid = TestRun.executor.run_in_background(
|
||||||
cli.remove_core_cmd(str(cache.cache_id), str(core.core_id))
|
cli.remove_core_cmd(str(cache.cache_id), str(core.core_id))
|
||||||
)
|
)
|
||||||
@ -258,42 +257,39 @@ def test_interrupt_core_remove(cache_mode, filesystem):
|
|||||||
percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id)
|
percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id)
|
||||||
TestRun.executor.run(f"kill -s SIGINT {flush_pid}")
|
TestRun.executor.run(f"kill -s SIGINT {flush_pid}")
|
||||||
|
|
||||||
with TestRun.step("Check md5 sum of test file after interruption."):
|
|
||||||
cache.set_cache_mode(CacheMode.WO)
|
|
||||||
test_file_md5sum_interrupt = test_file.md5sum()
|
|
||||||
cache.set_cache_mode(cache_mode)
|
|
||||||
|
|
||||||
with TestRun.step(
|
with TestRun.step(
|
||||||
"Check number of dirty data on exported object after core removal interruption."
|
"Check number of dirty data on exported object after core removal interruption"
|
||||||
):
|
):
|
||||||
cache_dirty_blocks_after = cache.get_dirty_blocks()
|
cache_dirty_blocks_after = cache.get_dirty_blocks()
|
||||||
if cache_dirty_blocks_after >= cache_dirty_blocks_before:
|
if cache_dirty_blocks_after >= cache_dirty_blocks_before:
|
||||||
TestRun.LOGGER.error(
|
TestRun.LOGGER.error(
|
||||||
"Quantity of dirty lines after core removal interruption " "should be lower."
|
"Quantity of dirty lines after core removal interruption should be lower."
|
||||||
)
|
)
|
||||||
if int(cache_dirty_blocks_after) == 0:
|
if int(cache_dirty_blocks_after) == 0:
|
||||||
TestRun.LOGGER.error(
|
TestRun.LOGGER.error(
|
||||||
"Quantity of dirty lines after core removal interruption " "should not be zero."
|
"Quantity of dirty lines after core removal interruption should not be zero."
|
||||||
)
|
)
|
||||||
|
|
||||||
with TestRun.step("Remove core from cache."):
|
with TestRun.step("Mount core and verify test file checksum after interruption"):
|
||||||
core.remove_core()
|
core.mount(mount_point)
|
||||||
|
|
||||||
with TestRun.step("Stop cache."):
|
if test_file.crc32sum() != test_file_crc32sum_before:
|
||||||
|
TestRun.LOGGER.error("Checksum after interrupting core removal is different.")
|
||||||
|
|
||||||
|
with TestRun.step("Unmount core"):
|
||||||
|
core.unmount()
|
||||||
|
|
||||||
|
with TestRun.step("Stop cache"):
|
||||||
cache.stop()
|
cache.stop()
|
||||||
|
|
||||||
with TestRun.step("Mount core device."):
|
with TestRun.step("Mount core device"):
|
||||||
core_part.mount(mount_point)
|
core_part.mount(mount_point)
|
||||||
|
|
||||||
with TestRun.step("Check md5 sum of test file again."):
|
with TestRun.step("Verify checksum of test file again"):
|
||||||
if test_file_md5sum_before != test_file.md5sum():
|
if test_file.crc32sum() != test_file_crc32sum_before:
|
||||||
TestRun.LOGGER.error("Md5 sum before interrupting core removal is different.")
|
TestRun.LOGGER.error("Checksum after core removal is different.")
|
||||||
|
|
||||||
is_sum_diff_after_interrupt = test_file_md5sum_interrupt != test_file.md5sum()
|
with TestRun.step("Unmount core device"):
|
||||||
if is_sum_diff_after_interrupt:
|
|
||||||
TestRun.LOGGER.error("Md5 sum after interrupting core removal is different.")
|
|
||||||
|
|
||||||
with TestRun.step("Unmount core device."):
|
|
||||||
core_part.unmount()
|
core_part.unmount()
|
||||||
|
|
||||||
|
|
||||||
@ -315,8 +311,19 @@ def test_interrupt_cache_mode_switch_parametrized(cache_mode, stop_percentage):
|
|||||||
- Md5sum are correct during all test steps.
|
- Md5sum are correct during all test steps.
|
||||||
- Dirty blocks quantity after interruption is lower but non-zero.
|
- Dirty blocks quantity after interruption is lower but non-zero.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
test_file_size = Size(1, Unit.GibiByte)
|
||||||
|
test_file_path = "/mnt/cas/test_file"
|
||||||
|
|
||||||
with TestRun.step("Prepare cache and core."):
|
with TestRun.step("Prepare cache and core."):
|
||||||
cache_part, core_part = prepare()
|
cache_dev = TestRun.disks["cache"]
|
||||||
|
core_dev = TestRun.disks["core"]
|
||||||
|
|
||||||
|
cache_dev.create_partitions([cache_size])
|
||||||
|
core_dev.create_partitions([cache_size * 2])
|
||||||
|
|
||||||
|
cache_part = cache_dev.partitions[0]
|
||||||
|
core_part = core_dev.partitions[0]
|
||||||
|
|
||||||
with TestRun.step("Disable udev"):
|
with TestRun.step("Disable udev"):
|
||||||
Udev.disable()
|
Udev.disable()
|
||||||
@ -325,23 +332,22 @@ def test_interrupt_cache_mode_switch_parametrized(cache_mode, stop_percentage):
|
|||||||
range(iterations_per_config), f"Reload cache configuration {iterations_per_config} times."
|
range(iterations_per_config), f"Reload cache configuration {iterations_per_config} times."
|
||||||
):
|
):
|
||||||
|
|
||||||
with TestRun.step("Start cache."):
|
with TestRun.step("Start cache"):
|
||||||
cache = casadm.start_cache(cache_part, cache_mode, force=True)
|
cache = casadm.start_cache(cache_part, cache_mode, force=True)
|
||||||
|
|
||||||
with TestRun.step("Set cleaning policy to NOP."):
|
with TestRun.step("Set cleaning policy to NOP"):
|
||||||
cache.set_cleaning_policy(CleaningPolicy.nop)
|
cache.set_cleaning_policy(CleaningPolicy.nop)
|
||||||
|
|
||||||
with TestRun.step(f"Add core device."):
|
with TestRun.step(f"Add core device"):
|
||||||
core = cache.add_core(core_part)
|
core = cache.add_core(core_part)
|
||||||
|
|
||||||
with TestRun.step(f"Create test file in mount point of exported object."):
|
with TestRun.step(f"Create test file in mount point of exported object"):
|
||||||
test_file_size = Size(4, Unit.GibiByte)
|
|
||||||
test_file = fs_utils.create_random_test_file(test_file_path, test_file_size)
|
test_file = fs_utils.create_random_test_file(test_file_path, test_file_size)
|
||||||
|
|
||||||
with TestRun.step("Check md5 sum of test file."):
|
with TestRun.step("Calculate md5sum of test file"):
|
||||||
test_file_md5_before = test_file.md5sum()
|
test_file_md5_before = test_file.md5sum()
|
||||||
|
|
||||||
with TestRun.step("Export file to CAS"):
|
with TestRun.step("Copy test data to core"):
|
||||||
dd = (
|
dd = (
|
||||||
Dd()
|
Dd()
|
||||||
.block_size(test_file_size)
|
.block_size(test_file_size)
|
||||||
@ -351,49 +357,58 @@ def test_interrupt_cache_mode_switch_parametrized(cache_mode, stop_percentage):
|
|||||||
)
|
)
|
||||||
dd.run()
|
dd.run()
|
||||||
|
|
||||||
with TestRun.step("Get number of dirty data on exported object before interruption."):
|
with TestRun.step("Get number of dirty data on exported object before interruption"):
|
||||||
os_utils.sync()
|
os_utils.sync()
|
||||||
os_utils.drop_caches(DropCachesMode.ALL)
|
os_utils.drop_caches(DropCachesMode.ALL)
|
||||||
cache_dirty_blocks_before = cache.get_dirty_blocks()
|
cache_dirty_blocks_before = cache.get_dirty_blocks()
|
||||||
|
|
||||||
with TestRun.step("Start switching cache mode."):
|
with TestRun.step("Start switching cache mode"):
|
||||||
flush_pid = TestRun.executor.run_in_background(
|
flush_pid = TestRun.executor.run_in_background(
|
||||||
cli.set_cache_mode_cmd(
|
cli.set_cache_mode_cmd(
|
||||||
str(CacheMode.DEFAULT.name.lower()), str(cache.cache_id), "yes"
|
cache_mode=str(CacheMode.DEFAULT.name.lower()),
|
||||||
|
cache_id=str(cache.cache_id),
|
||||||
|
flush_cache="yes",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
with TestRun.step("Send interruption signal."):
|
with TestRun.step("Kill flush process during cache flush operation"):
|
||||||
wait_for_flushing(cache, core)
|
wait_for_flushing(cache, core)
|
||||||
percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id)
|
percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id)
|
||||||
while percentage < stop_percentage:
|
while percentage < stop_percentage:
|
||||||
percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id)
|
percentage = casadm_parser.get_flushing_progress(cache.cache_id, core.core_id)
|
||||||
TestRun.executor.run(f"kill -s SIGINT {flush_pid}")
|
TestRun.executor.kill_process(flush_pid)
|
||||||
|
|
||||||
with TestRun.step("Check number of dirty data on exported object after interruption."):
|
with TestRun.step("Check number of dirty data on exported object after interruption"):
|
||||||
cache_dirty_blocks_after = cache.get_dirty_blocks()
|
cache_dirty_blocks_after = cache.get_dirty_blocks()
|
||||||
if cache_dirty_blocks_after >= cache_dirty_blocks_before:
|
if cache_dirty_blocks_after >= cache_dirty_blocks_before:
|
||||||
TestRun.LOGGER.error(
|
TestRun.LOGGER.error(
|
||||||
"Quantity of dirty lines after cache mode switching "
|
"Quantity of dirty lines after cache mode switching "
|
||||||
"interruption should be lower."
|
"interruption should be lower."
|
||||||
)
|
)
|
||||||
if int(cache_dirty_blocks_after) == 0:
|
if cache_dirty_blocks_after == Size.zero():
|
||||||
TestRun.LOGGER.error(
|
TestRun.LOGGER.error(
|
||||||
"Quantity of dirty lines after cache mode switching "
|
"Quantity of dirty lines after cache mode switching "
|
||||||
"interruption should not be zero."
|
"interruption should not be zero."
|
||||||
)
|
)
|
||||||
|
|
||||||
with TestRun.step("Check cache mode."):
|
with TestRun.step("Check cache mode"):
|
||||||
if cache.get_cache_mode() != cache_mode:
|
if cache.get_cache_mode() != cache_mode:
|
||||||
TestRun.LOGGER.error("Cache mode should remain the same.")
|
TestRun.LOGGER.error("Cache mode should remain the same.")
|
||||||
|
|
||||||
with TestRun.step("Stop cache."):
|
with TestRun.step("Stop cache"):
|
||||||
cache.stop()
|
cache.stop()
|
||||||
|
|
||||||
with TestRun.step("Check md5 sum of test file again."):
|
with TestRun.step("Copy test data from the exported object to a file"):
|
||||||
Dd().block_size(test_file_size).input(core.path).output(test_file.full_path).oflag(
|
dd = (
|
||||||
"direct"
|
Dd()
|
||||||
).run()
|
.block_size(test_file_size)
|
||||||
|
.input(core.path)
|
||||||
|
.output(test_file.full_path)
|
||||||
|
.oflag("direct")
|
||||||
|
)
|
||||||
|
dd.run()
|
||||||
|
|
||||||
|
with TestRun.step("Compare md5 sum of test files"):
|
||||||
target_file_md5 = test_file.md5sum()
|
target_file_md5 = test_file.md5sum()
|
||||||
compare_files(test_file_md5_before, target_file_md5)
|
compare_files(test_file_md5_before, target_file_md5)
|
||||||
|
|
||||||
@ -491,7 +506,7 @@ def create_test_file():
|
|||||||
bs = Size(512, Unit.KibiByte)
|
bs = Size(512, Unit.KibiByte)
|
||||||
cnt = int(cache_size.value / bs.value)
|
cnt = int(cache_size.value / bs.value)
|
||||||
test_file = File.create_file(test_file_path)
|
test_file = File.create_file(test_file_path)
|
||||||
dd = Dd().output(test_file_path).input("/dev/zero").block_size(bs).count(cnt)
|
dd = Dd().output(test_file_path).input("/dev/zero").block_size(bs).count(cnt).oflag("direct")
|
||||||
dd.run()
|
dd.run()
|
||||||
test_file.refresh_item()
|
test_file.refresh_item()
|
||||||
return test_file
|
return test_file
|
||||||
|
Loading…
Reference in New Issue
Block a user