github/open-cas-linux
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #197 from Ostrokrzew/last

Browse Source 
Add fault injection tests and update stress tests file
This commit is contained in:
Robert Baldyga 2019-12-30 18:58:23 +01:00 committed by GitHub
commit 8676dd2384
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 197 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
test/functional/.gitignore vendored
View File

@ -1,2 +1,3 @@
plugins/
results/
config/dut_config.yml

168
test/functional/tests/fault_injection/test_fault_injection_with_mounted_core.py Normal file
View File

@ -0,0 +1,168 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
import pytest
from api.cas import casadm, casadm_parser, cli
from api.cas.cache_config import CacheMode
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from core.test_run import TestRun
from test_tools import fs_utils
from test_tools.disk_utils import Filesystem
from test_utils.filesystem.file import File
from test_utils.size import Size, Unit
mount_point = "/mnt/cas"
test_file_path = f"{mount_point}/test_file"
@pytest.mark.parametrize("cache_mode", CacheMode)
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
def test_load_cache_with_mounted_core(cache_mode):
"""
title: Fault injection test for adding mounted core on cache load.
description: |
Negative test of the ability of CAS to add to cache while its loading
core device which is mounted.
pass_criteria:
- No system crash while loading cache.
- Adding mounted core while loading cache fails.
"""
with TestRun.step("Prepare cache and core devices. Start CAS."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(1, Unit.GibiByte)])
cache_dev = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(4, Unit.GibiByte)])
core_dev = core_dev.partitions[0]
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
with TestRun.step("Add core device with xfs filesystem and mount it."):
core_dev.create_filesystem(Filesystem.xfs)
core = cache.add_core(core_dev)
core.mount(mount_point)
with TestRun.step(f"Create test file in /tmp directory."):
file = fs_utils.create_test_file('/tmp/test_file')
with TestRun.step("Copy file to cache's exported object"):
copied_file = File.copy(file.full_path, test_file_path, force=True)
with TestRun.step("Unmount core device."):
core.unmount()
with TestRun.step("Stop cache."):
cache.stop()
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(f"Expected caches count: 0; Actual caches count: {caches_count}.")
with TestRun.step("Mount core device."):
core_dev.mount(mount_point)
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(f"Expected caches count: 1 Actual caches count: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 0:
TestRun.fail(f"Expected cores count: 0; Actual cores count: {cores_count}.")
with TestRun.step("Check properties of test file."):
if file.get_properties() != copied_file.get_properties():
TestRun.LOGGER.error("File properties before and after copying are different.")
core_dev.unmount()
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
@pytest.mark.parametrize("cache_mode", CacheMode)
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
def test_stop_cache_with_mounted_partition(cache_mode):
"""
title: Fault injection test for removing core and stopping cache with mounted core.
description: |
Negative test of the ability of CAS to remove core and stop cache while core
is still mounted.
pass_criteria:
- No system crash.
- Unable to stop CAS device.
- Unable to remove core when partition is mounted.
"""
with TestRun.step("Prepare cache and core devices. Start CAS."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(1, Unit.GibiByte)])
cache_dev = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(4, Unit.GibiByte)])
core_dev = core_dev.partitions[0]
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
with TestRun.step("Add core device with xfs filesystem and mount it."):
core_dev.create_filesystem(Filesystem.xfs)
core = cache.add_core(core_dev)
core.mount(mount_point)
with TestRun.step("Try to remove core from cache."):
output = TestRun.executor.run_expect_fail(cli.remove_core_cmd(cache_id=str(cache.cache_id),
core_id=str(core.core_id)))
if not output.stderr:
TestRun.fail("Removing core succeeded (should fail)!")
with TestRun.step("Try to stop CAS."):
output = TestRun.executor.run_expect_fail(cli.stop_cmd(cache_id=str(cache.cache_id)))
if not output.stderr:
TestRun.fail("Stopping CAS succeeded (should fail)!")
with TestRun.step("Unmount core device."):
core.unmount()
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
@pytest.mark.parametrize("cache_mode", CacheMode)
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
def test_add_occupied_core(cache_mode):
"""
title: Fault injection test for adding already used core to a cache.
description: |
Negative test of the ability to add core to cache
while the core is already used by the another cache instance.
pass_criteria:
- Adding already used core to another cache instance fails.
"""
with TestRun.step("Prepare two caches and one core device."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(2, Unit.GibiByte), Size(2, Unit.GibiByte)])
cache_dev1 = cache_dev.partitions[0]
cache_dev2 = cache_dev.partitions[1]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(4, Unit.GibiByte)])
core_dev = core_dev.partitions[0]
with TestRun.step("Start first cache instance"):
cache1 = casadm.start_cache(cache_dev1, cache_mode, force=True)
with TestRun.step("Add core device to first cache instance."):
core = cache1.add_core(core_dev)
with TestRun.step("Start second cache instance"):
cache2 = casadm.start_cache(cache_dev2, cache_mode, force=True)
with TestRun.step("Try adding the same core device to second cache instance."):
output = TestRun.executor.run_expect_fail(cli.add_core_cmd(cache_id=str(cache2.cache_id),
core_dev=str(core_dev),
core_id=str(core.core_id)))
if not output.stderr:
TestRun.fail("Adding same core to other cache succeeded (should fail)!")
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()

70
test/functional/tests/stress/test_stress_without_io.py
View File

@ -13,7 +13,6 @@ from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from core.test_run import TestRun
from test_tools import fs_utils
from test_tools.disk_utils import Filesystem
from test_utils.filesystem.file import File
from test_utils.size import Size, Unit
iterations_per_config = 50
@ -159,9 +158,8 @@ def test_stress_reload_cache(cache_mode):
- CAS device loads successfully.
- No data corruption.
"""
with TestRun.step("Prepare cache and core. Create test file and count it's checksum."):
cache, core, md5_before_load, size_before_load, permissions_before_load = \
prepare_with_file_creation(cache_mode)
with TestRun.step("Prepare cache and core. Create test file and count its checksum."):
cache, core, file, file_before = prepare_with_file_creation(cache_mode)
for _ in TestRun.iteration(range(0, iterations_per_config),
f"Stop and load cache {iterations_per_config} times."):
@ -180,7 +178,12 @@ def test_stress_reload_cache(cache_mode):
TestRun.fail(f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step("Check md5 of test file."):
check_files(core, size_before_load, permissions_before_load, md5_before_load)
core.mount(mount_point)
file_after = file.get_properties()
if file_after != file_before:
TestRun.LOGGER.error("File properties before and after are different.")
core.unmount()
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
@ -197,9 +200,8 @@ def test_stress_add_remove_core(cache_mode):
- Core is added and removed successfully.
- No data corruption.
"""
with TestRun.step("Prepare cache and core. Create test file and count it's checksum."):
cache, core, md5_before_load, size_before_load, permissions_before_load = \
prepare_with_file_creation(cache_mode)
with TestRun.step("Prepare cache and core. Create test file and count its checksum."):
cache, core, file, file_before = prepare_with_file_creation(cache_mode)
for _ in TestRun.iteration(range(0, iterations_per_config),
f"Add and remove core {iterations_per_config} times."):
@ -221,7 +223,12 @@ def test_stress_add_remove_core(cache_mode):
TestRun.fail(f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step("Check md5 of test file."):
check_files(core, size_before_load, permissions_before_load, md5_before_load)
core.mount(mount_point)
file_after = file.get_properties()
if file_after != file_before:
TestRun.LOGGER.error("File properties before and after are different.")
core.unmount()
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
@ -238,9 +245,9 @@ def test_stress_reload_module(cache_mode):
- CAS modules reloads with no errors.
- No data corruption.
"""
with TestRun.step("Prepare cache and core. Create test file and count it's checksum."):
cache, core, md5_before_load, size_before_load, permissions_before_load = \
prepare_with_file_creation(cache_mode)
with TestRun.step("Prepare cache and core. Create test file and count its checksum."):
cache, core, file, file_before = prepare_with_file_creation(cache_mode)
with TestRun.step("Save current cache configuration."):
cache_config = cache.get_cache_config()
@ -270,47 +277,26 @@ def test_stress_reload_module(cache_mode):
TestRun.fail("Cache configuration is different than before reloading modules.")
with TestRun.step("Check md5 of test file."):
check_files(core, size_before_load, permissions_before_load, md5_before_load)
core.mount(mount_point)
file_after = file.get_properties()
if file_after != file_before:
TestRun.LOGGER.error("File properties before and after are different.")
core.unmount()
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
def check_files(core, size_before, permissions_before, md5_before):
TestRun.LOGGER.info("Checking file md5.")
core.mount(mount_point)
file_after = fs_utils.parse_ls_output(fs_utils.ls(test_file_path))[0]
md5_after = file_after.md5sum()
if md5_before != md5_after:
TestRun.LOGGER.error(f"Md5 before ({md5_before}) and after ({md5_after}) are different.")
if permissions_before.user == file_after.permissions.user:
TestRun.LOGGER.error(f"User permissions before ({permissions_before.user}) "
f"and after ({file_after.permissions.user}) are different.")
if permissions_before.group != file_after.permissions.group:
TestRun.LOGGER.error(f"Group permissions before ({permissions_before.group}) "
f"and after ({file_after.permissions.group}) are different.")
if permissions_before.other != file_after.permissions.other:
TestRun.LOGGER.error(f"Other permissions before ({permissions_before.other}) "
f"and after ({file_after.permissions.other}) are different.")
if size_before != file_after.size:
TestRun.LOGGER.error(f"Size before ({size_before}) and after ({file_after.size}) "
f"are different.")
core.unmount()
def prepare_with_file_creation(config):
cache_dev, core_dev = prepare()
cache = casadm.start_cache(cache_dev, config, force=True)
core = cache.add_core(core_dev)
core.create_filesystem(Filesystem.ext3)
core.mount(mount_point)
file = File.create_file(test_file_path)
file.write("Test content")
md5_before_load = file.md5sum()
size_before_load = file.size
permissions_before_load = file.permissions
file = fs_utils.create_test_file(test_file_path)
file_properties = file.get_properties()
core.unmount()
return cache, core, md5_before_load, size_before_load, permissions_before_load
return cache, core, file, file_properties
def prepare():