github/open-cas-linux
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add IO class dss tests

Browse Source 
Signed-off-by: Katarzyna Lapinska <katarzyna.lapinska@intel.com>
This commit is contained in:
Katarzyna Lapinska 2022-04-26 09:05:59 +02:00
parent 66361bb64b
commit 8b8b2bcc26
5 changed files with 429 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
test/functional/api/cas/ioclass_config.py
View File

@ -56,7 +56,7 @@ class IoClass:
class_id=int(parts[0]),
rule=parts[1],
priority=int(parts[2]),
allocation=parts[3])
allocation="%.2f" % float(parts[3]))
@staticmethod
def list_to_csv(ioclass_list: [], add_default_rule: bool = True):

20
test/functional/tests/io_class/io_class_common.py
View File

@ -11,6 +11,7 @@ from api.cas.cache_config import (
CleaningPolicy,
SeqCutOffPolicy,
)
from api.cas.ioclass_config import IoClass
from core.test_run import TestRun
from test_tools.dd import Dd
from test_tools.fio.fio import Fio
@ -20,7 +21,8 @@ from test_utils.os_utils import drop_caches, DropCachesMode
from test_utils.size import Size, Unit
ioclass_config_path = "/tmp/opencas_ioclass.conf"
ioclass_config_path = "/etc/opencas/ioclass.conf"
template_config_path = "/etc/opencas/ioclass-config.csv"
mountpoint = "/tmp/cas1-1"
@ -87,6 +89,22 @@ def get_io_class_usage(cache, io_class_id, percent=False):
).usage_stats
def generate_and_load_random_io_class_config(cache):
random_list = IoClass.generate_random_ioclass_list(ioclass_config.MAX_IO_CLASS_ID + 1)
IoClass.save_list_to_config_file(random_list, add_default_rule=False)
cache.load_io_class(ioclass_config.default_config_file_path)
return random_list
def compare_io_classes_list(expected, actual):
if not IoClass.compare_ioclass_lists(expected, actual):
TestRun.LOGGER.error("IO classes configuration is not as expected.")
expected = '\n'.join(str(i) for i in expected)
TestRun.LOGGER.error(f"Expected IO classes:\n{expected}")
actual = '\n'.join(str(i) for i in actual)
TestRun.LOGGER.error(f"Actual IO classes:\n{actual}")
def run_io_dir(path, size_4k, offset=0):
dd = (
Dd()

79
test/functional/tests/io_class/test_io_class_preserve_config.py Normal file
View File

@ -0,0 +1,79 @@
#
# Copyright(c) 2022 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
import pytest
from api.cas import casadm, ioclass_config
from api.cas.ioclass_config import IoClass
from core.test_run_utils import TestRun
from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan
from test_utils.size import Size, Unit
from tests.io_class.io_class_common import compare_io_classes_list, \
generate_and_load_random_io_class_config
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
def test_io_class_preserve_configuration():
"""
title: Preserve IO class configuration after load.
description: |
Check Open CAS ability to preserve IO class configuration after starting CAS with
load option.
pass_criteria:
- No system crash
- Cache loads successfully
- IO class configuration is the same before and after reboot
"""
with TestRun.step("Prepare devices."):
cache_device = TestRun.disks['cache']
core_device = TestRun.disks['core']
cache_device.create_partitions([Size(150, Unit.MebiByte)])
core_device.create_partitions([Size(300, Unit.MebiByte)])
cache_device = cache_device.partitions[0]
core_device = core_device.partitions[0]
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_device, force=True)
with TestRun.step("Display IO class configuration – shall be only Unclassified IO class."):
default_io_class = [IoClass(
ioclass_config.DEFAULT_IO_CLASS_ID,
ioclass_config.DEFAULT_IO_CLASS_RULE,
ioclass_config.DEFAULT_IO_CLASS_PRIORITY,
allocation="1.00")]
actual = cache.list_io_classes()
compare_io_classes_list(default_io_class, actual)
with TestRun.step("Add core device."):
cache.add_core(core_device)
with TestRun.step("Create and load configuration file for 33 IO classes with random names, "
"allocation and priority values."):
generated_io_classes = generate_and_load_random_io_class_config(cache)
with TestRun.step("Display IO class configuration – shall be the same as created."):
actual = cache.list_io_classes()
compare_io_classes_list(generated_io_classes, actual)
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step(
"Load cache and check IO class configuration - shall be the same as created."):
cache = casadm.load_cache(cache_device)
actual = cache.list_io_classes()
compare_io_classes_list(generated_io_classes, actual)
with TestRun.step("Reboot platform."):
TestRun.executor.reboot()
with TestRun.step(
"Load cache and check IO class configuration - shall be the same as created."):
cache = casadm.load_cache(cache_device)
actual = cache.list_io_classes()
compare_io_classes_list(generated_io_classes, actual)

152
test/functional/tests/io_class/test_io_class_service_support.py Normal file
View File

@ -0,0 +1,152 @@
#
# Copyright(c) 2022 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
import os
import pytest
from datetime import timedelta
from api.cas import ioclass_config, casadm_parser
from api.cas.cache_config import CacheMode
from api.cas.casadm_params import StatsFilter
from api.cas.init_config import InitConfig
from api.cas.ioclass_config import IoClass
from core.test_run_utils import TestRun
from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan
from test_tools import fs_utils
from test_tools.disk_utils import Filesystem
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import IoEngine, ReadWrite
from test_utils import os_utils
from test_utils.os_utils import Runlevel
from test_utils.size import Size, Unit
from tests.io_class.io_class_common import prepare, mountpoint, ioclass_config_path, \
compare_io_classes_list, run_io_dir_read, template_config_path
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
@pytest.mark.parametrizex("runlevel", [Runlevel.runlevel3, Runlevel.runlevel5])
def test_io_class_service_load(runlevel):
"""
title: Open CAS service support for IO class - load.
description: |
Check Open CAS ability to load IO class configuration automatically on system start up.
pass_criteria:
- No system crash
- IO class configuration is the same before and after reboot
"""
with TestRun.step("Prepare devices."):
cache, core = prepare(core_size=Size(300, Unit.MebiByte),
cache_mode=CacheMode.WT)
with TestRun.step("Read the whole CAS device."):
run_io_dir_read(core.path)
with TestRun.step("Create ext4 filesystem on CAS device and mount it."):
core.create_filesystem(Filesystem.ext4)
core.mount(mountpoint)
with TestRun.step("Load IO class configuration file with rules that metadata will not be "
"cached and all other IO will be cached as unclassified."):
config_io_classes = prepare_and_load_io_class_config(cache, metadata_not_cached=True)
with TestRun.step("Run IO."):
run_io()
with TestRun.step("Save IO class usage and configuration statistic."):
saved_usage_stats = cache.get_io_class_statistics(io_class_id=0, stat_filter=[
StatsFilter.usage]).usage_stats
saved_conf_stats = cache.get_io_class_statistics(io_class_id=0, stat_filter=[
StatsFilter.conf]).config_stats
with TestRun.step("Create init config from running CAS configuration."):
InitConfig.create_init_config_from_running_configuration(
cache_extra_flags=f"ioclass_file={ioclass_config_path}")
os_utils.sync()
with TestRun.step(f"Reboot system to runlevel {runlevel}."):
os_utils.change_runlevel(runlevel)
TestRun.executor.reboot()
with TestRun.step("Check if CAS device loads properly - "
"IO class configuration and statistics shall not change"):
caches = casadm_parser.get_caches()
if len(caches) != 1:
TestRun.fail("Cache did not start at boot time.")
cache = caches[0]
cores = casadm_parser.get_cores(cache.cache_id)
if len(cores) != 1:
TestRun.fail(f"Actual number of cores: {len(cores)}\nExpected number of cores: 1")
core = cores[0]
output_io_classes = cache.list_io_classes()
compare_io_classes_list(config_io_classes, output_io_classes)
# Reads from core can invalidate some data so it is possible that occupancy after reboot
# is lower than before
reads_from_core = cache.get_statistics(stat_filter=[StatsFilter.blk]).block_stats.core.reads
read_usage_stats = cache.get_io_class_statistics(io_class_id=0, stat_filter=[
StatsFilter.usage]).usage_stats
read_conf_stats = cache.get_io_class_statistics(io_class_id=0, stat_filter=[
StatsFilter.conf]).config_stats
if read_conf_stats != saved_conf_stats:
TestRun.LOGGER.error(f"Statistics do not match. Before: {str(saved_conf_stats)} "
f"After: {str(read_conf_stats)}")
if read_usage_stats != saved_usage_stats and \
saved_usage_stats.occupancy - read_usage_stats.occupancy > reads_from_core:
TestRun.LOGGER.error(f"Statistics do not match. Before: {str(saved_usage_stats)} "
f"After: {str(read_usage_stats)}")
with TestRun.step("Mount CAS device and run IO again."):
core.mount(mountpoint)
run_io()
with TestRun.step("Check that data are mostly read from cache."):
cache_stats = cache.get_statistics()
read_hits = cache_stats.request_stats.read.hits
read_total = cache_stats.request_stats.read.total
read_hits_percentage = read_hits / read_total * 100
if read_hits_percentage <= 95:
TestRun.LOGGER.error(f"Read hits percentage too low: {read_hits_percentage}%\n"
f"Read hits: {read_hits}, read total: {read_total}")
def run_io():
fio = Fio() \
.create_command() \
.block_size(Size(1, Unit.Blocks4096)) \
.io_engine(IoEngine.libaio) \
.read_write(ReadWrite.read) \
.directory(os.path.join(mountpoint)) \
.sync() \
.do_verify() \
.num_jobs(32) \
.run_time(timedelta(minutes=1)) \
.time_based()\
.nr_files(30)\
.file_size(Size(250, Unit.KiB))
fio.run()
os_utils.sync()
os_utils.drop_caches()
def prepare_and_load_io_class_config(cache, metadata_not_cached=False):
ioclass_config.remove_ioclass_config()
if metadata_not_cached:
ioclass_config.create_ioclass_config(
add_default_rule=True, ioclass_config_path=ioclass_config_path
)
ioclass_config.add_ioclass(1, "metadata&done", 1, "0.00", ioclass_config_path)
else:
fs_utils.copy(template_config_path, ioclass_config_path)
config_io_classes = IoClass.csv_to_list(fs_utils.read_file(ioclass_config_path))
cache.load_io_class(ioclass_config_path)
output_io_classes = cache.list_io_classes()
if not IoClass.compare_ioclass_lists(config_io_classes, output_io_classes):
TestRun.fail("Initial IO class configuration not loaded correctly, aborting test.")
TestRun.LOGGER.info("Initial IO class configuration loaded correctly.")
return config_io_classes

178
test/functional/tests/stress/test_stress_change_io_class_config_io.py Normal file
View File

@ -0,0 +1,178 @@
#
# Copyright(c) 2022 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
import os
import random
import threading
import pytest
from datetime import timedelta
from time import sleep
from api.cas import casadm
from api.cas.cache_config import CacheMode
from api.cas.ioclass_config import IoClass
from core.test_run_utils import TestRun
from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan
from test_tools.disk_utils import Filesystem
from test_tools.fio.fio import Fio
from test_tools.fio.fio_param import IoEngine
from test_utils.asynchronous import start_async_func
from test_utils.size import Size, Unit
from tests.io_class.io_class_common import generate_and_load_random_io_class_config
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
def test_stress_io_class_change_config_during_io_raw():
"""
title: Set up IO class configuration from file during IO - stress.
description: |
Check Open CAS ability to change IO class configuration during running IO
on small cache and core devices.
pass_criteria:
- No system crash
- IO class configuration changes successfully
- No IO errors
"""
cores_per_cache = 4
with TestRun.step("Prepare devices."):
cache_device = TestRun.disks['cache']
core_device = TestRun.disks['core']
cache_device.create_partitions([Size(150, Unit.MebiByte)])
core_device.create_partitions([Size(256, Unit.MebiByte)] * cores_per_cache)
cache_device = cache_device.partitions[0]
with TestRun.step("Start cache in Write-Back mode and add core devices."):
cache = casadm.start_cache(cache_device, cache_mode=CacheMode.WB, force=True)
cores = [cache.add_core(part) for part in core_device.partitions]
with TestRun.step("Create IO class configuration file for 33 IO classes with random allocation "
"and priority value."):
generate_and_load_random_io_class_config(cache)
with TestRun.step("Run IO for all CAS devices."):
fio_task = start_async_func(run_io, cores, True)
with TestRun.step("In two-second time interval change IO class configuration "
"(using random values in allowed range) and cache mode "
"(between all supported). Check if Open CAS configuration has changed."):
change_mode_thread = threading.Thread(target=change_cache_mode, args=[cache, fio_task])
change_io_class_thread = threading.Thread(target=change_io_class_config,
args=[cache, fio_task])
change_mode_thread.start()
sleep(1)
change_io_class_thread.start()
while change_io_class_thread.is_alive() or change_mode_thread.is_alive():
sleep(10)
fio_result = fio_task.result()
if fio_result.exit_code != 0:
TestRun.fail("Fio ended with an error!")
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
@pytest.mark.asyncio
async def test_stress_io_class_change_config_during_io_fs():
"""
title: Set up IO class configuration from file for filesystems during IO - stress.
description: |
Check Intel CAS ability to change IO class configuration for different filesystems
during running IO on small cache and core devices.
pass_criteria:
- No system crash
- IO class configuration changes successfully
- No IO errors
"""
cores_per_cache = len(list(Filesystem))
with TestRun.step("Prepare devices."):
cache_device = TestRun.disks['cache']
core_device = TestRun.disks['core']
cache_device.create_partitions([Size(150, Unit.MebiByte)])
core_device.create_partitions([Size(3, Unit.GibiByte)] * cores_per_cache)
cache_device = cache_device.partitions[0]
with TestRun.step("Start cache in Write-Back mode and add core devices."):
cache = casadm.start_cache(cache_device, cache_mode=CacheMode.WB, force=True)
cores = [cache.add_core(part) for part in core_device.partitions]
with TestRun.step("Create IO class configuration file for 33 IO classes with random allocation "
"and priority value."):
generate_and_load_random_io_class_config(cache)
with TestRun.step("Create different filesystem on each CAS device."):
for core, fs in zip(cores, Filesystem):
core.create_filesystem(fs)
core.mount(os.path.join("/mnt", fs.name))
with TestRun.step("Run IO for all CAS devices."):
fio_task = start_async_func(run_io, cores)
with TestRun.step("In two-second time interval change IO class configuration "
"(using random values in allowed range) and cache mode "
"(between all supported). Check if Open CAS configuration has changed."):
change_mode_thread = threading.Thread(target=change_cache_mode, args=[cache, fio_task])
change_io_class_thread = threading.Thread(target=change_io_class_config,
args=[cache, fio_task])
change_mode_thread.start()
sleep(1)
change_io_class_thread.start()
while change_io_class_thread.is_alive() or change_mode_thread.is_alive():
sleep(10)
fio_result = fio_task.result()
if fio_result.exit_code != 0:
TestRun.fail("Fio ended with an error!")
def change_cache_mode(cache, fio_task):
while fio_task.done() is False:
sleep(2)
current_cache_mode = cache.get_cache_mode()
cache_modes = list(CacheMode)
cache_modes.remove(current_cache_mode)
new_cache_mode = random.choice(cache_modes)
cache.set_cache_mode(new_cache_mode, False)
def change_io_class_config(cache, fio_task):
while fio_task.done() is False:
sleep(2)
generated_io_classes = generate_and_load_random_io_class_config(cache)
loaded_io_classes = cache.list_io_classes()
if not IoClass.compare_ioclass_lists(generated_io_classes, loaded_io_classes):
TestRun.LOGGER.error("IO classes not changed correctly.")
generated_io_classes = '\n'.join(str(i) for i in generated_io_classes)
TestRun.LOGGER.error(f"Generated IO classes:\n{generated_io_classes}")
loaded_io_classes = '\n'.join(str(i) for i in loaded_io_classes)
TestRun.LOGGER.error(f"Loaded IO classes:\n{loaded_io_classes}")
def run_io(cores, direct=False):
fio = Fio().create_command() \
.io_engine(IoEngine.libaio) \
.time_based() \
.run_time(timedelta(hours=2)) \
.do_verify() \
.sync() \
.block_size(Size(1, Unit.Blocks4096)) \
.file_size(Size(2, Unit.GibiByte))
if direct:
fio.direct()
for core in cores:
fio.add_job().target(core.path)
else:
for core in cores:
fio.add_job().target(os.path.join(core.mount_point, "file"))
return fio.fio.run()