diff --git a/test/functional/api/cas/ioclass_config.py b/test/functional/api/cas/ioclass_config.py index 89bf82c..7a4ca4a 100644 --- a/test/functional/api/cas/ioclass_config.py +++ b/test/functional/api/cas/ioclass_config.py @@ -56,7 +56,7 @@ class IoClass: class_id=int(parts[0]), rule=parts[1], priority=int(parts[2]), - allocation=parts[3]) + allocation="%.2f" % float(parts[3])) @staticmethod def list_to_csv(ioclass_list: [], add_default_rule: bool = True): diff --git a/test/functional/tests/io_class/io_class_common.py b/test/functional/tests/io_class/io_class_common.py index d2359e2..16991be 100644 --- a/test/functional/tests/io_class/io_class_common.py +++ b/test/functional/tests/io_class/io_class_common.py @@ -11,6 +11,7 @@ from api.cas.cache_config import ( CleaningPolicy, SeqCutOffPolicy, ) +from api.cas.ioclass_config import IoClass from core.test_run import TestRun from test_tools.dd import Dd from test_tools.fio.fio import Fio @@ -20,7 +21,8 @@ from test_utils.os_utils import drop_caches, DropCachesMode from test_utils.size import Size, Unit -ioclass_config_path = "/tmp/opencas_ioclass.conf" +ioclass_config_path = "/etc/opencas/ioclass.conf" +template_config_path = "/etc/opencas/ioclass-config.csv" mountpoint = "/tmp/cas1-1" @@ -87,6 +89,22 @@ def get_io_class_usage(cache, io_class_id, percent=False): ).usage_stats +def generate_and_load_random_io_class_config(cache): + random_list = IoClass.generate_random_ioclass_list(ioclass_config.MAX_IO_CLASS_ID + 1) + IoClass.save_list_to_config_file(random_list, add_default_rule=False) + cache.load_io_class(ioclass_config.default_config_file_path) + return random_list + + +def compare_io_classes_list(expected, actual): + if not IoClass.compare_ioclass_lists(expected, actual): + TestRun.LOGGER.error("IO classes configuration is not as expected.") + expected = '\n'.join(str(i) for i in expected) + TestRun.LOGGER.error(f"Expected IO classes:\n{expected}") + actual = '\n'.join(str(i) for i in actual) + TestRun.LOGGER.error(f"Actual IO classes:\n{actual}") + + def run_io_dir(path, size_4k, offset=0): dd = ( Dd() diff --git a/test/functional/tests/io_class/test_io_class_preserve_config.py b/test/functional/tests/io_class/test_io_class_preserve_config.py new file mode 100644 index 0000000..ba41108 --- /dev/null +++ b/test/functional/tests/io_class/test_io_class_preserve_config.py @@ -0,0 +1,79 @@ +# +# Copyright(c) 2022 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +import pytest + +from api.cas import casadm, ioclass_config +from api.cas.ioclass_config import IoClass +from core.test_run_utils import TestRun +from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan +from test_utils.size import Size, Unit +from tests.io_class.io_class_common import compare_io_classes_list, \ + generate_and_load_random_io_class_config + + +@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand])) +@pytest.mark.require_disk("core", DiskTypeLowerThan("cache")) +def test_io_class_preserve_configuration(): + """ + title: Preserve IO class configuration after load. + description: | + Check Open CAS ability to preserve IO class configuration after starting CAS with + load option. + pass_criteria: + - No system crash + - Cache loads successfully + - IO class configuration is the same before and after reboot + """ + with TestRun.step("Prepare devices."): + cache_device = TestRun.disks['cache'] + core_device = TestRun.disks['core'] + + cache_device.create_partitions([Size(150, Unit.MebiByte)]) + core_device.create_partitions([Size(300, Unit.MebiByte)]) + + cache_device = cache_device.partitions[0] + core_device = core_device.partitions[0] + + with TestRun.step("Start cache."): + cache = casadm.start_cache(cache_device, force=True) + + with TestRun.step("Display IO class configuration – shall be only Unclassified IO class."): + default_io_class = [IoClass( + ioclass_config.DEFAULT_IO_CLASS_ID, + ioclass_config.DEFAULT_IO_CLASS_RULE, + ioclass_config.DEFAULT_IO_CLASS_PRIORITY, + allocation="1.00")] + actual = cache.list_io_classes() + compare_io_classes_list(default_io_class, actual) + + with TestRun.step("Add core device."): + cache.add_core(core_device) + + with TestRun.step("Create and load configuration file for 33 IO classes with random names, " + "allocation and priority values."): + generated_io_classes = generate_and_load_random_io_class_config(cache) + + with TestRun.step("Display IO class configuration – shall be the same as created."): + actual = cache.list_io_classes() + compare_io_classes_list(generated_io_classes, actual) + + with TestRun.step("Stop cache."): + cache.stop() + + with TestRun.step( + "Load cache and check IO class configuration - shall be the same as created."): + cache = casadm.load_cache(cache_device) + actual = cache.list_io_classes() + compare_io_classes_list(generated_io_classes, actual) + + with TestRun.step("Reboot platform."): + TestRun.executor.reboot() + + with TestRun.step( + "Load cache and check IO class configuration - shall be the same as created."): + cache = casadm.load_cache(cache_device) + actual = cache.list_io_classes() + compare_io_classes_list(generated_io_classes, actual) diff --git a/test/functional/tests/io_class/test_io_class_service_support.py b/test/functional/tests/io_class/test_io_class_service_support.py new file mode 100644 index 0000000..5d08605 --- /dev/null +++ b/test/functional/tests/io_class/test_io_class_service_support.py @@ -0,0 +1,152 @@ +# +# Copyright(c) 2022 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +import os +import pytest +from datetime import timedelta +from api.cas import ioclass_config, casadm_parser +from api.cas.cache_config import CacheMode +from api.cas.casadm_params import StatsFilter +from api.cas.init_config import InitConfig +from api.cas.ioclass_config import IoClass +from core.test_run_utils import TestRun +from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan +from test_tools import fs_utils +from test_tools.disk_utils import Filesystem +from test_tools.fio.fio import Fio +from test_tools.fio.fio_param import IoEngine, ReadWrite +from test_utils import os_utils +from test_utils.os_utils import Runlevel +from test_utils.size import Size, Unit +from tests.io_class.io_class_common import prepare, mountpoint, ioclass_config_path, \ + compare_io_classes_list, run_io_dir_read, template_config_path + + +@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand])) +@pytest.mark.require_disk("core", DiskTypeLowerThan("cache")) +@pytest.mark.parametrizex("runlevel", [Runlevel.runlevel3, Runlevel.runlevel5]) +def test_io_class_service_load(runlevel): + """ + title: Open CAS service support for IO class - load. + description: | + Check Open CAS ability to load IO class configuration automatically on system start up. + pass_criteria: + - No system crash + - IO class configuration is the same before and after reboot + """ + with TestRun.step("Prepare devices."): + cache, core = prepare(core_size=Size(300, Unit.MebiByte), + cache_mode=CacheMode.WT) + + with TestRun.step("Read the whole CAS device."): + run_io_dir_read(core.path) + + with TestRun.step("Create ext4 filesystem on CAS device and mount it."): + core.create_filesystem(Filesystem.ext4) + core.mount(mountpoint) + + with TestRun.step("Load IO class configuration file with rules that metadata will not be " + "cached and all other IO will be cached as unclassified."): + config_io_classes = prepare_and_load_io_class_config(cache, metadata_not_cached=True) + + with TestRun.step("Run IO."): + run_io() + + with TestRun.step("Save IO class usage and configuration statistic."): + saved_usage_stats = cache.get_io_class_statistics(io_class_id=0, stat_filter=[ + StatsFilter.usage]).usage_stats + saved_conf_stats = cache.get_io_class_statistics(io_class_id=0, stat_filter=[ + StatsFilter.conf]).config_stats + + with TestRun.step("Create init config from running CAS configuration."): + InitConfig.create_init_config_from_running_configuration( + cache_extra_flags=f"ioclass_file={ioclass_config_path}") + os_utils.sync() + + with TestRun.step(f"Reboot system to runlevel {runlevel}."): + os_utils.change_runlevel(runlevel) + TestRun.executor.reboot() + + with TestRun.step("Check if CAS device loads properly - " + "IO class configuration and statistics shall not change"): + caches = casadm_parser.get_caches() + if len(caches) != 1: + TestRun.fail("Cache did not start at boot time.") + cache = caches[0] + cores = casadm_parser.get_cores(cache.cache_id) + if len(cores) != 1: + TestRun.fail(f"Actual number of cores: {len(cores)}\nExpected number of cores: 1") + core = cores[0] + output_io_classes = cache.list_io_classes() + compare_io_classes_list(config_io_classes, output_io_classes) + + # Reads from core can invalidate some data so it is possible that occupancy after reboot + # is lower than before + reads_from_core = cache.get_statistics(stat_filter=[StatsFilter.blk]).block_stats.core.reads + read_usage_stats = cache.get_io_class_statistics(io_class_id=0, stat_filter=[ + StatsFilter.usage]).usage_stats + read_conf_stats = cache.get_io_class_statistics(io_class_id=0, stat_filter=[ + StatsFilter.conf]).config_stats + + if read_conf_stats != saved_conf_stats: + TestRun.LOGGER.error(f"Statistics do not match. Before: {str(saved_conf_stats)} " + f"After: {str(read_conf_stats)}") + if read_usage_stats != saved_usage_stats and \ + saved_usage_stats.occupancy - read_usage_stats.occupancy > reads_from_core: + TestRun.LOGGER.error(f"Statistics do not match. Before: {str(saved_usage_stats)} " + f"After: {str(read_usage_stats)}") + + with TestRun.step("Mount CAS device and run IO again."): + core.mount(mountpoint) + run_io() + + with TestRun.step("Check that data are mostly read from cache."): + cache_stats = cache.get_statistics() + read_hits = cache_stats.request_stats.read.hits + read_total = cache_stats.request_stats.read.total + read_hits_percentage = read_hits / read_total * 100 + if read_hits_percentage <= 95: + TestRun.LOGGER.error(f"Read hits percentage too low: {read_hits_percentage}%\n" + f"Read hits: {read_hits}, read total: {read_total}") + + +def run_io(): + fio = Fio() \ + .create_command() \ + .block_size(Size(1, Unit.Blocks4096)) \ + .io_engine(IoEngine.libaio) \ + .read_write(ReadWrite.read) \ + .directory(os.path.join(mountpoint)) \ + .sync() \ + .do_verify() \ + .num_jobs(32) \ + .run_time(timedelta(minutes=1)) \ + .time_based()\ + .nr_files(30)\ + .file_size(Size(250, Unit.KiB)) + fio.run() + + os_utils.sync() + os_utils.drop_caches() + + +def prepare_and_load_io_class_config(cache, metadata_not_cached=False): + ioclass_config.remove_ioclass_config() + + if metadata_not_cached: + ioclass_config.create_ioclass_config( + add_default_rule=True, ioclass_config_path=ioclass_config_path + ) + ioclass_config.add_ioclass(1, "metadata&done", 1, "0.00", ioclass_config_path) + else: + fs_utils.copy(template_config_path, ioclass_config_path) + + config_io_classes = IoClass.csv_to_list(fs_utils.read_file(ioclass_config_path)) + cache.load_io_class(ioclass_config_path) + output_io_classes = cache.list_io_classes() + if not IoClass.compare_ioclass_lists(config_io_classes, output_io_classes): + TestRun.fail("Initial IO class configuration not loaded correctly, aborting test.") + TestRun.LOGGER.info("Initial IO class configuration loaded correctly.") + return config_io_classes diff --git a/test/functional/tests/stress/test_stress_change_io_class_config_io.py b/test/functional/tests/stress/test_stress_change_io_class_config_io.py new file mode 100644 index 0000000..d770f4f --- /dev/null +++ b/test/functional/tests/stress/test_stress_change_io_class_config_io.py @@ -0,0 +1,178 @@ +# +# Copyright(c) 2022 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +import os +import random +import threading +import pytest + +from datetime import timedelta +from time import sleep +from api.cas import casadm +from api.cas.cache_config import CacheMode +from api.cas.ioclass_config import IoClass +from core.test_run_utils import TestRun +from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan +from test_tools.disk_utils import Filesystem +from test_tools.fio.fio import Fio +from test_tools.fio.fio_param import IoEngine +from test_utils.asynchronous import start_async_func +from test_utils.size import Size, Unit +from tests.io_class.io_class_common import generate_and_load_random_io_class_config + + +@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand])) +@pytest.mark.require_disk("core", DiskTypeLowerThan("cache")) +def test_stress_io_class_change_config_during_io_raw(): + """ + title: Set up IO class configuration from file during IO - stress. + description: | + Check Open CAS ability to change IO class configuration during running IO + on small cache and core devices. + pass_criteria: + - No system crash + - IO class configuration changes successfully + - No IO errors + """ + cores_per_cache = 4 + + with TestRun.step("Prepare devices."): + cache_device = TestRun.disks['cache'] + core_device = TestRun.disks['core'] + + cache_device.create_partitions([Size(150, Unit.MebiByte)]) + core_device.create_partitions([Size(256, Unit.MebiByte)] * cores_per_cache) + + cache_device = cache_device.partitions[0] + + with TestRun.step("Start cache in Write-Back mode and add core devices."): + cache = casadm.start_cache(cache_device, cache_mode=CacheMode.WB, force=True) + cores = [cache.add_core(part) for part in core_device.partitions] + + with TestRun.step("Create IO class configuration file for 33 IO classes with random allocation " + "and priority value."): + generate_and_load_random_io_class_config(cache) + + with TestRun.step("Run IO for all CAS devices."): + fio_task = start_async_func(run_io, cores, True) + + with TestRun.step("In two-second time interval change IO class configuration " + "(using random values in allowed range) and cache mode " + "(between all supported). Check if Open CAS configuration has changed."): + change_mode_thread = threading.Thread(target=change_cache_mode, args=[cache, fio_task]) + change_io_class_thread = threading.Thread(target=change_io_class_config, + args=[cache, fio_task]) + change_mode_thread.start() + sleep(1) + change_io_class_thread.start() + + while change_io_class_thread.is_alive() or change_mode_thread.is_alive(): + sleep(10) + + fio_result = fio_task.result() + if fio_result.exit_code != 0: + TestRun.fail("Fio ended with an error!") + + +@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand])) +@pytest.mark.require_disk("core", DiskTypeLowerThan("cache")) +@pytest.mark.asyncio +async def test_stress_io_class_change_config_during_io_fs(): + """ + title: Set up IO class configuration from file for filesystems during IO - stress. + description: | + Check Intel CAS ability to change IO class configuration for different filesystems + during running IO on small cache and core devices. + pass_criteria: + - No system crash + - IO class configuration changes successfully + - No IO errors + """ + cores_per_cache = len(list(Filesystem)) + + with TestRun.step("Prepare devices."): + cache_device = TestRun.disks['cache'] + core_device = TestRun.disks['core'] + + cache_device.create_partitions([Size(150, Unit.MebiByte)]) + core_device.create_partitions([Size(3, Unit.GibiByte)] * cores_per_cache) + + cache_device = cache_device.partitions[0] + + with TestRun.step("Start cache in Write-Back mode and add core devices."): + cache = casadm.start_cache(cache_device, cache_mode=CacheMode.WB, force=True) + cores = [cache.add_core(part) for part in core_device.partitions] + + with TestRun.step("Create IO class configuration file for 33 IO classes with random allocation " + "and priority value."): + generate_and_load_random_io_class_config(cache) + + with TestRun.step("Create different filesystem on each CAS device."): + for core, fs in zip(cores, Filesystem): + core.create_filesystem(fs) + core.mount(os.path.join("/mnt", fs.name)) + + with TestRun.step("Run IO for all CAS devices."): + fio_task = start_async_func(run_io, cores) + + with TestRun.step("In two-second time interval change IO class configuration " + "(using random values in allowed range) and cache mode " + "(between all supported). Check if Open CAS configuration has changed."): + change_mode_thread = threading.Thread(target=change_cache_mode, args=[cache, fio_task]) + change_io_class_thread = threading.Thread(target=change_io_class_config, + args=[cache, fio_task]) + change_mode_thread.start() + sleep(1) + change_io_class_thread.start() + + while change_io_class_thread.is_alive() or change_mode_thread.is_alive(): + sleep(10) + + fio_result = fio_task.result() + if fio_result.exit_code != 0: + TestRun.fail("Fio ended with an error!") + + +def change_cache_mode(cache, fio_task): + while fio_task.done() is False: + sleep(2) + current_cache_mode = cache.get_cache_mode() + cache_modes = list(CacheMode) + cache_modes.remove(current_cache_mode) + new_cache_mode = random.choice(cache_modes) + cache.set_cache_mode(new_cache_mode, False) + + +def change_io_class_config(cache, fio_task): + while fio_task.done() is False: + sleep(2) + generated_io_classes = generate_and_load_random_io_class_config(cache) + loaded_io_classes = cache.list_io_classes() + if not IoClass.compare_ioclass_lists(generated_io_classes, loaded_io_classes): + TestRun.LOGGER.error("IO classes not changed correctly.") + generated_io_classes = '\n'.join(str(i) for i in generated_io_classes) + TestRun.LOGGER.error(f"Generated IO classes:\n{generated_io_classes}") + loaded_io_classes = '\n'.join(str(i) for i in loaded_io_classes) + TestRun.LOGGER.error(f"Loaded IO classes:\n{loaded_io_classes}") + + +def run_io(cores, direct=False): + fio = Fio().create_command() \ + .io_engine(IoEngine.libaio) \ + .time_based() \ + .run_time(timedelta(hours=2)) \ + .do_verify() \ + .sync() \ + .block_size(Size(1, Unit.Blocks4096)) \ + .file_size(Size(2, Unit.GibiByte)) + if direct: + fio.direct() + for core in cores: + fio.add_job().target(core.path) + else: + for core in cores: + fio.add_job().target(os.path.join(core.mount_point, "file")) + + return fio.fio.run()