Add IO class dss tests

Signed-off-by: Katarzyna Lapinska <katarzyna.lapinska@intel.com>

test/functional/api/cas/ioclass_config.py

@@ -56,7 +56,7 @@ class IoClass:
             class_id=int(parts[0]),
             rule=parts[1],
             priority=int(parts[2]),
-            allocation=parts[3])
+            allocation="%.2f" % float(parts[3]))
 
     @staticmethod
     def list_to_csv(ioclass_list: [], add_default_rule: bool = True):

test/functional/tests/io_class/io_class_common.py

@@ -11,6 +11,7 @@ from api.cas.cache_config import (
     CleaningPolicy,
     SeqCutOffPolicy,
 )
+from api.cas.ioclass_config import IoClass
 from core.test_run import TestRun
 from test_tools.dd import Dd
 from test_tools.fio.fio import Fio
@@ -20,7 +21,8 @@ from test_utils.os_utils import drop_caches, DropCachesMode
 from test_utils.size import Size, Unit
 
 
-ioclass_config_path = "/tmp/opencas_ioclass.conf"
+ioclass_config_path = "/etc/opencas/ioclass.conf"
+template_config_path = "/etc/opencas/ioclass-config.csv"
 mountpoint = "/tmp/cas1-1"
 
 
@@ -87,6 +89,22 @@ def get_io_class_usage(cache, io_class_id, percent=False):
     ).usage_stats
 
 
+def generate_and_load_random_io_class_config(cache):
+    random_list = IoClass.generate_random_ioclass_list(ioclass_config.MAX_IO_CLASS_ID + 1)
+    IoClass.save_list_to_config_file(random_list, add_default_rule=False)
+    cache.load_io_class(ioclass_config.default_config_file_path)
+    return random_list
+
+
+def compare_io_classes_list(expected, actual):
+    if not IoClass.compare_ioclass_lists(expected, actual):
+        TestRun.LOGGER.error("IO classes configuration is not as expected.")
+        expected = '\n'.join(str(i) for i in expected)
+        TestRun.LOGGER.error(f"Expected IO classes:\n{expected}")
+        actual = '\n'.join(str(i) for i in actual)
+        TestRun.LOGGER.error(f"Actual IO classes:\n{actual}")
+
+
 def run_io_dir(path, size_4k, offset=0):
     dd = (
         Dd()

test/functional/tests/io_class/test_io_class_preserve_config.py

@@ -0,0 +1,79 @@
+#
+# Copyright(c) 2022 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+import pytest
+
+from api.cas import casadm, ioclass_config
+from api.cas.ioclass_config import IoClass
+from core.test_run_utils import TestRun
+from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan
+from test_utils.size import Size, Unit
+from tests.io_class.io_class_common import compare_io_classes_list, \
+    generate_and_load_random_io_class_config
+
+
+@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
+@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
+def test_io_class_preserve_configuration():
+    """
+    title: Preserve IO class configuration after load.
+    description: |
+        Check Open CAS ability to preserve IO class configuration after starting CAS with
+        load option.
+    pass_criteria:
+        - No system crash
+        - Cache loads successfully
+        - IO class configuration is the same before and after reboot
+    """
+    with TestRun.step("Prepare devices."):
+        cache_device = TestRun.disks['cache']
+        core_device = TestRun.disks['core']
+
+        cache_device.create_partitions([Size(150, Unit.MebiByte)])
+        core_device.create_partitions([Size(300, Unit.MebiByte)])
+
+        cache_device = cache_device.partitions[0]
+        core_device = core_device.partitions[0]
+
+    with TestRun.step("Start cache."):
+        cache = casadm.start_cache(cache_device, force=True)
+
+    with TestRun.step("Display IO class configuration – shall be only Unclassified IO class."):
+        default_io_class = [IoClass(
+            ioclass_config.DEFAULT_IO_CLASS_ID,
+            ioclass_config.DEFAULT_IO_CLASS_RULE,
+            ioclass_config.DEFAULT_IO_CLASS_PRIORITY,
+            allocation="1.00")]
+        actual = cache.list_io_classes()
+        compare_io_classes_list(default_io_class, actual)
+
+    with TestRun.step("Add core device."):
+        cache.add_core(core_device)
+
+    with TestRun.step("Create and load configuration file for 33 IO classes with random names, "
+                      "allocation and priority values."):
+        generated_io_classes = generate_and_load_random_io_class_config(cache)
+
+    with TestRun.step("Display IO class configuration – shall be the same as created."):
+        actual = cache.list_io_classes()
+        compare_io_classes_list(generated_io_classes, actual)
+
+    with TestRun.step("Stop cache."):
+        cache.stop()
+
+    with TestRun.step(
+            "Load cache and check IO class configuration - shall be the same as created."):
+        cache = casadm.load_cache(cache_device)
+        actual = cache.list_io_classes()
+        compare_io_classes_list(generated_io_classes, actual)
+
+    with TestRun.step("Reboot platform."):
+        TestRun.executor.reboot()
+
+    with TestRun.step(
+            "Load cache and check IO class configuration - shall be the same as created."):
+        cache = casadm.load_cache(cache_device)
+        actual = cache.list_io_classes()
+        compare_io_classes_list(generated_io_classes, actual)

test/functional/tests/io_class/test_io_class_service_support.py

@@ -0,0 +1,152 @@
+#
+# Copyright(c) 2022 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+import os
+import pytest
+from datetime import timedelta
+from api.cas import ioclass_config, casadm_parser
+from api.cas.cache_config import CacheMode
+from api.cas.casadm_params import StatsFilter
+from api.cas.init_config import InitConfig
+from api.cas.ioclass_config import IoClass
+from core.test_run_utils import TestRun
+from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan
+from test_tools import fs_utils
+from test_tools.disk_utils import Filesystem
+from test_tools.fio.fio import Fio
+from test_tools.fio.fio_param import IoEngine, ReadWrite
+from test_utils import os_utils
+from test_utils.os_utils import Runlevel
+from test_utils.size import Size, Unit
+from tests.io_class.io_class_common import prepare, mountpoint, ioclass_config_path, \
+    compare_io_classes_list, run_io_dir_read, template_config_path
+
+
+@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
+@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
+@pytest.mark.parametrizex("runlevel", [Runlevel.runlevel3, Runlevel.runlevel5])
+def test_io_class_service_load(runlevel):
+    """
+    title: Open CAS service support for IO class - load.
+    description: |
+        Check Open CAS ability to load IO class configuration automatically on system start up.
+    pass_criteria:
+        - No system crash
+        - IO class configuration is the same before and after reboot
+    """
+    with TestRun.step("Prepare devices."):
+        cache, core = prepare(core_size=Size(300, Unit.MebiByte),
+                              cache_mode=CacheMode.WT)
+
+    with TestRun.step("Read the whole CAS device."):
+        run_io_dir_read(core.path)
+
+    with TestRun.step("Create ext4 filesystem on CAS device and mount it."):
+        core.create_filesystem(Filesystem.ext4)
+        core.mount(mountpoint)
+
+    with TestRun.step("Load IO class configuration file with rules that metadata will not be "
+                      "cached and all other IO will be cached as unclassified."):
+        config_io_classes = prepare_and_load_io_class_config(cache, metadata_not_cached=True)
+
+    with TestRun.step("Run IO."):
+        run_io()
+
+    with TestRun.step("Save IO class usage and configuration statistic."):
+        saved_usage_stats = cache.get_io_class_statistics(io_class_id=0, stat_filter=[
+            StatsFilter.usage]).usage_stats
+        saved_conf_stats = cache.get_io_class_statistics(io_class_id=0, stat_filter=[
+            StatsFilter.conf]).config_stats
+
+    with TestRun.step("Create init config from running CAS configuration."):
+        InitConfig.create_init_config_from_running_configuration(
+            cache_extra_flags=f"ioclass_file={ioclass_config_path}")
+        os_utils.sync()
+
+    with TestRun.step(f"Reboot system to runlevel {runlevel}."):
+        os_utils.change_runlevel(runlevel)
+        TestRun.executor.reboot()
+
+    with TestRun.step("Check if CAS device loads properly - "
+                      "IO class configuration and statistics shall not change"):
+        caches = casadm_parser.get_caches()
+        if len(caches) != 1:
+            TestRun.fail("Cache did not start at boot time.")
+        cache = caches[0]
+        cores = casadm_parser.get_cores(cache.cache_id)
+        if len(cores) != 1:
+            TestRun.fail(f"Actual number of cores: {len(cores)}\nExpected number of cores: 1")
+        core = cores[0]
+        output_io_classes = cache.list_io_classes()
+        compare_io_classes_list(config_io_classes, output_io_classes)
+
+        # Reads from core can invalidate some data so it is possible that occupancy after reboot
+        # is lower than before
+        reads_from_core = cache.get_statistics(stat_filter=[StatsFilter.blk]).block_stats.core.reads
+        read_usage_stats = cache.get_io_class_statistics(io_class_id=0, stat_filter=[
+            StatsFilter.usage]).usage_stats
+        read_conf_stats = cache.get_io_class_statistics(io_class_id=0, stat_filter=[
+            StatsFilter.conf]).config_stats
+
+        if read_conf_stats != saved_conf_stats:
+            TestRun.LOGGER.error(f"Statistics do not match. Before: {str(saved_conf_stats)} "
+                                 f"After: {str(read_conf_stats)}")
+        if read_usage_stats != saved_usage_stats and \
+                saved_usage_stats.occupancy - read_usage_stats.occupancy > reads_from_core:
+            TestRun.LOGGER.error(f"Statistics do not match. Before: {str(saved_usage_stats)} "
+                                 f"After: {str(read_usage_stats)}")
+
+    with TestRun.step("Mount CAS device and run IO again."):
+        core.mount(mountpoint)
+        run_io()
+
+    with TestRun.step("Check that data are mostly read from cache."):
+        cache_stats = cache.get_statistics()
+        read_hits = cache_stats.request_stats.read.hits
+        read_total = cache_stats.request_stats.read.total
+        read_hits_percentage = read_hits / read_total * 100
+        if read_hits_percentage <= 95:
+            TestRun.LOGGER.error(f"Read hits percentage too low: {read_hits_percentage}%\n"
+                                 f"Read hits: {read_hits}, read total: {read_total}")
+
+
+def run_io():
+    fio = Fio() \
+        .create_command() \
+        .block_size(Size(1, Unit.Blocks4096)) \
+        .io_engine(IoEngine.libaio) \
+        .read_write(ReadWrite.read) \
+        .directory(os.path.join(mountpoint)) \
+        .sync() \
+        .do_verify() \
+        .num_jobs(32) \
+        .run_time(timedelta(minutes=1)) \
+        .time_based()\
+        .nr_files(30)\
+        .file_size(Size(250, Unit.KiB))
+    fio.run()
+
+    os_utils.sync()
+    os_utils.drop_caches()
+
+
+def prepare_and_load_io_class_config(cache, metadata_not_cached=False):
+    ioclass_config.remove_ioclass_config()
+
+    if metadata_not_cached:
+        ioclass_config.create_ioclass_config(
+            add_default_rule=True, ioclass_config_path=ioclass_config_path
+        )
+        ioclass_config.add_ioclass(1, "metadata&done", 1, "0.00", ioclass_config_path)
+    else:
+        fs_utils.copy(template_config_path, ioclass_config_path)
+
+    config_io_classes = IoClass.csv_to_list(fs_utils.read_file(ioclass_config_path))
+    cache.load_io_class(ioclass_config_path)
+    output_io_classes = cache.list_io_classes()
+    if not IoClass.compare_ioclass_lists(config_io_classes, output_io_classes):
+        TestRun.fail("Initial IO class configuration not loaded correctly, aborting test.")
+    TestRun.LOGGER.info("Initial IO class configuration loaded correctly.")
+    return config_io_classes

test/functional/tests/stress/test_stress_change_io_class_config_io.py

@@ -0,0 +1,178 @@
+#
+# Copyright(c) 2022 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+import os
+import random
+import threading
+import pytest
+
+from datetime import timedelta
+from time import sleep
+from api.cas import casadm
+from api.cas.cache_config import CacheMode
+from api.cas.ioclass_config import IoClass
+from core.test_run_utils import TestRun
+from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan
+from test_tools.disk_utils import Filesystem
+from test_tools.fio.fio import Fio
+from test_tools.fio.fio_param import IoEngine
+from test_utils.asynchronous import start_async_func
+from test_utils.size import Size, Unit
+from tests.io_class.io_class_common import generate_and_load_random_io_class_config
+
+
+@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
+@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
+def test_stress_io_class_change_config_during_io_raw():
+    """
+    title: Set up IO class configuration from file during IO - stress.
+    description: |
+        Check Open CAS ability to change IO class configuration during running IO
+        on small cache and core devices.
+    pass_criteria:
+        - No system crash
+        - IO class configuration changes successfully
+        - No IO errors
+    """
+    cores_per_cache = 4
+
+    with TestRun.step("Prepare devices."):
+        cache_device = TestRun.disks['cache']
+        core_device = TestRun.disks['core']
+
+        cache_device.create_partitions([Size(150, Unit.MebiByte)])
+        core_device.create_partitions([Size(256, Unit.MebiByte)] * cores_per_cache)
+
+        cache_device = cache_device.partitions[0]
+
+    with TestRun.step("Start cache in Write-Back mode and add core devices."):
+        cache = casadm.start_cache(cache_device, cache_mode=CacheMode.WB, force=True)
+        cores = [cache.add_core(part) for part in core_device.partitions]
+
+    with TestRun.step("Create IO class configuration file for 33 IO classes with random allocation "
+                      "and priority value."):
+        generate_and_load_random_io_class_config(cache)
+
+    with TestRun.step("Run IO for all CAS devices."):
+        fio_task = start_async_func(run_io, cores, True)
+
+    with TestRun.step("In two-second time interval change IO class configuration "
+                      "(using random values in allowed range) and cache mode "
+                      "(between all supported). Check if Open CAS configuration has changed."):
+        change_mode_thread = threading.Thread(target=change_cache_mode, args=[cache, fio_task])
+        change_io_class_thread = threading.Thread(target=change_io_class_config,
+                                                  args=[cache, fio_task])
+        change_mode_thread.start()
+        sleep(1)
+        change_io_class_thread.start()
+
+        while change_io_class_thread.is_alive() or change_mode_thread.is_alive():
+            sleep(10)
+
+        fio_result = fio_task.result()
+        if fio_result.exit_code != 0:
+            TestRun.fail("Fio ended with an error!")
+
+
+@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
+@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
+@pytest.mark.asyncio
+async def test_stress_io_class_change_config_during_io_fs():
+    """
+    title: Set up IO class configuration from file for filesystems during IO - stress.
+    description: |
+        Check Intel CAS ability to change IO class configuration for different filesystems
+        during running IO on small cache and core devices.
+    pass_criteria:
+        - No system crash
+        - IO class configuration changes successfully
+        - No IO errors
+    """
+    cores_per_cache = len(list(Filesystem))
+
+    with TestRun.step("Prepare devices."):
+        cache_device = TestRun.disks['cache']
+        core_device = TestRun.disks['core']
+
+        cache_device.create_partitions([Size(150, Unit.MebiByte)])
+        core_device.create_partitions([Size(3, Unit.GibiByte)] * cores_per_cache)
+
+        cache_device = cache_device.partitions[0]
+
+    with TestRun.step("Start cache in Write-Back mode and add core devices."):
+        cache = casadm.start_cache(cache_device, cache_mode=CacheMode.WB, force=True)
+        cores = [cache.add_core(part) for part in core_device.partitions]
+
+    with TestRun.step("Create IO class configuration file for 33 IO classes with random allocation "
+                      "and priority value."):
+        generate_and_load_random_io_class_config(cache)
+
+    with TestRun.step("Create different filesystem on each CAS device."):
+        for core, fs in zip(cores, Filesystem):
+            core.create_filesystem(fs)
+            core.mount(os.path.join("/mnt", fs.name))
+
+    with TestRun.step("Run IO for all CAS devices."):
+        fio_task = start_async_func(run_io, cores)
+
+    with TestRun.step("In two-second time interval change IO class configuration "
+                      "(using random values in allowed range) and cache mode "
+                      "(between all supported). Check if Open CAS configuration has changed."):
+        change_mode_thread = threading.Thread(target=change_cache_mode, args=[cache, fio_task])
+        change_io_class_thread = threading.Thread(target=change_io_class_config,
+                                                  args=[cache, fio_task])
+        change_mode_thread.start()
+        sleep(1)
+        change_io_class_thread.start()
+
+        while change_io_class_thread.is_alive() or change_mode_thread.is_alive():
+            sleep(10)
+
+        fio_result = fio_task.result()
+        if fio_result.exit_code != 0:
+            TestRun.fail("Fio ended with an error!")
+
+
+def change_cache_mode(cache, fio_task):
+    while fio_task.done() is False:
+        sleep(2)
+        current_cache_mode = cache.get_cache_mode()
+        cache_modes = list(CacheMode)
+        cache_modes.remove(current_cache_mode)
+        new_cache_mode = random.choice(cache_modes)
+        cache.set_cache_mode(new_cache_mode, False)
+
+
+def change_io_class_config(cache, fio_task):
+    while fio_task.done() is False:
+        sleep(2)
+        generated_io_classes = generate_and_load_random_io_class_config(cache)
+        loaded_io_classes = cache.list_io_classes()
+        if not IoClass.compare_ioclass_lists(generated_io_classes, loaded_io_classes):
+            TestRun.LOGGER.error("IO classes not changed correctly.")
+            generated_io_classes = '\n'.join(str(i) for i in generated_io_classes)
+            TestRun.LOGGER.error(f"Generated IO classes:\n{generated_io_classes}")
+            loaded_io_classes = '\n'.join(str(i) for i in loaded_io_classes)
+            TestRun.LOGGER.error(f"Loaded IO classes:\n{loaded_io_classes}")
+
+
+def run_io(cores, direct=False):
+    fio = Fio().create_command() \
+        .io_engine(IoEngine.libaio) \
+        .time_based() \
+        .run_time(timedelta(hours=2)) \
+        .do_verify() \
+        .sync() \
+        .block_size(Size(1, Unit.Blocks4096)) \
+        .file_size(Size(2, Unit.GibiByte))
+    if direct:
+        fio.direct()
+        for core in cores:
+            fio.add_job().target(core.path)
+    else:
+        for core in cores:
+            fio.add_job().target(os.path.join(core.mount_point, "file"))
+
+    return fio.fio.run()