Merge pull request #1190 from katlapinka/ioclass

Add IO class dss tests

test/functional/tests/io_class/io_class_common.py

@@ -11,6 +11,7 @@ from api.cas.cache_config import (
     CleaningPolicy,
     SeqCutOffPolicy,
 )
+from api.cas.ioclass_config import IoClass
 from core.test_run import TestRun
 from test_tools.dd import Dd
 from test_tools.fio.fio import Fio
@@ -20,7 +21,8 @@ from test_utils.os_utils import drop_caches, DropCachesMode
 from test_utils.size import Size, Unit
 
 
-ioclass_config_path = "/tmp/opencas_ioclass.conf"
+ioclass_config_path = "/etc/opencas/ioclass.conf"
+template_config_path = "/etc/opencas/ioclass-config.csv"
 mountpoint = "/tmp/cas1-1"
 
 
@@ -87,6 +89,22 @@ def get_io_class_usage(cache, io_class_id, percent=False):
     ).usage_stats
 
 
+def generate_and_load_random_io_class_config(cache):
+    random_list = IoClass.generate_random_ioclass_list(ioclass_config.MAX_IO_CLASS_ID + 1)
+    IoClass.save_list_to_config_file(random_list, add_default_rule=False)
+    cache.load_io_class(ioclass_config.default_config_file_path)
+    return random_list
+
+
+def compare_io_classes_list(expected, actual):
+    if not IoClass.compare_ioclass_lists(expected, actual):
+        TestRun.LOGGER.error("IO classes configuration is not as expected.")
+        expected = '\n'.join(str(i) for i in expected)
+        TestRun.LOGGER.error(f"Expected IO classes:\n{expected}")
+        actual = '\n'.join(str(i) for i in actual)
+        TestRun.LOGGER.error(f"Actual IO classes:\n{actual}")
+
+
 def run_io_dir(path, size_4k, offset=0):
     dd = (
         Dd()

test/functional/tests/io_class/test_io_class_preserve_config.py

@@ -0,0 +1,79 @@
+#
+# Copyright(c) 2022 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+import pytest
+
+from api.cas import casadm, ioclass_config
+from api.cas.ioclass_config import IoClass
+from core.test_run_utils import TestRun
+from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan
+from test_utils.size import Size, Unit
+from tests.io_class.io_class_common import compare_io_classes_list, \
+    generate_and_load_random_io_class_config
+
+
+@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
+@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
+def test_io_class_preserve_configuration():
+    """
+    title: Preserve IO class configuration after load.
+    description: |
+        Check Open CAS ability to preserve IO class configuration after starting CAS with
+        load option.
+    pass_criteria:
+        - No system crash
+        - Cache loads successfully
+        - IO class configuration is the same before and after reboot
+    """
+    with TestRun.step("Prepare devices."):
+        cache_device = TestRun.disks['cache']
+        core_device = TestRun.disks['core']
+
+        cache_device.create_partitions([Size(150, Unit.MebiByte)])
+        core_device.create_partitions([Size(300, Unit.MebiByte)])
+
+        cache_device = cache_device.partitions[0]
+        core_device = core_device.partitions[0]
+
+    with TestRun.step("Start cache."):
+        cache = casadm.start_cache(cache_device, force=True)
+
+    with TestRun.step("Display IO class configuration – shall be only Unclassified IO class."):
+        default_io_class = [IoClass(
+            ioclass_config.DEFAULT_IO_CLASS_ID,
+            ioclass_config.DEFAULT_IO_CLASS_RULE,
+            ioclass_config.DEFAULT_IO_CLASS_PRIORITY,
+            allocation="1.00")]
+        actual = cache.list_io_classes()
+        compare_io_classes_list(default_io_class, actual)
+
+    with TestRun.step("Add core device."):
+        cache.add_core(core_device)
+
+    with TestRun.step("Create and load configuration file for 33 IO classes with random names, "
+                      "allocation and priority values."):
+        generated_io_classes = generate_and_load_random_io_class_config(cache)
+
+    with TestRun.step("Display IO class configuration – shall be the same as created."):
+        actual = cache.list_io_classes()
+        compare_io_classes_list(generated_io_classes, actual)
+
+    with TestRun.step("Stop cache."):
+        cache.stop()
+
+    with TestRun.step(
+            "Load cache and check IO class configuration - shall be the same as created."):
+        cache = casadm.load_cache(cache_device)
+        actual = cache.list_io_classes()
+        compare_io_classes_list(generated_io_classes, actual)
+
+    with TestRun.step("Reboot platform."):
+        TestRun.executor.reboot()
+
+    with TestRun.step(
+            "Load cache and check IO class configuration - shall be the same as created."):
+        cache = casadm.load_cache(cache_device)
+        actual = cache.list_io_classes()
+        compare_io_classes_list(generated_io_classes, actual)

test/functional/tests/io_class/test_io_class_service_support.py

@@ -0,0 +1,152 @@
+#
+# Copyright(c) 2022 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+import os
+import pytest
+from datetime import timedelta
+from api.cas import ioclass_config, casadm_parser
+from api.cas.cache_config import CacheMode
+from api.cas.casadm_params import StatsFilter
+from api.cas.init_config import InitConfig
+from api.cas.ioclass_config import IoClass
+from core.test_run_utils import TestRun
+from storage_devices.disk import DiskTypeSet, DiskType, DiskTypeLowerThan
+from test_tools import fs_utils
+from test_tools.disk_utils import Filesystem
+from test_tools.fio.fio import Fio
+from test_tools.fio.fio_param import IoEngine, ReadWrite
+from test_utils import os_utils
+from test_utils.os_utils import Runlevel
+from test_utils.size import Size, Unit
+from tests.io_class.io_class_common import prepare, mountpoint, ioclass_config_path, \
+    compare_io_classes_list, run_io_dir_read, template_config_path
+
+
+@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
+@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
+@pytest.mark.parametrizex("runlevel", [Runlevel.runlevel3, Runlevel.runlevel5])
+def test_io_class_service_load(runlevel):
+    """
+    title: Open CAS service support for IO class - load.
+    description: |
+        Check Open CAS ability to load IO class configuration automatically on system start up.
+    pass_criteria:
+        - No system crash
+        - IO class configuration is the same before and after reboot
+    """
+    with TestRun.step("Prepare devices."):
+        cache, core = prepare(core_size=Size(300, Unit.MebiByte),
+                              cache_mode=CacheMode.WT)
+
+    with TestRun.step("Read the whole CAS device."):
+        run_io_dir_read(core.path)
+
+    with TestRun.step("Create ext4 filesystem on CAS device and mount it."):
+        core.create_filesystem(Filesystem.ext4)
+        core.mount(mountpoint)
+
+    with TestRun.step("Load IO class configuration file with rules that metadata will not be "
+                      "cached and all other IO will be cached as unclassified."):
+        config_io_classes = prepare_and_load_io_class_config(cache, metadata_not_cached=True)
+
+    with TestRun.step("Run IO."):
+        run_io()
+
+    with TestRun.step("Save IO class usage and configuration statistic."):
+        saved_usage_stats = cache.get_io_class_statistics(io_class_id=0, stat_filter=[
+            StatsFilter.usage]).usage_stats
+        saved_conf_stats = cache.get_io_class_statistics(io_class_id=0, stat_filter=[
+            StatsFilter.conf]).config_stats
+
+    with TestRun.step("Create init config from running CAS configuration."):
+        InitConfig.create_init_config_from_running_configuration(
+            cache_extra_flags=f"ioclass_file={ioclass_config_path}")
+        os_utils.sync()
+
+    with TestRun.step(f"Reboot system to runlevel {runlevel}."):
+        os_utils.change_runlevel(runlevel)
+        TestRun.executor.reboot()
+
+    with TestRun.step("Check if CAS device loads properly - "
+                      "IO class configuration and statistics shall not change"):
+        caches = casadm_parser.get_caches()
+        if len(caches) != 1:
+            TestRun.fail("Cache did not start at boot time.")
+        cache = caches[0]
+        cores = casadm_parser.get_cores(cache.cache_id)
+        if len(cores) != 1:
+            TestRun.fail(f"Actual number of cores: {len(cores)}\nExpected number of cores: 1")
+        core = cores[0]
+        output_io_classes = cache.list_io_classes()
+        compare_io_classes_list(config_io_classes, output_io_classes)
+
+        # Reads from core can invalidate some data so it is possible that occupancy after reboot
+        # is lower than before
+        reads_from_core = cache.get_statistics(stat_filter=[StatsFilter.blk]).block_stats.core.reads
+        read_usage_stats = cache.get_io_class_statistics(io_class_id=0, stat_filter=[
+            StatsFilter.usage]).usage_stats
+        read_conf_stats = cache.get_io_class_statistics(io_class_id=0, stat_filter=[
+            StatsFilter.conf]).config_stats
+
+        if read_conf_stats != saved_conf_stats:
+            TestRun.LOGGER.error(f"Statistics do not match. Before: {str(saved_conf_stats)} "
+                                 f"After: {str(read_conf_stats)}")
+        if read_usage_stats != saved_usage_stats and \
+                saved_usage_stats.occupancy - read_usage_stats.occupancy > reads_from_core:
+            TestRun.LOGGER.error(f"Statistics do not match. Before: {str(saved_usage_stats)} "
+                                 f"After: {str(read_usage_stats)}")
+
+    with TestRun.step("Mount CAS device and run IO again."):
+        core.mount(mountpoint)
+        run_io()
+
+    with TestRun.step("Check that data are mostly read from cache."):
+        cache_stats = cache.get_statistics()
+        read_hits = cache_stats.request_stats.read.hits
+        read_total = cache_stats.request_stats.read.total
+        read_hits_percentage = read_hits / read_total * 100
+        if read_hits_percentage <= 95:
+            TestRun.LOGGER.error(f"Read hits percentage too low: {read_hits_percentage}%\n"
+                                 f"Read hits: {read_hits}, read total: {read_total}")
+
+
+def run_io():
+    fio = Fio() \
+        .create_command() \
+        .block_size(Size(1, Unit.Blocks4096)) \
+        .io_engine(IoEngine.libaio) \
+        .read_write(ReadWrite.read) \
+        .directory(os.path.join(mountpoint)) \
+        .sync() \
+        .do_verify() \
+        .num_jobs(32) \
+        .run_time(timedelta(minutes=1)) \
+        .time_based()\
+        .nr_files(30)\
+        .file_size(Size(250, Unit.KiB))
+    fio.run()
+
+    os_utils.sync()
+    os_utils.drop_caches()
+
+
+def prepare_and_load_io_class_config(cache, metadata_not_cached=False):
+    ioclass_config.remove_ioclass_config()
+
+    if metadata_not_cached:
+        ioclass_config.create_ioclass_config(
+            add_default_rule=True, ioclass_config_path=ioclass_config_path
+        )
+        ioclass_config.add_ioclass(1, "metadata&done", 1, "0.00", ioclass_config_path)
+    else:
+        fs_utils.copy(template_config_path, ioclass_config_path)
+
+    config_io_classes = IoClass.csv_to_list(fs_utils.read_file(ioclass_config_path))
+    cache.load_io_class(ioclass_config_path)
+    output_io_classes = cache.list_io_classes()
+    if not IoClass.compare_ioclass_lists(config_io_classes, output_io_classes):
+        TestRun.fail("Initial IO class configuration not loaded correctly, aborting test.")
+    TestRun.LOGGER.info("Initial IO class configuration loaded correctly.")
+    return config_io_classes