From 1588deafd584c44a2d5c9a80b24216116eed7f1f Mon Sep 17 00:00:00 2001 From: Jan Musial Date: Tue, 10 Mar 2020 10:46:48 +0100 Subject: [PATCH 1/4] Update OCF Signed-off-by: Jan Musial --- ocf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocf b/ocf index 49d8f10..b08514c 160000 --- a/ocf +++ b/ocf @@ -1 +1 @@ -Subproject commit 49d8f10953250dea64af97bf37e17b9aa72dbfb8 +Subproject commit b08514c2db15fba4bb3bcc1a61d095f190a2ce6a From 1b0c0f6380f67a4149777c0622debad775787fc6 Mon Sep 17 00:00:00 2001 From: Jan Musial Date: Tue, 10 Mar 2020 09:20:48 +0100 Subject: [PATCH 2/4] Update test framework Signed-off-by: Jan Musial --- test/functional/test-framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/test-framework b/test/functional/test-framework index 3479438..bb5c9a4 160000 --- a/test/functional/test-framework +++ b/test/functional/test-framework @@ -1 +1 @@ -Subproject commit 3479438e60ee24fcd9ddb3f1be7616802a432646 +Subproject commit bb5c9a4e178bcf77a37dc56c04df329c78e162e1 From 8d86cdfa3b026be8173a648a68a87224ee563bf8 Mon Sep 17 00:00:00 2001 From: Jan Musial Date: Tue, 10 Mar 2020 09:20:57 +0100 Subject: [PATCH 3/4] Allow whole range of ALRU parameters in test API Signed-off-by: Jan Musial --- test/functional/api/cas/cache.py | 21 +++++++++++---------- test/functional/api/cas/cache_config.py | 4 ++-- test/functional/api/cas/casadm.py | 12 ++++++------ test/functional/api/cas/cli.py | 15 ++++++++------- 4 files changed, 27 insertions(+), 25 deletions(-) diff --git a/test/functional/api/cas/cache.py b/test/functional/api/cas/cache.py index 812a599..a876848 100644 --- a/test/functional/api/cas/cache.py +++ b/test/functional/api/cas/cache.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019 Intel Corporation +# Copyright(c) 2019-2020 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause-Clear # @@ -164,15 +164,16 @@ class Cache: if acp_params.flush_max_buffers else None) def set_params_alru(self, alru_params: FlushParametersAlru): - return casadm.set_param_cleaning_alru(self.cache_id, - int(alru_params.wake_up_time.total_seconds()) - if alru_params.wake_up_time else None, - int(alru_params.staleness_time.total_seconds()) - if alru_params.staleness_time else None, - alru_params.flush_max_buffers - if alru_params.flush_max_buffers else None, - alru_params.activity_threshold.total_milliseconds() - if alru_params.activity_threshold else None) + return casadm.set_param_cleaning_alru( + self.cache_id, + int(alru_params.wake_up_time.total_seconds()) + if alru_params.wake_up_time is not None else None, + int(alru_params.staleness_time.total_seconds()) + if alru_params.staleness_time is not None else None, + alru_params.flush_max_buffers + if alru_params.flush_max_buffers is not None else None, + alru_params.activity_threshold.total_milliseconds() + if alru_params.activity_threshold is not None else None) def get_cache_config(self): return CacheConfig(self.get_cache_line_size(), diff --git a/test/functional/api/cas/cache_config.py b/test/functional/api/cas/cache_config.py index 7be6c40..6eba5c8 100644 --- a/test/functional/api/cas/cache_config.py +++ b/test/functional/api/cas/cache_config.py @@ -158,10 +158,10 @@ class FlushParametersAlru: @staticmethod def alru_params_range(): alru_params = FlushParametersAlru() - alru_params.activity_threshold = (500, 1000000) + alru_params.activity_threshold = (0, 1000000) alru_params.flush_max_buffers = (1, 10000) alru_params.staleness_time = (1, 3600) - alru_params.wake_up_time = (1, 3600) + alru_params.wake_up_time = (0, 3600) return alru_params def __eq__(self, other): diff --git a/test/functional/api/cas/casadm.py b/test/functional/api/cas/casadm.py index 4111535..35759ad 100644 --- a/test/functional/api/cas/casadm.py +++ b/test/functional/api/cas/casadm.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019 Intel Corporation +# Copyright(c) 2019-2020 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause-Clear # @@ -274,11 +274,11 @@ def set_param_cleaning_alru(cache_id: int, wake_up: int = None, staleness_time: flush_max_buffers: int = None, activity_threshold: int = None): output = TestRun.executor.run( set_param_cleaning_alru_cmd( - cache_id=str(cache_id), - wake_up=str(wake_up) if wake_up else None, - staleness_time=str(staleness_time) if staleness_time else None, - flush_max_buffers=str(flush_max_buffers) if flush_max_buffers else None, - activity_threshold=str(activity_threshold) if activity_threshold else None)) + cache_id=cache_id, + wake_up=wake_up, + staleness_time=staleness_time, + flush_max_buffers=flush_max_buffers, + activity_threshold=activity_threshold)) if output.exit_code != 0: raise CmdException("Error while setting alru cleaning policy parameters.", output) return output diff --git a/test/functional/api/cas/cli.py b/test/functional/api/cas/cli.py index 4c3600e..3a43178 100644 --- a/test/functional/api/cas/cli.py +++ b/test/functional/api/cas/cli.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019 Intel Corporation +# Copyright(c) 2019-2020 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause-Clear # @@ -210,18 +210,19 @@ def set_param_cleaning_cmd(cache_id: str, policy: str, shortcut: bool = False): additional_params=add_params, shortcut=shortcut) -def set_param_cleaning_alru_cmd(cache_id: str, wake_up: str, staleness_time: str, - flush_max_buffers: str, activity_threshold: str, +def set_param_cleaning_alru_cmd(cache_id, wake_up, staleness_time, + flush_max_buffers, activity_threshold, shortcut: bool = False): add_param = "" if wake_up is not None: - add_param += (" -w " if shortcut else " --wake-up ") + wake_up + add_param += (" -w " if shortcut else " --wake-up ") + str(wake_up) if staleness_time is not None: - add_param += (" -s " if shortcut else " --staleness-time ") + staleness_time + add_param += (" -s " if shortcut else " --staleness-time ") + str(staleness_time) if flush_max_buffers is not None: - add_param += (" -b " if shortcut else " --flush-max-buffers ") + flush_max_buffers + add_param += (" -b " if shortcut else " --flush-max-buffers ") + str(flush_max_buffers) if activity_threshold is not None: - add_param += (" -t " if shortcut else " --activity-threshold ") + activity_threshold + add_param += (" -t " if shortcut else " --activity-threshold ") + str(activity_threshold) + return _set_param_cmd(namespace="cleaning-alru", cache_id=cache_id, additional_params=add_param, shortcut=shortcut) From 650dbf5d313f3ed33cedff06595ef2f04843476b Mon Sep 17 00:00:00 2001 From: Jan Musial Date: Tue, 10 Mar 2020 10:37:48 +0100 Subject: [PATCH 4/4] Add test for ALRU with activity threshold set to 0 Signed-off-by: Jan Musial --- test/functional/tests/cleaning/test_alru.py | 117 ++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 test/functional/tests/cleaning/test_alru.py diff --git a/test/functional/tests/cleaning/test_alru.py b/test/functional/tests/cleaning/test_alru.py new file mode 100644 index 0000000..4dd721a --- /dev/null +++ b/test/functional/tests/cleaning/test_alru.py @@ -0,0 +1,117 @@ +# +# Copyright(c) 2020 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +import time +from datetime import timedelta +import pytest + +from core.test_run import TestRun +from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan +from api.cas import casadm +from api.cas.cache_config import CacheMode, CleaningPolicy, FlushParametersAlru, SeqCutOffPolicy +from test_utils.os_utils import Udev, kill_all_io +from test_utils.time import Time +from test_utils.size import Size, Unit +from test_tools.fio.fio import Fio +from test_tools.fio.fio_param import ReadWrite, IoEngine + + +@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand])) +@pytest.mark.require_disk("core", DiskTypeLowerThan("cache")) +def test_alru_no_idle(): + """ + title: Test ALRU with activity threshold set to 0 + description: | + Verify that ALRU is able to perform cleaning if cache is under constant load and + activity threshold is set to 0. Constant load is performed by using fio instance running + in background. + pass_criteria: + - Dirty cache lines are cleaned successfuly. + """ + + with TestRun.step("Prepare configuration"): + cache, core = prepare() + + with TestRun.step("Prepare dirty data to be cleaned"): + bg_size = Size(2, Unit.MiB) + ( + Fio() + .create_command() + .io_engine(IoEngine.libaio) + .offset(bg_size) + .size(Size(10, Unit.MiB)) + .block_size(Size(4, Unit.KiB)) + .target(core) + .direct() + .read_write(ReadWrite.randwrite) + .run() + ) + + with TestRun.step("Run background fio"): + ( + Fio() + .create_command() + .io_engine(IoEngine.libaio) + .size(bg_size) + .block_size(Size(4, Unit.KiB)) + .target(core) + .direct() + .time_based(True) + .run_time(timedelta(hours=1)) + .read_write(ReadWrite.randwrite) + .run_in_background() + ) + + with TestRun.step("Verify that cache is dirty"): + # Wait for bg fio to dirty whole workset + time.sleep(5) + dirty_before = cache.get_statistics().usage_stats.dirty + + if dirty_before == Size(0): + TestRun.fail("Cache should be dirty") + + with TestRun.step("Check that cleaning doesn't occur under constant load"): + time.sleep(5) + + dirty_now = cache.get_statistics().usage_stats.dirty + + if dirty_before > dirty_now: + TestRun.fail( + f"Cleaning has run, while it shouldn't" + " (dirty down from {dirty_before} to {dirty_now}" + ) + + with TestRun.step("Set 0 idle time and wake up time for ALRU"): + cache.set_params_alru(FlushParametersAlru(activity_threshold=Time(0), wake_up_time=Time(0))) + + with TestRun.step("Check that cleaning is progressing"): + time.sleep(5) + + if dirty_before <= cache.get_statistics().usage_stats.dirty: + TestRun.fail("Cleaning didn't run") + + kill_all_io() + + +def prepare(): + cache_dev = TestRun.disks["cache"] + core_dev = TestRun.disks["core"] + + cache_dev.create_partitions([Size(100, Unit.MiB)]) + core_dev.create_partitions([Size(200, Unit.MiB)]) + + Udev.disable() + cache = casadm.start_cache(cache_dev.partitions[0], force=True, cache_mode=CacheMode.WB) + core = cache.add_core(core_dev.partitions[0]) + cache.set_seq_cutoff_policy(SeqCutOffPolicy.never) + cache.set_cleaning_policy(CleaningPolicy.alru) + cache.set_params_alru( + FlushParametersAlru( + activity_threshold=Time(seconds=100), + staleness_time=Time(seconds=1), + ) + ) + + return cache, core