diff --git a/test/functional/tests/io_class/test_io_class_directory.py b/test/functional/tests/io_class/test_io_class_directory.py index f3dfb56..8ac6284 100644 --- a/test/functional/tests/io_class/test_io_class_directory.py +++ b/test/functional/tests/io_class/test_io_class_directory.py @@ -5,6 +5,7 @@ import random from datetime import datetime +import time import pytest @@ -173,6 +174,7 @@ def test_ioclass_directory_file_operations(filesystem): Directory.create_directory(path=nested_dir_path, parents=True) sync() drop_caches(DropCachesMode.ALL) + time.sleep(ioclass_config.MAX_CLASSIFICATION_DELAY.seconds) with TestRun.step("Create test file."): classified_before = cache.get_io_class_statistics( @@ -180,6 +182,7 @@ def test_ioclass_directory_file_operations(filesystem): file_path = f"{test_dir_path}/test_file" (Dd().input("/dev/urandom").output(file_path).oflag("sync") .block_size(Size(1, Unit.MebiByte)).count(dd_blocks).run()) + time.sleep(ioclass_config.MAX_CLASSIFICATION_DELAY.seconds) sync() drop_caches(DropCachesMode.ALL) test_file = File(file_path).refresh_item() @@ -193,6 +196,7 @@ def test_ioclass_directory_file_operations(filesystem): classified_before = classified_after non_classified_before = cache.get_io_class_statistics(io_class_id=0).usage_stats.occupancy test_file.move(destination=mountpoint) + time.sleep(ioclass_config.MAX_CLASSIFICATION_DELAY.seconds) sync() drop_caches(DropCachesMode.ALL) @@ -208,7 +212,10 @@ def test_ioclass_directory_file_operations(filesystem): classified_before = classified_after non_classified_before = non_classified_after (Dd().input(test_file.full_path).output("/dev/null") - .block_size(Size(1, Unit.Blocks4096)).run()) + .iflag("sync").run()) + time.sleep(ioclass_config.MAX_CLASSIFICATION_DELAY.seconds) + sync() + drop_caches(DropCachesMode.ALL) with TestRun.step("Check classified occupancy."): classified_after = cache.get_io_class_statistics( @@ -222,6 +229,7 @@ def test_ioclass_directory_file_operations(filesystem): classified_before = classified_after non_classified_before = non_classified_after test_file.move(destination=nested_dir_path) + time.sleep(ioclass_config.MAX_CLASSIFICATION_DELAY.seconds) sync() drop_caches(DropCachesMode.ALL) @@ -238,6 +246,9 @@ def test_ioclass_directory_file_operations(filesystem): non_classified_before = non_classified_after (Dd().input(test_file.full_path).output("/dev/null") .block_size(Size(1, Unit.Blocks4096)).run()) + time.sleep(ioclass_config.MAX_CLASSIFICATION_DELAY.seconds) + sync() + drop_caches(DropCachesMode.ALL) with TestRun.step("Check classified occupancy."): classified_after = cache.get_io_class_statistics( diff --git a/test/functional/tests/io_class/test_io_class_occupancy.py b/test/functional/tests/io_class/test_io_class_occupancy.py index a5a3807..c5cbd79 100644 --- a/test/functional/tests/io_class/test_io_class_occupancy.py +++ b/test/functional/tests/io_class/test_io_class_occupancy.py @@ -96,42 +96,43 @@ def test_ioclass_occupancy_directory_write(io_size_multiplication, cache_mode, c f"To each directory perform IO with size of {io_size_multiplication} max io_class occupancy" ): for io_class in io_classes: - original_occupacies = {} + original_occupancies = {} tmp_io_class_list = [i for i in io_classes if i != io_class] for i in tmp_io_class_list: - original_occupacies[i.id] = get_io_class_occupancy(cache, i.id) + original_occupancies[i.id] = get_io_class_occupancy(cache, i.id) - run_io_dir( - f"{io_class.dir_path}/tmp_file", - int( - (io_class.max_occupancy * cache_size) / Unit.Blocks4096 * io_size_multiplication - ), - ) + io_count = get_io_count(io_class, cache_size, cache_line_size, io_size_multiplication) + run_io_dir(f"{io_class.dir_path}/tmp_file", io_count) - actuall_occupancy = get_io_class_occupancy(cache, io_class.id) - io_size = io_class.max_occupancy * cache_size + actual_occupancy = get_io_class_occupancy(cache, io_class.id) + expected_occupancy = io_class.max_occupancy * cache_size if io_size_multiplication < 1: - io_size *= io_size_multiplication - io_size.set_unit(Unit.Blocks4096) + expected_occupancy *= io_size_multiplication + expected_occupancy = expected_occupancy.align_down(cache_line_size.value.value) + expected_occupancy.set_unit(Unit.Blocks4096) - if not isclose(io_size.value, actuall_occupancy.value, rel_tol=0.1): + if not isclose(expected_occupancy.value, actual_occupancy.value, rel_tol=0.1): TestRun.LOGGER.error( - f"Occupancy for ioclass {i.id} should be equal {io_size} " - f"but is {actuall_occupancy} instead!" + f"Occupancy for ioclass {io_class.id} should be equal {expected_occupancy} " + f"but is {actual_occupancy} instead!" ) for i in tmp_io_class_list: - actuall_occupancy = get_io_class_occupancy(cache, i.id) - if original_occupacies[i.id] != actuall_occupancy: + actual_occupancy = get_io_class_occupancy(cache, i.id) + io_count = get_io_count(i, cache_size, cache_line_size, io_size_multiplication) + if ( + original_occupancies[i.id] != actual_occupancy + and io_count * Unit.Blocks4096.value < actual_occupancy.value + ): TestRun.LOGGER.error( f"Occupancy for ioclass {i.id} should not change " f"during IO to ioclass {io_class.id}. Original value: " - f"{original_occupacies[i.id]}, actuall: {actuall_occupancy}" + f"{original_occupancies[i.id]}, actual: {actual_occupancy}" ) with TestRun.step("Check if none of ioclasses did not exceed specified occupancy"): for io_class in io_classes: - actuall_occupancy = get_io_class_occupancy(cache, io_class.id) + actual_occupancy = get_io_class_occupancy(cache, io_class.id) occupancy_limit = ( (io_class.max_occupancy * cache_size) @@ -139,11 +140,11 @@ def test_ioclass_occupancy_directory_write(io_size_multiplication, cache_mode, c .set_unit(Unit.Blocks4096) ) - # Divergency may be casued be rounding max occupancy - if actuall_occupancy > occupancy_limit + Size(100, Unit.Blocks4096): + # Divergency may be caused by rounding max occupancy + if actual_occupancy > occupancy_limit * 1.01: TestRun.LOGGER.error( f"Occupancy for ioclass id exceeded: {io_class.id}. " - f"Limit: {occupancy_limit}, actuall: {actuall_occupancy}" + f"Limit: {occupancy_limit}, actual: {actual_occupancy}" ) @@ -193,12 +194,8 @@ def test_ioclass_occupancy_directory_read(io_size_multiplication, cache_line_siz f"max io_class occupancy for future read" ): for io_class in io_classes: - run_io_dir( - f"{io_class.dir_path}/tmp_file", - int( - (io_class.max_occupancy * cache_size) / Unit.Blocks4096 * io_size_multiplication - ), - ) + io_size = get_io_count(io_class, cache_size, cache_line_size, io_size_multiplication) + run_io_dir(f"{io_class.dir_path}/tmp_file", io_size) with TestRun.step("Remove old ioclass config"): ioclass_config.remove_ioclass_config() @@ -233,38 +230,42 @@ def test_ioclass_occupancy_directory_read(io_size_multiplication, cache_line_siz with TestRun.step(f"Read each file and check if data was inserted to appropriate ioclass"): for io_class in io_classes: - original_occupacies = {} + original_occupancies = {} tmp_io_class_list = [i for i in io_classes if i != io_class] for i in tmp_io_class_list: - original_occupacies[i.id] = get_io_class_occupancy(cache, i.id) + original_occupancies[i.id] = get_io_class_occupancy(cache, i.id) run_io_dir_read(f"{io_class.dir_path}/tmp_file") - actuall_occupancy = get_io_class_occupancy(cache, io_class.id) + actual_occupancy = get_io_class_occupancy(cache, io_class.id) - io_size = io_class.max_occupancy * cache_size + expected_occupancy = io_class.max_occupancy * cache_size if io_size_multiplication < 1: - io_size *= io_size_multiplication - io_size.set_unit(Unit.Blocks4096) + expected_occupancy *= io_size_multiplication + expected_occupancy.set_unit(Unit.Blocks4096) - if not isclose(io_size.value, actuall_occupancy.value, rel_tol=0.1): + if not isclose(expected_occupancy.value, actual_occupancy.value, rel_tol=0.1): TestRun.LOGGER.error( - f"Occupancy for ioclass {i.id} should be equal {io_size} " - f"but is {actuall_occupancy} instead!" + f"Occupancy for ioclass {i.id} should be equal {expected_occupancy} " + f"but is {actual_occupancy} instead!" ) for i in tmp_io_class_list: - actuall_occupancy = get_io_class_occupancy(cache, i.id) - if original_occupacies[i.id] != actuall_occupancy: + actual_occupancy = get_io_class_occupancy(cache, i.id) + io_count = get_io_count(i, cache_size, cache_line_size, io_size_multiplication) + if ( + original_occupancies[i.id] != actual_occupancy + and io_count * Unit.Blocks4096.value < actual_occupancy.value + ): TestRun.LOGGER.error( f"Occupancy for ioclass {i.id} should not change " f"during IO to ioclass {io_class.id}. Original value: " - f"{original_occupacies[i.id]}, actuall: {actuall_occupancy}" + f"{original_occupancies[i.id]}, actual: {actual_occupancy}" ) with TestRun.step("Check if none of ioclasses did not exceed specified occupancy"): for io_class in io_classes: - actuall_occupancy = get_io_class_occupancy(cache, io_class.id) + actual_occupancy = get_io_class_occupancy(cache, io_class.id) occupancy_limit = ( (io_class.max_occupancy * cache_size) @@ -272,11 +273,11 @@ def test_ioclass_occupancy_directory_read(io_size_multiplication, cache_line_siz .set_unit(Unit.Blocks4096) ) - # Divergency may be casued be rounding max occupancy - if actuall_occupancy > occupancy_limit + Size(100, Unit.Blocks4096): + # Divergency may be caused by rounding max occupancy + if actual_occupancy > occupancy_limit * 1.01: TestRun.LOGGER.error( f"Occupancy for ioclass id exceeded: {io_class.id}. " - f"Limit: {occupancy_limit}, actuall: {actuall_occupancy}" + f"Limit: {occupancy_limit}, actual: {actual_occupancy}" ) @@ -385,3 +386,11 @@ def test_ioclass_occupancy_sum_cache(): f"cache stats: {cache_stats}, sumed up stats {usage_stats_sum}\n" f"particular stats {[get_io_class_usage(cache, i.id) for i in io_classes]}" ) + + +def get_io_count(io_class, cache_size, cls, io_size_multiplication): + io_count = int((io_class.max_occupancy * cache_size) / Unit.Blocks4096 * io_size_multiplication) + # io size needs to be aligned to cache line size + io_count -= int(io_count % (cls.value / Unit.Blocks4096)) + + return io_count diff --git a/test/functional/tests/io_class/test_io_class_occupancy_resize.py b/test/functional/tests/io_class/test_io_class_occupancy_resize.py index 3a59cac..d27d430 100644 --- a/test/functional/tests/io_class/test_io_class_occupancy_resize.py +++ b/test/functional/tests/io_class/test_io_class_occupancy_resize.py @@ -100,7 +100,7 @@ def test_ioclass_resize(cache_line_size, new_occupancy): ) # Divergency may be casued be rounding max occupancy - if actuall_occupancy > occupancy_limit + Size(100, Unit.Blocks4096): + if actuall_occupancy > occupancy_limit * 1.01: TestRun.LOGGER.error( f"Occupancy for ioclass id exceeded: {io_class.id}. " f"Limit: {occupancy_limit}, actuall: {actuall_occupancy}" @@ -144,7 +144,7 @@ def test_ioclass_resize(cache_line_size, new_occupancy): ) # Divergency may be casued be rounding max occupancy - if actuall_occupancy > occupancy_limit + Size(100, Unit.Blocks4096): + if actuall_occupancy > occupancy_limit * 1.01: TestRun.LOGGER.error( f"Occupancy for ioclass id exceeded: {io_class.id}. " f"Limit: {occupancy_limit}, actuall: {actuall_occupancy}"