diff --git a/test/functional/tests/io_class/test_io_class_core_id.py b/test/functional/tests/io_class/test_io_class_core_id.py index 9d93a54..dff712f 100644 --- a/test/functional/tests/io_class/test_io_class_core_id.py +++ b/test/functional/tests/io_class/test_io_class_core_id.py @@ -49,14 +49,14 @@ def test_ioclass_core_id(filesystem): ioclass_config.add_ioclass( ioclass_id=cached_ioclass_id, eviction_priority=22, - allocation=True, + allocation="1.00", rule=f"core_id:eq:{core_1.core_id}&done", ioclass_config_path=ioclass_config.default_config_file_path, ) ioclass_config.add_ioclass( ioclass_id=not_cached_ioclass_id, eviction_priority=22, - allocation=False, + allocation="0.00", rule=f"core_id:eq:{core_2.core_id}&done", ioclass_config_path=ioclass_config.default_config_file_path, ) @@ -169,21 +169,21 @@ def prepare(filesystem, cores_number): ioclass_config.add_ioclass( ioclass_id=0, eviction_priority=22, - allocation=True, + allocation="1.00", rule="unclassified", ioclass_config_path=ioclass_config.default_config_file_path, ) ioclass_config.add_ioclass( ioclass_id=1, eviction_priority=22, - allocation=True, + allocation="0.00", rule="metadata", ioclass_config_path=ioclass_config.default_config_file_path, ) ioclass_config.add_ioclass( ioclass_id=2, eviction_priority=22, - allocation=False, + allocation="0.00", rule="direct", ioclass_config_path=ioclass_config.default_config_file_path, ) diff --git a/test/functional/tests/io_class/test_io_class_directory.py b/test/functional/tests/io_class/test_io_class_directory.py index 11b8102..bd8ce24 100644 --- a/test/functional/tests/io_class/test_io_class_directory.py +++ b/test/functional/tests/io_class/test_io_class_directory.py @@ -75,7 +75,7 @@ def test_ioclass_directory_depth(filesystem): ioclass_config.add_ioclass( ioclass_id=ioclass_id, eviction_priority=1, - allocation=True, + allocation="1.00", rule=f"directory:{base_dir_path}", ioclass_config_path=ioclass_config_path, ) @@ -94,7 +94,7 @@ def test_ioclass_directory_depth(filesystem): new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy if new_occupancy != base_occupancy + test_file_1.size: TestRun.LOGGER.error("Wrong occupancy after reading file!\n" - "Expected: {base_occupancy + test_file_1.size}, " + f"Expected: {base_occupancy + test_file_1.size}, " f"actual: {new_occupancy}") # Test classification in nested dir by creating a file @@ -104,8 +104,8 @@ def test_ioclass_directory_depth(filesystem): dd = ( Dd().input("/dev/urandom") .output(test_file_2.full_path) - .count(random.randint(1, 200)) - .block_size(Size(1, Unit.MebiByte)) + .count(random.randint(25600, 51200)) # 100MB to 200MB + .block_size(Size(1, Unit.Blocks4096)) ) dd.run() sync() @@ -114,9 +114,10 @@ def test_ioclass_directory_depth(filesystem): with TestRun.step("Check occupancy after creating the second file."): new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy + expected_occpuancy = (base_occupancy + test_file_2.size).set_unit(Unit.Blocks4096) if new_occupancy != base_occupancy + test_file_2.size: TestRun.LOGGER.error("Wrong occupancy after creating file!\n" - f"Expected: {base_occupancy + test_file_2.size}, " + f"Expected: {expected_occpuancy}, " f"actual: {new_occupancy}") @@ -149,7 +150,7 @@ def test_ioclass_directory_file_operations(filesystem): ioclass_config.add_ioclass( ioclass_id=ioclass_id, eviction_priority=1, - allocation=True, + allocation="1.00", rule=f"directory:{test_dir_path}", ioclass_config_path=ioclass_config_path, ) @@ -200,7 +201,7 @@ def test_ioclass_directory_file_operations(filesystem): classified_before = classified_after non_classified_before = non_classified_after (Dd().input(test_file.full_path).output("/dev/null") - .block_size(Size(1, Unit.MebiByte)).run()) + .block_size(Size(1, Unit.Blocks4096)).run()) with TestRun.step("Check classified occupancy."): classified_after = cache.get_io_class_statistics( @@ -229,7 +230,7 @@ def test_ioclass_directory_file_operations(filesystem): classified_before = classified_after non_classified_before = non_classified_after (Dd().input(test_file.full_path).output("/dev/null") - .block_size(Size(1, Unit.MebiByte)).run()) + .block_size(Size(1, Unit.Blocks4096)).run()) with TestRun.step("Check classified occupancy."): classified_after = cache.get_io_class_statistics( @@ -275,14 +276,14 @@ def test_ioclass_directory_dir_operations(filesystem): ioclass_config.add_ioclass( ioclass_id=ioclass_id_1, eviction_priority=1, - allocation=True, + allocation="1.00", rule=f"directory:{classified_dir_path_1}", ioclass_config_path=ioclass_config_path, ) ioclass_config.add_ioclass( ioclass_id=ioclass_id_2, eviction_priority=1, - allocation=True, + allocation="1.00", rule=f"directory:{classified_dir_path_2}", ioclass_config_path=ioclass_config_path, ) @@ -330,7 +331,7 @@ def test_ioclass_directory_dir_operations(filesystem): with TestRun.step("Read files with reclassification check."): read_files_with_reclassification_check(cache, target_ioclass_id=0, source_ioclass_id=ioclass_id_1, - directory=dir_2, with_delay=False) + directory=dir_2, with_delay=True) with TestRun.step(f"Remove {classified_dir_path_2}."): fs_utils.remove(path=classified_dir_path_2, force=True, recursive=True) @@ -377,7 +378,7 @@ def create_files_with_classification_delay_check(cache, directory: Directory, io unclassified_files.append(file_path) if len(unclassified_files) == file_counter: - pytest.xfail("No files were properly classified within max delay time!") + TestRun.LOGGER.error("No files were properly classified within max delay time!") if len(unclassified_files): TestRun.LOGGER.info("Rewriting unclassified test files...") @@ -393,42 +394,52 @@ def read_files_with_reclassification_check(cache, target_ioclass_id: int, source io_class_id=target_ioclass_id).usage_stats.occupancy source_occupancy_after = cache.get_io_class_statistics( io_class_id=source_ioclass_id).usage_stats.occupancy - unclassified_files = [] + files_to_reclassify = [] + target_ioclass_is_enabled = ioclass_is_enabled(cache, target_ioclass_id) for file in [item for item in directory.ls() if isinstance(item, File)]: target_occupancy_before = target_occupancy_after source_occupancy_before = source_occupancy_after time_from_start = datetime.now() - start_time - (Dd().input(file.full_path).output("/dev/null") - .block_size(Size(1, Unit.Blocks4096)).run()) + dd = Dd().input(file.full_path).output("/dev/null").block_size(Size(1, Unit.Blocks4096)) + dd.run() target_occupancy_after = cache.get_io_class_statistics( io_class_id=target_ioclass_id).usage_stats.occupancy source_occupancy_after = cache.get_io_class_statistics( io_class_id=source_ioclass_id).usage_stats.occupancy - if target_occupancy_after < target_occupancy_before: - pytest.xfail("Target IO class occupancy lowered!") - elif target_occupancy_after - target_occupancy_before < file.size: - unclassified_files.append(file) - if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY: - continue - pytest.xfail("Target IO class occupancy not changed properly!") - if source_occupancy_after >= source_occupancy_before: - if file not in unclassified_files: - unclassified_files.append(file) - if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY: - continue - pytest.xfail("Source IO class occupancy not changed properly!") - if len(unclassified_files): + if target_ioclass_is_enabled: + if target_occupancy_after < target_occupancy_before: + TestRun.LOGGER.error("Target IO class occupancy lowered!") + elif target_occupancy_after - target_occupancy_before < file.size: + files_to_reclassify.append(file) + if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY: + continue + TestRun.LOGGER.error("Target IO class occupancy not changed properly!") + elif target_occupancy_after > target_occupancy_before and with_delay: + files_to_reclassify.append(file) + + if source_occupancy_after >= source_occupancy_before: + if file not in files_to_reclassify: + files_to_reclassify.append(file) + if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY: + continue + TestRun.LOGGER.error("Source IO class occupancy not changed properly!") + + if len(files_to_reclassify): TestRun.LOGGER.info("Rereading unclassified test files...") sync() drop_caches(DropCachesMode.ALL) - for file in unclassified_files: + for file in files_to_reclassify: (Dd().input(file.full_path).output("/dev/null") .block_size(Size(1, Unit.Blocks4096)).run()) def check_occupancy(expected: Size, actual: Size): if expected != actual: - pytest.xfail("Occupancy check failed!\n" + TestRun.LOGGER.error("Occupancy check failed!\n" f"Expected: {expected}, actual: {actual}") + + +def ioclass_is_enabled(cache, ioclass_id: int): + return [i["allocation"] for i in cache.list_io_classes() if i["id"] == ioclass_id].pop() > 0.00 diff --git a/test/functional/tests/io_class/test_io_class_file.py b/test/functional/tests/io_class/test_io_class_file.py index 96b7955..ee4e004 100644 --- a/test/functional/tests/io_class/test_io_class_file.py +++ b/test/functional/tests/io_class/test_io_class_file.py @@ -47,7 +47,7 @@ def test_ioclass_file_extension(): ioclass_config.add_ioclass( ioclass_id=ioclass_id, eviction_priority=1, - allocation=True, + allocation="1.00", rule=f"extension:{tested_extension}&done", ioclass_config_path=ioclass_config_path, ) @@ -114,7 +114,7 @@ def test_ioclass_file_name_prefix(): ioclass_config.add_ioclass( ioclass_id=0, eviction_priority=255, - allocation=False, + allocation="0.00", rule=f"unclassified", ioclass_config_path=ioclass_config_path, ) @@ -122,7 +122,7 @@ def test_ioclass_file_name_prefix(): ioclass_config.add_ioclass( ioclass_id=ioclass_id, eviction_priority=1, - allocation=True, + allocation="1.00", rule=f"file_name_prefix:test&done", ioclass_config_path=ioclass_config_path, ) @@ -225,7 +225,7 @@ def test_ioclass_file_extension_preexisting_filesystem(): ioclass_config.add_ioclass( ioclass_id=ioclass_id, eviction_priority=1, - allocation=True, + allocation="1.00", rule=f"{rule}&done", ioclass_config_path=ioclass_config_path, ) @@ -279,7 +279,7 @@ def test_ioclass_file_offset(): ioclass_config.add_ioclass( ioclass_id=ioclass_id, eviction_priority=1, - allocation=True, + allocation="1.00", rule=f"file_offset:gt:{min_cached_offset}&file_offset:lt:{max_cached_offset}&done", ioclass_config_path=ioclass_config_path, ) @@ -405,7 +405,7 @@ def test_ioclass_file_size(filesystem): ioclass_config.add_ioclass( ioclass_id=0, eviction_priority=22, - allocation=False, + allocation="0.00", rule="unclassified", ioclass_config_path=ioclass_config_path, ) @@ -430,7 +430,7 @@ def test_ioclass_file_size(filesystem): ioclass_config.add_ioclass( ioclass_id=0, eviction_priority=22, - allocation=False, + allocation="0.00", rule="unclassified", ioclass_config_path=ioclass_config_path, ) @@ -464,35 +464,35 @@ def load_file_size_io_classes(cache, base_size): ioclass_config.add_ioclass( ioclass_id=1, eviction_priority=1, - allocation=True, + allocation="1.00", rule=f"file_size:eq:{base_size_bytes}", ioclass_config_path=ioclass_config_path, ) ioclass_config.add_ioclass( ioclass_id=2, eviction_priority=1, - allocation=True, + allocation="1.00", rule=f"file_size:lt:{base_size_bytes}", ioclass_config_path=ioclass_config_path, ) ioclass_config.add_ioclass( ioclass_id=3, eviction_priority=1, - allocation=True, + allocation="1.00", rule=f"file_size:gt:{base_size_bytes}", ioclass_config_path=ioclass_config_path, ) ioclass_config.add_ioclass( ioclass_id=4, eviction_priority=1, - allocation=True, + allocation="1.00", rule=f"file_size:le:{int(base_size_bytes / 2)}", ioclass_config_path=ioclass_config_path, ) ioclass_config.add_ioclass( ioclass_id=5, eviction_priority=1, - allocation=True, + allocation="1.00", rule=f"file_size:ge:{2 * base_size_bytes}", ioclass_config_path=ioclass_config_path, ) diff --git a/test/functional/tests/io_class/test_io_class_process.py b/test/functional/tests/io_class/test_io_class_process.py index 073cfc2..e05329b 100644 --- a/test/functional/tests/io_class/test_io_class_process.py +++ b/test/functional/tests/io_class/test_io_class_process.py @@ -38,7 +38,7 @@ def test_ioclass_process_name(): ioclass_config.add_ioclass( ioclass_id=ioclass_id, eviction_priority=1, - allocation=True, + allocation="1.00", rule=f"process_name:dd&done", ioclass_config_path=ioclass_config_path, ) @@ -113,7 +113,7 @@ def test_ioclass_pid(): ioclass_config.add_ioclass( ioclass_id=ioclass_id, eviction_priority=1, - allocation=True, + allocation="1.00", rule=f"pid:eq:{pid}&done", ioclass_config_path=ioclass_config_path, ) diff --git a/test/functional/tests/io_class/test_io_class_purge.py b/test/functional/tests/io_class/test_io_class_purge.py index a5fa970..abd8c8e 100644 --- a/test/functional/tests/io_class/test_io_class_purge.py +++ b/test/functional/tests/io_class/test_io_class_purge.py @@ -134,7 +134,7 @@ def add_io_class(class_id, eviction_prio, rule): ioclass_config.add_ioclass( ioclass_id=class_id, eviction_priority=eviction_prio, - allocation=True, + allocation="1.00", rule=rule, ioclass_config_path=ioclass_config_path, )