github/open-cas-linux
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tests: new ioclass config in existing tests

Browse Source 
Update existing tests to work with new ioclass config format

Signed-off-by: Michal Mielewczyk <michal.mielewczyk@intel.com>
This commit is contained in:
Michal Mielewczyk 2020-11-26 22:30:54 -05:00
parent 0bcd665818
commit fb1a1ab57f
5 changed files with 62 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
test/functional/tests/io_class/test_io_class_core_id.py
View File

@ -49,14 +49,14 @@ def test_ioclass_core_id(filesystem):
ioclass_config.add_ioclass(
ioclass_id=cached_ioclass_id,
eviction_priority=22,
allocation=True,
allocation="1.00",
rule=f"core_id:eq:{core_1.core_id}&done",
ioclass_config_path=ioclass_config.default_config_file_path,
)
ioclass_config.add_ioclass(
ioclass_id=not_cached_ioclass_id,
eviction_priority=22,
allocation=False,
allocation="0.00",
rule=f"core_id:eq:{core_2.core_id}&done",
ioclass_config_path=ioclass_config.default_config_file_path,
)
@ -169,21 +169,21 @@ def prepare(filesystem, cores_number):
ioclass_config.add_ioclass(
ioclass_id=0,
eviction_priority=22,
allocation=True,
allocation="1.00",
rule="unclassified",
ioclass_config_path=ioclass_config.default_config_file_path,
)
ioclass_config.add_ioclass(
ioclass_id=1,
eviction_priority=22,
allocation=True,
allocation="0.00",
rule="metadata",
ioclass_config_path=ioclass_config.default_config_file_path,
)
ioclass_config.add_ioclass(
ioclass_id=2,
eviction_priority=22,
allocation=False,
allocation="0.00",
rule="direct",
ioclass_config_path=ioclass_config.default_config_file_path,
)

73
test/functional/tests/io_class/test_io_class_directory.py
View File

@ -75,7 +75,7 @@ def test_ioclass_directory_depth(filesystem):
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
allocation="1.00",
rule=f"directory:{base_dir_path}",
ioclass_config_path=ioclass_config_path,
)
@ -94,7 +94,7 @@ def test_ioclass_directory_depth(filesystem):
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
if new_occupancy != base_occupancy + test_file_1.size:
TestRun.LOGGER.error("Wrong occupancy after reading file!\n"
"Expected: {base_occupancy + test_file_1.size}, "
f"Expected: {base_occupancy + test_file_1.size}, "
f"actual: {new_occupancy}")
# Test classification in nested dir by creating a file
@ -104,8 +104,8 @@ def test_ioclass_directory_depth(filesystem):
dd = (
Dd().input("/dev/urandom")
.output(test_file_2.full_path)
.count(random.randint(1, 200))
.block_size(Size(1, Unit.MebiByte))
.count(random.randint(25600, 51200)) # 100MB to 200MB
.block_size(Size(1, Unit.Blocks4096))
)
dd.run()
sync()
@ -114,9 +114,10 @@ def test_ioclass_directory_depth(filesystem):
with TestRun.step("Check occupancy after creating the second file."):
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
expected_occpuancy = (base_occupancy + test_file_2.size).set_unit(Unit.Blocks4096)
if new_occupancy != base_occupancy + test_file_2.size:
TestRun.LOGGER.error("Wrong occupancy after creating file!\n"
f"Expected: {base_occupancy + test_file_2.size}, "
f"Expected: {expected_occpuancy}, "
f"actual: {new_occupancy}")
@ -149,7 +150,7 @@ def test_ioclass_directory_file_operations(filesystem):
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
allocation="1.00",
rule=f"directory:{test_dir_path}",
ioclass_config_path=ioclass_config_path,
)
@ -200,7 +201,7 @@ def test_ioclass_directory_file_operations(filesystem):
classified_before = classified_after
non_classified_before = non_classified_after
(Dd().input(test_file.full_path).output("/dev/null")
.block_size(Size(1, Unit.MebiByte)).run())
.block_size(Size(1, Unit.Blocks4096)).run())
with TestRun.step("Check classified occupancy."):
classified_after = cache.get_io_class_statistics(
@ -229,7 +230,7 @@ def test_ioclass_directory_file_operations(filesystem):
classified_before = classified_after
non_classified_before = non_classified_after
(Dd().input(test_file.full_path).output("/dev/null")
.block_size(Size(1, Unit.MebiByte)).run())
.block_size(Size(1, Unit.Blocks4096)).run())
with TestRun.step("Check classified occupancy."):
classified_after = cache.get_io_class_statistics(
@ -275,14 +276,14 @@ def test_ioclass_directory_dir_operations(filesystem):
ioclass_config.add_ioclass(
ioclass_id=ioclass_id_1,
eviction_priority=1,
allocation=True,
allocation="1.00",
rule=f"directory:{classified_dir_path_1}",
ioclass_config_path=ioclass_config_path,
)
ioclass_config.add_ioclass(
ioclass_id=ioclass_id_2,
eviction_priority=1,
allocation=True,
allocation="1.00",
rule=f"directory:{classified_dir_path_2}",
ioclass_config_path=ioclass_config_path,
)
@ -330,7 +331,7 @@ def test_ioclass_directory_dir_operations(filesystem):
with TestRun.step("Read files with reclassification check."):
read_files_with_reclassification_check(cache,
target_ioclass_id=0, source_ioclass_id=ioclass_id_1,
directory=dir_2, with_delay=False)
directory=dir_2, with_delay=True)
with TestRun.step(f"Remove {classified_dir_path_2}."):
fs_utils.remove(path=classified_dir_path_2, force=True, recursive=True)
@ -377,7 +378,7 @@ def create_files_with_classification_delay_check(cache, directory: Directory, io
unclassified_files.append(file_path)
if len(unclassified_files) == file_counter:
pytest.xfail("No files were properly classified within max delay time!")
TestRun.LOGGER.error("No files were properly classified within max delay time!")
if len(unclassified_files):
TestRun.LOGGER.info("Rewriting unclassified test files...")
@ -393,42 +394,52 @@ def read_files_with_reclassification_check(cache, target_ioclass_id: int, source
io_class_id=target_ioclass_id).usage_stats.occupancy
source_occupancy_after = cache.get_io_class_statistics(
io_class_id=source_ioclass_id).usage_stats.occupancy
unclassified_files = []
files_to_reclassify = []
target_ioclass_is_enabled = ioclass_is_enabled(cache, target_ioclass_id)
for file in [item for item in directory.ls() if isinstance(item, File)]:
target_occupancy_before = target_occupancy_after
source_occupancy_before = source_occupancy_after
time_from_start = datetime.now() - start_time
(Dd().input(file.full_path).output("/dev/null")
.block_size(Size(1, Unit.Blocks4096)).run())
dd = Dd().input(file.full_path).output("/dev/null").block_size(Size(1, Unit.Blocks4096))
dd.run()
target_occupancy_after = cache.get_io_class_statistics(
io_class_id=target_ioclass_id).usage_stats.occupancy
source_occupancy_after = cache.get_io_class_statistics(
io_class_id=source_ioclass_id).usage_stats.occupancy
if target_occupancy_after < target_occupancy_before:
pytest.xfail("Target IO class occupancy lowered!")
elif target_occupancy_after - target_occupancy_before < file.size:
unclassified_files.append(file)
if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY:
continue
pytest.xfail("Target IO class occupancy not changed properly!")
if source_occupancy_after >= source_occupancy_before:
if file not in unclassified_files:
unclassified_files.append(file)
if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY:
continue
pytest.xfail("Source IO class occupancy not changed properly!")
if len(unclassified_files):
if target_ioclass_is_enabled:
if target_occupancy_after < target_occupancy_before:
TestRun.LOGGER.error("Target IO class occupancy lowered!")
elif target_occupancy_after - target_occupancy_before < file.size:
files_to_reclassify.append(file)
if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY:
continue
TestRun.LOGGER.error("Target IO class occupancy not changed properly!")
elif target_occupancy_after > target_occupancy_before and with_delay:
files_to_reclassify.append(file)
if source_occupancy_after >= source_occupancy_before:
if file not in files_to_reclassify:
files_to_reclassify.append(file)
if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY:
continue
TestRun.LOGGER.error("Source IO class occupancy not changed properly!")
if len(files_to_reclassify):
TestRun.LOGGER.info("Rereading unclassified test files...")
sync()
drop_caches(DropCachesMode.ALL)
for file in unclassified_files:
for file in files_to_reclassify:
(Dd().input(file.full_path).output("/dev/null")
.block_size(Size(1, Unit.Blocks4096)).run())
def check_occupancy(expected: Size, actual: Size):
if expected != actual:
pytest.xfail("Occupancy check failed!\n"
TestRun.LOGGER.error("Occupancy check failed!\n"
f"Expected: {expected}, actual: {actual}")
def ioclass_is_enabled(cache, ioclass_id: int):
return [i["allocation"] for i in cache.list_io_classes() if i["id"] == ioclass_id].pop() > 0.00

24
test/functional/tests/io_class/test_io_class_file.py
View File

@ -47,7 +47,7 @@ def test_ioclass_file_extension():
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
allocation="1.00",
rule=f"extension:{tested_extension}&done",
ioclass_config_path=ioclass_config_path,
)
@ -114,7 +114,7 @@ def test_ioclass_file_name_prefix():
ioclass_config.add_ioclass(
ioclass_id=0,
eviction_priority=255,
allocation=False,
allocation="0.00",
rule=f"unclassified",
ioclass_config_path=ioclass_config_path,
)
@ -122,7 +122,7 @@ def test_ioclass_file_name_prefix():
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
allocation="1.00",
rule=f"file_name_prefix:test&done",
ioclass_config_path=ioclass_config_path,
)
@ -225,7 +225,7 @@ def test_ioclass_file_extension_preexisting_filesystem():
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
allocation="1.00",
rule=f"{rule}&done",
ioclass_config_path=ioclass_config_path,
)
@ -279,7 +279,7 @@ def test_ioclass_file_offset():
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
allocation="1.00",
rule=f"file_offset:gt:{min_cached_offset}&file_offset:lt:{max_cached_offset}&done",
ioclass_config_path=ioclass_config_path,
)
@ -405,7 +405,7 @@ def test_ioclass_file_size(filesystem):
ioclass_config.add_ioclass(
ioclass_id=0,
eviction_priority=22,
allocation=False,
allocation="0.00",
rule="unclassified",
ioclass_config_path=ioclass_config_path,
)
@ -430,7 +430,7 @@ def test_ioclass_file_size(filesystem):
ioclass_config.add_ioclass(
ioclass_id=0,
eviction_priority=22,
allocation=False,
allocation="0.00",
rule="unclassified",
ioclass_config_path=ioclass_config_path,
)
@ -464,35 +464,35 @@ def load_file_size_io_classes(cache, base_size):
ioclass_config.add_ioclass(
ioclass_id=1,
eviction_priority=1,
allocation=True,
allocation="1.00",
rule=f"file_size:eq:{base_size_bytes}",
ioclass_config_path=ioclass_config_path,
)
ioclass_config.add_ioclass(
ioclass_id=2,
eviction_priority=1,
allocation=True,
allocation="1.00",
rule=f"file_size:lt:{base_size_bytes}",
ioclass_config_path=ioclass_config_path,
)
ioclass_config.add_ioclass(
ioclass_id=3,
eviction_priority=1,
allocation=True,
allocation="1.00",
rule=f"file_size:gt:{base_size_bytes}",
ioclass_config_path=ioclass_config_path,
)
ioclass_config.add_ioclass(
ioclass_id=4,
eviction_priority=1,
allocation=True,
allocation="1.00",
rule=f"file_size:le:{int(base_size_bytes / 2)}",
ioclass_config_path=ioclass_config_path,
)
ioclass_config.add_ioclass(
ioclass_id=5,
eviction_priority=1,
allocation=True,
allocation="1.00",
rule=f"file_size:ge:{2 * base_size_bytes}",
ioclass_config_path=ioclass_config_path,
)

4
test/functional/tests/io_class/test_io_class_process.py
View File

@ -38,7 +38,7 @@ def test_ioclass_process_name():
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
allocation="1.00",
rule=f"process_name:dd&done",
ioclass_config_path=ioclass_config_path,
)
@ -113,7 +113,7 @@ def test_ioclass_pid():
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
allocation="1.00",
rule=f"pid:eq:{pid}&done",
ioclass_config_path=ioclass_config_path,
)

2
test/functional/tests/io_class/test_io_class_purge.py
View File

@ -134,7 +134,7 @@ def add_io_class(class_id, eviction_prio, rule):
ioclass_config.add_ioclass(
ioclass_id=class_id,
eviction_priority=eviction_prio,
allocation=True,
allocation="1.00",
rule=rule,
ioclass_config_path=ioclass_config_path,
)