github/open-cas-linux
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor IO class tests

Browse Source 
Signed-off-by: Klaudia Jablonska <klaudia.jablonska@intel.com>
This commit is contained in:
Klaudia Jablonska
2022-08-29 16:04:22 +02:00
parent 2da9753a10
commit 1cf2af7ed4
17 changed files with 1100 additions and 883 deletions
Download Patch File Download Diff File Expand all files Collapse all files

526
test/functional/tests/io_class/test_io_classification.py
View File

@@ -1,5 +1,5 @@
#
# Copyright(c) 2019-2021 Intel Corporation
# Copyright(c) 2019-2022 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause
#
@@ -28,13 +28,12 @@ from tests.io_class.io_class_common import prepare, ioclass_config_path, mountpo
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
def test_ioclass_lba():
"""
title: Test IO classification by lba.
description: |
Write data to random lba and check if it is cached according to range
defined in ioclass rule
pass_criteria:
- No kernel bug.
- IO is classified properly based on lba range defined in config.
title: Test IO classification by lba.
description: |
Write data to random lba and check if it is cached according to range defined in ioclass rule.
pass_criteria:
- No kernel bug.
- IO is classified properly based on lba range defined in config.
"""
ioclass_id = 1
@@ -64,15 +63,16 @@ def test_ioclass_lba():
# '8' step is set to prevent writing cache line more than once
TestRun.LOGGER.info(f"Writing to one sector in each cache line from range.")
for lba in range(min_cached_lba, max_cached_lba, 8):
dd = (
Dd().input("/dev/zero")
.output(f"{core.path}")
.count(dd_count)
.block_size(dd_size)
.seek(lba)
.oflag("direct")
(
Dd()
.input("/dev/zero")
.output(f"{core.path}")
.count(dd_count)
.block_size(dd_size)
.seek(lba)
.oflag("direct")
.run()
)
dd.run()
dirty_count += 1
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
@@ -84,22 +84,24 @@ def test_ioclass_lba():
test_lba = [max_cached_lba + 1] + random.sample(
[
*range(0, min_cached_lba),
*range(max_cached_lba + 1, int(core.size.get_value(Unit.Blocks512)))
*range(max_cached_lba + 1, int(core.size.get_value(Unit.Blocks512))),
],
k=100)
k=100,
)
for lba in test_lba:
prev_dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
dd = (
Dd().input("/dev/zero")
.output(f"{core.path}")
.count(dd_count)
.block_size(dd_size)
.seek(lba)
.oflag("direct")
(
Dd()
.input("/dev/zero")
.output(f"{core.path}")
.count(dd_count)
.block_size(dd_size)
.seek(lba)
.oflag("direct")
.run()
)
dd.run()
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if prev_dirty != dirty:
@@ -110,11 +112,11 @@ def test_ioclass_lba():
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
def test_ioclass_request_size():
"""
title: Test IO classification by request size.
description: Check if requests with size within defined range are cached.
pass_criteria:
- No kernel bug.
- IO is classified properly based on request size range defined in config.
title: Test IO classification by request size.
description: Check if requests with size within defined range are cached.
pass_criteria:
- No kernel bug.
- IO is classified properly based on request size range defined in config.
"""
ioclass_id = 1
@@ -141,14 +143,15 @@ def test_ioclass_request_size():
for i in range(iterations):
cache.flush_cache()
req_size = random.choice(cached_req_sizes)
dd = (
Dd().input("/dev/zero")
.output(core.path)
.count(1)
.block_size(req_size)
.oflag("direct")
(
Dd()
.input("/dev/zero")
.output(core.path)
.count(1)
.block_size(req_size)
.oflag("direct")
.run()
)
dd.run()
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != req_size.value / Unit.Blocks4096.value:
TestRun.fail("Incorrect number of dirty blocks!")
@@ -164,14 +167,15 @@ def test_ioclass_request_size():
]
for i in range(iterations):
req_size = random.choice(not_cached_req_sizes)
dd = (
Dd().input("/dev/zero")
.output(core.path)
.count(1)
.block_size(req_size)
.oflag("direct")
(
Dd()
.input("/dev/zero")
.output(core.path)
.count(1)
.block_size(req_size)
.oflag("direct")
.run()
)
dd.run()
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != 0:
TestRun.fail("Dirty data present!")
@@ -183,13 +187,13 @@ def test_ioclass_request_size():
@pytest.mark.parametrizex("filesystem", list(Filesystem) + [False])
def test_ioclass_direct(filesystem):
"""
title: Direct IO classification.
description: Check if direct requests are properly cached.
pass_criteria:
- No kernel bug.
- Data from direct IO should be cached.
- Data from buffered IO should not be cached and if performed to/from already cached data
should cause reclassification to unclassified IO class.
title: Direct IO classification.
description: Check if direct requests are properly cached.
pass_criteria:
- No kernel bug.
- Data from direct IO should be cached.
- Data from buffered IO should not be cached and if performed to/from already cached data
should cause reclassification to unclassified IO class.
"""
ioclass_id = 1
@@ -211,11 +215,15 @@ def test_ioclass_direct(filesystem):
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
with TestRun.step("Prepare fio command."):
fio = Fio().create_command() \
.io_engine(IoEngine.libaio) \
.size(io_size).offset(io_size) \
.read_write(ReadWrite.write) \
fio = (
Fio()
.create_command()
.io_engine(IoEngine.libaio)
.size(io_size)
.offset(io_size)
.read_write(ReadWrite.write)
.target(f"{mountpoint}/tmp_file" if filesystem else core.path)
)
with TestRun.step("Prepare filesystem."):
if filesystem:
@@ -237,8 +245,10 @@ def test_ioclass_direct(filesystem):
with TestRun.step("Check if buffered writes are not cached."):
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
if new_occupancy != base_occupancy:
TestRun.fail("Buffered writes were cached!\n"
f"Expected: {base_occupancy}, actual: {new_occupancy}")
TestRun.fail(
"Buffered writes were cached!\n"
f"Expected: {base_occupancy}, actual: {new_occupancy}"
)
with TestRun.step(f"Run direct writes to {'file' if filesystem else 'device'}"):
fio.direct()
@@ -248,8 +258,10 @@ def test_ioclass_direct(filesystem):
with TestRun.step("Check if direct writes are cached."):
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
if new_occupancy != base_occupancy + io_size:
TestRun.fail("Wrong number of direct writes was cached!\n"
f"Expected: {base_occupancy + io_size}, actual: {new_occupancy}")
TestRun.fail(
"Wrong number of direct writes was cached!\n"
f"Expected: {base_occupancy + io_size}, actual: {new_occupancy}"
)
with TestRun.step(f"Run buffered reads from {'file' if filesystem else 'device'}"):
fio.remove_param("readwrite").remove_param("direct")
@@ -260,8 +272,10 @@ def test_ioclass_direct(filesystem):
with TestRun.step("Check if buffered reads caused reclassification."):
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
if new_occupancy != base_occupancy:
TestRun.fail("Buffered reads did not cause reclassification!"
f"Expected occupancy: {base_occupancy}, actual: {new_occupancy}")
TestRun.fail(
"Buffered reads did not cause reclassification!"
f"Expected occupancy: {base_occupancy}, actual: {new_occupancy}"
)
with TestRun.step(f"Run direct reads from {'file' if filesystem else 'device'}"):
fio.direct()
@@ -271,8 +285,10 @@ def test_ioclass_direct(filesystem):
with TestRun.step("Check if direct reads are cached."):
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
if new_occupancy != base_occupancy + io_size:
TestRun.fail("Wrong number of direct reads was cached!\n"
f"Expected: {base_occupancy + io_size}, actual: {new_occupancy}")
TestRun.fail(
"Wrong number of direct reads was cached!\n"
f"Expected: {base_occupancy + io_size}, actual: {new_occupancy}"
)
@pytest.mark.os_dependent
@@ -281,13 +297,13 @@ def test_ioclass_direct(filesystem):
@pytest.mark.parametrizex("filesystem", Filesystem)
def test_ioclass_metadata(filesystem):
"""
title: Metadata IO classification.
description: |
Determine if every operation on files that cause metadata update results in increased
writes to cached metadata.
pass_criteria:
- No kernel bug.
- Metadata is classified properly.
title: Metadata IO classification.
description: |
Determine if every operation on files that cause metadata update results in increased
writes to cached metadata.
pass_criteria:
- No kernel bug.
- Metadata is classified properly.
"""
# Exact values may not be tested as each file system has different metadata structure.
test_dir_path = f"{mountpoint}/test_dir"
@@ -308,31 +324,35 @@ def test_ioclass_metadata(filesystem):
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
with TestRun.step(f"Prepare {filesystem.name} filesystem and mount {core.path} "
f"at {mountpoint}."):
with TestRun.step(
f"Prepare {filesystem.name} filesystem and mount {core.path} " f"at {mountpoint}."
):
core.create_filesystem(filesystem)
core.mount(mountpoint)
sync()
with TestRun.step("Create 20 test files."):
requests_to_metadata_before = cache.get_io_class_statistics(
io_class_id=ioclass_id).request_stats.write
io_class_id=ioclass_id
).request_stats.write
files = []
for i in range(1, 21):
file_path = f"{mountpoint}/test_file_{i}"
dd = (
Dd().input("/dev/urandom")
.output(file_path)
.count(random.randint(5, 50))
.block_size(Size(1, Unit.MebiByte))
.oflag("sync")
(
Dd()
.input("/dev/urandom")
.output(file_path)
.count(random.randint(5, 50))
.block_size(Size(1, Unit.MebiByte))
.oflag("sync")
.run()
)
dd.run()
files.append(File(file_path))
with TestRun.step("Check requests to metadata."):
requests_to_metadata_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).request_stats.write
io_class_id=ioclass_id
).request_stats.write
if requests_to_metadata_after == requests_to_metadata_before:
TestRun.fail("No requests to metadata while creating files!")
@@ -344,7 +364,8 @@ def test_ioclass_metadata(filesystem):
with TestRun.step("Check requests to metadata."):
requests_to_metadata_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).request_stats.write
io_class_id=ioclass_id
).request_stats.write
if requests_to_metadata_after == requests_to_metadata_before:
TestRun.fail("No requests to metadata while renaming files!")
@@ -359,7 +380,8 @@ def test_ioclass_metadata(filesystem):
with TestRun.step("Check requests to metadata."):
requests_to_metadata_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).request_stats.write
io_class_id=ioclass_id
).request_stats.write
if requests_to_metadata_after == requests_to_metadata_before:
TestRun.fail("No requests to metadata while moving files!")
@@ -368,7 +390,8 @@ def test_ioclass_metadata(filesystem):
with TestRun.step("Check requests to metadata."):
requests_to_metadata_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).request_stats.write
io_class_id=ioclass_id
).request_stats.write
if requests_to_metadata_after == requests_to_metadata_before:
TestRun.fail("No requests to metadata while deleting directory with files!")
@@ -376,17 +399,16 @@ def test_ioclass_metadata(filesystem):
@pytest.mark.os_dependent
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
@pytest.mark.parametrizex("filesystem", Filesystem)
def test_ioclass_id_as_condition(filesystem):
def test_ioclass_id_as_condition():
"""
title: IO class as a condition.
description: |
Load config in which IO class ids are used as conditions in other IO class definitions.
pass_criteria:
- No kernel bug.
- IO is classified properly as described in IO class config.
title: IO class as a condition.
description: |
Load config in which IO class ids are used as conditions in other IO class definitions.
pass_criteria:
- No kernel bug.
- IO is classified properly as described in IO class config.
"""
filesystem = Filesystem.xfs
base_dir_path = f"{mountpoint}/base_dir"
ioclass_file_size = Size(random.randint(25, 50), Unit.MebiByte)
ioclass_file_size_bytes = int(ioclass_file_size.get_value(Unit.Byte))
@@ -448,8 +470,9 @@ def test_ioclass_id_as_condition(filesystem):
# CAS needs some time to resolve directory to inode
time.sleep(ioclass_config.MAX_CLASSIFICATION_DELAY.seconds)
with TestRun.step(f"Prepare {filesystem.name} filesystem "
f"and mount {core.path} at {mountpoint}."):
with TestRun.step(
f"Prepare {filesystem.name} filesystem " f"and mount {core.path} at {mountpoint}."
):
core.create_filesystem(filesystem)
core.mount(mountpoint)
fs_utils.create_directory(base_dir_path)
@@ -457,91 +480,118 @@ def test_ioclass_id_as_condition(filesystem):
# CAS needs some time to resolve directory to inode
time.sleep(ioclass_config.MAX_CLASSIFICATION_DELAY.seconds)
with TestRun.step("Run IO fulfilling IO class 1 condition (and not IO class 2) and check if "
"it is classified properly."):
with TestRun.step(
"Run IO fulfilling IO class 1 condition (and not IO class 2) and check if "
"it is classified properly."
):
# Should be classified as IO class 4
base_occupancy = cache.get_io_class_statistics(io_class_id=4).usage_stats.occupancy
non_ioclass_file_size = Size(random.randrange(1, 25), Unit.MebiByte)
(Fio().create_command()
.io_engine(IoEngine.libaio)
.size(non_ioclass_file_size)
.read_write(ReadWrite.write)
.target(f"{base_dir_path}/test_file_1")
.run())
(
Fio()
.create_command()
.io_engine(IoEngine.libaio)
.size(non_ioclass_file_size)
.read_write(ReadWrite.write)
.target(f"{base_dir_path}/test_file_1")
.run()
)
sync()
new_occupancy = cache.get_io_class_statistics(io_class_id=4).usage_stats.occupancy
if new_occupancy != base_occupancy + non_ioclass_file_size:
TestRun.fail("Writes were not properly cached!\n"
f"Expected: {base_occupancy + non_ioclass_file_size}, "
f"actual: {new_occupancy}")
TestRun.fail(
"Writes were not properly cached!\n"
f"Expected: {base_occupancy + non_ioclass_file_size}, "
f"actual: {new_occupancy}"
)
with TestRun.step("Run IO fulfilling IO class 2 condition (and not IO class 1) and check if "
"it is classified properly."):
with TestRun.step(
"Run IO fulfilling IO class 2 condition (and not IO class 1) and check if "
"it is classified properly."
):
# Should be classified as IO class 5
base_occupancy = cache.get_io_class_statistics(io_class_id=5).usage_stats.occupancy
(Fio().create_command()
.io_engine(IoEngine.libaio)
.size(ioclass_file_size)
.read_write(ReadWrite.write)
.target(f"{mountpoint}/test_file_2")
.run())
(
Fio()
.create_command()
.io_engine(IoEngine.libaio)
.size(ioclass_file_size)
.read_write(ReadWrite.write)
.target(f"{mountpoint}/test_file_2")
.run()
)
sync()
new_occupancy = cache.get_io_class_statistics(io_class_id=5).usage_stats.occupancy
if new_occupancy != base_occupancy + ioclass_file_size:
TestRun.fail("Writes were not properly cached!\n"
f"Expected: {base_occupancy + ioclass_file_size}, actual: {new_occupancy}")
TestRun.fail(
"Writes were not properly cached!\n"
f"Expected: {base_occupancy + ioclass_file_size}, actual: {new_occupancy}"
)
with TestRun.step("Run IO fulfilling IO class 1 and 2 conditions and check if "
"it is classified properly."):
with TestRun.step(
"Run IO fulfilling IO class 1 and 2 conditions and check if " "it is classified properly."
):
# Should be classified as IO class 5
base_occupancy = new_occupancy
(Fio().create_command()
.io_engine(IoEngine.libaio)
.size(ioclass_file_size)
.read_write(ReadWrite.write)
.target(f"{base_dir_path}/test_file_3")
.run())
(
Fio()
.create_command()
.io_engine(IoEngine.libaio)
.size(ioclass_file_size)
.read_write(ReadWrite.write)
.target(f"{base_dir_path}/test_file_3")
.run()
)
sync()
new_occupancy = cache.get_io_class_statistics(io_class_id=5).usage_stats.occupancy
if new_occupancy != base_occupancy + ioclass_file_size:
TestRun.fail("Writes were not properly cached!\n"
f"Expected: {base_occupancy + ioclass_file_size}, actual: {new_occupancy}")
TestRun.fail(
"Writes were not properly cached!\n"
f"Expected: {base_occupancy + ioclass_file_size}, actual: {new_occupancy}"
)
with TestRun.step("Run direct IO fulfilling IO class 1 and 2 conditions and check if "
"it is classified properly."):
with TestRun.step(
"Run direct IO fulfilling IO class 1 and 2 conditions and check if "
"it is classified properly."
):
# Should be classified as IO class 6
base_occupancy = cache.get_io_class_statistics(io_class_id=6).usage_stats.occupancy
(Fio().create_command()
.io_engine(IoEngine.libaio)
.size(ioclass_file_size)
.read_write(ReadWrite.write)
.target(f"{base_dir_path}/test_file_3")
.direct()
.run())
(
Fio()
.create_command()
.io_engine(IoEngine.libaio)
.size(ioclass_file_size)
.read_write(ReadWrite.write)
.target(f"{base_dir_path}/test_file_3")
.direct()
.run()
)
sync()
new_occupancy = cache.get_io_class_statistics(io_class_id=6).usage_stats.occupancy
if new_occupancy != base_occupancy + ioclass_file_size:
TestRun.fail("Writes were not properly cached!\n"
f"Expected: {base_occupancy + ioclass_file_size}, actual: {new_occupancy}")
TestRun.fail(
"Writes were not properly cached!\n"
f"Expected: {base_occupancy + ioclass_file_size}, actual: {new_occupancy}"
)
@pytest.mark.os_dependent
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
@pytest.mark.parametrizex("filesystem", Filesystem)
def test_ioclass_conditions_or(filesystem):
def test_ioclass_conditions_or():
"""
title: IO class condition 'or'.
description: |
Load config with IO class combining 5 contradicting conditions connected by OR operator.
pass_criteria:
- No kernel bug.
- Every IO fulfilling one condition is classified properly.
title: IO class condition 'or'.
description: |
Load config with IO class combining 5 contradicting conditions connected by OR operator.
pass_criteria:
- No kernel bug.
- Every IO fulfilling one condition is classified properly.
"""
filesystem = Filesystem.xfs
with TestRun.step("Prepare cache and core. Disable udev."):
cache, core = prepare()
@@ -553,14 +603,17 @@ def test_ioclass_conditions_or(filesystem):
ioclass_id=1,
eviction_priority=1,
allocation="1.00",
rule=f"directory:{mountpoint}/dir1|directory:{mountpoint}/dir2|directory:"
f"{mountpoint}/dir3|directory:{mountpoint}/dir4|directory:{mountpoint}/dir5",
rule=(
f"directory:{mountpoint}/dir1|directory:{mountpoint}/dir2|directory:"
f"{mountpoint}/dir3|directory:{mountpoint}/dir4|directory:{mountpoint}/dir5"
),
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
with TestRun.step(f"Prepare {filesystem.name} filesystem "
f"and mount {core.path} at {mountpoint}."):
with TestRun.step(
f"Prepare {filesystem.name} filesystem " f"and mount {core.path} at {mountpoint}."
):
core.create_filesystem(filesystem)
core.mount(mountpoint)
for i in range(1, 6):
@@ -571,35 +624,38 @@ def test_ioclass_conditions_or(filesystem):
for i in range(1, 6):
file_size = Size(random.randint(25, 50), Unit.MebiByte)
base_occupancy = cache.get_io_class_statistics(io_class_id=1).usage_stats.occupancy
(Fio().create_command()
.io_engine(IoEngine.libaio)
.size(file_size)
.read_write(ReadWrite.write)
.target(f"{mountpoint}/dir{i}/test_file")
.run())
(
Fio()
.create_command()
.io_engine(IoEngine.libaio)
.size(file_size)
.read_write(ReadWrite.write)
.target(f"{mountpoint}/dir{i}/test_file")
.run()
)
sync()
new_occupancy = cache.get_io_class_statistics(io_class_id=1).usage_stats.occupancy
if new_occupancy != base_occupancy + file_size:
TestRun.fail("Occupancy has not increased correctly!\n"
f"Expected: {base_occupancy + file_size}, actual: {new_occupancy}")
TestRun.fail(
"Occupancy has not increased correctly!\n"
f"Expected: {base_occupancy + file_size}, actual: {new_occupancy}"
)
@pytest.mark.os_dependent
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
@pytest.mark.parametrizex("filesystem", Filesystem)
def test_ioclass_conditions_and(filesystem):
def test_ioclass_conditions_and():
"""
title: IO class condition 'and'.
description: |
Load config with IO class combining 5 conditions contradicting
at least one other condition.
pass_criteria:
- No kernel bug.
- Every IO fulfilling one of the conditions is not classified.
title: IO class condition 'and'.
description: |
Load config with IO class combining 5 conditions contradicting at least one other condition.
pass_criteria:
- No kernel bug.
- Every IO fulfilling one of the conditions is not classified.
"""
filesystem = Filesystem.xfs
file_size = Size(random.randint(25, 50), Unit.MebiByte)
file_size_bytes = int(file_size.get_value(Unit.Byte))
@@ -613,63 +669,79 @@ def test_ioclass_conditions_and(filesystem):
ioclass_id=1,
eviction_priority=1,
allocation="1.00",
rule=f"file_size:gt:{file_size_bytes}&file_size:lt:{file_size_bytes}&"
f"file_size:ge:{file_size_bytes}&file_size:le:{file_size_bytes}&"
f"file_size:eq:{file_size_bytes}",
rule=(
f"file_size:gt:{file_size_bytes}&file_size:lt:{file_size_bytes}&"
f"file_size:ge:{file_size_bytes}&file_size:le:{file_size_bytes}&"
f"file_size:eq:{file_size_bytes}"
),
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
TestRun.LOGGER.info(f"Preparing {filesystem.name} filesystem "
f"and mounting {core.path} at {mountpoint}")
TestRun.LOGGER.info(
f"Preparing {filesystem.name} filesystem " f"and mounting {core.path} at {mountpoint}"
)
core.create_filesystem(filesystem)
core.mount(mountpoint)
sync()
base_occupancy = cache.get_io_class_statistics(io_class_id=1).usage_stats.occupancy
# Perform IO
for size in [file_size, file_size + Size(1, Unit.MebiByte), file_size - Size(1, Unit.MebiByte)]:
(Fio().create_command()
.io_engine(IoEngine.libaio)
.size(size)
.read_write(ReadWrite.write)
.target(f"{mountpoint}/test_file")
.run())
for size in [
file_size,
file_size + Size(1, Unit.MebiByte),
file_size - Size(1, Unit.MebiByte),
]:
(
Fio()
.create_command()
.io_engine(IoEngine.libaio)
.size(size)
.read_write(ReadWrite.write)
.target(f"{mountpoint}/test_file")
.run()
)
sync()
new_occupancy = cache.get_io_class_statistics(io_class_id=1).usage_stats.occupancy
if new_occupancy != base_occupancy:
TestRun.fail("Unexpected occupancy increase!\n"
f"Expected: {base_occupancy}, actual: {new_occupancy}")
TestRun.fail(
"Unexpected occupancy increase!\n"
f"Expected: {base_occupancy}, actual: {new_occupancy}"
)
@pytest.mark.os_dependent
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@pytest.mark.require_disk("core", DiskTypeLowerThan("cache"))
@pytest.mark.parametrizex("filesystem", Filesystem)
def test_ioclass_effective_ioclass(filesystem):
def test_ioclass_effective_ioclass():
"""
title: Effective IO class with multiple non-exclusive conditions
description: |
Test CAS ability to properly classify IO fulfilling multiple conditions based on
IO class ids and presence of '&done' annotation in IO class rules
pass_criteria:
- In every iteration first IO is classified to the last in order IO class
- In every iteration second IO is classified to the IO class with '&done' annotation
title: Effective IO class with multiple non-exclusive conditions
description: |
Test CAS ability to properly classify IO fulfilling multiple conditions based on
IO class ids and presence of '&done' annotation in IO class rules
pass_criteria:
- In every iteration first IO is classified to the last in order IO class
- In every iteration second IO is classified to the IO class with '&done' annotation
"""
filesystem = Filesystem.xfs
with TestRun.LOGGER.step(f"Test prepare"):
cache, core = prepare(default_allocation="1.00")
Udev.disable()
file_size = Size(10, Unit.Blocks4096)
file_size_bytes = int(file_size.get_value(Unit.Byte))
test_dir = f"{mountpoint}/test"
rules = ["direct", # rule contradicting other rules
f"directory:{test_dir}",
f"file_size:le:{2 * file_size_bytes}",
f"file_size:ge:{file_size_bytes // 2}"]
rules = [
"direct", # rule contradicting other rules
f"directory:{test_dir}",
f"file_size:le:{2 * file_size_bytes}",
f"file_size:ge:{file_size_bytes // 2}",
]
with TestRun.LOGGER.step(f"Preparing {filesystem.name} filesystem "
f"and mounting {core.path} at {mountpoint}"):
with TestRun.LOGGER.step(
f"Preparing {filesystem.name} filesystem " f"and mounting {core.path} at {mountpoint}"
):
core.create_filesystem(filesystem)
core.mount(mountpoint)
fs_utils.create_directory(test_dir)
@@ -682,41 +754,54 @@ def test_ioclass_effective_ioclass(filesystem):
with TestRun.LOGGER.step("Perform IO fulfilling the non-contradicting conditions"):
base_occupancy = cache.get_io_class_statistics(
io_class_id=io_class_id).usage_stats.occupancy
fio = (Fio().create_command()
.io_engine(IoEngine.libaio)
.size(file_size)
.read_write(ReadWrite.write)
.target(f"{test_dir}/test_file{i}"))
io_class_id=io_class_id
).usage_stats.occupancy
fio = (
Fio()
.create_command()
.io_engine(IoEngine.libaio)
.size(file_size)
.read_write(ReadWrite.write)
.target(f"{test_dir}/test_file{i}")
)
fio.run()
sync()
with TestRun.LOGGER.step("Check if IO was properly classified "
"(to the last non-contradicting IO class)"):
with TestRun.LOGGER.step(
"Check if IO was properly classified " "(to the last non-contradicting IO class)"
):
new_occupancy = cache.get_io_class_statistics(
io_class_id=io_class_id).usage_stats.occupancy
io_class_id=io_class_id
).usage_stats.occupancy
if new_occupancy != base_occupancy + file_size:
TestRun.LOGGER.error("Wrong IO classification!\n"
f"Expected: {base_occupancy + file_size}, "
f"actual: {new_occupancy}")
TestRun.LOGGER.error(
"Wrong IO classification!\n"
f"Expected: {base_occupancy + file_size}, "
f"actual: {new_occupancy}"
)
with TestRun.LOGGER.step("Add '&done' to the second in order non-contradicting condition"):
io_class_id = add_done_to_second_non_exclusive_condition(rules, permutation, cache)
with TestRun.LOGGER.step("Repeat IO"):
base_occupancy = cache.get_io_class_statistics(
io_class_id=io_class_id).usage_stats.occupancy
io_class_id=io_class_id
).usage_stats.occupancy
fio.run()
sync()
with TestRun.LOGGER.step("Check if IO was properly classified "
"(to the IO class with '&done' annotation)"):
with TestRun.LOGGER.step(
"Check if IO was properly classified " "(to the IO class with '&done' annotation)"
):
new_occupancy = cache.get_io_class_statistics(
io_class_id=io_class_id).usage_stats.occupancy
io_class_id=io_class_id
).usage_stats.occupancy
if new_occupancy != base_occupancy + file_size:
TestRun.LOGGER.error("Wrong IO classification!\n"
f"Expected: {base_occupancy + file_size}, "
f"actual: {new_occupancy}")
TestRun.LOGGER.error(
"Wrong IO classification!\n"
f"Expected: {base_occupancy + file_size}, "
f"actual: {new_occupancy}"
)
def load_io_classes_in_permutation_order(rules, permutation, cache):
@@ -729,9 +814,9 @@ def load_io_classes_in_permutation_order(rules, permutation, cache):
ioclass_list = [IoClass.default(allocation="0.0")]
for n in range(len(rules)):
ioclass_list.append(IoClass(class_id=permutation[n], rule=rules[n]))
IoClass.save_list_to_config_file(ioclass_list,
add_default_rule=False,
ioclass_config_path=ioclass_config_path)
IoClass.save_list_to_config_file(
ioclass_list, add_default_rule=False, ioclass_config_path=ioclass_config_path
)
casadm.load_io_classes(cache.cache_id, file=ioclass_config_path)
@@ -745,8 +830,7 @@ def add_done_to_second_non_exclusive_condition(rules, permutation, cache):
if non_exclusive_conditions == 2:
break
second_class_id += 1
fs_utils.replace_first_pattern_occurrence(ioclass_config_path,
rules[idx], f"{rules[idx]}&done")
fs_utils.replace_first_pattern_occurrence(ioclass_config_path, rules[idx], f"{rules[idx]}&done")
sync()
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
return second_class_id