github/open-cas-linux
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #299 from Deixx/stats-flat

Browse Source 
Refactor statistics
This commit is contained in:
Katarzyna Łapińska 2020-01-24 08:57:54 +01:00 committed by GitHub
commit ed6cf8a76b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 126 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
test/functional/api/cas/cache.py
View File

@ -3,12 +3,9 @@
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from api.cas.cache_config import *
from api.cas.casadm_params import *
from api.cas.casadm_parser import *
from api.cas.cli import *
from api.cas.statistics import CacheStats, IoClassStats
from core.test_run import TestRun
from storage_devices.device import Device
from test_utils.os_utils import *
@ -108,10 +105,7 @@ class Cache:
stat_filter, percentage_val)
return CacheStats(stats)
# TODO: Get rid of this method below by tuning 'stats' and 'io_class' tests
# to utilize new statistics API with method above.
def get_statistics_deprecated(self,
def get_statistics_flat(self,
io_class_id: int = None,
stat_filter: List[StatsFilter] = None,
percentage_val: bool = False):

8
test/functional/api/cas/core.py
View File

@ -4,9 +4,6 @@
#
from datetime import timedelta
from api.cas.cache import Device
from api.cas.casadm_parser import *
from api.cas.cli import *
from api.cas.statistics import CoreStats, IoClassStats
@ -67,10 +64,7 @@ class Core(Device):
stat_filter, percentage_val)
return CoreStats(stats)
# TODO: Get rid of this method below by tuning 'stats' and 'io_class' tests
# to utilize new statistics API with method above.
def get_statistics_deprecated(self,
def get_statistics_flat(self,
io_class_id: int = None,
stat_filter: List[StatsFilter] = None,
percentage_val: bool = False):

54
test/functional/tests/io_class/test_io_class_directory.py
View File

@ -8,13 +8,13 @@ from datetime import datetime
import pytest
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools import fs_utils
from test_tools.dd import Dd
from test_tools.disk_utils import Filesystem
from test_utils.filesystem.directory import Directory
from test_utils.filesystem.file import File
from test_utils.os_utils import drop_caches, DropCachesMode, sync, Udev
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from .io_class_common import *
@ -72,7 +72,7 @@ def test_ioclass_directory_depth(filesystem):
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
base_occupancy = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
base_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
TestRun.LOGGER.info("Reading the file in the nested directory")
dd = (
Dd()
@ -82,7 +82,7 @@ def test_ioclass_directory_depth(filesystem):
)
dd.run()
new_occupancy = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
assert new_occupancy == base_occupancy + test_file_1.size, \
"Wrong occupancy after reading file!\n" \
f"Expected: {base_occupancy + test_file_1.size}, actual: {new_occupancy}"
@ -103,7 +103,7 @@ def test_ioclass_directory_depth(filesystem):
drop_caches(DropCachesMode.ALL)
test_file_2.refresh_item()
new_occupancy = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
assert new_occupancy == base_occupancy + test_file_2.size, \
"Wrong occupancy after creating file!\n" \
f"Expected: {base_occupancy + test_file_2.size}, actual: {new_occupancy}"
@ -122,7 +122,8 @@ def test_ioclass_directory_dir_operations(filesystem):
"""
def create_files_with_classification_delay_check(directory: Directory, ioclass_id: int):
start_time = datetime.now()
occupancy_after = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
occupancy_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
dd_blocks = 10
dd_size = Size(dd_blocks, Unit.Blocks4096)
file_counter = 0
@ -135,7 +136,8 @@ def test_ioclass_directory_dir_operations(filesystem):
time_from_start = datetime.now() - start_time
(Dd().input("/dev/zero").output(file_path).oflag("sync")
.block_size(Size(1, Unit.Blocks4096)).count(dd_blocks).run())
occupancy_after = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
occupancy_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
if occupancy_after - occupancy_before < dd_size:
unclassified_files.append(file_path)
@ -151,10 +153,10 @@ def test_ioclass_directory_dir_operations(filesystem):
def read_files_with_reclassification_check(
target_ioclass_id: int, source_ioclass_id: int, directory: Directory, with_delay: bool):
start_time = datetime.now()
target_occupancy_after = cache.get_statistics_deprecated(
io_class_id=target_ioclass_id)["occupancy"]
source_occupancy_after = cache.get_statistics_deprecated(
io_class_id=source_ioclass_id)["occupancy"]
target_occupancy_after = cache.get_io_class_statistics(
io_class_id=target_ioclass_id).usage_stats.occupancy
source_occupancy_after = cache.get_io_class_statistics(
io_class_id=source_ioclass_id).usage_stats.occupancy
unclassified_files = []
for file in [item for item in directory.ls() if isinstance(item, File)]:
@ -163,10 +165,10 @@ def test_ioclass_directory_dir_operations(filesystem):
time_from_start = datetime.now() - start_time
(Dd().input(file.full_path).output("/dev/null")
.block_size(Size(1, Unit.Blocks4096)).run())
target_occupancy_after = cache.get_statistics_deprecated(
io_class_id=target_ioclass_id)["occupancy"]
source_occupancy_after = cache.get_statistics_deprecated(
io_class_id=source_ioclass_id)["occupancy"]
target_occupancy_after = cache.get_io_class_statistics(
io_class_id=target_ioclass_id).usage_stats.occupancy
source_occupancy_after = cache.get_io_class_statistics(
io_class_id=source_ioclass_id).usage_stats.occupancy
if target_occupancy_after < target_occupancy_before:
pytest.xfail("Target IO class occupancy lowered!")
elif target_occupancy_after - target_occupancy_before < file.size:
@ -323,7 +325,8 @@ def test_ioclass_directory_file_operations(filesystem):
drop_caches(DropCachesMode.ALL)
TestRun.LOGGER.info("Creating test file")
classified_before = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
classified_before = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
file_path = f"{test_dir_path}/test_file"
(Dd().input("/dev/urandom").output(file_path).oflag("sync")
.block_size(Size(1, Unit.MebiByte)).count(dd_blocks).run())
@ -332,21 +335,22 @@ def test_ioclass_directory_file_operations(filesystem):
test_file = File(file_path).refresh_item()
TestRun.LOGGER.info("Checking classified occupancy")
classified_after = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
classified_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before + test_file.size, classified_after)
TestRun.LOGGER.info("Moving test file out of classified directory")
classified_before = classified_after
non_classified_before = cache.get_statistics_deprecated(io_class_id=0)["occupancy"]
non_classified_before = cache.get_io_class_statistics(io_class_id=0).usage_stats.occupancy
test_file.move(destination=mountpoint)
sync()
drop_caches(DropCachesMode.ALL)
TestRun.LOGGER.info("Checking classified occupancy")
classified_after = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
classified_after = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before, classified_after)
TestRun.LOGGER.info("Checking non-classified occupancy")
non_classified_after = cache.get_statistics_deprecated(io_class_id=0)["occupancy"]
non_classified_after = cache.get_io_class_statistics(io_class_id=0).usage_stats.occupancy
check_occupancy(non_classified_before, non_classified_after)
TestRun.LOGGER.info("Reading test file")
@ -356,10 +360,10 @@ def test_ioclass_directory_file_operations(filesystem):
.block_size(Size(1, Unit.MebiByte)).run())
TestRun.LOGGER.info("Checking classified occupancy")
classified_after = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
classified_after = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before - test_file.size, classified_after)
TestRun.LOGGER.info("Checking non-classified occupancy")
non_classified_after = cache.get_statistics_deprecated(io_class_id=0)["occupancy"]
non_classified_after = cache.get_io_class_statistics(io_class_id=0).usage_stats.occupancy
check_occupancy(non_classified_before + test_file.size, non_classified_after)
TestRun.LOGGER.info(f"Moving test file to {nested_dir_path}")
@ -370,10 +374,10 @@ def test_ioclass_directory_file_operations(filesystem):
drop_caches(DropCachesMode.ALL)
TestRun.LOGGER.info("Checking classified occupancy")
classified_after = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
classified_after = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before, classified_after)
TestRun.LOGGER.info("Checking non-classified occupancy")
non_classified_after = cache.get_statistics_deprecated(io_class_id=0)["occupancy"]
non_classified_after = cache.get_io_class_statistics(io_class_id=0).usage_stats.occupancy
check_occupancy(non_classified_before, non_classified_after)
TestRun.LOGGER.info("Reading test file")
@ -383,8 +387,8 @@ def test_ioclass_directory_file_operations(filesystem):
.block_size(Size(1, Unit.MebiByte)).run())
TestRun.LOGGER.info("Checking classified occupancy")
classified_after = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
classified_after = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before + test_file.size, classified_after)
TestRun.LOGGER.info("Checking non-classified occupancy")
non_classified_after = cache.get_statistics_deprecated(io_class_id=0)["occupancy"]
non_classified_after = cache.get_io_class_statistics(io_class_id=0).usage_stats.occupancy
check_occupancy(non_classified_before - test_file.size, non_classified_after)

50
test/functional/tests/io_class/test_io_class_file.py
View File

@ -7,11 +7,11 @@ import random
import pytest
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools.dd import Dd
from test_tools.disk_utils import Filesystem
from test_utils.filesystem.file import File
from test_utils.os_utils import sync, Udev, DropCachesMode, drop_caches
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from .io_class_common import *
@ -56,8 +56,9 @@ def test_ioclass_file_extension():
for i in range(iterations):
dd.run()
sync()
stats = cache.get_statistics_deprecated(io_class_id=ioclass_id)
assert stats["dirty"].get_value(Unit.Blocks4096) == (i + 1) * dd_count
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != (i + 1) * dd_count:
TestRun.LOGGER.error(f"Wrong amount of dirty data ({dirty}).")
cache.flush_cache()
@ -73,8 +74,9 @@ def test_ioclass_file_extension():
)
dd.run()
sync()
stats = cache.get_statistics_deprecated(io_class_id=ioclass_id)
assert stats["dirty"].get_value(Unit.Blocks4096) == 0
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != 0:
TestRun.LOGGER.error(f"Wrong amount of dirty data ({dirty}).")
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@ -213,11 +215,9 @@ def test_ioclass_file_extension_preexisting_filesystem():
)
dd.run()
sync()
stats = cache.get_statistics_deprecated(io_class_id=ioclass_id)
assert (
stats["dirty"].get_value(Unit.Blocks4096)
== (extensions.index(ext) + 1) * dd_count
)
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != (extensions.index(ext) + 1) * dd_count:
TestRun.LOGGER.error(f"Wrong amount of dirty data ({dirty}).")
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@ -269,10 +269,9 @@ def test_ioclass_file_offset():
)
dd.run()
sync()
stats = cache.get_statistics_deprecated(io_class_id=ioclass_id)
assert (
stats["dirty"].get_value(Unit.Blocks4096) == 1
), f"Offset not cached: {file_offset}"
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != 1:
TestRun.LOGGER.error(f"Offset not cached: {file_offset}")
cache.flush_cache()
min_seek = 0
@ -290,10 +289,9 @@ def test_ioclass_file_offset():
)
dd.run()
sync()
stats = cache.get_statistics_deprecated(io_class_id=ioclass_id)
assert (
stats["dirty"].get_value(Unit.Blocks4096) == 0
), f"Inappropriately cached offset: {file_offset}"
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != 0:
TestRun.LOGGER.error(f"Inappropriately cached offset: {file_offset}")
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@ -349,10 +347,12 @@ def test_ioclass_file_size(filesystem):
TestRun.LOGGER.info("Creating files belonging to different IO classes "
"(classification by writes).")
for size, ioclass_id in size_to_class.items():
occupancy_before = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
occupancy_before = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
file_path = f"{mountpoint}/test_file_{size.get_value()}"
Dd().input("/dev/zero").output(file_path).oflag("sync").block_size(size).count(1).run()
occupancy_after = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
occupancy_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
if occupancy_after != occupancy_before + size:
pytest.xfail("File not cached properly!\n"
f"Expected {occupancy_before + size}\n"
@ -366,9 +366,11 @@ def test_ioclass_file_size(filesystem):
"(classification by reads).")
for file in test_files:
ioclass_id = size_to_class[file.size]
occupancy_before = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
occupancy_before = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
Dd().input(file.full_path).output("/dev/null").block_size(file.size).run()
occupancy_after = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
occupancy_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
if occupancy_after != occupancy_before + file.size:
pytest.xfail("File not reclassified properly!\n"
f"Expected {occupancy_before + file.size}\n"
@ -390,10 +392,10 @@ def test_ioclass_file_size(filesystem):
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
occupancy_before = cache.get_statistics_deprecated(io_class_id=0)["occupancy"]
occupancy_before = cache.get_io_class_statistics(io_class_id=0).usage_stats.occupancy
for file in test_files:
Dd().input(file.full_path).output("/dev/null").block_size(file.size).run()
occupancy_after = cache.get_statistics_deprecated(io_class_id=0)["occupancy"]
occupancy_after = cache.get_io_class_statistics(io_class_id=0).usage_stats.occupancy
if occupancy_after != occupancy_before + file.size:
pytest.xfail("File not reclassified properly!\n"
f"Expected {occupancy_before + file.size}\n"

13
test/functional/tests/io_class/test_io_class_process.py
View File

@ -7,9 +7,9 @@ import time
import pytest
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools.dd import Dd
from test_utils.os_utils import sync, Udev
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from .io_class_common import *
@ -50,8 +50,9 @@ def test_ioclass_process_name():
dd.run()
sync()
time.sleep(0.1)
stats = cache.get_statistics_deprecated(io_class_id=ioclass_id)
assert stats["dirty"].get_value(Unit.Blocks4096) == (i + 1) * dd_count
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != (i + 1) * dd_count:
TestRun.LOGGER.error(f"Wrong amount of dirty data ({dirty}).")
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@ -109,7 +110,7 @@ def test_ioclass_pid():
f"stdout: {output.stdout} \n stderr :{output.stderr}"
)
sync()
stats = cache.get_statistics_deprecated(io_class_id=ioclass_id)
assert stats["dirty"].get_value(Unit.Blocks4096) == dd_count
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != dd_count:
TestRun.LOGGER.error(f"Wrong amount of dirty data ({dirty}).")
ioclass_config.remove_ioclass(ioclass_id)

80
test/functional/tests/io_class/test_io_classification.py
View File

@ -64,9 +64,9 @@ def test_ioclass_lba():
sync()
dirty_count += 1
stats = cache.get_statistics_deprecated(io_class_id=ioclass_id)
if stats["dirty"].get_value(Unit.Blocks4096) != dirty_count:
TestRun.fail(f"LBA {lba} not cached")
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != dirty_count:
TestRun.LOGGER.error(f"LBA {lba} not cached")
cache.flush_cache()
@ -87,9 +87,9 @@ def test_ioclass_lba():
dd.run()
sync()
stats = cache.get_statistics_deprecated(io_class_id=ioclass_id)
if stats["dirty"].get_value(Unit.Blocks4096) != 0:
TestRun.fail(f"Inappropriately cached lba: {rand_lba}")
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != 0:
TestRun.LOGGER.error(f"Inappropriately cached lba: {rand_lba}")
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@ -128,8 +128,8 @@ def test_ioclass_request_size():
.oflag("direct")
)
dd.run()
stats = cache.get_statistics_deprecated(io_class_id=ioclass_id)
if stats["dirty"].get_value(Unit.Blocks4096) != req_size.value / Unit.Blocks4096.value:
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != req_size.value / Unit.Blocks4096.value:
TestRun.fail("Incorrect number of dirty blocks!")
cache.flush_cache()
@ -154,8 +154,8 @@ def test_ioclass_request_size():
.oflag("direct")
)
dd.run()
stats = cache.get_statistics_deprecated(io_class_id=ioclass_id)
if stats["dirty"].get_value(Unit.Blocks4096) != 0:
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != 0:
TestRun.fail("Dirty data present!")
@ -205,12 +205,12 @@ def test_ioclass_direct(filesystem):
else:
TestRun.LOGGER.info("Testing on raw exported object")
base_occupancy = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
base_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
TestRun.LOGGER.info(f"Buffered writes to {'file' if filesystem else 'device'}")
fio.run()
sync()
new_occupancy = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
if new_occupancy != base_occupancy:
TestRun.fail("Buffered writes were cached!\n"
f"Expected: {base_occupancy}, actual: {new_occupancy}")
@ -219,7 +219,7 @@ def test_ioclass_direct(filesystem):
fio.direct()
fio.run()
sync()
new_occupancy = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
if new_occupancy != base_occupancy + io_size:
TestRun.fail("Wrong number of direct writes was cached!\n"
f"Expected: {base_occupancy + io_size}, actual: {new_occupancy}")
@ -229,7 +229,7 @@ def test_ioclass_direct(filesystem):
fio.read_write(ReadWrite.read)
fio.run()
sync()
new_occupancy = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
if new_occupancy != base_occupancy:
TestRun.fail("Buffered reads did not cause reclassification!"
f"Expected occupancy: {base_occupancy}, actual: {new_occupancy}")
@ -238,7 +238,7 @@ def test_ioclass_direct(filesystem):
fio.direct()
fio.run()
sync()
new_occupancy = cache.get_statistics_deprecated(io_class_id=ioclass_id)["occupancy"]
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
if new_occupancy != base_occupancy + io_size:
TestRun.fail("Wrong number of direct reads was cached!\n"
f"Expected: {base_occupancy + io_size}, actual: {new_occupancy}")
@ -273,8 +273,8 @@ def test_ioclass_metadata(filesystem):
core.mount(mountpoint)
sync()
requests_to_metadata_before = cache.get_statistics_deprecated(
io_class_id=ioclass_id)["write total"]
requests_to_metadata_before = cache.get_io_class_statistics(
io_class_id=ioclass_id).request_stats.write
TestRun.LOGGER.info("Creating 20 test files")
files = []
for i in range(1, 21):
@ -291,10 +291,10 @@ def test_ioclass_metadata(filesystem):
files.append(File(file_path))
TestRun.LOGGER.info("Checking requests to metadata")
requests_to_metadata_after = cache.get_statistics_deprecated(
io_class_id=ioclass_id)["write total"]
requests_to_metadata_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).request_stats.write
if requests_to_metadata_after == requests_to_metadata_before:
pytest.xfail("No requests to metadata while creating files!")
TestRun.fail("No requests to metadata while creating files!")
requests_to_metadata_before = requests_to_metadata_after
TestRun.LOGGER.info("Renaming all test files")
@ -303,10 +303,10 @@ def test_ioclass_metadata(filesystem):
sync()
TestRun.LOGGER.info("Checking requests to metadata")
requests_to_metadata_after = cache.get_statistics_deprecated(
io_class_id=ioclass_id)["write total"]
requests_to_metadata_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).request_stats.write
if requests_to_metadata_after == requests_to_metadata_before:
pytest.xfail("No requests to metadata while renaming files!")
TestRun.fail("No requests to metadata while renaming files!")
requests_to_metadata_before = requests_to_metadata_after
test_dir_path = f"{mountpoint}/test_dir"
@ -319,19 +319,19 @@ def test_ioclass_metadata(filesystem):
sync()
TestRun.LOGGER.info("Checking requests to metadata")
requests_to_metadata_after = cache.get_statistics_deprecated(
io_class_id=ioclass_id)["write total"]
requests_to_metadata_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).request_stats.write
if requests_to_metadata_after == requests_to_metadata_before:
pytest.xfail("No requests to metadata while moving files!")
TestRun.fail("No requests to metadata while moving files!")
TestRun.LOGGER.info(f"Removing {test_dir_path}")
fs_utils.remove(path=test_dir_path, force=True, recursive=True)
TestRun.LOGGER.info("Checking requests to metadata")
requests_to_metadata_after = cache.get_statistics_deprecated(
io_class_id=ioclass_id)["write total"]
requests_to_metadata_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).request_stats.write
if requests_to_metadata_after == requests_to_metadata_before:
pytest.xfail("No requests to metadata while deleting directory with files!")
TestRun.fail("No requests to metadata while deleting directory with files!")
@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.optane, DiskType.nand]))
@ -408,7 +408,7 @@ def test_ioclass_id_as_condition(filesystem):
# IO fulfilling IO class 1 condition (and not IO class 2)
# Should be classified as IO class 4
base_occupancy = cache.get_statistics_deprecated(io_class_id=4)["occupancy"]
base_occupancy = cache.get_io_class_statistics(io_class_id=4).usage_stats.occupancy
non_ioclass_file_size = Size(random.randrange(1, 25), Unit.MebiByte)
(Fio().create_command()
.io_engine(IoEngine.libaio)
@ -417,7 +417,7 @@ def test_ioclass_id_as_condition(filesystem):
.target(f"{base_dir_path}/test_file_1")
.run())
sync()
new_occupancy = cache.get_statistics_deprecated(io_class_id=4)["occupancy"]
new_occupancy = cache.get_io_class_statistics(io_class_id=4).usage_stats.occupancy
if new_occupancy != base_occupancy + non_ioclass_file_size:
TestRun.fail("Writes were not properly cached!\n"
@ -425,7 +425,7 @@ def test_ioclass_id_as_condition(filesystem):
# IO fulfilling IO class 2 condition (and not IO class 1)
# Should be classified as IO class 5
base_occupancy = cache.get_statistics_deprecated(io_class_id=5)["occupancy"]
base_occupancy = cache.get_io_class_statistics(io_class_id=5).usage_stats.occupancy
(Fio().create_command()
.io_engine(IoEngine.libaio)
.size(ioclass_file_size)
@ -433,7 +433,7 @@ def test_ioclass_id_as_condition(filesystem):
.target(f"{mountpoint}/test_file_2")
.run())
sync()
new_occupancy = cache.get_statistics_deprecated(io_class_id=5)["occupancy"]
new_occupancy = cache.get_io_class_statistics(io_class_id=5).usage_stats.occupancy
if new_occupancy != base_occupancy + ioclass_file_size:
TestRun.fail("Writes were not properly cached!\n"
@ -449,7 +449,7 @@ def test_ioclass_id_as_condition(filesystem):
.target(f"{base_dir_path}/test_file_3")
.run())
sync()
new_occupancy = cache.get_statistics_deprecated(io_class_id=5)["occupancy"]
new_occupancy = cache.get_io_class_statistics(io_class_id=5).usage_stats.occupancy
if new_occupancy != base_occupancy + ioclass_file_size:
TestRun.fail("Writes were not properly cached!\n"
@ -457,7 +457,7 @@ def test_ioclass_id_as_condition(filesystem):
# Same IO but direct
# Should be classified as IO class 6
base_occupancy = cache.get_statistics_deprecated(io_class_id=6)["occupancy"]
base_occupancy = cache.get_io_class_statistics(io_class_id=6).usage_stats.occupancy
(Fio().create_command()
.io_engine(IoEngine.libaio)
.size(ioclass_file_size)
@ -466,7 +466,7 @@ def test_ioclass_id_as_condition(filesystem):
.direct()
.run())
sync()
new_occupancy = cache.get_statistics_deprecated(io_class_id=6)["occupancy"]
new_occupancy = cache.get_io_class_statistics(io_class_id=6).usage_stats.occupancy
if new_occupancy != base_occupancy + ioclass_file_size:
TestRun.fail("Writes were not properly cached!\n"
@ -506,7 +506,7 @@ def test_ioclass_conditions_or(filesystem):
# Perform IO fulfilling each condition and check if occupancy raises
for i in range(1, 6):
file_size = Size(random.randint(25, 50), Unit.MebiByte)
base_occupancy = cache.get_statistics_deprecated(io_class_id=1)["occupancy"]
base_occupancy = cache.get_io_class_statistics(io_class_id=1).usage_stats.occupancy
(Fio().create_command()
.io_engine(IoEngine.libaio)
.size(file_size)
@ -514,7 +514,7 @@ def test_ioclass_conditions_or(filesystem):
.target(f"{mountpoint}/dir{i}/test_file")
.run())
sync()
new_occupancy = cache.get_statistics_deprecated(io_class_id=1)["occupancy"]
new_occupancy = cache.get_io_class_statistics(io_class_id=1).usage_stats.occupancy
if new_occupancy != base_occupancy + file_size:
TestRun.fail("Occupancy has not increased correctly!\n"
@ -553,7 +553,7 @@ def test_ioclass_conditions_and(filesystem):
core.mount(mountpoint)
sync()
base_occupancy = cache.get_statistics_deprecated(io_class_id=1)["occupancy"]
base_occupancy = cache.get_io_class_statistics(io_class_id=1).usage_stats.occupancy
# Perform IO
for size in [file_size, file_size + Size(1, Unit.MebiByte), file_size - Size(1, Unit.MebiByte)]:
(Fio().create_command()
@ -563,7 +563,7 @@ def test_ioclass_conditions_and(filesystem):
.target(f"{mountpoint}/test_file")
.run())
sync()
new_occupancy = cache.get_statistics_deprecated(io_class_id=1)["occupancy"]
new_occupancy = cache.get_io_class_statistics(io_class_id=1).usage_stats.occupancy
if new_occupancy != base_occupancy:
TestRun.fail("Unexpected occupancy increase!\n"

17
test/functional/tests/stats/test_block_stats.py
View File

@ -5,15 +5,16 @@
import pytest
from api.cas.casadm import StatsFilter
from api.cas import casadm
from api.cas import ioclass_config
from test_tools.dd import Dd
from api.cas.cache_config import CacheMode, CleaningPolicy
from api.cas.casadm import StatsFilter
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_utils.size import Size, Unit
from test_tools.dd import Dd
from test_utils.os_utils import Udev
from test_utils.size import Size, Unit
ioclass_config_path = "/tmp/opencas_ioclass.conf"
mountpoint = "/tmp/cas1-1"
@ -112,8 +113,8 @@ def test_block_stats_write(cache_mode, zero_stats):
for i in range(iterations):
dd.seek(dd_seek)
dd.run()
cache_stats = cache.get_statistics_deprecated(stat_filter=[StatsFilter.blk])
core_stats = core.get_statistics_deprecated(stat_filter=[StatsFilter.blk])
cache_stats = cache.get_statistics_flat(stat_filter=[StatsFilter.blk])
core_stats = core.get_statistics_flat(stat_filter=[StatsFilter.blk])
# Check cache stats
assumed_value = (dd_size.get_value(Unit.Blocks4096) * dd_count) * (i + 1)
@ -237,8 +238,8 @@ def test_block_stats_read(cache_mode, zero_stats):
for i in range(iterations):
dd.skip(dd_skip)
dd.run()
cache_stats = cache.get_statistics_deprecated(stat_filter=[StatsFilter.blk])
core_stats = core.get_statistics_deprecated(stat_filter=[StatsFilter.blk])
cache_stats = cache.get_statistics_flat(stat_filter=[StatsFilter.blk])
core_stats = core.get_statistics_flat(stat_filter=[StatsFilter.blk])
# Check cache stats
assumed_value = (dd_size.get_value(Unit.Blocks4096) * dd_count) * (i + 1)
@ -283,7 +284,7 @@ def test_block_stats_read(cache_mode, zero_stats):
def flush(cache):
cache.flush_cache()
cache.reset_counters()
stats = cache.get_statistics_deprecated(stat_filter=[StatsFilter.blk])
stats = cache.get_statistics_flat(stat_filter=[StatsFilter.blk])
for key, value in stats.items():
assert value.get_value(Unit.Blocks4096) == 0

13
test/functional/tests/stats/test_ioclass_stats.py
View File

@ -5,17 +5,18 @@
import pytest
from api.cas.casadm import StatsFilter
from api.cas import casadm
from api.cas import ioclass_config
from api.cas import casadm_parser
from api.cas import ioclass_config
from api.cas.cache_config import CleaningPolicy
from api.cas.casadm import StatsFilter
from core.test_run import TestRun
from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
from test_tools.disk_utils import Filesystem
from test_utils.size import Size, Unit
from test_utils.os_utils import sync, Udev
from test_utils.filesystem.file import File
from test_utils.os_utils import sync, Udev
from test_utils.size import Size, Unit
ioclass_config_path = "/tmp/opencas_ioclass.conf"
mountpoint = "/tmp/cas1-1"
@ -105,11 +106,11 @@ def test_ioclass_stats_sum():
core.unmount()
sync()
cache_stats = cache.get_statistics_deprecated(
cache_stats = cache.get_statistics_flat(
stat_filter=[StatsFilter.usage, StatsFilter.req, StatsFilter.blk]
)
for ioclass_id in ioclass_id_list:
ioclass_stats = cache.get_statistics_deprecated(
ioclass_stats = cache.get_statistics_flat(
stat_filter=[StatsFilter.usage, StatsFilter.req, StatsFilter.blk],
io_class_id=ioclass_id,
)