github/ocf
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

switch to volume I/O interface in functional tests

Browse Source 
... instead of core::new_io()

Signed-off-by: Adam Rutkowski <adam.j.rutkowski@intel.com>
This commit is contained in:
Adam Rutkowski
2022-03-22 15:41:50 +01:00
parent 9e88afad62
commit 05f5d49be9
13 changed files with 252 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
tests/functional/tests/management/test_add_remove.py
View File

@@ -9,9 +9,11 @@ from ctypes import c_int
from random import randint
from pyocf.types.cache import Cache, CacheMode
from pyocf.types.core import Core
from pyocf.types.volume import RamVolume
from pyocf.types.volume import RamVolume, Volume
from pyocf.types.volume_core import CoreVolume
from pyocf.types.data import Data
from pyocf.types.io import IoDir
from pyocf.types.queue import Queue
from pyocf.utils import Size as S
from pyocf.types.shared import OcfError, OcfCompletion, CacheLineSize
@@ -79,11 +81,14 @@ def test_remove_dirty_no_flush(pyocf_ctx, cache_mode, cls):
core = Core.using_device(core_device)
cache.add_core(core)
vol = CoreVolume(core, open=True)
queue = core.cache.get_default_queue()
# Prepare data
core_size = core.get_stats()["size"]
data = Data(core_size.B)
_io_to_core(core, data)
_io_to_core(vol, queue, data)
# Remove core from cache
cache.remove_core(core)
@@ -122,11 +127,12 @@ def test_10add_remove_with_io(pyocf_ctx):
# Add and remove core 10 times in a loop with io in between
for i in range(0, 10):
cache.add_core(core)
vol = CoreVolume(core, open=True)
stats = cache.get_stats()
assert stats["conf"]["core_count"] == 1
write_data = Data.from_string("Test data")
io = core.new_io(
io = vol.new_io(
cache.get_default_queue(), S.from_sector(1).B, write_data.size,
IoDir.WRITE, 0, 0
)
@@ -303,8 +309,8 @@ def test_add_remove_incrementally(pyocf_ctx, cache_mode, cls):
assert stats["conf"]["core_count"] == core_amount
def _io_to_core(exported_obj: Core, data: Data):
io = exported_obj.new_io(exported_obj.cache.get_default_queue(), 0, data.size,
def _io_to_core(vol: Volume, queue: Queue, data: Data):
io = vol.new_io(queue, 0, data.size,
IoDir.WRITE, 0, 0)
io.set_data(data)

26
tests/functional/tests/management/test_attach_cache.py
View File

@@ -28,6 +28,7 @@ from pyocf.types.shared import (
SeqCutOffPolicy,
)
from pyocf.types.volume import RamVolume
from pyocf.types.volume_core import CoreVolume
from pyocf.utils import Size
logger = logging.getLogger(__name__)
@@ -49,6 +50,9 @@ def test_attach_different_size(
core = Core.using_device(core_device)
cache.add_core(core)
vol = CoreVolume(core, open=True)
queue = cache.get_default_queue()
cache.configure_partition(
part_id=1, name="test_part", max_size=50, priority=1
)
@@ -61,7 +65,7 @@ def test_attach_different_size(
data = bytes(block_size)
for i in range(cache_size.blocks_4k):
io_to_exp_obj(core, block_size * i, block_size, data, 0, IoDir.WRITE, 1, 0)
io_to_exp_obj(vol, queue, block_size * i, block_size, data, 0, IoDir.WRITE, 1, 0)
part_current_size = CacheLines(
cache.get_partition_info(part_id=1)["_curr_size"], cls
@@ -76,7 +80,7 @@ def test_attach_different_size(
cache_size = cache.get_stats()["conf"]["size"]
for i in range(cache_size.blocks_4k):
io_to_exp_obj(core, block_size * i, block_size, data, 0, IoDir.WRITE, 1, 0)
io_to_exp_obj(vol, queue, block_size * i, block_size, data, 0, IoDir.WRITE, 1, 0)
part_current_size = CacheLines(
cache.get_partition_info(part_id=1)["_curr_size"], cls
@@ -85,22 +89,8 @@ def test_attach_different_size(
assert part_current_size.blocks_4k == cache_size.blocks_4k * 0.5
def io_to_exp_obj(core, address, size, data, offset, direction, target_ioclass, flags):
return _io(
core.new_io,
core.cache.get_default_queue(),
address,
size,
data,
offset,
direction,
target_ioclass,
flags,
)
def _io(new_io, queue, address, size, data, offset, direction, target_ioclass, flags):
io = new_io(queue, address, size, direction, target_ioclass, flags)
def io_to_exp_obj(vol, queue, address, size, data, offset, direction, target_ioclass, flags):
io = vol.new_io(queue, address, size, direction, target_ioclass, flags)
if direction == IoDir.READ:
_data = Data.from_bytes(bytes(size))
else:

134
tests/functional/tests/management/test_start_stop.py
View File

@@ -24,8 +24,10 @@ from pyocf.types.core import Core
from pyocf.types.ctx import OcfCtx
from pyocf.types.data import Data
from pyocf.types.io import IoDir
from pyocf.types.queue import Queue
from pyocf.types.shared import OcfError, OcfCompletion, CacheLineSize, SeqCutOffPolicy
from pyocf.types.volume import Volume, RamVolume
from pyocf.types.volume_core import CoreVolume
from pyocf.utils import Size
logger = logging.getLogger(__name__)
@@ -62,21 +64,23 @@ def test_start_write_first_and_check_mode(pyocf_ctx, mode: CacheMode, cls: Cache
cache_device = RamVolume(Size.from_MiB(50))
core_device = RamVolume(Size.from_MiB(10))
cache = Cache.start_on_device(cache_device, cache_mode=mode, cache_line_size=cls)
core_exported = Core.using_device(core_device)
core = Core.using_device(core_device)
cache.add_core(core_exported)
cache.add_core(core)
vol = CoreVolume(core, open=True)
queue = cache.get_default_queue()
logger.info("[STAGE] Initial write to exported object")
cache_device.reset_stats()
core_device.reset_stats()
test_data = Data.from_string("This is test data")
io_to_core(core_exported, test_data, Size.from_sector(1).B)
check_stats_write_empty(core_exported, mode, cls)
io_to_core(vol, queue, test_data, Size.from_sector(1).B)
check_stats_write_empty(core, mode, cls)
logger.info("[STAGE] Read from exported object after initial write")
io_from_exported_object(core_exported, test_data.size, Size.from_sector(1).B)
check_stats_read_after_write(core_exported, mode, cls, True)
io_from_exported_object(vol, queue, test_data.size, Size.from_sector(1).B)
check_stats_read_after_write(core, mode, cls, True)
logger.info("[STAGE] Write to exported object after read")
cache_device.reset_stats()
@@ -84,10 +88,10 @@ def test_start_write_first_and_check_mode(pyocf_ctx, mode: CacheMode, cls: Cache
test_data = Data.from_string("Changed test data")
io_to_core(core_exported, test_data, Size.from_sector(1).B)
check_stats_write_after_read(core_exported, mode, cls)
io_to_core(vol, queue, test_data, Size.from_sector(1).B)
check_stats_write_after_read(core, mode, cls)
check_md5_sums(core_exported, mode)
check_md5_sums(core, mode)
@pytest.mark.parametrize("cls", CacheLineSize)
@@ -100,20 +104,23 @@ def test_start_read_first_and_check_mode(pyocf_ctx, mode: CacheMode, cls: CacheL
cache_device = RamVolume(Size.from_MiB(50))
core_device = RamVolume(Size.from_MiB(5))
cache = Cache.start_on_device(cache_device, cache_mode=mode, cache_line_size=cls)
core_exported = Core.using_device(core_device)
core = Core.using_device(core_device)
cache.add_core(core_exported)
cache.add_core(core)
front_vol = CoreVolume(core, open=True)
bottom_vol = core.get_volume()
queue = cache.get_default_queue()
logger.info("[STAGE] Initial write to core device")
test_data = Data.from_string("This is test data")
io_to_core(core_exported, test_data, Size.from_sector(1).B, True)
io_to_core(bottom_vol, queue, test_data, Size.from_sector(1).B)
cache_device.reset_stats()
core_device.reset_stats()
logger.info("[STAGE] Initial read from exported object")
io_from_exported_object(core_exported, test_data.size, Size.from_sector(1).B)
check_stats_read_empty(core_exported, mode, cls)
io_from_exported_object(front_vol, queue, test_data.size, Size.from_sector(1).B)
check_stats_read_empty(core, mode, cls)
logger.info("[STAGE] Write to exported object after initial read")
cache_device.reset_stats()
@@ -121,15 +128,15 @@ def test_start_read_first_and_check_mode(pyocf_ctx, mode: CacheMode, cls: CacheL
test_data = Data.from_string("Changed test data")
io_to_core(core_exported, test_data, Size.from_sector(1).B)
io_to_core(front_vol, queue, test_data, Size.from_sector(1).B)
check_stats_write_after_read(core_exported, mode, cls, True)
check_stats_write_after_read(core, mode, cls, True)
logger.info("[STAGE] Read from exported object after write")
io_from_exported_object(core_exported, test_data.size, Size.from_sector(1).B)
check_stats_read_after_write(core_exported, mode, cls)
io_from_exported_object(front_vol, queue, test_data.size, Size.from_sector(1).B)
check_stats_read_after_write(core, mode, cls)
check_md5_sums(core_exported, mode)
check_md5_sums(core, mode)
@pytest.mark.parametrize("cls", CacheLineSize)
@@ -180,17 +187,21 @@ def test_stop(pyocf_ctx, mode: CacheMode, cls: CacheLineSize, with_flush: bool):
cache_device = RamVolume(Size.from_MiB(50))
core_device = RamVolume(Size.from_MiB(5))
cache = Cache.start_on_device(cache_device, cache_mode=mode, cache_line_size=cls)
core_exported = Core.using_device(core_device)
cache.add_core(core_exported)
core = Core.using_device(core_device)
cache.add_core(core)
front_vol = CoreVolume(core, open=True)
queue = cache.get_default_queue()
cls_no = 10
run_io_and_cache_data_if_possible(core_exported, mode, cls, cls_no)
run_io_and_cache_data_if_possible(core, mode, cls, cls_no)
stats = cache.get_stats()
assert int(stats["conf"]["dirty"]) == (cls_no if mode.lazy_write() else 0),\
"Dirty data before MD5"
md5_exported_core = core_exported.exp_obj_md5()
md5_exported_core = core.exp_obj_md5()
if with_flush:
cache.flush()
@@ -417,27 +428,29 @@ def test_start_stop_noqueue(pyocf_ctx):
assert not c.results["error"], "Failed to stop cache: {}".format(c.results["error"])
def run_io_and_cache_data_if_possible(exported_obj, mode, cls, cls_no):
def run_io_and_cache_data_if_possible(core, mode, cls, cls_no):
front_vol = core.get_front_volume()
bottom_vol = core.get_volume()
queue = core.cache.get_default_queue()
test_data = Data(cls_no * cls)
if mode in {CacheMode.WI, CacheMode.WA}:
logger.info("[STAGE] Write to core device")
io_to_core(exported_obj, test_data, 0, True)
io_to_core(bottom_vol, queue, test_data, 0)
logger.info("[STAGE] Read from exported object")
io_from_exported_object(exported_obj, test_data.size, 0)
io_from_exported_object(front_vol, queue, test_data.size, 0)
else:
logger.info("[STAGE] Write to exported object")
io_to_core(exported_obj, test_data, 0)
io_to_core(front_vol, queue, test_data, 0)
stats = exported_obj.cache.get_stats()
stats = core.cache.get_stats()
assert stats["usage"]["occupancy"]["value"] == \
((cls_no * cls / CacheLineSize.LINE_4KiB) if mode != CacheMode.PT else 0), "Occupancy"
def io_to_core(exported_obj: Core, data: Data, offset: int, to_core_device=False):
new_io = exported_obj.new_core_io if to_core_device else exported_obj.new_io
io = new_io(exported_obj.cache.get_default_queue(), offset, data.size,
IoDir.WRITE, 0, 0)
def io_to_core(vol: Volume, queue: Queue, data: Data, offset: int):
io = vol.new_io(queue, offset, data.size, IoDir.WRITE, 0, 0)
io.set_data(data)
completion = OcfCompletion([("err", c_int)])
@@ -448,10 +461,9 @@ def io_to_core(exported_obj: Core, data: Data, offset: int, to_core_device=False
assert completion.results["err"] == 0, "IO to exported object completion"
def io_from_exported_object(exported_obj: Core, buffer_size: int, offset: int):
def io_from_exported_object(vol: Volume, queue: Queue, buffer_size: int, offset: int):
read_buffer = Data(buffer_size)
io = exported_obj.new_io(exported_obj.cache.get_default_queue(), offset,
read_buffer.size, IoDir.READ, 0, 0)
io = vol.new_io(queue, offset, read_buffer.size, IoDir.READ, 0, 0)
io.set_data(read_buffer)
completion = OcfCompletion([("err", c_int)])
@@ -463,28 +475,28 @@ def io_from_exported_object(exported_obj: Core, buffer_size: int, offset: int):
return read_buffer
def check_stats_read_empty(exported_obj: Core, mode: CacheMode, cls: CacheLineSize):
exported_obj.cache.settle()
stats = exported_obj.cache.get_stats()
def check_stats_read_empty(core: Core, mode: CacheMode, cls: CacheLineSize):
core.cache.settle()
stats = core.cache.get_stats()
assert stats["conf"]["cache_mode"] == mode, "Cache mode"
assert exported_obj.cache.device.get_stats()[IoDir.WRITE] == (1 if mode.read_insert() else 0), \
assert core.cache.device.get_stats()[IoDir.WRITE] == (1 if mode.read_insert() else 0), \
"Writes to cache device"
assert exported_obj.device.get_stats()[IoDir.READ] == 1, "Reads from core device"
assert core.device.get_stats()[IoDir.READ] == 1, "Reads from core device"
assert stats["req"]["rd_full_misses"]["value"] == (0 if mode == CacheMode.PT else 1), \
"Read full misses"
assert stats["usage"]["occupancy"]["value"] == \
((cls / CacheLineSize.LINE_4KiB) if mode.read_insert() else 0), "Occupancy"
def check_stats_write_empty(exported_obj: Core, mode: CacheMode, cls: CacheLineSize):
exported_obj.cache.settle()
stats = exported_obj.cache.get_stats()
def check_stats_write_empty(core: Core, mode: CacheMode, cls: CacheLineSize):
core.cache.settle()
stats = core.cache.get_stats()
assert stats["conf"]["cache_mode"] == mode, "Cache mode"
# TODO(ajrutkow): why 1 for WT ??
assert exported_obj.cache.device.get_stats()[IoDir.WRITE] == \
assert core.cache.device.get_stats()[IoDir.WRITE] == \
(2 if mode.lazy_write() else (1 if mode == CacheMode.WT else 0)), \
"Writes to cache device"
assert exported_obj.device.get_stats()[IoDir.WRITE] == (0 if mode.lazy_write() else 1), \
assert core.device.get_stats()[IoDir.WRITE] == (0 if mode.lazy_write() else 1), \
"Writes to core device"
assert stats["req"]["wr_full_misses"]["value"] == (1 if mode.write_insert() else 0), \
"Write full misses"
@@ -493,17 +505,17 @@ def check_stats_write_empty(exported_obj: Core, mode: CacheMode, cls: CacheLineS
"Occupancy"
def check_stats_write_after_read(exported_obj: Core,
def check_stats_write_after_read(core: Core,
mode: CacheMode,
cls: CacheLineSize,
read_from_empty=False):
exported_obj.cache.settle()
stats = exported_obj.cache.get_stats()
assert exported_obj.cache.device.get_stats()[IoDir.WRITE] == \
core.cache.settle()
stats = core.cache.get_stats()
assert core.cache.device.get_stats()[IoDir.WRITE] == \
(0 if mode in {CacheMode.WI, CacheMode.PT} else
(2 if read_from_empty and mode.lazy_write() else 1)), \
"Writes to cache device"
assert exported_obj.device.get_stats()[IoDir.WRITE] == (0 if mode.lazy_write() else 1), \
assert core.device.get_stats()[IoDir.WRITE] == (0 if mode.lazy_write() else 1), \
"Writes to core device"
assert stats["req"]["wr_hits"]["value"] == \
(1 if (mode.read_insert() and mode != CacheMode.WI)
@@ -514,17 +526,17 @@ def check_stats_write_after_read(exported_obj: Core,
"Occupancy"
def check_stats_read_after_write(exported_obj, mode, cls, write_to_empty=False):
exported_obj.cache.settle()
stats = exported_obj.cache.get_stats()
assert exported_obj.cache.device.get_stats()[IoDir.WRITE] == \
def check_stats_read_after_write(core, mode, cls, write_to_empty=False):
core.cache.settle()
stats = core.cache.get_stats()
assert core.cache.device.get_stats()[IoDir.WRITE] == \
(2 if mode.lazy_write() else (0 if mode == CacheMode.PT else 1)), \
"Writes to cache device"
assert exported_obj.cache.device.get_stats()[IoDir.READ] == \
assert core.cache.device.get_stats()[IoDir.READ] == \
(1 if mode in {CacheMode.WT, CacheMode.WB, CacheMode.WO}
or (mode == CacheMode.WA and not write_to_empty) else 0), \
"Reads from cache device"
assert exported_obj.device.get_stats()[IoDir.READ] == \
assert core.device.get_stats()[IoDir.READ] == \
(0 if mode in {CacheMode.WB, CacheMode.WO, CacheMode.WT}
or (mode == CacheMode.WA and not write_to_empty) else 1), \
"Reads from core device"
@@ -540,15 +552,15 @@ def check_stats_read_after_write(exported_obj, mode, cls, write_to_empty=False):
(0 if mode == CacheMode.PT else (cls / CacheLineSize.LINE_4KiB)), "Occupancy"
def check_md5_sums(exported_obj: Core, mode: CacheMode):
def check_md5_sums(core: Core, mode: CacheMode):
if mode.lazy_write():
assert exported_obj.device.md5() != exported_obj.exp_obj_md5(), \
assert core.device.md5() != core.exp_obj_md5(), \
"MD5 check: core device vs exported object without flush"
exported_obj.cache.flush()
assert exported_obj.device.md5() == exported_obj.exp_obj_md5(), \
core.cache.flush()
assert core.device.md5() == core.exp_obj_md5(), \
"MD5 check: core device vs exported object after flush"
else:
assert exported_obj.device.md5() == exported_obj.exp_obj_md5(), \
assert core.device.md5() == core.exp_obj_md5(), \
"MD5 check: core device vs exported object"