diff --git a/example/simple/src/main.c b/example/simple/src/main.c index cd503e7..1a0b07f 100644 --- a/example/simple/src/main.c +++ b/example/simple/src/main.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2019-2021 Intel Corporation + * Copyright(c) 2019-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ @@ -260,11 +260,12 @@ int submit_io(ocf_core_t core, struct volume_data *data, uint64_t addr, uint64_t len, int dir, ocf_end_io_t cmpl) { ocf_cache_t cache = ocf_core_get_cache(core); + ocf_volume_t core_vol = ocf_core_get_front_volume(core); struct cache_priv *cache_priv = ocf_cache_get_priv(cache); struct ocf_io *io; /* Allocate new io */ - io = ocf_core_new_io(core, cache_priv->io_queue, addr, len, dir, 0, 0); + io = ocf_volume_new_io(core_vol, cache_priv->io_queue, addr, len, dir, 0, 0); if (!io) return -ENOMEM; diff --git a/inc/ocf_core.h b/inc/ocf_core.h index 04fff9b..e372369 100644 --- a/inc/ocf_core.h +++ b/inc/ocf_core.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ @@ -146,29 +146,6 @@ const char *ocf_core_get_name(ocf_core_t core); */ ocf_core_state_t ocf_core_get_state(ocf_core_t core); -/** - * @brief Allocate new ocf_io - * - * @param[in] core Core object - * @param[in] queue IO queue handle - * @param[in] addr OCF IO destination address - * @param[in] bytes OCF IO size in bytes - * @param[in] dir OCF IO direction - * @param[in] io_class OCF IO destination class - * @param[in] flags OCF IO flags - * - * @retval ocf_io object - */ -static inline struct ocf_io *ocf_core_new_io(ocf_core_t core, ocf_queue_t queue, - uint64_t addr, uint32_t bytes, uint32_t dir, - uint32_t io_class, uint64_t flags) -{ - ocf_volume_t volume = ocf_core_get_front_volume(core); - - return ocf_volume_new_io(volume, queue, addr, bytes, dir, - io_class, flags); -} - /** * @brief Submit ocf_io * diff --git a/tests/functional/pyocf/rio.py b/tests/functional/pyocf/rio.py index 6231b42..1462c16 100644 --- a/tests/functional/pyocf/rio.py +++ b/tests/functional/pyocf/rio.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2021 Intel Corporation +# Copyright(c) 2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # diff --git a/tests/functional/pyocf/types/cache.py b/tests/functional/pyocf/types/cache.py index f8709a6..0619ddb 100644 --- a/tests/functional/pyocf/types/cache.py +++ b/tests/functional/pyocf/types/cache.py @@ -35,11 +35,11 @@ from ..utils import Size, struct_to_dict from .core import Core from .queue import Queue from .stats.cache import CacheInfo +from .io import IoDir from .ioclass import IoClassesInfo, IoClassInfo from .stats.shared import UsageStats, RequestsStats, BlocksStats, ErrorsStats from .ctx import OcfCtx -from .volume import Volume - +from .volume import RamVolume class Backfill(Structure): _fields_ = [("_max_queue_size", c_uint32), ("_queue_unblock_size", c_uint32)] @@ -697,7 +697,7 @@ class Cache: raise OcfError("Failed getting core by name", result) uuid = self.owner.lib.ocf_core_get_uuid_wrapper(core_handle) - device = Volume.get_by_uuid(uuid.contents._data.decode("ascii")) + device = RamVolume.get_by_uuid(uuid.contents._data.decode("ascii")) core = Core(device) core.cache = self core.handle = core_handle @@ -749,6 +749,12 @@ class Cache: self.cores.remove(core) + def get_front_volume(self): + return Volume.get_instance(lib.ocf_cache_get_front_volume(self.cache_handle)) + + def get_volume(self): + return Volume.get_instance(lib.ocf_cache_get_volume(self.cache_handle)) + def get_stats(self): cache_info = CacheInfo() usage = UsageStats() @@ -897,6 +903,10 @@ lib.ocf_mngt_cache_remove_core.argtypes = [c_void_p, c_void_p, c_void_p] lib.ocf_mngt_cache_add_core.argtypes = [c_void_p, c_void_p, c_void_p, c_void_p] lib.ocf_cache_get_name.argtypes = [c_void_p] lib.ocf_cache_get_name.restype = c_char_p +lib.ocf_cache_get_front_volume.argtypes = [c_void_p] +lib.ocf_cache_get_front_volume.restype = c_void_p +lib.ocf_cache_get_volume.argtypes = [c_void_p] +lib.ocf_cache_get_volume.restype = c_void_p lib.ocf_mngt_cache_cleaning_set_policy.argtypes = [ c_void_p, c_uint32, diff --git a/tests/functional/pyocf/types/core.py b/tests/functional/pyocf/types/core.py index 201706a..9ee0bdf 100644 --- a/tests/functional/pyocf/types/core.py +++ b/tests/functional/pyocf/types/core.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2019-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -99,30 +99,14 @@ class Core: def get_handle(self): return self.handle - def new_io( - self, queue: Queue, addr: int, length: int, direction: IoDir, - io_class: int, flags: int - ): - if not self.cache: - raise Exception("Core isn't attached to any cache") + def get_front_volume(self): + return Volume.get_instance(lib.ocf_core_get_front_volume(self.handle)) - io = OcfLib.getInstance().ocf_core_new_io_wrapper( - self.handle, queue.handle, addr, length, direction, io_class, flags) + def get_volume(self): + return Volume.get_instance(lib.ocf_core_get_volume(self.handle)) - if io is None: - raise Exception("Failed to create io!") - - return Io.from_pointer(io) - - def new_core_io( - self, queue: Queue, addr: int, length: int, direction: IoDir, - io_class: int, flags: int - ): - lib = OcfLib.getInstance() - volume = lib.ocf_core_get_volume(self.handle) - io = lib.ocf_volume_new_io( - volume, queue.handle, addr, length, direction, io_class, flags) - return Io.from_pointer(io) + def get_default_queue(self): + return self.cache.get_default_queue() def get_stats(self): core_info = CoreInfo() @@ -191,52 +175,13 @@ class Core: def reset_stats(self): self.cache.owner.lib.ocf_core_stats_initialize(self.handle) - def exp_obj_md5(self): - logging.getLogger("pyocf").warning( - "Reading whole exported object! This disturbs statistics values" - ) - - cache_line_size = int(self.cache.get_stats()['conf']['cache_line_size']) - read_buffer_all = Data(self.device.size) - - read_buffer = Data(cache_line_size) - - position = 0 - while position < read_buffer_all.size: - io = self.new_io(self.cache.get_default_queue(), position, - cache_line_size, IoDir.READ, 0, 0) - io.set_data(read_buffer) - - cmpl = OcfCompletion([("err", c_int)]) - io.callback = cmpl.callback - io.submit() - cmpl.wait() - - if cmpl.results["err"]: - raise Exception("Error reading whole exported object") - - read_buffer_all.copy(read_buffer, position, 0, cache_line_size) - position += cache_line_size - - return read_buffer_all.md5() - - lib = OcfLib.getInstance() lib.ocf_core_get_uuid_wrapper.restype = POINTER(Uuid) lib.ocf_core_get_uuid_wrapper.argtypes = [c_void_p] -lib.ocf_core_get_volume.restype = c_void_p -lib.ocf_volume_new_io.argtypes = [ - c_void_p, - c_void_p, - c_uint64, - c_uint32, - c_uint32, - c_uint32, - c_uint64, -] -lib.ocf_volume_new_io.restype = c_void_p lib.ocf_core_get_volume.argtypes = [c_void_p] lib.ocf_core_get_volume.restype = c_void_p +lib.ocf_core_get_front_volume.argtypes = [c_void_p] +lib.ocf_core_get_front_volume.restype = c_void_p lib.ocf_mngt_core_set_seq_cutoff_policy.argtypes = [c_void_p, c_uint32] lib.ocf_mngt_core_set_seq_cutoff_policy.restype = c_int lib.ocf_mngt_core_set_seq_cutoff_threshold.argtypes = [c_void_p, c_uint32] @@ -247,13 +192,3 @@ lib.ocf_stats_collect_core.argtypes = [c_void_p, c_void_p, c_void_p, c_void_p, c lib.ocf_stats_collect_core.restype = c_int lib.ocf_core_get_info.argtypes = [c_void_p, c_void_p] lib.ocf_core_get_info.restype = c_int -lib.ocf_core_new_io_wrapper.argtypes = [ - c_void_p, - c_void_p, - c_uint64, - c_uint32, - c_uint32, - c_uint32, - c_uint64, -] -lib.ocf_core_new_io_wrapper.restype = c_void_p diff --git a/tests/functional/pyocf/types/ctx.py b/tests/functional/pyocf/types/ctx.py index 030adbf..d26ec35 100644 --- a/tests/functional/pyocf/types/ctx.py +++ b/tests/functional/pyocf/types/ctx.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2019-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -12,8 +12,6 @@ from .cleaner import CleanerOps, Cleaner from .shared import OcfError from ..ocf import OcfLib from .queue import Queue -from .volume import Volume - class OcfCtxOps(Structure): _fields_ = [ diff --git a/tests/functional/pyocf/types/io.py b/tests/functional/pyocf/types/io.py index 467e3a8..e314a80 100644 --- a/tests/functional/pyocf/types/io.py +++ b/tests/functional/pyocf/types/io.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2019-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -96,7 +96,13 @@ class Io(Structure): self.del_object() def submit(self): - return OcfLib.getInstance().ocf_core_submit_io_wrapper(byref(self)) + return OcfLib.getInstance().ocf_volume_submit_io(byref(self)) + + def submit_flush(self): + return OcfLib.getInstance().ocf_volume_submit_flush(byref(self)) + + def submit_discard(self): + return OcfLib.getInstance().ocf_volume_submit_discard(byref(self)) def set_data(self, data: Data, offset: int = 0): self.data = data @@ -111,8 +117,14 @@ IoOps._fields_ = [("_set_data", IoOps.SET_DATA), ("_get_data", IoOps.GET_DATA)] lib = OcfLib.getInstance() lib.ocf_io_set_cmpl_wrapper.argtypes = [POINTER(Io), c_void_p, c_void_p, Io.END] -lib.ocf_core_new_io_wrapper.argtypes = [c_void_p] -lib.ocf_core_new_io_wrapper.restype = c_void_p - lib.ocf_io_set_data.argtypes = [POINTER(Io), c_void_p, c_uint32] lib.ocf_io_set_data.restype = c_int + +lib.ocf_volume_submit_io.argtypes = [POINTER(Io)] +lib.ocf_volume_submit_io.restype = None + +lib.ocf_volume_submit_flush.argtypes = [POINTER(Io)] +lib.ocf_volume_submit_flush.restype = None + +lib.ocf_volume_submit_discard.argtypes = [POINTER(Io)] +lib.ocf_volume_submit_discard.restype = None diff --git a/tests/functional/pyocf/types/volume.py b/tests/functional/pyocf/types/volume.py index 4fbac92..db344bb 100644 --- a/tests/functional/pyocf/types/volume.py +++ b/tests/functional/pyocf/types/volume.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2019-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -24,10 +24,12 @@ from hashlib import md5 import weakref from .io import Io, IoOps, IoDir +from .queue import Queue from .shared import OcfErrorCode, Uuid from ..ocf import OcfLib from ..utils import print_buffer, Size as S from .data import Data +from .queue import Queue class VolumeCaps(Structure): @@ -66,75 +68,137 @@ class VolumeProperties(Structure): ("_caps", VolumeCaps), ("_io_ops", IoOps), ("_deinit", c_char_p), - ("_ops", VolumeOps), + ("_ops_", VolumeOps), ] - class VolumeIoPriv(Structure): _fields_ = [("_data", c_void_p), ("_offset", c_uint64)] -class Volume(Structure): - VOLUME_POISON = 0x13 +VOLUME_POISON = 0x13 - _fields_ = [("_storage", c_void_p)] + +class Volume: _instances_ = weakref.WeakValueDictionary() _uuid_ = weakref.WeakValueDictionary() + _ops_ = {} + _props_ = {} - props = None + @classmethod + def get_ops(cls): + if cls in Volume._ops_: + return Volume._ops_[cls] - def __init__(self, size: S, uuid=None): - super().__init__() - self.size = size - if uuid: - if uuid in type(self)._uuid_: - raise Exception( - "Volume with uuid {} already created".format(uuid) - ) - self.uuid = uuid - else: - self.uuid = str(id(self)) + @VolumeOps.SUBMIT_IO + def _submit_io(io): + io_structure = cast(io, POINTER(Io)) + volume = Volume.get_instance( + OcfLib.getInstance().ocf_io_get_volume(io_structure) + ) - type(self)._uuid_[self.uuid] = self + volume.submit_io(io_structure) - self.data = create_string_buffer(int(self.size)) - memset(self.data, self.VOLUME_POISON, self.size) - self._storage = cast(self.data, c_void_p) + @VolumeOps.SUBMIT_FLUSH + def _submit_flush(flush): + io_structure = cast(flush, POINTER(Io)) + volume = Volume.get_instance( + OcfLib.getInstance().ocf_io_get_volume(io_structure) + ) - self.reset_stats() - self.opened = False + volume.submit_flush(io_structure) - def get_copy(self): - new_volume = Volume(self.size) - memmove(new_volume.data, self.data, self.size) - return new_volume + @VolumeOps.SUBMIT_METADATA + def _submit_metadata(meta): + raise NotImplementedError + + @VolumeOps.SUBMIT_DISCARD + def _submit_discard(discard): + io_structure = cast(discard, POINTER(Io)) + volume = Volume.get_instance( + OcfLib.getInstance().ocf_io_get_volume(io_structure) + ) + + volume.submit_discard(io_structure) + + @VolumeOps.SUBMIT_WRITE_ZEROES + def _submit_write_zeroes(write_zeroes): + raise NotImplementedError + + @VolumeOps.OPEN + def _open(ref): + uuid_ptr = cast( + OcfLib.getInstance().ocf_volume_get_uuid(ref), POINTER(Uuid) + ) + uuid = str(uuid_ptr.contents._data, encoding="ascii") + try: + volume = Volume.get_by_uuid(uuid) + except: # noqa E722 TODO:Investigate whether this really should be so broad + print("Tried to access unallocated volume {}".format(uuid)) + print("{}".format(Volume._uuid_)) + return -1 + + return Volume.open(ref, volume) + + @VolumeOps.CLOSE + def _close(ref): + volume = Volume.get_instance(ref) + volume.close() + volume.opened = False + + @VolumeOps.GET_MAX_IO_SIZE + def _get_max_io_size(ref): + return Volume.get_instance(ref).get_max_io_size() + + @VolumeOps.GET_LENGTH + def _get_length(ref): + return Volume.get_instance(ref).get_length() + + Volume._ops_[cls] = VolumeOps( + _submit_io=_submit_io, + _submit_flush=_submit_flush, + _submit_metadata=_submit_metadata, + _submit_discard=_submit_discard, + _submit_write_zeroes=_submit_write_zeroes, + _open=_open, + _close=_close, + _get_max_io_size=_get_max_io_size, + _get_length=_get_length, + ) + + return Volume._ops_[cls] + + @staticmethod + def open(ref, volume): + if volume.opened: + return -OcfErrorCode.OCF_ERR_NOT_OPEN_EXC + + Volume._instances_[ref] = volume + volume.handle = ref + + return volume.do_open() + + @classmethod + def get_io_ops(cls): + return IoOps(_set_data=cls._io_set_data, _get_data=cls._io_get_data) @classmethod def get_props(cls): - if not cls.props: - cls.props = VolumeProperties( - _name=str(cls.__name__).encode("ascii"), - _io_priv_size=sizeof(VolumeIoPriv), - _volume_priv_size=0, - _caps=VolumeCaps(_atomic_writes=0), - _ops=VolumeOps( - _submit_io=cls._submit_io, - _submit_flush=cls._submit_flush, - _submit_metadata=cls._submit_metadata, - _submit_discard=cls._submit_discard, - _submit_write_zeroes=cls._submit_write_zeroes, - _open=cls._open, - _close=cls._close, - _get_max_io_size=cls._get_max_io_size, - _get_length=cls._get_length, - ), - _io_ops=IoOps( - _set_data=cls._io_set_data, _get_data=cls._io_get_data - ), - _deinit=0, - ) + if cls in Volume._props_: + return Volume._props_[cls] - return cls.props + Volume._props_[cls] = VolumeProperties( + _name=str(cls.__name__).encode("ascii"), + _io_priv_size=sizeof(VolumeIoPriv), + _volume_priv_size=0, + _caps=VolumeCaps(_atomic_writes=0), + _ops_=cls.get_ops(), + _io_ops=cls.get_io_ops(), + _deinit=0, + ) + return Volume._props_[cls] + + def get_copy(self): + raise NotImplementedError @classmethod def get_instance(cls, ref): @@ -148,84 +212,6 @@ class Volume(Structure): def get_by_uuid(cls, uuid): return cls._uuid_[uuid] - @staticmethod - @VolumeOps.SUBMIT_IO - def _submit_io(io): - io_structure = cast(io, POINTER(Io)) - volume = Volume.get_instance( - OcfLib.getInstance().ocf_io_get_volume(io_structure) - ) - - volume.submit_io(io_structure) - - @staticmethod - @VolumeOps.SUBMIT_FLUSH - def _submit_flush(flush): - io_structure = cast(flush, POINTER(Io)) - volume = Volume.get_instance( - OcfLib.getInstance().ocf_io_get_volume(io_structure) - ) - - volume.submit_flush(io_structure) - - @staticmethod - @VolumeOps.SUBMIT_METADATA - def _submit_metadata(meta): - pass - - @staticmethod - @VolumeOps.SUBMIT_DISCARD - def _submit_discard(discard): - io_structure = cast(discard, POINTER(Io)) - volume = Volume.get_instance( - OcfLib.getInstance().ocf_io_get_volume(io_structure) - ) - - volume.submit_discard(io_structure) - - @staticmethod - @VolumeOps.SUBMIT_WRITE_ZEROES - def _submit_write_zeroes(write_zeroes): - pass - - @staticmethod - @CFUNCTYPE(c_int, c_void_p) - def _open(ref): - uuid_ptr = cast( - OcfLib.getInstance().ocf_volume_get_uuid(ref), POINTER(Uuid) - ) - uuid = str(uuid_ptr.contents._data, encoding="ascii") - try: - volume = Volume.get_by_uuid(uuid) - except: # noqa E722 TODO:Investigate whether this really should be so broad - print("Tried to access unallocated volume {}".format(uuid)) - print("{}".format(Volume._uuid_)) - return -1 - - if volume.opened: - return -OcfErrorCode.OCF_ERR_NOT_OPEN_EXC - - Volume._instances_[ref] = volume - - return volume.open() - - @staticmethod - @VolumeOps.CLOSE - def _close(ref): - volume = Volume.get_instance(ref) - volume.close() - volume.opened = False - - @staticmethod - @VolumeOps.GET_MAX_IO_SIZE - def _get_max_io_size(ref): - return Volume.get_instance(ref).get_max_io_size() - - @staticmethod - @VolumeOps.GET_LENGTH - def _get_length(ref): - return Volume.get_instance(ref).get_length() - @staticmethod @IoOps.SET_DATA def _io_set_data(io, data, offset): @@ -246,36 +232,40 @@ class Volume(Structure): ) return io_priv.contents._data - def open(self): + def __init__(self, uuid=None): + if uuid: + if uuid in type(self)._uuid_: + raise Exception( + "Volume with uuid {} already created".format(uuid) + ) + self.uuid = uuid + else: + self.uuid = str(id(self)) + + type(self)._uuid_[self.uuid] = self + + self.reset_stats() + self.is_online = True + self.opened = False + + def do_open(self): self.opened = True return 0 def close(self): - pass + self.opened = False def get_length(self): - return self.size - - def resize(self, size): - self.size = size - self.data = create_string_buffer(int(self.size)) - memset(self.data, self.VOLUME_POISON, self.size) - self._storage = cast(self.data, c_void_p) + raise NotImplementedError def get_max_io_size(self): - return S.from_KiB(128) + raise NotImplementedError - def submit_flush(self, flush): - flush.contents._end(flush, 0) + def do_submit_flush(self, flush): + raise NotImplementedError - def submit_discard(self, discard): - try: - dst = self._storage + discard.contents._addr - memset(dst, 0, discard.contents._bytes) - - discard.contents._end(discard, 0) - except: # noqa E722 - discard.contents._end(discard, -OcfErrorCode.OCF_ERR_NOT_SUPP) + def do_submit_discard(self, discard): + raise NotImplementedError def get_stats(self): return self.stats @@ -283,10 +273,103 @@ class Volume(Structure): def reset_stats(self): self.stats = {IoDir.WRITE: 0, IoDir.READ: 0} - def submit_io(self, io): - try: - self.stats[IoDir(io.contents._dir)] += 1 + def inc_stats(self, _dir): + self.stats[_dir] += 1 + def do_submit_io(self, io): + raise NotImplementedError + + def dump(self, offset=0, size=0, ignore=VOLUME_POISON, **kwargs): + raise NotImplementedError + + def md5(self): + raise NotImplementedError + + def offline(self): + self.is_online = False + + def online(self): + self.is_online = True + + def _reject_io(self, io): + cast(io, POINTER(Io)).contents._end(io, -OcfErrorCode.OCF_ERR_IO) + + def submit_flush(self, io): + if self.is_online: + self.do_submit_flush(io) + else: + self._reject_io(io) + + def submit_io(self, io): + if self.is_online: + self.inc_stats(IoDir(io.contents._dir)) + self.do_submit_io(io) + else: + self._reject_io(io) + + def submit_discard(self, io): + if self.is_online: + self.do_submit_discard(io) + else: + self._reject_io(io) + + def new_io( + self, + queue: Queue, + addr: int, + length: int, + direction: IoDir, + io_class: int, + flags: int, + ): + lib = OcfLib.getInstance() + io = lib.ocf_volume_new_io( + self.handle, queue.handle, addr, length, direction, io_class, flags + ) + return Io.from_pointer(io) + + +class RamVolume(Volume): + props = None + + def __init__(self, size: S, uuid=None): + super().__init__(uuid) + self.size = size + self.data = create_string_buffer(int(self.size)) + memset(self.data, VOLUME_POISON, self.size) + self.data_ptr = cast(self.data, c_void_p).value + + def get_copy(self): + new_volume = RamVolume(self.size) + memmove(new_volume.data, self.data, self.size) + return new_volume + + def get_length(self): + return self.size + + def resize(self, size): + self.size = size + self.data = create_string_buffer(int(self.size)) + memset(self.data, VOLUME_POISON, self.size) + self.data_ptr = cast(self.data, c_void_p).value + + def get_max_io_size(self): + return S.from_KiB(128) + + def do_submit_flush(self, flush): + flush.contents._end(flush, 0) + + def do_submit_discard(self, discard): + try: + dst = self.data_ptr + discard.contents._addr + memset(dst, 0, discard.contents._bytes) + + discard.contents._end(discard, 0) + except: # noqa E722 + discard.contents._end(discard, -OcfErrorCode.OCF_ERR_NOT_SUPP) + + def do_submit_io(self, io): + try: io_priv = cast( OcfLib.getInstance().ocf_io_get_priv(io), POINTER(VolumeIoPriv)) offset = io_priv.contents._offset @@ -294,11 +377,11 @@ class Volume(Structure): if io.contents._dir == IoDir.WRITE: src_ptr = cast(OcfLib.getInstance().ocf_io_get_data(io), c_void_p) src = Data.get_instance(src_ptr.value).handle.value + offset - dst = self._storage + io.contents._addr + dst = self.data_ptr + io.contents._addr elif io.contents._dir == IoDir.READ: dst_ptr = cast(OcfLib.getInstance().ocf_io_get_data(io), c_void_p) dst = Data.get_instance(dst_ptr.value).handle.value + offset - src = self._storage + io.contents._addr + src = self.data_ptr + io.contents._addr memmove(dst, src, io.contents._bytes) io_priv.contents._offset += io.contents._bytes @@ -311,18 +394,18 @@ class Volume(Structure): if size == 0: size = int(self.size) - int(offset) - print_buffer(self._storage, size, ignore=ignore, **kwargs) + print_buffer(self.data_ptr, size, ignore=ignore, **kwargs) def md5(self): m = md5() - m.update(string_at(self._storage, self.size)) + m.update(string_at(self.data_ptr, self.size)) return m.hexdigest() def get_bytes(self): - return string_at(self._storage, self.size) + return string_at(self.data_ptr, self.size) -class ErrorDevice(Volume): +class ErrorDevice(RamVolume): def __init__( self, size, @@ -341,9 +424,9 @@ class ErrorDevice(Volume): def set_mapping(self, error_sectors: set): self.error_sectors = error_sectors - def submit_io(self, io): + def do_submit_io(self, io): if not self.armed: - super().submit_io(io) + super().do_submit_io(io) return direction = IoDir(io.contents._dir) @@ -368,7 +451,7 @@ class ErrorDevice(Volume): io.contents._end(io, -OcfErrorCode.OCF_ERR_IO) self.stats["errors"][direction] += 1 else: - super().submit_io(io) + super().do_submit_io(io) def arm(self): self.armed = True @@ -384,24 +467,19 @@ class ErrorDevice(Volume): self.stats["errors"] = {IoDir.WRITE: 0, IoDir.READ: 0} -class TraceDevice(Volume): - def __init__(self, size, trace_fcn=None, uuid=None): - super().__init__(size, uuid) - self.trace_fcn = trace_fcn - - def submit_io(self, io): - submit = True - - if self.trace_fcn: - submit = self.trace_fcn(self, io) - - if submit: - super().submit_io(io) - - lib = OcfLib.getInstance() lib.ocf_io_get_priv.restype = POINTER(VolumeIoPriv) lib.ocf_io_get_volume.argtypes = [c_void_p] lib.ocf_io_get_volume.restype = c_void_p lib.ocf_io_get_data.argtypes = [c_void_p] lib.ocf_io_get_data.restype = c_void_p +lib.ocf_volume_new_io.argtypes = [ + c_void_p, + c_void_p, + c_uint64, + c_uint32, + c_uint32, + c_uint32, + c_uint64, +] +lib.ocf_volume_new_io.restype = c_void_p diff --git a/tests/functional/pyocf/types/volume_cache.py b/tests/functional/pyocf/types/volume_cache.py new file mode 100644 index 0000000..d44e683 --- /dev/null +++ b/tests/functional/pyocf/types/volume_cache.py @@ -0,0 +1,31 @@ +# +# Copyright(c) 2022 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause +# + +from ctypes import cast, POINTER + +from .cache import Cache +from .io import Io +from .io import IoDir +from .volume_exp_obj import ExpObjVolume +from .volume import Volume + + +class CacheVolume(ExpObjVolume): + def __init__(self, cache, open=False, uuid=None): + super().__init__(cache, uuid) + self.cache = cache + self.lib = cache.owner.lib + if open: + self.open() + + def open(self): + return Volume.open( + self.lib.ocf_cache_get_front_volume(self.cache.handle), + self + ) + + def md5(self): + cache_line_size = int(self.cache.get_stats()['conf']['cache_line_size']) + return self._exp_obj_md5(cache_line_size) diff --git a/tests/functional/pyocf/types/volume_core.py b/tests/functional/pyocf/types/volume_core.py new file mode 100644 index 0000000..af0badd --- /dev/null +++ b/tests/functional/pyocf/types/volume_core.py @@ -0,0 +1,27 @@ +# +# Copyright(c) 2022 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause +# + +from .core import Core +from .volume_exp_obj import ExpObjVolume +from .io import IoDir +from .volume import Volume + + +class CoreVolume(ExpObjVolume): + def __init__(self, core, open=False, uuid=None): + super().__init__(core, uuid) + self.core = core + self.lib = core.cache.owner.lib + if open: + self.open() + + def open(self): + return Volume.open( + self.lib.ocf_core_get_front_volume(self.core.handle), + self + ) + + def md5(self): + return self._exp_obj_md5(4096) diff --git a/tests/functional/pyocf/types/volume_exp_obj.py b/tests/functional/pyocf/types/volume_exp_obj.py new file mode 100644 index 0000000..20771c5 --- /dev/null +++ b/tests/functional/pyocf/types/volume_exp_obj.py @@ -0,0 +1,126 @@ +# +# Copyright(c) 2022 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause +# + +import logging +from ctypes import c_int, c_void_p, CFUNCTYPE, byref, c_uint32, c_uint64, cast, POINTER + +from ..ocf import OcfLib +from .volume import Volume, VOLUME_POISON +from pyocf.utils import Size +from pyocf.types.data import Data +from pyocf.types.io import IoDir, Io +from pyocf.types.shared import OcfCompletion + + +class ExpObjVolume(Volume): + def __init__(self, parent, uuid=None): + super().__init__(uuid) + self.parent = parent + + def __alloc_io(self, addr, _bytes, _dir, _class, _flags): + vol = self.parent.get_front_volume() + queue = self.parent.get_default_queue() # TODO multiple queues? + return vol.new_io( + queue, addr, _bytes, _dir, _class, _flags + ) + + def _alloc_io(self, io): + exp_obj_io = self.__alloc_io( + io.contents._addr, + io.contents._bytes, + io.contents._dir, + io.contents._class, + io.contents._flags, + ) + + lib = OcfLib.getInstance() + cdata = OcfLib.getInstance().ocf_io_get_data(io) + OcfLib.getInstance().ocf_io_set_data(byref(exp_obj_io), cdata, 0) + + def cb(error): + nonlocal io + io = cast(io, POINTER(Io)) + io.contents._end(io, error) + + exp_obj_io.callback = cb + + return exp_obj_io + + def get_length(self): + return Size.from_B(OcfLib.getInstance().ocf_volume_get_length(self.c_vol)) + + def get_max_io_size(self): + return Size.from_B(OcfLib.getInstance().ocf_volume_get_max_io_size(self.c_vol)) + + def do_submit_io(self, io): + io = self._alloc_io(io) + io.submit() + + def do_submit_flush(self, flush): + io = self._alloc_io(flush) + io.submit_flush() + + def do_submit_discard(self, discard): + io = self._alloc_io(discard) + io.submit_discard() + + def _read(self, offset=0, size=0): + if size == 0: + size = self.get_length().B - offset + exp_obj_io = self.__alloc_io(offset, size, IoDir.READ, 0, 0) + completion = OcfCompletion([("err", c_int)]) + exp_obj_io.callback = completion + data = Data.from_bytes(bytes(size)) + exp_obj_io.set_data(data) + exp_obj_io.submit() + completion.wait() + error = completion.results["err"] + if error: + raise Exception("error reading exported object for dump") + return data + + def dump(self, offset=0, size=0, ignore=VOLUME_POISON, **kwargs): + data = self._read(offset, size) + data.dump(ignore=ifnore, **kwargs) + + def md5(self): + raise NotImplementedError + + def _exp_obj_md5(self, read_size): + logging.getLogger("pyocf").warning( + "Reading whole exported object! This disturbs statistics values" + ) + + read_buffer_all = Data(self.parent.device.size) + + read_buffer = Data(read_size) + + position = 0 + while position < read_buffer_all.size: + io = self.new_io(self.parent.get_default_queue(), position, + read_size, IoDir.READ, 0, 0) + io.set_data(read_buffer) + + cmpl = OcfCompletion([("err", c_int)]) + io.callback = cmpl.callback + io.submit() + cmpl.wait() + + if cmpl.results["err"]: + raise Exception("Error reading whole exported object") + + read_buffer_all.copy(read_buffer, position, 0, read_size) + position += read_size + + return read_buffer_all.md5() + + +lib = OcfLib.getInstance() +lib.ocf_volume_get_max_io_size.argtypes = [c_void_p] +lib.ocf_volume_get_max_io_size.restype = c_uint32 +lib.ocf_volume_get_length.argtypes = [c_void_p] +lib.ocf_volume_get_length.restype = c_uint64 +lib.ocf_io_get_data.argtypes = [POINTER(Io)] +lib.ocf_io_get_data.restype = c_void_p diff --git a/tests/functional/pyocf/types/volume_replicated.py b/tests/functional/pyocf/types/volume_replicated.py new file mode 100644 index 0000000..cc0959c --- /dev/null +++ b/tests/functional/pyocf/types/volume_replicated.py @@ -0,0 +1,92 @@ +# +# Copyright(c) 2022 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause +# + +from threading import Lock +from .volume import Volume, VOLUME_POISON +from .io import Io, IoDir +from ctypes import cast, c_void_p, CFUNCTYPE, c_int, POINTER, memmove, sizeof, pointer + + +class ReplicatedVolume(Volume): + def __init__(self, primary: Volume, secondary: Volume, uuid=None): + super().__init__(uuid) + self.primary = primary + self.secondary = secondary + + if secondary.get_max_io_size() < primary.get_max_io_size(): + raise Exception("secondary volume max io size too small") + if secondary.get_length() < primary.get_length(): + raise Exception("secondary volume size too small") + + def do_open(self): + ret = self.primary.do_open() + if ret: + return ret + ret = self.secondary.do_open() + if ret: + self.primary.close() + return ret + + def close(self): + self.primary.close() + self.secondary.close() + + def get_length(self): + return self.primary.get_length() + + def get_max_io_size(self): + return self.primary.get_max_io_size() + + def _prepare_io(self, io): + original_cb = Io.END() + pointer(original_cb)[0] = io.contents._end + lock = Lock() + error = 0 + io_remaining = 2 + + @CFUNCTYPE(None, c_void_p, c_int) + def cb(io, err): + nonlocal io_remaining + nonlocal error + nonlocal original_cb + nonlocal lock + io = cast(io, POINTER(Io)) + + with lock: + if err: + error = err + io_remaining -= 1 + finished = True if io_remaining == 0 else False + if finished: + io.contents._end = original_cb + original_cb(io, error) + + io.contents._end = cb + + def do_submit_io(self, io): + if io.contents._dir == IoDir.WRITE: + self._prepare_io(io) + self.primary.submit_io(io) + self.secondary.submit_io(io) + else: + # for read just pass through down to primary + # with original completion + self.primary.submit_io(io) + + def do_submit_flush(self, flush): + self._prepare_io(flush) + self.primary.submit_flush(flush) + self.secondary.submit_flush(flush) + + def do_submit_discard(self, discard): + self._prepare_io(discard) + self.primary.submit_discard(discard) + self.secondary.submit_discard(discard) + + def dump(self, offset=0, size=0, ignore=VOLUME_POISON, **kwargs): + self.primary.dump() + + def md5(self): + return self.primary.md5() diff --git a/tests/functional/pyocf/wrappers/ocf_io_wrappers.c b/tests/functional/pyocf/wrappers/ocf_io_wrappers.c index 776de72..87e095d 100644 --- a/tests/functional/pyocf/wrappers/ocf_io_wrappers.c +++ b/tests/functional/pyocf/wrappers/ocf_io_wrappers.c @@ -1,18 +1,11 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ #include "ocf/ocf_io.h" #include "ocf/ocf_core.h" -struct ocf_io *ocf_core_new_io_wrapper(ocf_core_t core, ocf_queue_t queue, - uint64_t addr, uint32_t bytes, uint32_t dir, - uint32_t io_class, uint64_t flags) -{ - return ocf_core_new_io(core, queue, addr, bytes, dir, io_class, flags); -} - void ocf_io_set_cmpl_wrapper(struct ocf_io *io, void *context, void *context2, ocf_end_io_t fn) { @@ -34,3 +27,13 @@ void ocf_core_submit_io_wrapper(struct ocf_io *io) ocf_core_submit_io(io); } + +void ocf_core_submit_flush_wrapper(struct ocf_io *io) +{ + ocf_core_submit_flush(io); +} + +void ocf_core_submit_discard_wrapper(struct ocf_io *io) +{ + ocf_core_submit_discard(io); +} diff --git a/tests/functional/tests/basic/test_pyocf.py b/tests/functional/tests/basic/test_pyocf.py index 7a36b1d..4468078 100644 --- a/tests/functional/tests/basic/test_pyocf.py +++ b/tests/functional/tests/basic/test_pyocf.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2019-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -8,7 +8,8 @@ from ctypes import c_int from pyocf.types.cache import Cache from pyocf.types.core import Core -from pyocf.types.volume import Volume, ErrorDevice +from pyocf.types.volume import RamVolume, ErrorDevice +from pyocf.types.volume_core import CoreVolume from pyocf.types.data import Data from pyocf.types.io import IoDir from pyocf.utils import Size as S @@ -21,25 +22,27 @@ def test_ctx_fixture(pyocf_ctx): def test_simple_wt_write(pyocf_ctx): - cache_device = Volume(S.from_MiB(50)) - core_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) + core_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device) core = Core.using_device(core_device) + queue = cache.get_default_queue() cache.add_core(core) + vol = CoreVolume(core, open=True) cache_device.reset_stats() core_device.reset_stats() - r = Rio().target(core).readwrite(ReadWrite.WRITE).size(S.from_sector(1)).run() + r = Rio().target(vol).readwrite(ReadWrite.WRITE).size(S.from_sector(1)).run([queue]) assert cache_device.get_stats()[IoDir.WRITE] == 1 cache.settle() stats = cache.get_stats() assert stats["req"]["wr_full_misses"]["value"] == 1 assert stats["usage"]["occupancy"]["value"] == 1 - assert core.exp_obj_md5() == core_device.md5() + assert vol.md5() == core_device.md5() cache.stop() @@ -51,14 +54,14 @@ def test_start_corrupted_metadata_lba(pyocf_ctx): def test_load_cache_no_preexisting_data(pyocf_ctx): - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) with pytest.raises(OcfError, match="OCF_ERR_NO_METADATA"): cache = Cache.load_from_device(cache_device) def test_load_cache(pyocf_ctx): - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device) cache.stop() @@ -67,7 +70,7 @@ def test_load_cache(pyocf_ctx): def test_load_cache_recovery(pyocf_ctx): - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device) @@ -80,16 +83,17 @@ def test_load_cache_recovery(pyocf_ctx): @pytest.mark.parametrize("open_cores", [True, False]) def test_load_cache_with_cores(pyocf_ctx, open_cores): - cache_device = Volume(S.from_MiB(40)) - core_device = Volume(S.from_MiB(40)) + cache_device = RamVolume(S.from_MiB(40)) + core_device = RamVolume(S.from_MiB(40)) cache = Cache.start_on_device(cache_device) core = Core.using_device(core_device, name="test_core") cache.add_core(core) + vol = CoreVolume(core, open=True) write_data = Data.from_string("This is test data") - io = core.new_io(cache.get_default_queue(), S.from_sector(3).B, + io = vol.new_io(cache.get_default_queue(), S.from_sector(3).B, write_data.size, IoDir.WRITE, 0, 0) io.set_data(write_data) @@ -106,8 +110,10 @@ def test_load_cache_with_cores(pyocf_ctx, open_cores): else: core = cache.get_core_by_name("test_core") + vol = CoreVolume(core, open=True) + read_data = Data(write_data.size) - io = core.new_io(cache.get_default_queue(), S.from_sector(3).B, + io = vol.new_io(cache.get_default_queue(), S.from_sector(3).B, read_data.size, IoDir.READ, 0, 0) io.set_data(read_data) @@ -117,4 +123,4 @@ def test_load_cache_with_cores(pyocf_ctx, open_cores): cmpl.wait() assert read_data.md5() == write_data.md5() - assert core.exp_obj_md5() == core_device.md5() + assert vol.md5() == core_device.md5() diff --git a/tests/functional/tests/conftest.py b/tests/functional/tests/conftest.py index f0989d9..52fe7ff 100644 --- a/tests/functional/tests/conftest.py +++ b/tests/functional/tests/conftest.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2019-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -10,7 +10,10 @@ import gc sys.path.append(os.path.join(os.path.dirname(__file__), os.path.pardir)) from pyocf.types.logger import LogLevel, DefaultLogger, BufferLogger -from pyocf.types.volume import Volume, ErrorDevice +from pyocf.types.volume import RamVolume, ErrorDevice +from pyocf.types.volume_cache import CacheVolume +from pyocf.types.volume_core import CoreVolume +from pyocf.types.volume_replicated import ReplicatedVolume from pyocf.types.ctx import OcfCtx @@ -21,8 +24,11 @@ def pytest_configure(config): @pytest.fixture() def pyocf_ctx(): c = OcfCtx.with_defaults(DefaultLogger(LogLevel.WARN)) - c.register_volume_type(Volume) + c.register_volume_type(RamVolume) c.register_volume_type(ErrorDevice) + c.register_volume_type(CacheVolume) + c.register_volume_type(CoreVolume) + c.register_volume_type(ReplicatedVolume) yield c c.exit() gc.collect() @@ -32,8 +38,11 @@ def pyocf_ctx(): def pyocf_ctx_log_buffer(): logger = BufferLogger(LogLevel.DEBUG) c = OcfCtx.with_defaults(logger) - c.register_volume_type(Volume) + c.register_volume_type(RamVolume) c.register_volume_type(ErrorDevice) + c.register_volume_type(CacheVolume) + c.register_volume_type(CoreVolume) + c.register_volume_type(ReplicatedVolume) yield logger c.exit() gc.collect() diff --git a/tests/functional/tests/engine/test_io_flags.py b/tests/functional/tests/engine/test_io_flags.py index 8f7d22f..d8efbe4 100644 --- a/tests/functional/tests/engine/test_io_flags.py +++ b/tests/functional/tests/engine/test_io_flags.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2020-2021 Intel Corporation +# Copyright(c) 2020-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -12,7 +12,8 @@ import pytest from pyocf.types.cache import Cache, CacheMode from pyocf.types.core import Core -from pyocf.types.volume import Volume +from pyocf.types.volume import RamVolume +from pyocf.types.volume_core import CoreVolume from pyocf.types.data import Data from pyocf.types.io import IoDir from pyocf.utils import Size @@ -28,8 +29,10 @@ def __io(io, queue, address, size, data, direction): return int(completion.results["err"]) -def _io(new_io, queue, address, size, data, offset, direction, flags): - io = new_io(queue, address, size, direction, 0, flags) +def io_to_exp_obj(core, address, size, data, offset, direction, flags): + vol = core.get_front_volume() + queue = core.cache.get_default_queue() + io = vol.new_io(queue, address, size, direction, 0, flags) if direction == IoDir.READ: _data = Data.from_bytes(bytes(size)) else: @@ -40,20 +43,7 @@ def _io(new_io, queue, address, size, data, offset, direction, flags): return ret -def io_to_exp_obj(core, address, size, data, offset, direction, flags): - return _io( - core.new_io, - core.cache.get_default_queue(), - address, - size, - data, - offset, - direction, - flags, - ) - - -class FlagsValVolume(Volume): +class FlagsValVolume(RamVolume): def __init__(self, size, flags): self.flags = flags self.check = False @@ -91,6 +81,7 @@ def test_io_flags(pyocf_ctx, cache_mode): core = Core.using_device(core_device) cache.add_core(core) + vol = CoreVolume(core, open=True) cache_device.set_check(True) core_device.set_check(True) diff --git a/tests/functional/tests/engine/test_pp.py b/tests/functional/tests/engine/test_pp.py index 5fab028..bd827cd 100644 --- a/tests/functional/tests/engine/test_pp.py +++ b/tests/functional/tests/engine/test_pp.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2019-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -10,7 +10,8 @@ from datetime import timedelta from pyocf.types.cache import Cache, PromotionPolicy, NhitParams from pyocf.types.core import Core -from pyocf.types.volume import Volume +from pyocf.types.volume import RamVolume +from pyocf.types.volume_core import CoreVolume from pyocf.types.data import Data from pyocf.types.io import IoDir from pyocf.utils import Size @@ -28,8 +29,8 @@ def test_init_nhit(pyocf_ctx, promotion_policy): * verify that promotion policy type is properly reflected in stats """ - cache_device = Volume(Size.from_MiB(50)) - core_device = Volume(Size.from_MiB(50)) + cache_device = RamVolume(Size.from_MiB(50)) + core_device = RamVolume(Size.from_MiB(50)) cache = Cache.start_on_device(cache_device, promotion_policy=promotion_policy) core = Core.using_device(core_device) @@ -55,18 +56,20 @@ def test_change_to_nhit_and_back_io_in_flight(pyocf_ctx): """ # Step 1 - cache_device = Volume(Size.from_MiB(50)) - core_device = Volume(Size.from_MiB(50)) + cache_device = RamVolume(Size.from_MiB(50)) + core_device = RamVolume(Size.from_MiB(50)) cache = Cache.start_on_device(cache_device) core = Core.using_device(core_device) cache.add_core(core) + vol = CoreVolume(core, open=True) + queue = cache.get_default_queue() # Step 2 r = ( Rio() - .target(core) + .target(vol) .njobs(10) .bs(Size.from_KiB(4)) .readwrite(ReadWrite.RANDWRITE) @@ -74,7 +77,7 @@ def test_change_to_nhit_and_back_io_in_flight(pyocf_ctx): .time_based() .time(timedelta(minutes=1)) .qd(10) - .run_async() + .run_async([queue]) ) # Step 3 @@ -85,7 +88,7 @@ def test_change_to_nhit_and_back_io_in_flight(pyocf_ctx): assert r.error_count == 0, "No IO's should fail when turning NHIT policy on" # Step 5 - r.run_async() + r.run_async([queue]) # Step 6 cache.set_promotion_policy(PromotionPolicy.ALWAYS) @@ -107,15 +110,17 @@ def fill_cache(cache, fill_ratio): bytes_to_fill = Size(round(cache_lines.bytes * fill_ratio)) core = cache.cores[0] + vol = CoreVolume(core, open=True) + queue = cache.get_default_queue() r = ( Rio() - .target(core) + .target(vol) .readwrite(ReadWrite.RANDWRITE) .size(bytes_to_fill) .bs(Size(512)) .qd(10) - .run() + .run([queue]) ) @@ -137,12 +142,14 @@ def test_promoted_after_hits_various_thresholds( """ # Step 1 - cache_device = Volume(Size.from_MiB(50)) - core_device = Volume(Size.from_MiB(50)) + cache_device = RamVolume(Size.from_MiB(50)) + core_device = RamVolume(Size.from_MiB(50)) cache = Cache.start_on_device(cache_device, promotion_policy=PromotionPolicy.NHIT) core = Core.using_device(core_device) cache.add_core(core) + vol = CoreVolume(core, open=True) + queue = cache.get_default_queue() # Step 2 cache.set_promotion_policy_param( @@ -167,12 +174,12 @@ def test_promoted_after_hits_various_thresholds( .readwrite(ReadWrite.WRITE) .bs(Size(4096)) .offset(last_core_line) - .target(core) + .target(vol) .size(Size(4096) + last_core_line) ) for i in range(insertion_threshold - 1): - r.run() + r.run([queue]) cache.settle() stats = cache.get_stats() @@ -183,7 +190,7 @@ def test_promoted_after_hits_various_thresholds( ) # Step 5 - r.run() + r.run([queue]) cache.settle() stats = cache.get_stats() @@ -207,15 +214,17 @@ def test_partial_hit_promotion(pyocf_ctx): """ # Step 1 - cache_device = Volume(Size.from_MiB(50)) - core_device = Volume(Size.from_MiB(50)) + cache_device = RamVolume(Size.from_MiB(50)) + core_device = RamVolume(Size.from_MiB(50)) cache = Cache.start_on_device(cache_device) core = Core.using_device(core_device) cache.add_core(core) + vol = CoreVolume(core, open=True) + queue = cache.get_default_queue() # Step 2 - r = Rio().readwrite(ReadWrite.READ).bs(Size(512)).size(Size(512)).target(core).run() + r = Rio().readwrite(ReadWrite.READ).bs(Size(512)).size(Size(512)).target(vol).run([queue]) stats = cache.get_stats() cache_lines = stats["conf"]["size"] @@ -232,7 +241,7 @@ def test_partial_hit_promotion(pyocf_ctx): # Step 4 req_size = Size(2 * cache_lines.line_size) - r.size(req_size).bs(req_size).readwrite(ReadWrite.WRITE).run() + r.size(req_size).bs(req_size).readwrite(ReadWrite.WRITE).run([queue]) cache.settle() stats = cache.get_stats() diff --git a/tests/functional/tests/engine/test_read.py b/tests/functional/tests/engine/test_read.py index 48c2e59..e14ce6a 100644 --- a/tests/functional/tests/engine/test_read.py +++ b/tests/functional/tests/engine/test_read.py @@ -1,4 +1,5 @@ -# Copyright(c) 2019-2021 Intel Corporation +# +# Copyright(c) 2019-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -13,7 +14,8 @@ from datetime import datetime from pyocf.types.cache import Cache, CacheMode from pyocf.types.core import Core -from pyocf.types.volume import Volume +from pyocf.types.volume import RamVolume +from pyocf.types.volume_core import CoreVolume from pyocf.types.data import Data from pyocf.types.io import IoDir from pyocf.utils import Size @@ -37,8 +39,8 @@ def __io(io, queue, address, size, data, direction): return int(completion.results["err"]) -def _io(new_io, queue, address, size, data, offset, direction): - io = new_io(queue, address, size, direction, 0, 0) +def io_to_exp_obj(vol, queue, address, size, data, offset, direction): + io = vol.new_io(queue, address, size, direction, 0, 0) if direction == IoDir.READ: _data = Data.from_bytes(bytes(size)) else: @@ -49,30 +51,6 @@ def _io(new_io, queue, address, size, data, offset, direction): return ret -def io_to_core(core, address, size, data, offset, direction): - return _io( - core.new_core_io, - core.cache.get_default_queue(), - address, - size, - data, - offset, - direction, - ) - - -def io_to_exp_obj(core, address, size, data, offset, direction): - return _io( - core.new_io, - core.cache.get_default_queue(), - address, - size, - data, - offset, - direction, - ) - - def sector_to_region(sector, region_start): num_regions = len(region_start) i = 0 @@ -259,15 +237,17 @@ def test_read_data_consistency(pyocf_ctx, cacheline_size, cache_mode, rand_seed) result_b = bytes(WORKSET_SIZE) - cache_device = Volume(Size.from_MiB(50)) - core_device = Volume(Size.from_MiB(50)) + cache_device = RamVolume(Size.from_MiB(50)) + core_device = RamVolume(Size.from_MiB(50)) cache = Cache.start_on_device( cache_device, cache_mode=CacheMode.WO, cache_line_size=cacheline_size ) - core = Core.using_device(core_device) + core = Core.using_device(core_device) cache.add_core(core) + queue = cache.get_default_queue() + vol = CoreVolume(core, open=True) insert_order = list(range(CACHELINE_COUNT)) @@ -310,7 +290,8 @@ def test_read_data_consistency(pyocf_ctx, cacheline_size, cache_mode, rand_seed) # write data to core and invalidate all CL and write data pattern to core cache.change_cache_mode(cache_mode=CacheMode.PT) io_to_exp_obj( - core, + vol, + queue, WORKSET_OFFSET, len(data[SectorStatus.INVALID]), data[SectorStatus.INVALID], @@ -331,7 +312,8 @@ def test_read_data_consistency(pyocf_ctx, cacheline_size, cache_mode, rand_seed) region = sector_to_region(sec, region_start) if region_state[region] != SectorStatus.INVALID: io_to_exp_obj( - core, + vol, + queue, WORKSET_OFFSET + SECTOR_SIZE * sec, SECTOR_SIZE, data[SectorStatus.CLEAN], @@ -345,7 +327,8 @@ def test_read_data_consistency(pyocf_ctx, cacheline_size, cache_mode, rand_seed) region = sector_to_region(sec, region_start) if region_state[region] == SectorStatus.DIRTY: io_to_exp_obj( - core, + vol, + queue, WORKSET_OFFSET + SECTOR_SIZE * sec, SECTOR_SIZE, data[SectorStatus.DIRTY], @@ -372,7 +355,7 @@ def test_read_data_consistency(pyocf_ctx, cacheline_size, cache_mode, rand_seed) END = end * SECTOR_SIZE size = (end - start + 1) * SECTOR_SIZE assert 0 == io_to_exp_obj( - core, WORKSET_OFFSET + START, size, result_b, START, IoDir.READ + vol, queue, WORKSET_OFFSET + START, size, result_b, START, IoDir.READ ), "error reading in {}: region_state={}, start={}, end={}, insert_order={}".format( cache_mode, region_state, start, end, insert_order ) diff --git a/tests/functional/tests/engine/test_seq_cutoff.py b/tests/functional/tests/engine/test_seq_cutoff.py index fca4ffa..d204f28 100644 --- a/tests/functional/tests/engine/test_seq_cutoff.py +++ b/tests/functional/tests/engine/test_seq_cutoff.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2020-2021 Intel Corporation +# Copyright(c) 2020-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -10,7 +10,8 @@ import pytest from pyocf.types.cache import Cache, CacheMode from pyocf.types.core import Core -from pyocf.types.volume import Volume +from pyocf.types.volume import RamVolume +from pyocf.types.volume_core import CoreVolume from pyocf.types.data import Data from pyocf.types.io import IoDir from pyocf.utils import Size @@ -27,11 +28,11 @@ class Stream: return f"{self.last} {self.length} {self.direction}" -def _io(core, addr, size, direction, context): +def _io(vol, queue, addr, size, direction, context): comp = OcfCompletion([("error", c_int)], context=context) data = Data(size) - io = core.new_io(core.cache.get_default_queue(), addr, size, direction, 0, 0) + io = vol.new_io(queue, addr, size, direction, 0, 0) io.set_data(data) io.callback = comp.callback io.submit() @@ -39,11 +40,11 @@ def _io(core, addr, size, direction, context): return comp -def io_to_streams(core, streams, io_size): +def io_to_streams(vol, queue, streams, io_size): completions = [] for stream in streams: completions.append( - _io(core, stream.last, io_size, stream.direction, context=(io_size, stream)) + _io(vol, queue, stream.last, io_size, stream.direction, context=(io_size, stream)) ) for c in completions: @@ -90,10 +91,12 @@ def test_seq_cutoff_max_streams(pyocf_ctx): non_active_stream = choice(streams) streams.remove(non_active_stream) - cache = Cache.start_on_device(Volume(Size.from_MiB(200)), cache_mode=CacheMode.WT) - core = Core.using_device(Volume(core_size), seq_cutoff_promotion_count=1) + cache = Cache.start_on_device(RamVolume(Size.from_MiB(200)), cache_mode=CacheMode.WT) + core = Core.using_device(RamVolume(core_size), seq_cutoff_promotion_count=1) cache.add_core(core) + vol = CoreVolume(core, open=True) + queue = cache.get_default_queue() cache.set_seq_cut_off_policy(SeqCutOffPolicy.ALWAYS) cache.set_seq_cut_off_threshold(threshold) @@ -101,7 +104,7 @@ def test_seq_cutoff_max_streams(pyocf_ctx): # STEP 1 shuffle(streams) io_size = threshold - Size.from_sector(1) - io_to_streams(core, streams, io_size) + io_to_streams(vol, queue, streams, io_size) stats = cache.get_stats() assert ( @@ -115,7 +118,7 @@ def test_seq_cutoff_max_streams(pyocf_ctx): streams.remove(lru_stream) shuffle(streams) - io_to_streams(core, streams, Size.from_sector(1)) + io_to_streams(vol, queue, streams, Size.from_sector(1)) stats = cache.get_stats() assert ( @@ -126,7 +129,7 @@ def test_seq_cutoff_max_streams(pyocf_ctx): ), "All streams should be handled in PT - cutoff engaged for all streams" # STEP 3 - io_to_streams(core, [non_active_stream], Size.from_sector(1)) + io_to_streams(vol, queue, [non_active_stream], Size.from_sector(1)) stats = cache.get_stats() assert ( @@ -134,7 +137,7 @@ def test_seq_cutoff_max_streams(pyocf_ctx): ), "This request should be serviced by cache - no cutoff for inactive stream" # STEP 4 - io_to_streams(core, [lru_stream], Size.from_sector(1)) + io_to_streams(vol, queue, [lru_stream], Size.from_sector(1)) stats = cache.get_stats() assert ( diff --git a/tests/functional/tests/eviction/test_eviction.py b/tests/functional/tests/eviction/test_eviction.py index d2cc1e8..ea4613e 100644 --- a/tests/functional/tests/eviction/test_eviction.py +++ b/tests/functional/tests/eviction/test_eviction.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2019-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -14,7 +14,8 @@ from pyocf.types.core import Core from pyocf.types.data import Data from pyocf.types.io import IoDir from pyocf.types.shared import OcfCompletion, CacheLineSize, SeqCutOffPolicy, CacheLines -from pyocf.types.volume import Volume +from pyocf.types.volume import RamVolume +from pyocf.types.volume_core import CoreVolume from pyocf.utils import Size logger = logging.getLogger(__name__) @@ -24,25 +25,27 @@ logger = logging.getLogger(__name__) @pytest.mark.parametrize("mode", [CacheMode.WT]) def test_eviction_two_cores(pyocf_ctx, mode: CacheMode, cls: CacheLineSize): """Test if eviction works correctly when remapping cachelines between distinct cores.""" - cache_device = Volume(Size.from_MiB(50)) + cache_device = RamVolume(Size.from_MiB(50)) - core_device1 = Volume(Size.from_MiB(40)) - core_device2 = Volume(Size.from_MiB(40)) + core_device1 = RamVolume(Size.from_MiB(40)) + core_device2 = RamVolume(Size.from_MiB(40)) cache = Cache.start_on_device(cache_device, cache_mode=mode, cache_line_size=cls) cache.set_seq_cut_off_policy(SeqCutOffPolicy.NEVER) cache_size = cache.get_stats()["conf"]["size"] - core_exported1 = Core.using_device(core_device1, name="core1") - core_exported2 = Core.using_device(core_device2, name="core2") - cache.add_core(core_exported1) - cache.add_core(core_exported2) + core1 = Core.using_device(core_device1, name="core1") + core2 = Core.using_device(core_device2, name="core2") + cache.add_core(core1) + vol1 = CoreVolume(core1, open=True) + cache.add_core(core2) + vol2 = CoreVolume(core2, open=True) valid_io_size = Size.from_B(cache_size.B) test_data = Data(valid_io_size) - send_io(core_exported1, test_data) - send_io(core_exported2, test_data) + send_io(core1, test_data) + send_io(core2, test_data) - stats1 = core_exported1.get_stats() - stats2 = core_exported2.get_stats() + stats1 = core1.get_stats() + stats2 = core2.get_stats() # IO to the second core should evict all the data from the first core assert stats1["usage"]["occupancy"]["value"] == 0 assert stats2["usage"]["occupancy"]["value"] == valid_io_size.blocks_4k @@ -52,20 +55,21 @@ def test_eviction_two_cores(pyocf_ctx, mode: CacheMode, cls: CacheLineSize): @pytest.mark.parametrize("mode", [CacheMode.WT, CacheMode.WB, CacheMode.WO]) def test_write_size_greater_than_cache(pyocf_ctx, mode: CacheMode, cls: CacheLineSize): """Test if eviction does not occur when IO greater than cache size is submitted.""" - cache_device = Volume(Size.from_MiB(50)) + cache_device = RamVolume(Size.from_MiB(50)) - core_device = Volume(Size.from_MiB(200)) + core_device = RamVolume(Size.from_MiB(200)) cache = Cache.start_on_device(cache_device, cache_mode=mode, cache_line_size=cls) cache_size = cache.get_stats()["conf"]["size"] - core_exported = Core.using_device(core_device) - cache.add_core(core_exported) + core = Core.using_device(core_device) + cache.add_core(core) + vol = CoreVolume(core, open=True) cache.set_seq_cut_off_policy(SeqCutOffPolicy.NEVER) valid_io_size = Size.from_B(cache_size.B // 2) test_data = Data(valid_io_size) - send_io(core_exported, test_data) + send_io(core, test_data) - stats = core_exported.cache.get_stats() + stats = core.cache.get_stats() first_block_sts = stats["block"] first_usage_sts = stats["usage"] pt_writes_first = stats["req"]["wr_pt"] @@ -80,12 +84,12 @@ def test_write_size_greater_than_cache(pyocf_ctx, mode: CacheMode, cls: CacheLin io_size_bigger_than_cache = Size.from_MiB(100) io_offset = valid_io_size test_data = Data(io_size_bigger_than_cache) - send_io(core_exported, test_data, io_offset) + send_io(core, test_data, io_offset) if mode is not CacheMode.WT: # Flush first write cache.flush() - stats = core_exported.cache.get_stats() + stats = core.cache.get_stats() second_block_sts = stats["block"] second_usage_sts = stats["usage"] pt_writes_second = stats["req"]["wr_pt"] @@ -106,13 +110,14 @@ def test_write_size_greater_than_cache(pyocf_ctx, mode: CacheMode, cls: CacheLin @pytest.mark.parametrize("cls", CacheLineSize) def test_evict_overflown_pinned(pyocf_ctx, cls: CacheLineSize): """ Verify if overflown pinned ioclass is evicted """ - cache_device = Volume(Size.from_MiB(50)) - core_device = Volume(Size.from_MiB(100)) + cache_device = RamVolume(Size.from_MiB(50)) + core_device = RamVolume(Size.from_MiB(100)) cache = Cache.start_on_device( cache_device, cache_mode=CacheMode.WT, cache_line_size=cls ) core = Core.using_device(core_device) cache.add_core(core) + vol = CoreVolume(core, open=True) test_ioclass_id = 1 pinned_ioclass_id = 2 @@ -176,9 +181,10 @@ def test_evict_overflown_pinned(pyocf_ctx, cls: CacheLineSize): ), "Overflown part has not been evicted" -def send_io(exported_obj: Core, data: Data, addr: int = 0, target_ioclass: int = 0): - io = exported_obj.new_io( - exported_obj.cache.get_default_queue(), +def send_io(core: Core, data: Data, addr: int = 0, target_ioclass: int = 0): + vol = core.get_front_volume() + io = vol.new_io( + core.cache.get_default_queue(), addr, data.size, IoDir.WRITE, diff --git a/tests/functional/tests/flush/test_flush_after_mngmt.py b/tests/functional/tests/flush/test_flush_after_mngmt.py index e9aa185..da80983 100644 --- a/tests/functional/tests/flush/test_flush_after_mngmt.py +++ b/tests/functional/tests/flush/test_flush_after_mngmt.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2020-2021 Intel Corporation +# Copyright(c) 2020-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -12,7 +12,8 @@ import pytest from pyocf.types.cache import Cache, CacheMode from pyocf.types.core import Core -from pyocf.types.volume import Volume +from pyocf.types.volume import RamVolume +from pyocf.types.volume_core import CoreVolume from pyocf.types.data import Data from pyocf.types.io import IoDir from pyocf.utils import Size @@ -28,8 +29,8 @@ def __io(io, queue, address, size, data, direction): return int(completion.results["err"]) -def _io(new_io, queue, address, size, data, offset, direction, flags): - io = new_io(queue, address, size, direction, 0, flags) +def io_to_exp_obj(vol, queue, address, size, data, offset, direction, flags): + io = vol.new_io(queue, address, size, direction, 0, flags) if direction == IoDir.READ: _data = Data.from_bytes(bytes(size)) else: @@ -40,20 +41,7 @@ def _io(new_io, queue, address, size, data, offset, direction, flags): return ret -def io_to_exp_obj(core, address, size, data, offset, direction, flags): - return _io( - core.new_io, - core.cache.get_default_queue(), - address, - size, - data, - offset, - direction, - flags, - ) - - -class FlushValVolume(Volume): +class FlushValVolume(RamVolume): def __init__(self, size): self.flush_last = False super().__init__(size) @@ -87,12 +75,15 @@ def test_flush_after_mngmt(pyocf_ctx): cache.add_core(core) assert cache_device.flush_last + vol = CoreVolume(core, open=True) + queue = cache.get_default_queue() + # WT I/O to write data to core and cache VC - io_to_exp_obj(core, block_size * 0, block_size, data, 0, IoDir.WRITE, 0) + io_to_exp_obj(vol, queue, block_size * 0, block_size, data, 0, IoDir.WRITE, 0) # WB I/O to produce dirty cachelines in CAS cache.change_cache_mode(CacheMode.WB) - io_to_exp_obj(core, block_size * 1, block_size, data, 0, IoDir.WRITE, 0) + io_to_exp_obj(vol, queue, block_size * 1, block_size, data, 0, IoDir.WRITE, 0) # after cache flush VCs are expected to be cleared cache.flush() @@ -100,14 +91,14 @@ def test_flush_after_mngmt(pyocf_ctx): assert core_device.flush_last # I/O to write data to cache device VC - io_to_exp_obj(core, block_size * 0, block_size, data, 0, IoDir.WRITE, 0) + io_to_exp_obj(vol, queue, block_size * 0, block_size, data, 0, IoDir.WRITE, 0) # cache save must flush VC cache.save() assert cache_device.flush_last # I/O to write data to cache device VC - io_to_exp_obj(core, block_size * 0, block_size, data, 0, IoDir.WRITE, 0) + io_to_exp_obj(vol, queue, block_size * 0, block_size, data, 0, IoDir.WRITE, 0) # cache stop must flush VC cache.stop() diff --git a/tests/functional/tests/management/test_add_remove.py b/tests/functional/tests/management/test_add_remove.py index b95cfe3..2228ca7 100644 --- a/tests/functional/tests/management/test_add_remove.py +++ b/tests/functional/tests/management/test_add_remove.py @@ -1,4 +1,5 @@ -# Copyright(c) 2019-2021 Intel Corporation +# +# Copyright(c) 2019-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -8,9 +9,11 @@ from ctypes import c_int from random import randint from pyocf.types.cache import Cache, CacheMode from pyocf.types.core import Core -from pyocf.types.volume import Volume +from pyocf.types.volume import RamVolume, Volume +from pyocf.types.volume_core import CoreVolume from pyocf.types.data import Data from pyocf.types.io import IoDir +from pyocf.types.queue import Queue from pyocf.utils import Size as S from pyocf.types.shared import OcfError, OcfCompletion, CacheLineSize @@ -19,13 +22,13 @@ from pyocf.types.shared import OcfError, OcfCompletion, CacheLineSize @pytest.mark.parametrize("cls", CacheLineSize) def test_adding_core(pyocf_ctx, cache_mode, cls): # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device( cache_device, cache_mode=cache_mode, cache_line_size=cls ) # Create core device - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core.using_device(core_device) # Check statistics before adding core @@ -44,13 +47,13 @@ def test_adding_core(pyocf_ctx, cache_mode, cls): @pytest.mark.parametrize("cls", CacheLineSize) def test_removing_core(pyocf_ctx, cache_mode, cls): # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device( cache_device, cache_mode=cache_mode, cache_line_size=cls ) # Create core device - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core.using_device(core_device) # Add core to cache @@ -68,21 +71,24 @@ def test_removing_core(pyocf_ctx, cache_mode, cls): @pytest.mark.parametrize("cls", CacheLineSize) def test_remove_dirty_no_flush(pyocf_ctx, cache_mode, cls): # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device( cache_device, cache_mode=cache_mode, cache_line_size=cls ) # Create core device - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core.using_device(core_device) cache.add_core(core) + vol = CoreVolume(core, open=True) + queue = core.cache.get_default_queue() + # Prepare data core_size = core.get_stats()["size"] data = Data(core_size.B) - _io_to_core(core, data) + _io_to_core(vol, queue, data) # Remove core from cache cache.remove_core(core) @@ -90,11 +96,11 @@ def test_remove_dirty_no_flush(pyocf_ctx, cache_mode, cls): def test_30add_remove(pyocf_ctx): # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device) # Create core device - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core.using_device(core_device) # Add and remove core device in a loop 100 times @@ -111,21 +117,22 @@ def test_30add_remove(pyocf_ctx): def test_10add_remove_with_io(pyocf_ctx): # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device) # Create core device - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core.using_device(core_device) # Add and remove core 10 times in a loop with io in between for i in range(0, 10): cache.add_core(core) + vol = CoreVolume(core, open=True) stats = cache.get_stats() assert stats["conf"]["core_count"] == 1 write_data = Data.from_string("Test data") - io = core.new_io( + io = vol.new_io( cache.get_default_queue(), S.from_sector(1).B, write_data.size, IoDir.WRITE, 0, 0 ) @@ -143,7 +150,7 @@ def test_10add_remove_with_io(pyocf_ctx): def test_add_remove_30core(pyocf_ctx): # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device) core_devices = [] core_amount = 30 @@ -152,7 +159,7 @@ def test_add_remove_30core(pyocf_ctx): for i in range(0, core_amount): stats = cache.get_stats() assert stats["conf"]["core_count"] == i - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core.using_device(core_device, name=f"core{i}") core_devices.append(core) cache.add_core(core) @@ -176,13 +183,13 @@ def test_adding_to_random_cache(pyocf_ctx): # Create 5 cache devices for i in range(0, cache_amount): - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, name=f"cache{i}") cache_devices.append(cache) # Create 50 core devices and add to random cache for i in range(0, core_amount): - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core.using_device(core_device, name=f"core{i}") core_devices[core] = randint(0, cache_amount - 1) cache_devices[core_devices[core]].add_core(core) @@ -202,13 +209,13 @@ def test_adding_to_random_cache(pyocf_ctx): @pytest.mark.parametrize("cls", CacheLineSize) def test_adding_core_twice(pyocf_ctx, cache_mode, cls): # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device( cache_device, cache_mode=cache_mode, cache_line_size=cls ) # Create core device - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core.using_device(core_device) # Add core @@ -227,19 +234,19 @@ def test_adding_core_twice(pyocf_ctx, cache_mode, cls): @pytest.mark.parametrize("cls", CacheLineSize) def test_adding_core_already_used(pyocf_ctx, cache_mode, cls): # Start first cache device - cache_device1 = Volume(S.from_MiB(50)) + cache_device1 = RamVolume(S.from_MiB(50)) cache1 = Cache.start_on_device( cache_device1, cache_mode=cache_mode, cache_line_size=cls, name="cache1" ) # Start second cache device - cache_device2 = Volume(S.from_MiB(50)) + cache_device2 = RamVolume(S.from_MiB(50)) cache2 = Cache.start_on_device( cache_device2, cache_mode=cache_mode, cache_line_size=cls, name="cache2" ) # Create core device - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core.using_device(core_device) # Add core to first cache @@ -261,7 +268,7 @@ def test_adding_core_already_used(pyocf_ctx, cache_mode, cls): @pytest.mark.parametrize("cls", CacheLineSize) def test_add_remove_incrementally(pyocf_ctx, cache_mode, cls): # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device( cache_device, cache_mode=cache_mode, cache_line_size=cls ) @@ -270,7 +277,7 @@ def test_add_remove_incrementally(pyocf_ctx, cache_mode, cls): # Create 5 core devices and add to cache for i in range(0, core_amount): - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core.using_device(core_device, name=f"core{i}") core_devices.append(core) cache.add_core(core) @@ -302,8 +309,8 @@ def test_add_remove_incrementally(pyocf_ctx, cache_mode, cls): assert stats["conf"]["core_count"] == core_amount -def _io_to_core(exported_obj: Core, data: Data): - io = exported_obj.new_io(exported_obj.cache.get_default_queue(), 0, data.size, +def _io_to_core(vol: Volume, queue: Queue, data: Data): + io = vol.new_io(queue, 0, data.size, IoDir.WRITE, 0, 0) io.set_data(data) @@ -325,13 +332,13 @@ def test_try_add_core_with_changed_size(pyocf_ctx, cache_mode, cls): :param cls: cache line size we start with """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device( cache_device, cache_mode=cache_mode, cache_line_size=cls ) # Add core to cache - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core.using_device(core_device) cache.add_core(core) @@ -359,13 +366,13 @@ def test_load_with_changed_core_size(pyocf_ctx, cache_mode, cls): :param cls: cache line size we start with """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device( cache_device, cache_mode=cache_mode, cache_line_size=cls ) # Add core to cache - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core.using_device(core_device) cache.add_core(core) diff --git a/tests/functional/tests/management/test_attach_cache.py b/tests/functional/tests/management/test_attach_cache.py index 6915ee7..812a1f4 100644 --- a/tests/functional/tests/management/test_attach_cache.py +++ b/tests/functional/tests/management/test_attach_cache.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2019-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -27,7 +27,8 @@ from pyocf.types.shared import ( CacheLineSize, SeqCutOffPolicy, ) -from pyocf.types.volume import Volume +from pyocf.types.volume import RamVolume +from pyocf.types.volume_core import CoreVolume from pyocf.utils import Size logger = logging.getLogger(__name__) @@ -43,12 +44,15 @@ def test_attach_different_size( attach cache with different size and trigger IO. Verify if occupancy thresold is respected with both original and new cache device. """ - cache_device = Volume(Size.from_MiB(100)) - core_device = Volume(Size.from_MiB(100)) + cache_device = RamVolume(Size.from_MiB(100)) + core_device = RamVolume(Size.from_MiB(100)) cache = Cache.start_on_device(cache_device, cache_mode=mode, cache_line_size=cls) core = Core.using_device(core_device) cache.add_core(core) + vol = CoreVolume(core, open=True) + queue = cache.get_default_queue() + cache.configure_partition( part_id=1, name="test_part", max_size=50, priority=1 ) @@ -61,7 +65,7 @@ def test_attach_different_size( data = bytes(block_size) for i in range(cache_size.blocks_4k): - io_to_exp_obj(core, block_size * i, block_size, data, 0, IoDir.WRITE, 1, 0) + io_to_exp_obj(vol, queue, block_size * i, block_size, data, 0, IoDir.WRITE, 1, 0) part_current_size = CacheLines( cache.get_partition_info(part_id=1)["_curr_size"], cls @@ -70,13 +74,13 @@ def test_attach_different_size( assert part_current_size.blocks_4k == cache_size.blocks_4k * 0.5 cache.detach_device() - new_cache_device = Volume(Size.from_MiB(new_cache_size)) + new_cache_device = RamVolume(Size.from_MiB(new_cache_size)) cache.attach_device(new_cache_device, force=True) cache_size = cache.get_stats()["conf"]["size"] for i in range(cache_size.blocks_4k): - io_to_exp_obj(core, block_size * i, block_size, data, 0, IoDir.WRITE, 1, 0) + io_to_exp_obj(vol, queue, block_size * i, block_size, data, 0, IoDir.WRITE, 1, 0) part_current_size = CacheLines( cache.get_partition_info(part_id=1)["_curr_size"], cls @@ -85,22 +89,8 @@ def test_attach_different_size( assert part_current_size.blocks_4k == cache_size.blocks_4k * 0.5 -def io_to_exp_obj(core, address, size, data, offset, direction, target_ioclass, flags): - return _io( - core.new_io, - core.cache.get_default_queue(), - address, - size, - data, - offset, - direction, - target_ioclass, - flags, - ) - - -def _io(new_io, queue, address, size, data, offset, direction, target_ioclass, flags): - io = new_io(queue, address, size, direction, target_ioclass, flags) +def io_to_exp_obj(vol, queue, address, size, data, offset, direction, target_ioclass, flags): + io = vol.new_io(queue, address, size, direction, target_ioclass, flags) if direction == IoDir.READ: _data = Data.from_bytes(bytes(size)) else: diff --git a/tests/functional/tests/management/test_change_params.py b/tests/functional/tests/management/test_change_params.py index 40b6847..631ba94 100644 --- a/tests/functional/tests/management/test_change_params.py +++ b/tests/functional/tests/management/test_change_params.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2019-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -7,7 +7,7 @@ import pytest from pyocf.types.cache import Cache, CacheMode, CleaningPolicy, SeqCutOffPolicy from pyocf.types.core import Core -from pyocf.types.volume import Volume +from pyocf.types.volume import RamVolume from pyocf.utils import Size as S from pyocf.types.shared import CacheLineSize @@ -17,7 +17,7 @@ from pyocf.types.shared import CacheLineSize @pytest.mark.parametrize("cls", CacheLineSize) def test_change_cache_mode(pyocf_ctx, from_cm, to_cm, cls): # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device( cache_device, cache_mode=from_cm, cache_line_size=cls ) @@ -32,7 +32,7 @@ def test_change_cache_mode(pyocf_ctx, from_cm, to_cm, cls): @pytest.mark.parametrize("cls", CacheLineSize) def test_change_cleaning_policy(pyocf_ctx, cm, cls): # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device( cache_device, cache_mode=cm, cache_line_size=cls ) @@ -57,15 +57,15 @@ def test_change_cleaning_policy(pyocf_ctx, cm, cls): @pytest.mark.parametrize("cls", CacheLineSize) def test_cache_change_seq_cut_off_policy(pyocf_ctx, cm, cls): # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device( cache_device, cache_mode=cm, cache_line_size=cls ) # Create 2 core devices - core_device1 = Volume(S.from_MiB(10)) + core_device1 = RamVolume(S.from_MiB(10)) core1 = Core.using_device(core_device1, name="core1") - core_device2 = Volume(S.from_MiB(10)) + core_device2 = RamVolume(S.from_MiB(10)) core2 = Core.using_device(core_device2, name="core2") # Add cores @@ -96,15 +96,15 @@ def test_cache_change_seq_cut_off_policy(pyocf_ctx, cm, cls): @pytest.mark.parametrize("cls", CacheLineSize) def test_core_change_seq_cut_off_policy(pyocf_ctx, cm, cls): # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device( cache_device, cache_mode=cm, cache_line_size=cls ) # Create 2 core devices - core_device1 = Volume(S.from_MiB(10)) + core_device1 = RamVolume(S.from_MiB(10)) core1 = Core.using_device(core_device1, name="core1") - core_device2 = Volume(S.from_MiB(10)) + core_device2 = RamVolume(S.from_MiB(10)) core2 = Core.using_device(core_device2, name="core2") # Add cores diff --git a/tests/functional/tests/management/test_start_stop.py b/tests/functional/tests/management/test_start_stop.py index 60b515e..626fa32 100644 --- a/tests/functional/tests/management/test_start_stop.py +++ b/tests/functional/tests/management/test_start_stop.py @@ -21,10 +21,13 @@ from pyocf.types.cache import ( Backfill ) from pyocf.types.core import Core +from pyocf.types.ctx import OcfCtx from pyocf.types.data import Data from pyocf.types.io import IoDir +from pyocf.types.queue import Queue from pyocf.types.shared import OcfError, OcfCompletion, CacheLineSize, SeqCutOffPolicy -from pyocf.types.volume import Volume +from pyocf.types.volume import Volume, RamVolume +from pyocf.types.volume_core import CoreVolume from pyocf.utils import Size logger = logging.getLogger(__name__) @@ -34,8 +37,8 @@ def test_start_check_default(pyocf_ctx): """Test if default values are correct after start. """ - cache_device = Volume(Size.from_MiB(50)) - core_device = Volume(Size.from_MiB(10)) + cache_device = RamVolume(Size.from_MiB(50)) + core_device = RamVolume(Size.from_MiB(10)) cache = Cache.start_on_device(cache_device) core = Core.using_device(core_device) @@ -58,24 +61,26 @@ def test_start_write_first_and_check_mode(pyocf_ctx, mode: CacheMode, cls: Cache After start check proper cache mode behaviour, starting with write operation. """ - cache_device = Volume(Size.from_MiB(50)) - core_device = Volume(Size.from_MiB(10)) + cache_device = RamVolume(Size.from_MiB(50)) + core_device = RamVolume(Size.from_MiB(10)) cache = Cache.start_on_device(cache_device, cache_mode=mode, cache_line_size=cls) - core_exported = Core.using_device(core_device) + core = Core.using_device(core_device) - cache.add_core(core_exported) + cache.add_core(core) + vol = CoreVolume(core, open=True) + queue = cache.get_default_queue() logger.info("[STAGE] Initial write to exported object") cache_device.reset_stats() core_device.reset_stats() test_data = Data.from_string("This is test data") - io_to_core(core_exported, test_data, Size.from_sector(1).B) - check_stats_write_empty(core_exported, mode, cls) + io_to_core(vol, queue, test_data, Size.from_sector(1).B) + check_stats_write_empty(core, mode, cls) logger.info("[STAGE] Read from exported object after initial write") - io_from_exported_object(core_exported, test_data.size, Size.from_sector(1).B) - check_stats_read_after_write(core_exported, mode, cls, True) + io_from_exported_object(vol, queue, test_data.size, Size.from_sector(1).B) + check_stats_read_after_write(core, mode, cls, True) logger.info("[STAGE] Write to exported object after read") cache_device.reset_stats() @@ -83,10 +88,10 @@ def test_start_write_first_and_check_mode(pyocf_ctx, mode: CacheMode, cls: Cache test_data = Data.from_string("Changed test data") - io_to_core(core_exported, test_data, Size.from_sector(1).B) - check_stats_write_after_read(core_exported, mode, cls) + io_to_core(vol, queue, test_data, Size.from_sector(1).B) + check_stats_write_after_read(core, mode, cls) - check_md5_sums(core_exported, mode) + check_md5_sums(core, mode) @pytest.mark.parametrize("cls", CacheLineSize) @@ -96,23 +101,26 @@ def test_start_read_first_and_check_mode(pyocf_ctx, mode: CacheMode, cls: CacheL After start check proper cache mode behaviour, starting with read operation. """ - cache_device = Volume(Size.from_MiB(50)) - core_device = Volume(Size.from_MiB(5)) + cache_device = RamVolume(Size.from_MiB(50)) + core_device = RamVolume(Size.from_MiB(5)) cache = Cache.start_on_device(cache_device, cache_mode=mode, cache_line_size=cls) - core_exported = Core.using_device(core_device) + core = Core.using_device(core_device) - cache.add_core(core_exported) + cache.add_core(core) + front_vol = CoreVolume(core, open=True) + bottom_vol = core.get_volume() + queue = cache.get_default_queue() logger.info("[STAGE] Initial write to core device") test_data = Data.from_string("This is test data") - io_to_core(core_exported, test_data, Size.from_sector(1).B, True) + io_to_core(bottom_vol, queue, test_data, Size.from_sector(1).B) cache_device.reset_stats() core_device.reset_stats() logger.info("[STAGE] Initial read from exported object") - io_from_exported_object(core_exported, test_data.size, Size.from_sector(1).B) - check_stats_read_empty(core_exported, mode, cls) + io_from_exported_object(front_vol, queue, test_data.size, Size.from_sector(1).B) + check_stats_read_empty(core, mode, cls) logger.info("[STAGE] Write to exported object after initial read") cache_device.reset_stats() @@ -120,15 +128,15 @@ def test_start_read_first_and_check_mode(pyocf_ctx, mode: CacheMode, cls: CacheL test_data = Data.from_string("Changed test data") - io_to_core(core_exported, test_data, Size.from_sector(1).B) + io_to_core(front_vol, queue, test_data, Size.from_sector(1).B) - check_stats_write_after_read(core_exported, mode, cls, True) + check_stats_write_after_read(core, mode, cls, True) logger.info("[STAGE] Read from exported object after write") - io_from_exported_object(core_exported, test_data.size, Size.from_sector(1).B) - check_stats_read_after_write(core_exported, mode, cls) + io_from_exported_object(front_vol, queue, test_data.size, Size.from_sector(1).B) + check_stats_read_after_write(core, mode, cls) - check_md5_sums(core_exported, mode) + check_md5_sums(core, mode) @pytest.mark.parametrize("cls", CacheLineSize) @@ -139,7 +147,7 @@ def test_start_params(pyocf_ctx, mode: CacheMode, cls: CacheLineSize, layout: Me Check if cache starts without errors. If possible check whether cache reports properly set parameters. """ - cache_device = Volume(Size.from_MiB(50)) + cache_device = RamVolume(Size.from_MiB(50)) queue_size = randrange(60000, 2**32) unblock_size = randrange(1, queue_size) volatile_metadata = randrange(2) == 1 @@ -176,20 +184,24 @@ def test_stop(pyocf_ctx, mode: CacheMode, cls: CacheLineSize, with_flush: bool): Check if cache is stopped properly in different modes with or without preceding flush operation. """ - cache_device = Volume(Size.from_MiB(50)) - core_device = Volume(Size.from_MiB(5)) + cache_device = RamVolume(Size.from_MiB(50)) + core_device = RamVolume(Size.from_MiB(5)) cache = Cache.start_on_device(cache_device, cache_mode=mode, cache_line_size=cls) - core_exported = Core.using_device(core_device) - cache.add_core(core_exported) + core = Core.using_device(core_device) + + cache.add_core(core) + front_vol = CoreVolume(core, open=True) + queue = cache.get_default_queue() + cls_no = 10 - run_io_and_cache_data_if_possible(core_exported, mode, cls, cls_no) + run_io_and_cache_data_if_possible(core, mode, cls, cls_no) stats = cache.get_stats() assert int(stats["conf"]["dirty"]) == (cls_no if mode.lazy_write() else 0),\ "Dirty data before MD5" - md5_exported_core = core_exported.exp_obj_md5() + md5_exported_core = front_vol.md5() if with_flush: cache.flush() @@ -211,7 +223,7 @@ def test_start_stop_multiple(pyocf_ctx): caches = [] caches_no = randrange(6, 11) for i in range(1, caches_no): - cache_device = Volume(Size.from_MiB(50)) + cache_device = RamVolume(Size.from_MiB(50)) cache_name = f"cache{i}" cache_mode = CacheMode(randrange(0, len(CacheMode))) size = 4096 * 2**randrange(0, len(CacheLineSize)) @@ -243,7 +255,7 @@ def test_100_start_stop(pyocf_ctx): """ for i in range(1, 101): - cache_device = Volume(Size.from_MiB(50)) + cache_device = RamVolume(Size.from_MiB(50)) cache_name = f"cache{i}" cache_mode = CacheMode(randrange(0, len(CacheMode))) size = 4096 * 2**randrange(0, len(CacheLineSize)) @@ -278,7 +290,7 @@ def test_start_stop_incrementally(pyocf_ctx): while run: if add: for i in range(0, randrange(3, 5) if increase else randrange(1, 3)): - cache_device = Volume(Size.from_MiB(50)) + cache_device = RamVolume(Size.from_MiB(50)) cache_name = f"cache{next(counter)}" cache_mode = CacheMode(randrange(0, len(CacheMode))) size = 4096 * 2**randrange(0, len(CacheLineSize)) @@ -318,8 +330,8 @@ def test_start_cache_same_id(pyocf_ctx, mode, cls): Check that OCF does not allow for 2 caches to be started with the same cache_name """ - cache_device1 = Volume(Size.from_MiB(50)) - cache_device2 = Volume(Size.from_MiB(50)) + cache_device1 = RamVolume(Size.from_MiB(50)) + cache_device2 = RamVolume(Size.from_MiB(50)) cache_name = "cache" cache = Cache.start_on_device(cache_device1, cache_mode=mode, @@ -349,7 +361,9 @@ def test_start_cache_huge_device(pyocf_ctx_log_buffer, cls): def submit_io(self, io): io.contents._end(io, 0) - cache_device = HugeDevice(Size.from_MiB(50)) + OcfCtx.get_default().register_volume_type(HugeDevice) + + cache_device = HugeDevice() with pytest.raises(OcfError, match="OCF_ERR_INVAL_CACHE_DEV"): cache = Cache.start_on_device(cache_device, cache_line_size=cls, metadata_volatile=True) @@ -367,7 +381,7 @@ def test_start_cache_same_device(pyocf_ctx, mode, cls): Check that OCF does not allow for 2 caches using the same cache device to be started """ - cache_device = Volume(Size.from_MiB(50)) + cache_device = RamVolume(Size.from_MiB(50)) cache = Cache.start_on_device( cache_device, cache_mode=mode, cache_line_size=cls, name="cache1" ) @@ -387,7 +401,7 @@ def test_start_too_small_device(pyocf_ctx, mode, cls): Check if starting cache with device below minimum size is blocked """ - cache_device = Volume(Size.from_B(20 * 1024 * 1024 - 1)) + cache_device = RamVolume(Size.from_B(20 * 1024 * 1024 - 1)) with pytest.raises(OcfError, match="OCF_ERR_INVAL_CACHE_DEV"): Cache.start_on_device(cache_device, cache_mode=mode, cache_line_size=cls) @@ -414,27 +428,29 @@ def test_start_stop_noqueue(pyocf_ctx): assert not c.results["error"], "Failed to stop cache: {}".format(c.results["error"]) -def run_io_and_cache_data_if_possible(exported_obj, mode, cls, cls_no): +def run_io_and_cache_data_if_possible(core, mode, cls, cls_no): + front_vol = core.get_front_volume() + bottom_vol = core.get_volume() + queue = core.cache.get_default_queue() + test_data = Data(cls_no * cls) if mode in {CacheMode.WI, CacheMode.WA}: logger.info("[STAGE] Write to core device") - io_to_core(exported_obj, test_data, 0, True) + io_to_core(bottom_vol, queue, test_data, 0) logger.info("[STAGE] Read from exported object") - io_from_exported_object(exported_obj, test_data.size, 0) + io_from_exported_object(front_vol, queue, test_data.size, 0) else: logger.info("[STAGE] Write to exported object") - io_to_core(exported_obj, test_data, 0) + io_to_core(front_vol, queue, test_data, 0) - stats = exported_obj.cache.get_stats() + stats = core.cache.get_stats() assert stats["usage"]["occupancy"]["value"] == \ ((cls_no * cls / CacheLineSize.LINE_4KiB) if mode != CacheMode.PT else 0), "Occupancy" -def io_to_core(exported_obj: Core, data: Data, offset: int, to_core_device=False): - new_io = exported_obj.new_core_io if to_core_device else exported_obj.new_io - io = new_io(exported_obj.cache.get_default_queue(), offset, data.size, - IoDir.WRITE, 0, 0) +def io_to_core(vol: Volume, queue: Queue, data: Data, offset: int): + io = vol.new_io(queue, offset, data.size, IoDir.WRITE, 0, 0) io.set_data(data) completion = OcfCompletion([("err", c_int)]) @@ -445,10 +461,9 @@ def io_to_core(exported_obj: Core, data: Data, offset: int, to_core_device=False assert completion.results["err"] == 0, "IO to exported object completion" -def io_from_exported_object(exported_obj: Core, buffer_size: int, offset: int): +def io_from_exported_object(vol: Volume, queue: Queue, buffer_size: int, offset: int): read_buffer = Data(buffer_size) - io = exported_obj.new_io(exported_obj.cache.get_default_queue(), offset, - read_buffer.size, IoDir.READ, 0, 0) + io = vol.new_io(queue, offset, read_buffer.size, IoDir.READ, 0, 0) io.set_data(read_buffer) completion = OcfCompletion([("err", c_int)]) @@ -460,28 +475,28 @@ def io_from_exported_object(exported_obj: Core, buffer_size: int, offset: int): return read_buffer -def check_stats_read_empty(exported_obj: Core, mode: CacheMode, cls: CacheLineSize): - exported_obj.cache.settle() - stats = exported_obj.cache.get_stats() +def check_stats_read_empty(core: Core, mode: CacheMode, cls: CacheLineSize): + core.cache.settle() + stats = core.cache.get_stats() assert stats["conf"]["cache_mode"] == mode, "Cache mode" - assert exported_obj.cache.device.get_stats()[IoDir.WRITE] == (1 if mode.read_insert() else 0), \ + assert core.cache.device.get_stats()[IoDir.WRITE] == (1 if mode.read_insert() else 0), \ "Writes to cache device" - assert exported_obj.device.get_stats()[IoDir.READ] == 1, "Reads from core device" + assert core.device.get_stats()[IoDir.READ] == 1, "Reads from core device" assert stats["req"]["rd_full_misses"]["value"] == (0 if mode == CacheMode.PT else 1), \ "Read full misses" assert stats["usage"]["occupancy"]["value"] == \ ((cls / CacheLineSize.LINE_4KiB) if mode.read_insert() else 0), "Occupancy" -def check_stats_write_empty(exported_obj: Core, mode: CacheMode, cls: CacheLineSize): - exported_obj.cache.settle() - stats = exported_obj.cache.get_stats() +def check_stats_write_empty(core: Core, mode: CacheMode, cls: CacheLineSize): + core.cache.settle() + stats = core.cache.get_stats() assert stats["conf"]["cache_mode"] == mode, "Cache mode" # TODO(ajrutkow): why 1 for WT ?? - assert exported_obj.cache.device.get_stats()[IoDir.WRITE] == \ + assert core.cache.device.get_stats()[IoDir.WRITE] == \ (2 if mode.lazy_write() else (1 if mode == CacheMode.WT else 0)), \ "Writes to cache device" - assert exported_obj.device.get_stats()[IoDir.WRITE] == (0 if mode.lazy_write() else 1), \ + assert core.device.get_stats()[IoDir.WRITE] == (0 if mode.lazy_write() else 1), \ "Writes to core device" assert stats["req"]["wr_full_misses"]["value"] == (1 if mode.write_insert() else 0), \ "Write full misses" @@ -490,17 +505,17 @@ def check_stats_write_empty(exported_obj: Core, mode: CacheMode, cls: CacheLineS "Occupancy" -def check_stats_write_after_read(exported_obj: Core, +def check_stats_write_after_read(core: Core, mode: CacheMode, cls: CacheLineSize, read_from_empty=False): - exported_obj.cache.settle() - stats = exported_obj.cache.get_stats() - assert exported_obj.cache.device.get_stats()[IoDir.WRITE] == \ + core.cache.settle() + stats = core.cache.get_stats() + assert core.cache.device.get_stats()[IoDir.WRITE] == \ (0 if mode in {CacheMode.WI, CacheMode.PT} else (2 if read_from_empty and mode.lazy_write() else 1)), \ "Writes to cache device" - assert exported_obj.device.get_stats()[IoDir.WRITE] == (0 if mode.lazy_write() else 1), \ + assert core.device.get_stats()[IoDir.WRITE] == (0 if mode.lazy_write() else 1), \ "Writes to core device" assert stats["req"]["wr_hits"]["value"] == \ (1 if (mode.read_insert() and mode != CacheMode.WI) @@ -511,17 +526,17 @@ def check_stats_write_after_read(exported_obj: Core, "Occupancy" -def check_stats_read_after_write(exported_obj, mode, cls, write_to_empty=False): - exported_obj.cache.settle() - stats = exported_obj.cache.get_stats() - assert exported_obj.cache.device.get_stats()[IoDir.WRITE] == \ +def check_stats_read_after_write(core, mode, cls, write_to_empty=False): + core.cache.settle() + stats = core.cache.get_stats() + assert core.cache.device.get_stats()[IoDir.WRITE] == \ (2 if mode.lazy_write() else (0 if mode == CacheMode.PT else 1)), \ "Writes to cache device" - assert exported_obj.cache.device.get_stats()[IoDir.READ] == \ + assert core.cache.device.get_stats()[IoDir.READ] == \ (1 if mode in {CacheMode.WT, CacheMode.WB, CacheMode.WO} or (mode == CacheMode.WA and not write_to_empty) else 0), \ "Reads from cache device" - assert exported_obj.device.get_stats()[IoDir.READ] == \ + assert core.device.get_stats()[IoDir.READ] == \ (0 if mode in {CacheMode.WB, CacheMode.WO, CacheMode.WT} or (mode == CacheMode.WA and not write_to_empty) else 1), \ "Reads from core device" @@ -537,15 +552,15 @@ def check_stats_read_after_write(exported_obj, mode, cls, write_to_empty=False): (0 if mode == CacheMode.PT else (cls / CacheLineSize.LINE_4KiB)), "Occupancy" -def check_md5_sums(exported_obj: Core, mode: CacheMode): +def check_md5_sums(core: Core, mode: CacheMode): if mode.lazy_write(): - assert exported_obj.device.md5() != exported_obj.exp_obj_md5(), \ + assert core.device.md5() != core.get_front_volume().md5(), \ "MD5 check: core device vs exported object without flush" - exported_obj.cache.flush() - assert exported_obj.device.md5() == exported_obj.exp_obj_md5(), \ + core.cache.flush() + assert core.device.md5() == core.get_front_volume().md5(), \ "MD5 check: core device vs exported object after flush" else: - assert exported_obj.device.md5() == exported_obj.exp_obj_md5(), \ + assert core.device.md5() == core.get_front_volume().md5(), \ "MD5 check: core device vs exported object" diff --git a/tests/functional/tests/security/test_management_fuzzy.py b/tests/functional/tests/security/test_management_fuzzy.py index 0abff54..77dfc0f 100644 --- a/tests/functional/tests/security/test_management_fuzzy.py +++ b/tests/functional/tests/security/test_management_fuzzy.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2019-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -18,7 +18,7 @@ from pyocf.types.cache import ( ConfValidValues, ) from pyocf.types.core import Core -from pyocf.types.volume import Volume +from pyocf.types.volume import RamVolume from pyocf.utils import Size as S from tests.utils.random import ( Range, @@ -41,7 +41,7 @@ def test_neg_change_cache_mode(pyocf_ctx, cm, cls): :param cls: cache line size we start with """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, cache_mode=cm, cache_line_size=cls) # Change cache mode to invalid one and check if failed @@ -65,7 +65,7 @@ def test_neg_set_cleaning_policy(pyocf_ctx, cm, cls): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, cache_mode=cm, cache_line_size=cls) # Set cleaning policy to invalid one and check if failed @@ -90,7 +90,7 @@ def test_neg_attach_cls(pyocf_ctx, cm, cls): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache(owner=cache_device.owner, cache_mode=cm, cache_line_size=cls) cache.start_cache() @@ -115,13 +115,13 @@ def test_neg_cache_set_seq_cut_off_policy(pyocf_ctx, cm, cls): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, cache_mode=cm, cache_line_size=cls) # Create 2 core devices - core_device1 = Volume(S.from_MiB(10)) + core_device1 = RamVolume(S.from_MiB(10)) core1 = Core.using_device(core_device1, name="core1") - core_device2 = Volume(S.from_MiB(10)) + core_device2 = RamVolume(S.from_MiB(10)) core2 = Core.using_device(core_device2, name="core2") # Add cores @@ -149,13 +149,13 @@ def test_neg_cache_set_seq_cut_off_promotion(pyocf_ctx, cm, cls): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, cache_mode=cm, cache_line_size=cls) # Create 2 core devices - core_device1 = Volume(S.from_MiB(10)) + core_device1 = RamVolume(S.from_MiB(10)) core1 = Core.using_device(core_device1, name="core1") - core_device2 = Volume(S.from_MiB(10)) + core_device2 = RamVolume(S.from_MiB(10)) core2 = Core.using_device(core_device2, name="core2") # Add cores @@ -185,11 +185,11 @@ def test_neg_core_set_seq_cut_off_promotion(pyocf_ctx, cm, cls): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, cache_mode=cm, cache_line_size=cls) # Create core device - core_device1 = Volume(S.from_MiB(10)) + core_device1 = RamVolume(S.from_MiB(10)) core1 = Core.using_device(core_device1, name="core1") # Add core @@ -218,13 +218,13 @@ def test_neg_cache_set_seq_cut_off_threshold(pyocf_ctx, cm, cls): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, cache_mode=cm, cache_line_size=cls) # Create 2 core devices - core_device1 = Volume(S.from_MiB(10)) + core_device1 = RamVolume(S.from_MiB(10)) core1 = Core.using_device(core_device1, name="core1") - core_device2 = Volume(S.from_MiB(10)) + core_device2 = RamVolume(S.from_MiB(10)) core2 = Core.using_device(core_device2, name="core2") # Add cores @@ -254,11 +254,11 @@ def test_neg_core_set_seq_cut_off_threshold(pyocf_ctx, cm, cls): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, cache_mode=cm, cache_line_size=cls) # Create core device - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core.using_device(core_device, name="core") # Add core @@ -287,11 +287,11 @@ def test_neg_core_set_seq_cut_off_policy(pyocf_ctx, cm, cls): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, cache_mode=cm, cache_line_size=cls) # Create core device - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core.using_device(core_device) # Add core @@ -318,7 +318,7 @@ def test_neg_set_alru_param(pyocf_ctx, cm, cls): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, cache_mode=cm, cache_line_size=cls) # Change invalid alru param and check if failed @@ -355,7 +355,7 @@ def test_neg_set_alru_param_value(pyocf_ctx, cm, cls, param): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, cache_mode=cm, cache_line_size=cls) cache.set_cleaning_policy(CleaningPolicy.ALRU) @@ -382,7 +382,7 @@ def test_neg_set_acp_param(pyocf_ctx, cm, cls): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, cache_mode=cm, cache_line_size=cls) # Change invalid acp param and check if failed @@ -415,7 +415,7 @@ def test_neg_set_acp_param_value(pyocf_ctx, cm, cls, param): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, cache_mode=cm, cache_line_size=cls) cache.set_cleaning_policy(CleaningPolicy.ACP) @@ -442,7 +442,7 @@ def test_neg_set_promotion_policy(pyocf_ctx, cm, cls): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, cache_mode=cm, cache_line_size=cls) # Change to invalid promotion policy and check if failed @@ -466,7 +466,7 @@ def test_neg_set_nhit_promotion_policy_param(pyocf_ctx, cm, cls): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device( cache_device, cache_mode=cm, @@ -496,7 +496,7 @@ def test_neg_set_nhit_promotion_policy_param_trigger(pyocf_ctx, cm, cls): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device( cache_device, cache_mode=cm, @@ -528,7 +528,7 @@ def test_neg_set_nhit_promotion_policy_param_threshold(pyocf_ctx, cm, cls): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device( cache_device, cache_mode=cm, @@ -559,7 +559,7 @@ def test_neg_set_ioclass_max_size(pyocf_ctx, cm, cls): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, cache_mode=cm, cache_line_size=cls) # Set invalid max size and check if failed @@ -589,7 +589,7 @@ def test_neg_set_ioclass_priority(pyocf_ctx, cm, cls): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, cache_mode=cm, cache_line_size=cls) # Set invalid priority and check if failed @@ -619,7 +619,7 @@ def test_neg_set_ioclass_cache_mode(pyocf_ctx, cm, cls): :return: """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, cache_mode=cm, cache_line_size=cls) # Set invalid cache mode and check if failed @@ -644,7 +644,7 @@ def test_neg_set_ioclass_name(pyocf_ctx): invalid_chars += [",", '"'] # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device( cache_device, cache_mode=CacheMode.WT, cache_line_size=CacheLineSize.LINE_4KiB ) @@ -669,7 +669,7 @@ def test_neg_set_ioclass_name_len(pyocf_ctx): """ # Start cache device - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device( cache_device, cache_mode=CacheMode.WT, cache_line_size=CacheLineSize.LINE_4KiB ) diff --git a/tests/functional/tests/security/test_management_start_fuzzy.py b/tests/functional/tests/security/test_management_start_fuzzy.py index 87296c3..6f8c540 100644 --- a/tests/functional/tests/security/test_management_start_fuzzy.py +++ b/tests/functional/tests/security/test_management_start_fuzzy.py @@ -9,7 +9,7 @@ import pytest from pyocf.types.cache import Cache, CacheMode, MetadataLayout, PromotionPolicy from pyocf.types.shared import OcfError, CacheLineSize -from pyocf.types.volume import Volume +from pyocf.types.volume import RamVolume from pyocf.utils import Size from tests.utils.random import RandomGenerator, DefaultRanges, Range @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) def try_start_cache(**config): - cache_device = Volume(Size.from_MiB(50)) + cache_device = RamVolume(Size.from_MiB(50)) cache = Cache.start_on_device(cache_device, **config) cache.stop() @@ -58,7 +58,7 @@ def test_fuzzy_start_name(pyocf_ctx, string_randomize, cm, cls): :param cm: cache mode value to start cache with :param cls: cache line size value to start cache with """ - cache_device = Volume(Size.from_MiB(50)) + cache_device = RamVolume(Size.from_MiB(50)) incorrect_values = [''] try: cache = Cache.start_on_device(cache_device, name=string_randomize, cache_mode=cm, diff --git a/tests/functional/tests/security/test_negative_io.py b/tests/functional/tests/security/test_negative_io.py index 3d1e3bb..0a80cf1 100644 --- a/tests/functional/tests/security/test_negative_io.py +++ b/tests/functional/tests/security/test_negative_io.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2019-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -11,8 +11,10 @@ import pytest from pyocf.types.cache import Cache, Core from pyocf.types.data import Data from pyocf.types.io import IoDir +from pyocf.types.queue import Queue from pyocf.types.shared import OcfCompletion -from pyocf.types.volume import Volume +from pyocf.types.volume import Volume, RamVolume +from pyocf.types.volume_core import CoreVolume from pyocf.utils import Size @@ -22,9 +24,9 @@ def test_neg_write_too_long_data(pyocf_ctx, c_uint16_randomize): Check if writing data larger than exported object size is properly blocked """ - core = prepare_cache_and_core(Size.from_MiB(1)) + vol, queue = prepare_cache_and_core(Size.from_MiB(1)) data = Data(int(Size.from_KiB(c_uint16_randomize))) - completion = io_operation(core, data, IoDir.WRITE) + completion = io_operation(vol, queue, data, IoDir.WRITE) if c_uint16_randomize > 1024: assert completion.results["err"] != 0 @@ -38,9 +40,9 @@ def test_neg_read_too_long_data(pyocf_ctx, c_uint16_randomize): Check if reading data larger than exported object size is properly blocked """ - core = prepare_cache_and_core(Size.from_MiB(1)) + vol, queue = prepare_cache_and_core(Size.from_MiB(1)) data = Data(int(Size.from_KiB(c_uint16_randomize))) - completion = io_operation(core, data, IoDir.READ) + completion = io_operation(vol, queue, data, IoDir.READ) if c_uint16_randomize > 1024: assert completion.results["err"] != 0 @@ -56,9 +58,9 @@ def test_neg_write_too_far(pyocf_ctx, c_uint16_randomize): """ limited_size = c_uint16_randomize % (int(Size.from_KiB(4)) + 1) - core = prepare_cache_and_core(Size.from_MiB(4)) + vol, queue = prepare_cache_and_core(Size.from_MiB(4)) data = Data(int(Size.from_KiB(limited_size))) - completion = io_operation(core, data, IoDir.WRITE, int(Size.from_MiB(3))) + completion = io_operation(vol, queue, data, IoDir.WRITE, int(Size.from_MiB(3))) if limited_size > 1024: assert completion.results["err"] != 0 @@ -74,9 +76,9 @@ def test_neg_read_too_far(pyocf_ctx, c_uint16_randomize): """ limited_size = c_uint16_randomize % (int(Size.from_KiB(4)) + 1) - core = prepare_cache_and_core(Size.from_MiB(4)) + vol, queue = prepare_cache_and_core(Size.from_MiB(4)) data = Data(int(Size.from_KiB(limited_size))) - completion = io_operation(core, data, IoDir.READ, offset=(Size.from_MiB(3))) + completion = io_operation(vol, queue, data, IoDir.READ, offset=(Size.from_MiB(3))) if limited_size > 1024: assert completion.results["err"] != 0 @@ -91,9 +93,9 @@ def test_neg_write_offset_outside_of_device(pyocf_ctx, c_int_sector_randomize): IO offset is located outside of device range """ - core = prepare_cache_and_core(Size.from_MiB(2)) + vol, queue = prepare_cache_and_core(Size.from_MiB(2)) data = Data(int(Size.from_KiB(1))) - completion = io_operation(core, data, IoDir.WRITE, offset=c_int_sector_randomize) + completion = io_operation(vol, queue, data, IoDir.WRITE, offset=c_int_sector_randomize) if 0 <= c_int_sector_randomize <= int(Size.from_MiB(2)) - int(Size.from_KiB(1)): assert completion.results["err"] == 0 @@ -108,9 +110,9 @@ def test_neg_read_offset_outside_of_device(pyocf_ctx, c_int_sector_randomize): IO offset is located outside of device range """ - core = prepare_cache_and_core(Size.from_MiB(2)) + vol, queue = prepare_cache_and_core(Size.from_MiB(2)) data = Data(int(Size.from_KiB(1))) - completion = io_operation(core, data, IoDir.READ, offset=c_int_sector_randomize) + completion = io_operation(vol, queue, data, IoDir.READ, offset=c_int_sector_randomize) if 0 <= c_int_sector_randomize <= int(Size.from_MiB(2)) - int(Size.from_KiB(1)): assert completion.results["err"] == 0 @@ -125,11 +127,11 @@ def test_neg_offset_unaligned(pyocf_ctx, c_int_randomize): IO offset is not aligned """ - core = prepare_cache_and_core(Size.from_MiB(2)) + vol, queue = prepare_cache_and_core(Size.from_MiB(2)) data = Data(int(Size.from_KiB(1))) if c_int_randomize % 512 != 0: with pytest.raises(Exception, match="Failed to create io!"): - core.new_io(core.cache.get_default_queue(), c_int_randomize, data.size, + vol.new_io(queue, c_int_randomize, data.size, IoDir.WRITE, 0, 0) @@ -140,11 +142,11 @@ def test_neg_size_unaligned(pyocf_ctx, c_uint16_randomize): IO size is not aligned """ - core = prepare_cache_and_core(Size.from_MiB(2)) + vol, queue = prepare_cache_and_core(Size.from_MiB(2)) data = Data(int(Size.from_B(c_uint16_randomize))) if c_uint16_randomize % 512 != 0: with pytest.raises(Exception, match="Failed to create io!"): - core.new_io(core.cache.get_default_queue(), 0, data.size, + vol.new_io(queue, 0, data.size, IoDir.WRITE, 0, 0) @@ -155,9 +157,9 @@ def test_neg_io_class(pyocf_ctx, c_int_randomize): number is not in allowed values {0, ..., 32} """ - core = prepare_cache_and_core(Size.from_MiB(2)) + vol, queue = prepare_cache_and_core(Size.from_MiB(2)) data = Data(int(Size.from_MiB(1))) - completion = io_operation(core, data, randrange(0, 2), io_class=c_int_randomize) + completion = io_operation(vol, queue, data, randrange(0, 2), io_class=c_int_randomize) if 0 <= c_int_randomize <= 32: assert completion.results["err"] == 0 @@ -172,9 +174,9 @@ def test_neg_io_direction(pyocf_ctx, c_int_randomize): that is when IO direction value is not in allowed values {0, 1} """ - core = prepare_cache_and_core(Size.from_MiB(2)) + vol, queue = prepare_cache_and_core(Size.from_MiB(2)) data = Data(int(Size.from_MiB(1))) - completion = io_operation(core, data, c_int_randomize) + completion = io_operation(vol, queue, data, c_int_randomize) if c_int_randomize in [0, 1]: assert completion.results["err"] == 0 @@ -183,19 +185,28 @@ def test_neg_io_direction(pyocf_ctx, c_int_randomize): def prepare_cache_and_core(core_size: Size, cache_size: Size = Size.from_MiB(50)): - cache_device = Volume(cache_size) - core_device = Volume(core_size) + cache_device = RamVolume(cache_size) + core_device = RamVolume(core_size) cache = Cache.start_on_device(cache_device) core = Core.using_device(core_device) cache.add_core(core) - return core + vol = CoreVolume(core, open=True) + queue = cache.get_default_queue() + + return vol, queue -def io_operation(core: Core, data: Data, io_direction: int, offset: int = 0, io_class: int = 0): - io = core.new_io(core.cache.get_default_queue(), offset, data.size, - io_direction, io_class, 0) +def io_operation( + vol: Volume, + queue: Queue, + data: Data, + io_direction: int, + offset: int = 0, + io_class: int = 0, +): + io = vol.new_io(queue, offset, data.size, io_direction, io_class, 0) io.set_data(data) completion = OcfCompletion([("err", c_int)]) diff --git a/tests/functional/tests/security/test_secure_erase.py b/tests/functional/tests/security/test_secure_erase.py index 8123f6c..ab51e2a 100644 --- a/tests/functional/tests/security/test_secure_erase.py +++ b/tests/functional/tests/security/test_secure_erase.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2019-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -8,7 +8,8 @@ from ctypes import c_int from pyocf.types.cache import Cache, CacheMode from pyocf.types.core import Core -from pyocf.types.volume import Volume +from pyocf.types.volume import Volume, RamVolume +from pyocf.types.volume_core import CoreVolume from pyocf.utils import Size as S from pyocf.types.data import Data, DataOps from pyocf.types.ctx import OcfCtx @@ -75,18 +76,20 @@ def test_secure_erase_simple_io_read_misses(cache_mode): Cleaner, ) - ctx.register_volume_type(Volume) + ctx.register_volume_type(RamVolume) - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, cache_mode=cache_mode) - core_device = Volume(S.from_MiB(50)) + core_device = RamVolume(S.from_MiB(50)) core = Core.using_device(core_device) cache.add_core(core) + vol = CoreVolume(core, open=True) + queue = cache.get_default_queue() write_data = DataCopyTracer(S.from_sector(1)) - io = core.new_io( - cache.get_default_queue(), + io = vol.new_io( + queue, S.from_sector(1).B, write_data.size, IoDir.WRITE, @@ -103,8 +106,8 @@ def test_secure_erase_simple_io_read_misses(cache_mode): cmpls = [] for i in range(100): read_data = DataCopyTracer(S.from_sector(1)) - io = core.new_io( - cache.get_default_queue(), + io = vol.new_io( + queue, i * S.from_sector(1).B, read_data.size, IoDir.READ, @@ -122,9 +125,7 @@ def test_secure_erase_simple_io_read_misses(cache_mode): c.wait() write_data = DataCopyTracer.from_string("TEST DATA" * 100) - io = core.new_io( - cache.get_default_queue(), S.from_sector(1), write_data.size, IoDir.WRITE, 0, 0 - ) + io = vol.new_io(queue, S.from_sector(1), write_data.size, IoDir.WRITE, 0, 0) io.set_data(write_data) cmpl = OcfCompletion([("err", c_int)]) @@ -147,7 +148,6 @@ def test_secure_erase_simple_io_read_misses(cache_mode): + stats["req"]["rd_full_misses"]["value"] ) > 0 - @pytest.mark.security def test_secure_erase_simple_io_cleaning(): """ @@ -168,19 +168,19 @@ def test_secure_erase_simple_io_cleaning(): Cleaner, ) - ctx.register_volume_type(Volume) + ctx.register_volume_type(RamVolume) - cache_device = Volume(S.from_MiB(50)) + cache_device = RamVolume(S.from_MiB(50)) cache = Cache.start_on_device(cache_device, cache_mode=CacheMode.WB) - core_device = Volume(S.from_MiB(100)) + core_device = RamVolume(S.from_MiB(100)) core = Core.using_device(core_device) cache.add_core(core) + vol = CoreVolume(core, open=True) + queue = cache.get_default_queue() read_data = Data(S.from_sector(1).B) - io = core.new_io( - cache.get_default_queue(), S.from_sector(1).B, read_data.size, IoDir.WRITE, 0, 0 - ) + io = vol.new_io(queue, S.from_sector(1).B, read_data.size, IoDir.WRITE, 0, 0) io.set_data(read_data) cmpl = OcfCompletion([("err", c_int)]) @@ -189,9 +189,7 @@ def test_secure_erase_simple_io_cleaning(): cmpl.wait() read_data = Data(S.from_sector(8).B) - io = core.new_io( - cache.get_default_queue(), S.from_sector(1).B, read_data.size, IoDir.READ, 0, 0 - ) + io = vol.new_io(queue, S.from_sector(1).B, read_data.size, IoDir.READ, 0, 0) io.set_data(read_data) cmpl = OcfCompletion([("err", c_int)]) diff --git a/tests/functional/tests/surprise_shutdown/test_management_surprise_shutdown.py b/tests/functional/tests/surprise_shutdown/test_management_surprise_shutdown.py index 4dc3483..1fb059a 100644 --- a/tests/functional/tests/surprise_shutdown/test_management_surprise_shutdown.py +++ b/tests/functional/tests/surprise_shutdown/test_management_surprise_shutdown.py @@ -17,7 +17,8 @@ from pyocf.types.cache import ( ) from pyocf.types.data import Data from pyocf.types.core import Core -from pyocf.types.volume import ErrorDevice, Volume +from pyocf.types.volume import ErrorDevice, RamVolume, VOLUME_POISON +from pyocf.types.volume_core import CoreVolume from pyocf.types.io import IoDir from pyocf.types.ioclass import IoClassesInfo, IoClassInfo from pyocf.utils import Size as S @@ -34,20 +35,20 @@ mngmt_op_surprise_shutdown_test_cache_size = S.from_MiB(40) mngmt_op_surprise_shutdown_test_io_offset = S.from_MiB(4).B -def ocf_write(cache, core, val, offset): +def ocf_write(vol, queue, val, offset): data = Data.from_bytes(bytes([val] * 512)) comp = OcfCompletion([("error", c_int)]) - io = core.new_io(cache.get_default_queue(), offset, 512, IoDir.WRITE, 0, 0) + io = vol.new_io(queue, offset, 512, IoDir.WRITE, 0, 0) io.set_data(data) io.callback = comp.callback io.submit() comp.wait() -def ocf_read(cache, core, offset): +def ocf_read(vol, queue, offset): data = Data(byte_count=512) comp = OcfCompletion([("error", c_int)]) - io = core.new_io(cache.get_default_queue(), offset, 512, IoDir.READ, 0, 0) + io = vol.new_io(queue, offset, 512, IoDir.READ, 0, 0) io.set_data(data) io.callback = comp.callback io.submit() @@ -118,7 +119,7 @@ def mngmt_op_surprise_shutdown_test( # power failure during core insert @pytest.mark.security def test_surprise_shutdown_add_core(pyocf_ctx): - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) def check_core(cache, error_triggered): stats = cache.get_stats() @@ -138,7 +139,7 @@ def test_surprise_shutdown_add_core(pyocf_ctx): @pytest.mark.security @pytest.mark.long def test_surprise_shutdown_remove_core(pyocf_ctx): - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core.using_device(core_device) def prepare_func(cache): @@ -158,12 +159,13 @@ def test_surprise_shutdown_remove_core(pyocf_ctx): @pytest.mark.long def test_surprise_shutdown_remove_core_with_data(pyocf_ctx): io_offset = mngmt_op_surprise_shutdown_test_io_offset - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core.using_device(core_device, name="core1") def prepare_func(cache): cache.add_core(core) - ocf_write(cache, core, 0xAA, io_offset) + vol = CoreVolume(core, open=True) + ocf_write(vol, cache.get_default_queue(), 0xAA, io_offset) def tested_func(cache): cache.flush() @@ -175,7 +177,8 @@ def test_surprise_shutdown_remove_core_with_data(pyocf_ctx): assert core_device.get_bytes()[io_offset] == 0xAA else: core = cache.get_core_by_name("core1") - assert ocf_read(cache, core, io_offset) == 0xAA + vol = CoreVolume(core, open=True) + assert ocf_read(vol, cache.get_default_queue(), io_offset) == 0xAA mngmt_op_surprise_shutdown_test(pyocf_ctx, tested_func, prepare_func, check_func) @@ -184,8 +187,8 @@ def test_surprise_shutdown_remove_core_with_data(pyocf_ctx): @pytest.mark.security @pytest.mark.long def test_surprise_shutdown_swap_core(pyocf_ctx): - core_device_1 = Volume(S.from_MiB(10), uuid="dev1") - core_device_2 = Volume(S.from_MiB(10), uuid="dev2") + core_device_1 = RamVolume(S.from_MiB(10), uuid="dev1") + core_device_2 = RamVolume(S.from_MiB(10), uuid="dev2") core1 = Core.using_device(core_device_1, name="core1") core2 = Core.using_device(core_device_2, name="core2") @@ -219,15 +222,16 @@ def test_surprise_shutdown_swap_core(pyocf_ctx): @pytest.mark.security @pytest.mark.long def test_surprise_shutdown_swap_core_with_data(pyocf_ctx): - core_device_1 = Volume(S.from_MiB(10), uuid="dev1") - core_device_2 = Volume(S.from_MiB(10), uuid="dev2") + core_device_1 = RamVolume(S.from_MiB(10), uuid="dev1") + core_device_2 = RamVolume(S.from_MiB(10), uuid="dev2") core1 = Core.using_device(core_device_1, name="core1") core2 = Core.using_device(core_device_2, name="core2") def prepare(cache): cache.add_core(core1) + vol = CoreVolume(core1, open=True) cache.save() - ocf_write(cache, core1, 0xAA, mngmt_op_surprise_shutdown_test_io_offset) + ocf_write(vol, cache.get_default_queue(), 0xAA, mngmt_op_surprise_shutdown_test_io_offset) cache.remove_core(core1) cache.save() @@ -249,10 +253,11 @@ def test_surprise_shutdown_swap_core_with_data(pyocf_ctx): core2 = cache.get_core_by_name("core2") if core2 is not None: + vol2 = CoreVolume(core2, open=True) assert core2.device.uuid == "dev2" assert ( - ocf_read(cache, core2, mngmt_op_surprise_shutdown_test_io_offset) - == Volume.VOLUME_POISON + ocf_read(vol2, cache.get_default_queue(), mngmt_op_surprise_shutdown_test_io_offset) + == VOLUME_POISON ) mngmt_op_surprise_shutdown_test(pyocf_ctx, tested_func, prepare, check_func) @@ -312,7 +317,7 @@ def test_surprise_shutdown_start_cache(pyocf_ctx): @pytest.mark.security @pytest.mark.long def test_surprise_shutdown_stop_cache(pyocf_ctx): - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) error_triggered = True error_io_seq_no = 0 io_offset = mngmt_op_surprise_shutdown_test_io_offset @@ -328,7 +333,8 @@ def test_surprise_shutdown_stop_cache(pyocf_ctx): cache = Cache.start_on_device(device, cache_mode=CacheMode.WB) core = Core(device=core_device) cache.add_core(core) - ocf_write(cache, core, 0xAA, io_offset) + vol = CoreVolume(core, open=True) + ocf_write(vol, cache.get_default_queue(), 0xAA, io_offset) # start error injection device.arm() @@ -353,7 +359,7 @@ def test_surprise_shutdown_stop_cache(pyocf_ctx): device.disarm() cache = None - assert core_device.get_bytes()[io_offset] == Volume.VOLUME_POISON + assert core_device.get_bytes()[io_offset] == VOLUME_POISON cache = Cache.load_from_device(device, open_cores=False) stats = cache.get_stats() @@ -361,7 +367,8 @@ def test_surprise_shutdown_stop_cache(pyocf_ctx): assert stats["usage"]["occupancy"]["value"] == 1 core = Core(device=core_device) cache.add_core(core, try_add=True) - assert ocf_read(cache, core, io_offset) == 0xAA + vol = CoreVolume(core, open=True) + assert ocf_read(vol, cache.get_default_queue(), io_offset) == 0xAA cache.stop() @@ -371,7 +378,7 @@ def test_surprise_shutdown_stop_cache(pyocf_ctx): @pytest.mark.security def test_surprise_shutdown_cache_reinit(pyocf_ctx): - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) error_io = {IoDir.WRITE: 0} @@ -388,13 +395,15 @@ def test_surprise_shutdown_cache_reinit(pyocf_ctx): cache = Cache.start_on_device(device, cache_mode=CacheMode.WB) core = Core(device=core_device) cache.add_core(core) + vol = CoreVolume(core, open=True) + queue = cache.get_default_queue() # insert dirty cacheline - ocf_write(cache, core, 0xAA, io_offset) + ocf_write(vol, queue, 0xAA, io_offset) cache.stop() - assert core_device.get_bytes()[io_offset] == Volume.VOLUME_POISON + assert core_device.get_bytes()[io_offset] == VOLUME_POISON # start error injection device.arm() @@ -432,7 +441,8 @@ def test_surprise_shutdown_cache_reinit(pyocf_ctx): if stats["conf"]["core_count"] == 0: assert stats["usage"]["occupancy"]["value"] == 0 cache.add_core(core) - assert ocf_read(cache, core, io_offset) == Volume.VOLUME_POISON + vol = CoreVolume(core, open=True) + assert ocf_read(vol, cache.get_default_queue(), io_offset) == VOLUME_POISON cache.stop() @@ -440,7 +450,7 @@ def test_surprise_shutdown_cache_reinit(pyocf_ctx): def _test_surprise_shutdown_mngmt_generic(pyocf_ctx, func): - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core(device=core_device) def prepare(cache): @@ -464,7 +474,7 @@ def test_surprise_shutdown_change_cache_mode(pyocf_ctx): @pytest.mark.security @pytest.mark.long def test_surprise_shutdown_set_cleaning_policy(pyocf_ctx): - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core(device=core_device) for c1 in CleaningPolicy: @@ -485,7 +495,7 @@ def test_surprise_shutdown_set_cleaning_policy(pyocf_ctx): @pytest.mark.security @pytest.mark.long def test_surprise_shutdown_set_seq_cut_off_policy(pyocf_ctx): - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core(device=core_device) for s1 in SeqCutOffPolicy: @@ -522,7 +532,7 @@ def test_surprise_shutdown_set_seq_cut_off_threshold(pyocf_ctx): @pytest.mark.security @pytest.mark.long def test_surprise_shutdown_set_cleaning_policy_param(pyocf_ctx): - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core(device=core_device) for pol in CleaningPolicy: @@ -574,7 +584,7 @@ def test_surprise_shutdown_set_cleaning_policy_param(pyocf_ctx): @pytest.mark.security @pytest.mark.long def test_surprise_shutdown_set_promotion_policy(pyocf_ctx): - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core(device=core_device) for pp1 in PromotionPolicy: @@ -595,7 +605,7 @@ def test_surprise_shutdown_set_promotion_policy(pyocf_ctx): @pytest.mark.security @pytest.mark.long def test_surprise_shutdown_set_promotion_policy_param(pyocf_ctx): - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core(device=core_device) for pp in PromotionPolicy: @@ -633,7 +643,7 @@ def test_surprise_shutdown_set_promotion_policy_param(pyocf_ctx): @pytest.mark.security @pytest.mark.long def test_surprise_shutdown_set_io_class_config(pyocf_ctx): - core_device = Volume(S.from_MiB(10)) + core_device = RamVolume(S.from_MiB(10)) core = Core(device=core_device) class_range = range(0, IoClassesInfo.MAX_IO_CLASSES)