Implement pyocf adapter with sample OCF test

PyOCF is a tool written with testing OCF functionality in mind.
It is a Python3 (3.6 version required) package which wraps OCF
by providing Python objects in place of OCF objects (volumes, queues,
etc). Thin layer of translation between OCF objects and PyOCF objects
enables using customized behaviors for OCF primitives by subclassing
PyOCF classes.

This initial version implements only WT and WI modes and single,
synchronously operating Queue.

TO DO:

  - Queues/Cleaner/MetadataUpdater implemented as Python threads
  - Loading of caches from PyOCF Volumes (fix bugs in OCF)
  - Make sure it works multi-threaded for more sophisticated tests

Co-authored-by: Jan Musial <jan.musial@intel.com>
Signed-off-by: Michal Rakowski <michal.rakowski@intel.com>
Signed-off-by: Jan Musial <jan.musial@intel.com>

tests/functional/pyocf/ocf.py

@@ -0,0 +1,25 @@
+#
+# Copyright(c) 2019 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+from ctypes import *
+
+lib = None
+
+
+class OcfLib:
+    __lib__ = None
+
+    @classmethod
+    def getInstance(cls):
+        if cls.__lib__ is None:
+            lib = cdll.LoadLibrary("./pyocf/libocf.so")
+            lib.ocf_volume_get_uuid.restype = c_void_p
+            lib.ocf_volume_get_uuid.argtypes = [c_void_p]
+
+            lib.ocf_core_get_front_volume.restype = c_void_p
+            lib.ocf_core_get_front_volume.argtypes = [c_void_p]
+
+            cls.__lib__ = lib
+
+        return cls.__lib__

tests/functional/pyocf/types/cache.py

@@ -0,0 +1,312 @@
+#
+# Copyright(c) 2019 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+from ctypes import *
+from enum import IntEnum
+import logging
+from datetime import timedelta
+
+from .shared import Uuid, OcfError, CacheLineSize, CacheLines
+from ..utils import Size, struct_to_dict
+from .core import Core
+from .stats.cache import *
+from .stats.shared import *
+
+
+class Backfill(Structure):
+    _fields_ = [("_max_queue_size", c_uint32), ("_queue_unblock_size", c_uint32)]
+
+
+class CacheConfig(Structure):
+    _fields_ = [
+        ("_id", c_uint16),
+        ("_name", c_char_p),
+        ("_cache_mode", c_uint32),
+        ("_eviction_policy", c_uint32),
+        ("_cache_line_size", c_uint64),
+        ("_metadata_layout", c_uint32),
+        ("_metadata_volatile", c_bool),
+        ("_backfill", Backfill),
+        ("_locked", c_bool),
+        ("_pt_unaligned_io", c_bool),
+        ("_use_submit_io_fast", c_bool),
+    ]
+
+
+class CacheDeviceConfig(Structure):
+    _fields_ = [
+        ("_uuid", Uuid),
+        ("_volume_type", c_uint8),
+        ("_cache_line_size", c_uint64),
+        ("_force", c_bool),
+        ("_min_free_ram", c_uint64),
+        ("_perform_test", c_bool),
+        ("_discard_on_start", c_bool),
+    ]
+
+
+class CacheMode(IntEnum):
+    WT = 0
+    WB = 1
+    WA = 2
+    PT = 3
+    WI = 4
+    DEFAULT = WT
+
+
+class EvictionPolicy(IntEnum):
+    LRU = 0
+    DEFAULT = LRU
+
+
+class CleaningPolicy(IntEnum):
+    NOP = 0
+    ALRU = 1
+    ACP = 2
+    DEFAULT = ALRU
+
+
+class MetadataLayout(IntEnum):
+    STRIPING = 0
+    SEQUENTIAL = 1
+    DEFAULT = STRIPING
+
+
+class Cache:
+    DEFAULT_ID = 0
+    DEFAULT_BACKFILL_QUEUE_SIZE = 65536
+    DEFAULT_BACKFILL_UNBLOCK = 60000
+    DEFAULT_PT_UNALIGNED_IO = False
+    DEFAULT_USE_SUBMIT_FAST = False
+
+    def __init__(
+        self,
+        owner,
+        cache_id: int = DEFAULT_ID,
+        name: str = "",
+        cache_mode: CacheMode = CacheMode.DEFAULT,
+        eviction_policy: EvictionPolicy = EvictionPolicy.DEFAULT,
+        cache_line_size: CacheLineSize = CacheLineSize.DEFAULT,
+        metadata_layout: MetadataLayout = MetadataLayout.DEFAULT,
+        metadata_volatile: bool = False,
+        max_queue_size: int = DEFAULT_BACKFILL_QUEUE_SIZE,
+        queue_unblock_size: int = DEFAULT_BACKFILL_UNBLOCK,
+        locked: bool = True,
+        pt_unaligned_io: bool = DEFAULT_PT_UNALIGNED_IO,
+        use_submit_fast: bool = DEFAULT_USE_SUBMIT_FAST,
+    ):
+
+        self.owner = owner
+        self.cache_line_size = cache_line_size
+        self.cfg = CacheConfig(
+            _id=cache_id,
+            _name=name.encode("ascii") if name else None,
+            _cache_mode=cache_mode,
+            _eviction_policy=eviction_policy,
+            _cache_line_size=cache_line_size,
+            _metadata_layout=metadata_layout,
+            _metadata_volatile=metadata_volatile,
+            _backfill=Backfill(
+                _max_queue_size=max_queue_size, _queue_unblock_size=queue_unblock_size
+            ),
+            _locked=locked,
+            _pt_unaligned_io=pt_unaligned_io,
+            _use_submit_fast=use_submit_fast,
+        )
+        self.cache_handle = c_void_p()
+        self.queues = []
+
+    def start_cache(self):
+        status = self.owner.lib.ocf_mngt_cache_start(
+            self.owner.ctx_handle, byref(self.cache_handle), byref(self.cfg)
+        )
+        if status:
+            raise OcfError("Creating cache instance failed", status)
+        self.owner.caches += [self]
+
+    def configure_device(
+        self, device, force=False, perform_test=False, cache_line_size=None
+    ):
+        self.device_name = device.uuid
+        self.dev_cfg = CacheDeviceConfig(
+            _uuid=Uuid(
+                _data=cast(
+                    create_string_buffer(self.device_name.encode("ascii")), c_char_p
+                ),
+                _size=len(self.device_name) + 1,
+            ),
+            _volume_type=device.type_id,
+            _cache_line_size=cache_line_size
+            if cache_line_size
+            else self.cache_line_size,
+            _force=force,
+            _min_free_ram=0,
+            _perform_test=perform_test,
+            _discard_on_start=False,
+        )
+
+    def attach_device(
+        self, device, force=False, perform_test=False, cache_line_size=None,
+    ):
+        self.configure_device(device, force, perform_test, cache_line_size)
+
+        status = device.owner.lib.ocf_mngt_cache_attach(
+            self.cache_handle, byref(self.dev_cfg)
+        )
+        if status:
+            raise OcfError("Attaching cache device failed", status)
+
+    def load_cache(self, device):
+        self.configure_device(device)
+
+        status = device.owner.lib.ocf_mngt_cache_load(
+            self.owner.ctx_handle,
+            byref(self.cache_handle),
+            byref(self.cfg),
+            byref(self.dev_cfg),
+        )
+        if status:
+            raise OcfError("Loading cache device failed", status)
+
+    @classmethod
+    def load_from_device(cls, device, name=""):
+        c = cls(name=name, owner=device.owner)
+        c.load_cache(device)
+        return c
+
+    @classmethod
+    def start_on_device(cls, device, **kwargs):
+        c = cls(locked=True, owner=device.owner, **kwargs)
+
+        c.start_cache()
+        c.attach_device(device, force=True)
+        return c
+
+    def _get_and_lock(self, read=True):
+        status = self.owner.lib.ocf_mngt_cache_get(self.cache_handle)
+        if status:
+            raise OcfError("Couldn't get cache instance", status)
+
+        if read:
+            status = self.owner.lib.ocf_mngt_cache_read_lock(self.cache_handle)
+        else:
+            status = self.owner.lib.ocf_mngt_cache_lock(self.cache_handle)
+
+        if status:
+            self.owner.lib.ocf_mngt_cache_put(self.cache_handle)
+            raise OcfError("Couldn't lock cache instance", status)
+
+    def _put_and_unlock(self, read=True):
+        if read:
+            self.owner.lib.ocf_mngt_cache_read_unlock(self.cache_handle)
+        else:
+            self.owner.lib.ocf_mngt_cache_unlock(self.cache_handle)
+
+        self.owner.lib.ocf_mngt_cache_put(self.cache_handle)
+
+    def get_and_read_lock(self):
+        self._get_and_lock(True)
+
+    def get_and_write_lock(self):
+        self._get_and_lock(False)
+
+    def put_and_read_unlock(self):
+        self._put_and_unlock(True)
+
+    def put_and_write_unlock(self):
+        self._put_and_unlock(False)
+
+    def add_core(self, core: Core):
+        self.get_and_write_lock()
+
+        status = self.owner.lib.ocf_mngt_cache_add_core(
+            self.cache_handle, byref(core.get_handle()), byref(core.get_cfg())
+        )
+
+        if status:
+            self.put_and_write_unlock()
+            raise OcfError("Failed adding core", status)
+
+        core.cache = self
+
+        self.put_and_write_unlock()
+
+    def get_stats(self):
+        cache_info = CacheInfo()
+        usage = UsageStats()
+        req = RequestsStats()
+        block = BlocksStats()
+        errors = ErrorsStats()
+
+        self.get_and_read_lock()
+
+        status = self.owner.lib.ocf_cache_get_info(self.cache_handle, byref(cache_info))
+        if status:
+            self.put_and_read_unlock()
+            raise OcfError("Failed getting cache info", status)
+
+        status = self.owner.lib.ocf_stats_collect_cache(
+            self.cache_handle, byref(usage), byref(req), byref(block), byref(errors)
+        )
+        if status:
+            self.put_and_read_unlock()
+            raise OcfError("Failed getting stats", status)
+
+        line_size = CacheLineSize(cache_info.cache_line_size)
+
+        self.put_and_read_unlock()
+        return {
+            "conf": {
+                "attached": cache_info.attached,
+                "volume_type": self.owner.volume_types[cache_info.volume_type],
+                "size": CacheLines(cache_info.size, line_size),
+                "inactive": {
+                    "occupancy": CacheLines(cache_info.inactive.occupancy, line_size),
+                    "dirty": CacheLines(cache_info.inactive.dirty, line_size),
+                },
+                "occupancy": CacheLines(cache_info.occupancy, line_size),
+                "dirty": CacheLines(cache_info.dirty, line_size),
+                "dirty_initial": CacheLines(cache_info.dirty_initial, line_size),
+                "dirty_for": timedelta(seconds=cache_info.dirty_for),
+                "cache_mode": CacheMode(cache_info.cache_mode),
+                "fallback_pt": {
+                    "error_counter": cache_info.fallback_pt.error_counter,
+                    "status": cache_info.fallback_pt.status,
+                },
+                "state": cache_info.state,
+                "eviction_policy": EvictionPolicy(cache_info.eviction_policy),
+                "cleaning_policy": CleaningPolicy(cache_info.cleaning_policy),
+                "cache_line_size": line_size,
+                "flushed": CacheLines(cache_info.flushed, line_size),
+                "core_count": cache_info.core_count,
+                "metadata_footprint": Size(cache_info.metadata_footprint),
+                "metadata_end_offset": Size(cache_info.metadata_end_offset),
+            },
+            "block": struct_to_dict(block),
+            "req": struct_to_dict(req),
+            "usage": struct_to_dict(usage),
+            "errors": struct_to_dict(errors),
+        }
+
+    def reset_stats(self):
+        self.owner.lib.ocf_core_stats_initialize_all(self.cache_handle)
+
+    def get_default_queue(self):
+        if not self.queues:
+            raise Exception("No queues added for cache")
+
+        return self.queues[0]
+
+
+    def stop(self):
+        self.get_and_write_lock()
+
+        status = self.owner.lib.ocf_mngt_cache_stop(self.cache_handle)
+        if status:
+            self.put_and_write_unlock()
+            raise OcfError("Failed stopping cache", status)
+
+        self.put_and_write_unlock()
+        self.owner.caches.remove(self)

tests/functional/pyocf/types/cleaner.py

@@ -0,0 +1,37 @@
+#
+# Copyright(c) 2019 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+from ctypes import *
+from .shared import SharedOcfObject
+
+
+class CleanerOps(Structure):
+    INIT = CFUNCTYPE(c_int, c_void_p)
+    STOP = CFUNCTYPE(None, c_void_p)
+
+    _fields_ = [("init", INIT), ("stop", STOP)]
+
+
+class Cleaner(SharedOcfObject):
+    _instances_ = {}
+    _fields_ = [("cleaner", c_void_p)]
+
+    def __init__(self):
+        self._as_parameter_ = self.cleaner
+        super().__init__()
+
+    @classmethod
+    def get_ops(cls):
+        return CleanerOps(init=cls._init, stop=cls._stop)
+
+    @staticmethod
+    @CleanerOps.INIT
+    def _init(cleaner):
+        return 0
+
+    @staticmethod
+    @CleanerOps.STOP
+    def _stop(cleaner):
+        pass

tests/functional/pyocf/types/core.py

@@ -0,0 +1,145 @@
+#
+# Copyright(c) 2019 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+from ctypes import *
+import logging
+from datetime import timedelta
+
+from ..ocf import OcfLib
+from .shared import Uuid
+from .volume import Volume
+from .data import Data
+from .io import Io, IoDir
+from .stats.shared import *
+from .stats.core import *
+from ..utils import Size, struct_to_dict
+from .queue import Queue
+
+
+class UserMetadata(Structure):
+    _fields_ = [("data", c_void_p), ("size", c_size_t)]
+
+
+class CoreConfig(Structure):
+    _fields_ = [
+        ("_uuid", Uuid),
+        ("_volume_type", c_uint8),
+        ("_core_id", c_uint16),
+        ("_name", c_char_p),
+        ("_cache_id", c_uint16),
+        ("_try_add", c_bool),
+        ("_seq_cutoff_threshold", c_uint32),
+        ("_user_metadata", UserMetadata),
+    ]
+
+
+class Core:
+    DEFAULT_ID = 4096
+    DEFAULT_SEQ_CUTOFF_THRESHOLD = 1024 * 1024
+
+    def __init__(
+        self,
+        device: Volume,
+        try_add: bool,
+        name: str = "",
+        core_id: int = DEFAULT_ID,
+        seq_cutoff_threshold: int = DEFAULT_SEQ_CUTOFF_THRESHOLD,
+    ):
+
+        self.device = device
+        self.device_name = device.uuid
+        self.core_id = core_id
+        self.handle = c_void_p()
+        self.cfg = CoreConfig(
+            _uuid=Uuid(
+                _data=cast(
+                    create_string_buffer(self.device_name.encode("ascii")), c_char_p
+                ),
+                _size=len(self.device_name) + 1,
+            ),
+            _core_id=self.core_id,
+            _name=name.encode("ascii") if name else None,
+            _volume_type=self.device.type_id,
+            _try_add=try_add,
+            _seq_cutoff_threshold=seq_cutoff_threshold,
+            _user_metadata=UserMetadata(_data=None, _size=0),
+        )
+
+    @classmethod
+    def using_device(cls, device, **kwargs):
+        c = cls(device=device, try_add=False, **kwargs)
+
+        return c
+
+    def get_cfg(self):
+        return self.cfg
+
+    def get_handle(self):
+        return self.handle
+
+    def new_io(self):
+        if not self.cache:
+            raise Exception("Core isn't attached to any cache")
+
+        io = OcfLib.getInstance().ocf_core_new_io_wrapper(self.handle)
+        return Io.from_pointer(io)
+
+    def new_core_io(self):
+        lib = OcfLib.getInstance()
+        core = lib.ocf_core_get_volume(self.handle)
+        io = lib.ocf_volume_new_io(core)
+        return Io.from_pointer(io)
+
+    def get_stats(self):
+        core_stats = CoreStats()
+        usage = UsageStats()
+        req = RequestsStats()
+        blocks = BlocksStats()
+        errors = ErrorsStats()
+
+        self.cache.get_and_lock(True)
+
+        status = self.cache.owner.lib.ocf_stats_collect_core(
+            self.handle, byref(usage), byref(req), byref(blocks), byref(errors)
+        )
+        if status:
+            self.cache.put_and_unlock(True)
+            raise OcfError("Failed collecting core stats", status)
+
+        status = self.cache.owner.lib.ocf_core_get_stats(self.handle, byref(core_stats))
+        if status:
+            self.cache.put_and_unlock(True)
+            raise OcfError("Failed getting core stats", status)
+
+        self.cache.put_and_unlock(True)
+        return {
+            "size": Size(core_stats.core_size_bytes),
+            "dirty_for": timedelta(seconds=core_stats.dirty_for),
+            "usage": struct_to_dict(usage),
+            "req": struct_to_dict(req),
+            "blocks": struct_to_dict(blocks),
+            "errors": struct_to_dict(errors),
+        }
+
+    def reset_stats(self):
+        self.cache.owner.lib.ocf_core_stats_initialize(self.handle)
+
+    def exp_obj_md5(self):
+        logging.getLogger("pyocf").warning(
+            "Reading whole exported object! This disturbs statistics values"
+        )
+        read_buffer = Data(self.device.size)
+        io = self.new_io()
+        io.configure(0, read_buffer.size, IoDir.READ, 0, 0)
+        io.set_data(read_buffer)
+        io.set_queue(self.cache.get_default_queue())
+        io.submit()
+        return read_buffer.md5()
+
+
+lib = OcfLib.getInstance()
+lib.ocf_core_get_volume.restype = c_void_p
+lib.ocf_volume_new_io.argtypes = [c_void_p]
+lib.ocf_volume_new_io.restype = c_void_p

tests/functional/pyocf/types/ctx.py

@@ -0,0 +1,85 @@
+#
+# Copyright(c) 2019 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+from ctypes import *
+from enum import IntEnum
+
+from .logger import LoggerOps, Logger
+from .data import DataOps, Data
+from .queue import Queue
+from .cleaner import CleanerOps, Cleaner
+from .metadata_updater import MetadataUpdaterOps, MetadataUpdater
+from .shared import OcfError
+from ..ocf import OcfLib
+
+
+class OcfCtxOps(Structure):
+    _fields_ = [
+        ("data", DataOps),
+        ("cleaner", CleanerOps),
+        ("metadata_updater", MetadataUpdaterOps),
+        ("logger", LoggerOps),
+    ]
+
+
+class OcfCtxCfg(Structure):
+    _fields_ = [("name", c_char_p), ("ops", OcfCtxOps), ("logger_priv", c_void_p)]
+
+
+class OcfCtx:
+    def __init__(self, lib, name, logger, data, mu, cleaner):
+        self.logger = logger
+        self.data = data
+        self.mu = mu
+        self.cleaner = cleaner
+        self.ctx_handle = c_void_p()
+        self.lib = lib
+        self.volume_types_count = 1
+        self.volume_types = {}
+        self.caches = []
+
+        self.cfg = OcfCtxCfg(
+            name=name,
+            ops=OcfCtxOps(
+                data=self.data.get_ops(),
+                cleaner=self.cleaner.get_ops(),
+                metadata_updater=self.mu.get_ops(),
+                logger=logger.get_ops(),
+            ),
+            logger_priv=cast(pointer(logger.get_priv()), c_void_p),
+        )
+
+        result = self.lib.ocf_ctx_init(byref(self.ctx_handle), byref(self.cfg))
+        if result != 0:
+            raise OcfError("Context initialization failed", result)
+
+    def register_volume_type(self, volume_type):
+        self.volume_types[self.volume_types_count] = volume_type.get_props()
+        volume_type.type_id = self.volume_types_count
+        volume_type.owner = self
+
+        result = self.lib.ocf_ctx_register_volume_type(
+            self.ctx_handle,
+            self.volume_types_count,
+            byref(self.volume_types[self.volume_types_count]),
+        )
+        if result != 0:
+            raise OcfError("Data object registration failed", result)
+
+        self.volume_types_count += 1
+
+    def exit(self):
+        self.lib.ocf_ctx_exit(self.ctx_handle)
+
+
+def get_default_ctx(logger):
+    return OcfCtx(
+        OcfLib.getInstance(),
+        b"PyOCF default ctx",
+        logger,
+        Data,
+        MetadataUpdater,
+        Cleaner,
+    )

tests/functional/pyocf/types/data.py

@@ -0,0 +1,203 @@
+#
+# Copyright(c) 2019 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+from ctypes import *
+from enum import IntEnum
+from hashlib import md5
+
+from .shared import SharedOcfObject
+from ..utils import print_buffer
+
+
+class DataSeek(IntEnum):
+    BEGIN = 0
+    CURRENT = 1
+
+
+class DataOps(Structure):
+    ALLOC = CFUNCTYPE(c_void_p, c_uint32)
+    FREE = CFUNCTYPE(None, c_void_p)
+    MLOCK = CFUNCTYPE(c_int, c_void_p)
+    MUNLOCK = CFUNCTYPE(None, c_void_p)
+    READ = CFUNCTYPE(c_uint32, c_void_p, c_void_p, c_uint32)
+    WRITE = CFUNCTYPE(c_uint32, c_void_p, c_void_p, c_uint32)
+    ZERO = CFUNCTYPE(c_uint32, c_void_p, c_uint32)
+    SEEK = CFUNCTYPE(c_uint32, c_void_p, c_uint32, c_uint32)
+    COPY = CFUNCTYPE(c_uint64, c_void_p, c_void_p, c_uint64, c_uint64, c_uint64)
+    SECURE_ERASE = CFUNCTYPE(None, c_void_p)
+
+    _fields_ = [
+        ("_alloc", ALLOC),
+        ("_free", FREE),
+        ("_mlock", MLOCK),
+        ("_munlock", MUNLOCK),
+        ("_read", READ),
+        ("_write", WRITE),
+        ("_zero", ZERO),
+        ("_seek", SEEK),
+        ("_copy", COPY),
+        ("_secure_erase", SECURE_ERASE),
+    ]
+
+
+class Data(SharedOcfObject):
+    PAGE_SIZE = 4096
+
+    _instances_ = {}
+
+    _fields_ = [("data", c_void_p)]
+
+    def __init__(self, byte_count: int):
+        self.size = byte_count
+        self.position = 0
+        self.buffer = create_string_buffer(int(self.size))
+        self.data = cast(self.buffer, c_void_p)
+        memset(self.data, 0, self.size)
+        type(self)._instances_[self.data] = self
+        self._as_parameter_ = self.data
+
+        super().__init__()
+
+    @classmethod
+    def get_ops(cls):
+        return DataOps(
+            _alloc=cls._alloc,
+            _free=cls._free,
+            _mlock=cls._mlock,
+            _munlock=cls._munlock,
+            _read=cls._read,
+            _write=cls._write,
+            _zero=cls._zero,
+            _seek=cls._seek,
+            _copy=cls._copy,
+            _secure_erase=cls._secure_erase,
+        )
+
+    @classmethod
+    def pages(cls, pages: int):
+        return cls(pages * Data.PAGE_SIZE)
+
+    @classmethod
+    def from_bytes(cls, source: bytes):
+        d = cls(len(source))
+
+        memmove(d.data, cast(source, c_void_p), len(source))
+
+        return d
+
+    @classmethod
+    def from_string(cls, source: str, encoding: str = "ascii"):
+        return cls.from_bytes(bytes(source, encoding))
+
+    def __str__(self):
+        char_array = cast(self.data, c_char_p)
+        return str(char_array.value, "ascii")
+
+    def __wstr__(self):
+        char_array = cast(self.data, c_wchar_p)
+        return str(char_array.value, "utf-8")
+
+    def set_data(self, contents):
+        if len(contents) > self.size:
+            raise Exception("Data too big to fit into allocated buffer")
+
+        memmove(self.data, cast(contents, c_void_p), len(contents))
+        self.position = 0
+
+    @staticmethod
+    @DataOps.ALLOC
+    def _alloc(pages):
+        data = Data.pages(pages)
+        return data.data
+
+    @staticmethod
+    @DataOps.FREE
+    def _free(data):
+        Data.del_object(data)
+
+    @staticmethod
+    @DataOps.MLOCK
+    def _mlock(ref):
+        return Data.get_instance(ref).mlock()
+
+    @staticmethod
+    @DataOps.MUNLOCK
+    def _munlock(ref):
+        Data.get_instance(ref).munlock()
+
+    @staticmethod
+    @DataOps.READ
+    def _read(dst, src, size):
+        return Data.get_instance(src).read(dst, size)
+
+    @staticmethod
+    @DataOps.WRITE
+    def _write(dst, src, size):
+        return Data.get_instance(dst).write(src, size)
+
+    @staticmethod
+    @DataOps.ZERO
+    def _zero(dst, size):
+        return Data.get_instance(dst).zero(size)
+
+    @staticmethod
+    @DataOps.SEEK
+    def _seek(dst, seek, size):
+        return Data.get_instance(dst).seek(DataSeek(seek), size)
+
+    @staticmethod
+    @DataOps.COPY
+    def _copy(dst, src, end, start, size):
+        return Data.get_instance(dst).copy(Data.get_instance(src), end, start, size)
+
+    @staticmethod
+    @DataOps.SECURE_ERASE
+    def _secure_erase(dst):
+        Data.get_instance(dst).secure_erase()
+
+    def read(self, dst, size):
+        to_read = min(self.size - self.position, size)
+        memmove(dst, self.data + self.position, to_read)
+        return to_read
+
+    def write(self, src, size):
+        to_write = min(self.size - self.position, size)
+        memmove(self.data + self.position, src, to_write)
+        return to_write
+
+    def mlock(self):
+        return 0
+
+    def munlock(self):
+        pass
+
+    def zero(self, size):
+        to_zero = min(self.size - self.position, size)
+        memset(self.data + self.position, 0, to_zero)
+        return to_zero
+
+    def seek(self, seek, size):
+        if seek == DataSeek.CURRENT:
+            to_move = min(self.size - self.position, size)
+            self.position += to_move
+        else:
+            to_move = min(self.size, size)
+            self.position = to_move
+
+        return to_move
+
+    def copy(self, src, end, start, size):
+        return size
+
+    def secure_erase(self):
+        pass
+
+    def dump(self):
+        print_buffer(self.buffer, self.size)
+
+    def md5(self):
+        m = md5()
+        m.update(string_at(self.data, self.size))
+        return m.hexdigest()

tests/functional/pyocf/types/io.py

@@ -0,0 +1,130 @@
+#
+# Copyright(c) 2019 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+from ctypes import *
+from enum import IntEnum, IntFlag, auto
+
+from ..ocf import OcfLib
+from .data import Data
+from .queue import Queue
+
+
+class IoDir(IntEnum):
+    READ = 0
+    WRITE = 1
+
+
+class IoOps(Structure):
+    pass
+
+
+class Io(Structure):
+    START = CFUNCTYPE(None, c_void_p)
+    HANDLE = CFUNCTYPE(None, c_void_p, c_void_p)
+    END = CFUNCTYPE(None, c_void_p, c_int)
+
+    _instances_ = {}
+    _fields_ = [
+        ("_volume", c_void_p),
+        ("_ops", POINTER(IoOps)),
+        ("_addr", c_uint64),
+        ("_flags", c_uint64),
+        ("_bytes", c_uint32),
+        ("_class", c_uint32),
+        ("_dir", c_uint32),
+        ("_io_queue", c_void_p),
+        ("_start", START),
+        ("_handle", HANDLE),
+        ("_end", END),
+        ("_priv1", c_void_p),
+        ("_priv2", c_void_p),
+    ]
+
+    @classmethod
+    def from_pointer(cls, ref):
+        c = cls.from_address(ref)
+        cls._instances_[ref] = c
+        OcfLib.getInstance().ocf_io_set_cmpl_wrapper(byref(c), None, None, c.c_end)
+        return c
+
+    @classmethod
+    def get_instance(cls, ref):
+        return cls._instances_[cast(ref, c_void_p).value]
+
+    def del_object(self):
+        del type(self)._instances_[cast(byref(self), c_void_p).value]
+
+    def put(self):
+        OcfLib.getInstance().ocf_io_put(byref(self))
+
+    def get(self):
+        OcfLib.getInstance().ocf_io_get(byref(self))
+
+    @staticmethod
+    @END
+    def c_end(io, err):
+        Io.get_instance(io).end(err)
+
+    @staticmethod
+    @START
+    def c_start(io):
+        Io.get_instance(io).start()
+
+    @staticmethod
+    @HANDLE
+    def c_handle(io, opaque):
+        Io.get_instance(io).handle(opaque)
+
+    def end(self, err):
+        if err:
+            print("IO err {}".format(err))
+
+        self.put()
+        self.del_object()
+
+    def submit(self):
+        return OcfLib.getInstance().ocf_core_submit_io_wrapper(byref(self))
+
+    def configure(
+        self, addr: int, length: int, direction: IoDir, io_class: int, flags: int
+    ):
+        OcfLib.getInstance().ocf_io_configure_wrapper(
+            byref(self), addr, length, direction, io_class, flags
+        )
+
+    def set_data(self, data: Data):
+        self.data = data
+        OcfLib.getInstance().ocf_io_set_data_wrapper(byref(self), data, 0)
+
+    def set_queue(self, queue: Queue):
+        self.queue = queue
+        OcfLib.getInstance().ocf_io_set_queue_wrapper(byref(self), queue.handle)
+
+
+IoOps.SET_DATA = CFUNCTYPE(c_int, POINTER(Io), c_void_p, c_uint32)
+IoOps.GET_DATA = CFUNCTYPE(c_void_p, POINTER(Io))
+
+IoOps._fields_ = [("_set_data", IoOps.SET_DATA), ("_get_data", IoOps.GET_DATA)]
+
+lib = OcfLib.getInstance()
+lib.ocf_core_new_io_wrapper.restype = POINTER(Io)
+lib.ocf_io_set_cmpl_wrapper.argtypes = [POINTER(Io), c_void_p, c_void_p, Io.END]
+lib.ocf_io_configure_wrapper.argtypes = [
+    POINTER(Io),
+    c_uint64,
+    c_uint32,
+    c_uint32,
+    c_uint32,
+    c_uint64,
+]
+lib.ocf_io_set_queue_wrapper.argtypes = [POINTER(Io), c_uint32]
+
+lib.ocf_core_new_io_wrapper.argtypes = [c_void_p]
+lib.ocf_core_new_io_wrapper.restype = c_void_p
+
+lib.ocf_io_set_data_wrapper.argtypes = [POINTER(Io), c_void_p, c_uint32]
+lib.ocf_io_set_data_wrapper.restype = c_int
+
+lib.ocf_io_set_queue_wrapper.argtypes = [POINTER(Io), c_void_p]

tests/functional/pyocf/types/logger.py

@@ -0,0 +1,168 @@
+#
+# Copyright(c) 2019 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+from ctypes import *
+from enum import IntEnum
+import logging
+from io import StringIO
+
+from ..ocf import OcfLib
+
+logger = logging.getLogger("pyocf")
+logger.setLevel(logging.DEBUG)
+
+
+class LogLevel(IntEnum):
+    EMERG = 0
+    ALERT = 1
+    CRIT = 2
+    ERR = 3
+    WARN = 4
+    NOTICE = 5
+    INFO = 6
+    DEBUG = 7
+
+
+LevelMapping = {
+    LogLevel.EMERG: logging.CRITICAL,
+    LogLevel.ALERT: logging.CRITICAL,
+    LogLevel.CRIT: logging.CRITICAL,
+    LogLevel.ERR: logging.ERROR,
+    LogLevel.WARN: logging.WARNING,
+    LogLevel.NOTICE: logging.INFO,
+    LogLevel.INFO: logging.INFO,
+    LogLevel.DEBUG: logging.DEBUG,
+}
+
+
+class LoggerOps(Structure):
+    OPEN = CFUNCTYPE(c_int, c_void_p)
+    CLOSE = CFUNCTYPE(None, c_void_p)
+    # PRINTF ommited - we cannot make variadic function call in ctypes
+    LOG = CFUNCTYPE(c_int, c_void_p, c_uint, c_char_p)
+    PRINTF_RL = CFUNCTYPE(c_int, c_void_p, c_char_p)
+    DUMP_STACK = CFUNCTYPE(c_int, c_void_p)
+
+    _fields_ = [
+        ("_open", OPEN),
+        ("_close", CLOSE),
+        ("_printf", c_void_p),
+        ("_printf_rl", PRINTF_RL),
+        ("_dump_stack", DUMP_STACK),
+    ]
+
+
+class LoggerPriv(Structure):
+    _fields_ = [("_log", LoggerOps.LOG)]
+
+
+class Logger(Structure):
+    _instances_ = {}
+
+    _fields_ = [("logger", c_void_p)]
+
+    def __init__(self):
+        self.ops = LoggerOps(
+            _open=self._open,
+            _printf=cast(OcfLib.getInstance().pyocf_printf_helper, c_void_p),
+            _close=self._close,
+        )
+        self.priv = LoggerPriv(_log=self._log)
+        self._as_parameter_ = cast(pointer(self.priv), c_void_p).value
+        self._instances_[self._as_parameter_] = self
+
+    def get_ops(self):
+        return self.ops
+
+    def get_priv(self):
+        return self.priv
+
+    @classmethod
+    def get_instance(cls, ctx: int):
+        priv = OcfLib.getInstance().ocf_logger_get_priv(ctx)
+        return cls._instances_[priv]
+
+    @staticmethod
+    @LoggerOps.LOG
+    def _log(ref, lvl, msg):
+        Logger.get_instance(ref).log(lvl, str(msg, "ascii").strip())
+        return 0
+
+    @staticmethod
+    @LoggerOps.OPEN
+    def _open(ref):
+        if hasattr(Logger.get_instance(ref), "open"):
+            return Logger.get_instance(ref).open()
+        else:
+            return 0
+
+    @staticmethod
+    @LoggerOps.CLOSE
+    def _close(ref):
+        if hasattr(Logger.get_instance(ref), "close"):
+            return Logger.get_instance(ref).close()
+        else:
+            return 0
+
+
+class DefaultLogger(Logger):
+    def __init__(self, level: LogLevel = LogLevel.WARN):
+        super().__init__()
+        self.level = level
+
+        ch = logging.StreamHandler()
+        fmt = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+        ch.setFormatter(fmt)
+        ch.setLevel(LevelMapping[level])
+        logger.addHandler(ch)
+
+    def log(self, lvl: int, msg: str):
+        logger.log(LevelMapping[lvl], msg)
+
+    def close(self):
+        logger.handlers = []
+
+
+class FileLogger(Logger):
+    def __init__(self, f, console_level=None):
+        super().__init__()
+        fmt = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+
+        fh = logging.FileHandler(f)
+        fh.setLevel(logging.DEBUG)
+        fh.setFormatter(fmt)
+
+        logger.addHandler(fh)
+
+        if console_level:
+            sh = logging.StreamHandler()
+            sh.setLevel(LevelMapping[console_level])
+            sh.setFormatter(fmt)
+            logger.addHandler(sh)
+
+    def log(self, lvl, msg):
+        logger.log(LevelMapping[lvl], msg)
+
+    def close(self):
+        logger.handlers = []
+
+
+class BufferLogger(Logger):
+    def __init__(self, level: LogLevel):
+        super().__init__()
+        self.level = level
+        self.buffer = StringIO()
+
+    def log(self, lvl, msg):
+        if lvl < self.level:
+            self.buffer.write(msg + "\n")
+
+    def get_lines(self):
+        return self.buffer.getvalue().split("\n")
+
+
+lib = OcfLib.getInstance()
+lib.ocf_logger_get_priv.restype = c_void_p
+lib.ocf_logger_get_priv.argtypes = [c_void_p]

tests/functional/pyocf/types/metadata_updater.py

@@ -0,0 +1,48 @@
+#
+# Copyright(c) 2019 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+from ctypes import *
+from .shared import SharedOcfObject
+
+
+class MetadataUpdaterOps(Structure):
+    INIT = CFUNCTYPE(c_int, c_void_p)
+    KICK = CFUNCTYPE(None, c_void_p)
+    STOP = CFUNCTYPE(None, c_void_p)
+
+    _fields_ = [("_init", INIT), ("_kick", KICK), ("_stop", STOP)]
+
+
+class MetadataUpdater(SharedOcfObject):
+    _instances_ = {}
+    _fields_ = [("mu", c_void_p)]
+    ops = None
+
+    def __init__(self):
+        self._as_parameter_ = self.mu
+        super().__init__()
+
+    @classmethod
+    def get_ops(cls):
+        if not cls.ops:
+            cls.ops = MetadataUpdaterOps(
+                _init=cls._init, _kick=cls._kick, _stop=cls._stop
+            )
+        return cls.ops
+
+    @staticmethod
+    @MetadataUpdaterOps.INIT
+    def _init(ref):
+        return 0
+
+    @staticmethod
+    @MetadataUpdaterOps.KICK
+    def _kick(ref):
+        pass
+
+    @staticmethod
+    @MetadataUpdaterOps.STOP
+    def _stop(ref):
+        pass

tests/functional/pyocf/types/queue.py

@@ -0,0 +1,56 @@
+#
+# Copyright(c) 2019 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+from ctypes import *
+
+from ..ocf import OcfLib
+from .shared import SharedOcfObject, OcfError
+
+
+class QueueOps(Structure):
+    KICK = CFUNCTYPE(None, c_void_p)
+    KICK_SYNC = CFUNCTYPE(None, c_void_p)
+    STOP = CFUNCTYPE(None, c_void_p)
+
+    _fields_ = [
+        ("kick", KICK),
+        ("kick_sync", KICK_SYNC),
+        ("stop", STOP),
+    ]
+
+
+class Queue:
+    _instances_ = {}
+
+    def __init__(self, cache):
+        self.ops = QueueOps(kick_sync=type(self)._kick_sync, stop=type(self)._stop)
+        self.handle = c_void_p()
+
+        status = OcfLib.getInstance().ocf_queue_create(cache.cache_handle, byref(self.handle), byref(self.ops))
+        if status:
+            raise OcfError("Couldn't create queue object", status)
+
+        Queue._instances_[self.handle.value] = self
+        cache.queues += [self]
+
+    @classmethod
+    def get_instance(cls, ref):
+        return cls._instances_[ref]
+
+    @staticmethod
+    @QueueOps.KICK_SYNC
+    def _kick_sync(ref):
+        Queue.get_instance(ref).kick_sync()
+
+    @staticmethod
+    @QueueOps.STOP
+    def _stop(ref):
+        Queue.get_instance(ref).stop()
+
+    def kick_sync(self):
+        OcfLib.getInstance().ocf_queue_run(self.handle)
+
+    def stop(self):
+        pass

tests/functional/pyocf/types/shared.py

@@ -0,0 +1,100 @@
+#
+# Copyright(c) 2019 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+from ctypes import *
+from enum import IntEnum, auto
+
+from ..utils import Size as S
+
+
+class OcfErrorCode(IntEnum):
+    OCF_ERR_INVAL = 1000000
+    OCF_ERR_INTR = auto()
+    OCF_ERR_NO_MEM = auto()
+    OCF_ERR_NO_LOCK = auto()
+    OCF_ERR_INVAL_VOLUME_TYPE = auto()
+    OCF_ERR_UNKNOWN = auto()
+    OCF_ERR_TOO_MANY_CACHES = auto()
+    OCF_ERR_NO_FREE_RAM = auto()
+    OCF_ERR_START_CACHE_FAIL = auto()
+    OCF_ERR_CACHE_IN_USE = auto()
+    OCF_ERR_CACHE_NOT_EXIST = auto()
+    OCF_ERR_CACHE_EXIST = auto()
+    OCF_ERR_TOO_MANY_CORES = auto()
+    OCF_ERR_CORE_NOT_AVAIL = auto()
+    OCF_ERR_NOT_OPEN_EXC = auto()
+    OCF_ERR_CACHE_NOT_AVAIL = auto()
+    OCF_ERR_IO_CLASS_NOT_EXIST = auto()
+    OCF_ERR_WRITE_CACHE = auto()
+    OCF_ERR_WRITE_CORE = auto()
+    OCF_ERR_DIRTY_SHUTDOWN = auto()
+    OCF_ERR_DIRTY_EXISTS = auto()
+    OCF_ERR_FLUSHING_INTERRUPTED = auto()
+    OCF_ERR_CANNOT_ADD_CORE_TO_POOL = auto()
+    OCF_ERR_CACHE_IN_INCOMPLETE_STATE = auto()
+    OCF_ERR_CORE_IN_INACTIVE_STATE = auto()
+    OCF_ERR_INVALID_CACHE_MODE = auto()
+    OCF_ERR_INVALID_CACHE_LINE_SIZE = auto()
+
+
+class OcfError(BaseException):
+    def __init__(self, msg, error_code):
+        super().__init__(self, msg)
+        self.error_code = OcfErrorCode(abs(error_code))
+        self.msg = msg
+
+    def __str__(self):
+        return "{} ({})".format(self.msg, repr(self.error_code))
+
+
+class SharedOcfObject(Structure):
+    _instances_ = {}
+
+    def __init__(self):
+        super().__init__()
+        type(self)._instances_[self._as_parameter_] = self
+
+    @classmethod
+    def get_instance(cls, ref: int):
+        try:
+            return cls._instances_[ref]
+        except:
+            print(
+                "OcfSharedObject corruption. wanted: {} instances: {}".format(
+                    ref, cls._instances_
+                )
+            )
+            return None
+
+    @classmethod
+    def del_object(cls, ref: int):
+        del cls._instances_[ref]
+
+
+class Uuid(Structure):
+    _fields_ = [("_size", c_size_t), ("_data", c_char_p)]
+
+
+class CacheLineSize(IntEnum):
+    LINE_4KiB = S.from_KiB(4)
+    LINE_8KiB = S.from_KiB(8)
+    LINE_16KiB = S.from_KiB(16)
+    LINE_32KiB = S.from_KiB(32)
+    LINE_64KiB = S.from_KiB(64)
+    DEFAULT = LINE_4KiB
+
+
+class CacheLines(S):
+    def __init__(self, count: int, line_size: CacheLineSize):
+        self.bytes = count * line_size
+        self.line_size = line_size
+
+    def __int__(self):
+        return int(self.bytes / self.line_size)
+
+    def __str__(self):
+        return "{} ({})".format(int(self), super().__str__())
+
+    __repr__ = __str__

tests/functional/pyocf/types/stats/cache.py

@@ -0,0 +1,37 @@
+#
+# Copyright(c) 2019 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+from ctypes import *
+
+
+class _Inactive(Structure):
+    _fields_ = [("occupancy", c_uint32), ("dirty", c_uint32)]
+
+
+class _FallbackPt(Structure):
+    _fields_ = [("error_counter", c_int), ("status", c_bool)]
+
+
+class CacheInfo(Structure):
+    _fields_ = [
+        ("attached", c_bool),
+        ("volume_type", c_uint8),
+        ("size", c_uint32),
+        ("inactive", _Inactive),
+        ("occupancy", c_uint32),
+        ("dirty", c_uint32),
+        ("dirty_initial", c_uint32),
+        ("dirty_for", c_uint32),
+        ("cache_mode", c_uint32),
+        ("fallback_pt", _FallbackPt),
+        ("state", c_uint8),
+        ("eviction_policy", c_uint32),
+        ("cleaning_policy", c_uint32),
+        ("cache_line_size", c_uint64),
+        ("flushed", c_uint32),
+        ("core_count", c_uint32),
+        ("metadata_footprint", c_uint64),
+        ("metadata_end_offset", c_uint32),
+    ]

tests/functional/pyocf/types/stats/core.py

@@ -0,0 +1,30 @@
+
+#
+# Copyright(c) 2019 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+from ctypes import *
+
+from .shared import *
+
+
+class CoreStats(Structure):
+    _fields_ = [
+        ("core_size", c_uint64),
+        ("core_size_bytes", c_uint64),
+        ("cache_occupancy", c_uint32),
+        ("dirty", c_uint32),
+        ("flushed", c_uint32),
+        ("dirty_for", c_uint32),
+        ("read_reqs", OcfStatsReq),
+        ("write_reqs", OcfStatsReq),
+        ("cache_volume", OcfStatsBlock),
+        ("core_volume", OcfStatsBlock),
+        ("core", OcfStatsBlock),
+        ("cache_errors", OcfStatsError),
+        ("core_errors", OcfStatsError),
+        ("debug_stat", OcfStatsDebug),
+        ("seq_cutoff_threshold", c_uint32),
+        ("seq_cutoff_policy", c_uint32),
+    ]

tests/functional/pyocf/types/stats/shared.py

@@ -0,0 +1,88 @@
+#
+# Copyright(c) 2019 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+from ctypes import *
+
+
+class _Stat(Structure):
+    _fields_ = [("value", c_uint64), ("permil", c_uint64)]
+
+
+class OcfStatsReq(Structure):
+    _fields_ = [
+        ("partial_miss", c_uint64),
+        ("full_miss", c_uint64),
+        ("total", c_uint64),
+        ("pass_through", c_uint64),
+    ]
+
+
+class OcfStatsBlock(Structure):
+    _fields_ = [("read", c_uint64), ("write", c_uint64)]
+
+
+class OcfStatsError(Structure):
+    _fields_ = [("read", c_uint32), ("write", c_uint32)]
+
+
+class OcfStatsDebug(Structure):
+    _fields_ = [
+        ("read_size", c_uint64 * 12),
+        ("write_size", c_uint64 * 12),
+        ("read_align", c_uint64 * 4),
+        ("write_align", c_uint64 * 4),
+    ]
+
+
+class UsageStats(Structure):
+    _fields_ = [
+        ("occupancy", _Stat),
+        ("free", _Stat),
+        ("clean", _Stat),
+        ("dirty", _Stat),
+    ]
+
+
+class RequestsStats(Structure):
+    _fields_ = [
+        ("rd_hits", _Stat),
+        ("rd_partial_misses", _Stat),
+        ("rd_full_misses", _Stat),
+        ("rd_total", _Stat),
+        ("wr_hits", _Stat),
+        ("wr_partial_misses", _Stat),
+        ("wr_full_misses", _Stat),
+        ("wr_total", _Stat),
+        ("rd_pt", _Stat),
+        ("wr_pt", _Stat),
+        ("serviced", _Stat),
+        ("total", _Stat),
+    ]
+
+
+class BlocksStats(Structure):
+    _fields_ = [
+        ("core_volume_rd", _Stat),
+        ("core_volume_wr", _Stat),
+        ("core_volume_total", _Stat),
+        ("cache_volume_rd", _Stat),
+        ("cache_volume_wr", _Stat),
+        ("cache_volume_total", _Stat),
+        ("volume_rd", _Stat),
+        ("volume_wr", _Stat),
+        ("volume_total", _Stat),
+    ]
+
+
+class ErrorsStats(Structure):
+    _fields_ = [
+        ("core_volume_rd", _Stat),
+        ("core_volume_wr", _Stat),
+        ("core_volume_total", _Stat),
+        ("cache_volume_rd", _Stat),
+        ("cache_volume_wr", _Stat),
+        ("cache_volume_total", _Stat),
+        ("total", _Stat),
+    ]

tests/functional/pyocf/types/volume.py

@@ -0,0 +1,260 @@
+#
+# Copyright(c) 2019 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+
+from ctypes import *
+from hashlib import md5
+from collections import defaultdict
+
+from .io import Io, IoOps, IoDir
+from .shared import OcfError
+from ..ocf import OcfLib
+from ..utils import print_buffer, Size as S
+from .data import Data
+
+
+class VolumeCaps(Structure):
+    _fields_ = [("_atomic_writes", c_uint32, 1)]
+
+
+class VolumeOps(Structure):
+    SUBMIT_IO = CFUNCTYPE(None, POINTER(Io))
+    SUBMIT_FLUSH = CFUNCTYPE(None, c_void_p)
+    SUBMIT_METADATA = CFUNCTYPE(None, c_void_p)
+    SUBMIT_DISCARD = CFUNCTYPE(None, c_void_p)
+    SUBMIT_WRITE_ZEROES = CFUNCTYPE(None, c_void_p)
+    OPEN = CFUNCTYPE(c_int, c_void_p)
+    CLOSE = CFUNCTYPE(None, c_void_p)
+    GET_MAX_IO_SIZE = CFUNCTYPE(c_uint, c_void_p)
+    GET_LENGTH = CFUNCTYPE(c_uint64, c_void_p)
+
+    _fields_ = [
+        ("_submit_io", SUBMIT_IO),
+        ("_submit_flush", SUBMIT_FLUSH),
+        ("_submit_metadata", SUBMIT_METADATA),
+        ("_submit_discard", SUBMIT_DISCARD),
+        ("_submit_write_zeroes", SUBMIT_WRITE_ZEROES),
+        ("_open", OPEN),
+        ("_close", CLOSE),
+        ("_get_max_io_size", GET_MAX_IO_SIZE),
+        ("_get_length", GET_LENGTH),
+    ]
+
+
+class VolumeProperties(Structure):
+    _fields_ = [
+        ("_name", c_char_p),
+        ("_io_priv_size", c_uint32),
+        ("_volume_priv_size", c_uint32),
+        ("_caps", VolumeCaps),
+        ("_ops", VolumeOps),
+        ("_io_ops", IoOps),
+    ]
+
+
+class VolumeIoPriv(Structure):
+    _fields_ = [("_data", c_void_p)]
+
+
+class Volume(Structure):
+    _fields_ = [("_storage", c_void_p)]
+    _instances_ = {}
+    _uuid_ = {}
+
+    def __init__(self, size: S, uuid=None):
+        self.size = size
+        if uuid:
+            if uuid in type(self)._uuid_:
+                raise Exception("Volume with uuid {} already created".format(uuid))
+            self.uuid = uuid
+        else:
+            self.uuid = str(id(self))
+
+        type(self)._uuid_[self.uuid] = self
+
+        self.data = create_string_buffer(int(self.size))
+        self._storage = cast(self.data, c_void_p)
+        self.reset_stats()
+
+    @classmethod
+    def get_props(cls):
+        return VolumeProperties(
+            _name=str(cls.__name__).encode("ascii"),
+            _io_priv_size=sizeof(VolumeIoPriv),
+            _volume_priv_size=0,
+            _caps=VolumeCaps(_atomic_writes=0),
+            _ops=VolumeOps(
+                _submit_io=cls._submit_io,
+                _submit_flush=cls._submit_flush,
+                _submit_metadata=cls._submit_metadata,
+                _submit_discard=cls._submit_discard,
+                _submit_write_zeroes=cls._submit_write_zeroes,
+                _open=cls._open,
+                _close=cls._close,
+                _get_max_io_size=cls._get_max_io_size,
+                _get_length=cls._get_length,
+            ),
+            _io_ops=IoOps(_set_data=cls._io_set_data, _get_data=cls._io_get_data),
+        )
+
+    @classmethod
+    def get_instance(cls, ref):
+        return cls._instances_[ref]
+
+    @classmethod
+    def get_by_uuid(cls, uuid):
+        return cls._uuid_[uuid]
+
+    @staticmethod
+    @VolumeOps.SUBMIT_IO
+    def _submit_io(io):
+        io_structure = cast(io, POINTER(Io))
+        volume = Volume.get_instance(io_structure.contents._volume)
+
+        volume.submit_io(io_structure)
+
+    @staticmethod
+    @VolumeOps.SUBMIT_FLUSH
+    def _submit_flush(flush):
+        io_structure = cast(io, POINTER(Io))
+        volume = Volume.get_instance(io_structure.contents._volume)
+
+        volume.submit_flush(io_structure)
+
+    @staticmethod
+    @VolumeOps.SUBMIT_METADATA
+    def _submit_metadata(meta):
+        pass
+
+    @staticmethod
+    @VolumeOps.SUBMIT_DISCARD
+    def _submit_discard(discard):
+        io_structure = cast(io, POINTER(Io))
+        volume = Volume.get_instance(io_structure.contents._volume)
+
+        volume.submit_discard(io_structure)
+
+    @staticmethod
+    @VolumeOps.SUBMIT_WRITE_ZEROES
+    def _submit_write_zeroes(write_zeroes):
+        pass
+
+    @staticmethod
+    @CFUNCTYPE(c_int, c_void_p)
+    def _open(ref):
+        uuid_ptr = cast(OcfLib.getInstance().ocf_volume_get_uuid(ref), c_void_p)
+        uuid_str = cast(
+            OcfLib.getInstance().ocf_uuid_to_str_wrapper(uuid_ptr), c_char_p
+        )
+        uuid = str(uuid_str.value, encoding="ascii")
+        try:
+            volume = Volume.get_by_uuid(uuid)
+        except:
+            print("Tried to access unallocated volume {}".format(uuid))
+            print("{}".format(Volume._uuid_))
+            return -1
+
+        type(volume)._instances_[ref] = volume
+
+        return volume.open()
+
+    @staticmethod
+    @VolumeOps.CLOSE
+    def _close(ref):
+        Volume.get_instance(ref).close()
+        del Volume._instances_[ref]
+
+    @staticmethod
+    @VolumeOps.GET_MAX_IO_SIZE
+    def _get_max_io_size(ref):
+        return S.from_KiB(128)
+
+    @staticmethod
+    @VolumeOps.GET_LENGTH
+    def _get_length(ref):
+        return Volume.get_instance(ref).get_length()
+
+    @staticmethod
+    @IoOps.SET_DATA
+    def _io_set_data(io, data, offset):
+        io_priv = cast(OcfLib.getInstance().ocf_io_get_priv(io), POINTER(VolumeIoPriv))
+        data = Data.get_instance(data)
+        data.position = offset
+        io_priv.contents._data = cast(data, c_void_p)
+        return 0
+
+    @staticmethod
+    @IoOps.GET_DATA
+    def _io_get_data(io):
+        io_priv = cast(OcfLib.getInstance().ocf_io_get_priv(io), POINTER(VolumeIoPriv))
+        return io_priv.contents._data
+
+    def open(self):
+        return 0
+
+    def close(self):
+        pass
+
+    def get_length(self):
+        return self.size
+
+    def submit_flush(self, flush):
+        flush.contents._end(io, 0)
+
+    def submit_discard(self, discard):
+        discard.contents._end(io, 0)
+
+    def get_stats(self):
+        return self.stats
+
+    def reset_stats(self):
+        self.stats = {IoDir.WRITE: 0, IoDir.READ: 0}
+
+    def submit_io(self, io):
+        try:
+            self.stats[io.contents._dir] += 1
+            if io.contents._dir == IoDir.WRITE:
+                src_ptr = cast(io.contents._ops.contents._get_data(io), c_void_p)
+                src = Data.get_instance(src_ptr.value)
+                dst = self._storage + io.contents._addr
+            elif io.contents._dir == IoDir.READ:
+                dst_ptr = cast(io.contents._ops.contents._get_data(io), c_void_p)
+                dst = Data.get_instance(dst_ptr.value)
+                src = self._storage + io.contents._addr
+
+            memmove(dst, src, io.contents._bytes)
+
+            io.contents._end(io, 0)
+        except:
+            io.contents._end(io, -5)
+
+    def dump_contents(self, stop_after_zeros=0, offset=0, size=0):
+        if size == 0:
+            size = self.size
+        print_buffer(self._storage + offset, size, stop_after_zeros=stop_after_zeros)
+
+    def md5(self):
+        m = md5()
+        m.update(string_at(self._storage, self.size))
+        return m.hexdigest()
+
+
+class ErrorDevice(Volume):
+    def __init__(self, size, error_sectors: set = None, uuid=None):
+        super().__init__(size, uuid)
+        self.error_sectors = error_sectors or set()
+
+    def set_mapping(self, error_sectors: set):
+        self.error_sectors = error_sectors
+
+    def submit_io(self, io):
+        if io.contents._addr in self.error_sectors:
+            io.contents._end(io, -5)
+            self.stats["errors"][io.contents._dir] += 1
+        else:
+            super().submit_io(io)
+
+    def reset_stats(self):
+        super().reset_stats()
+        self.stats["errors"] = {IoDir.WRITE: 0, IoDir.READ: 0}

tests/functional/pyocf/utils.py

@@ -0,0 +1,139 @@
+#
+# Copyright(c) 2019 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+#
+from ctypes import *
+
+
+def print_buffer(buf, length, offset=0, width=16, stop_after_zeros=0):
+    end = offset + length
+    zero_lines = 0
+    buf = string_at(buf, length)
+    whole_buffer_empty = True
+    stop_after_zeros = int(stop_after_zeros / width)
+
+    for addr in range(offset, end, width):
+        cur_line = buf[addr : min(end, addr + width)]
+        all_zeros = True
+        byteline = ""
+        asciiline = ""
+        if not any(cur_line):
+            if stop_after_zeros and zero_lines > stop_after_zeros:
+                print(
+                    "<{} bytes of empty space encountered, stopping>".format(
+                        stop_after_zeros * width
+                    )
+                )
+                return
+            zero_lines += 1
+            continue
+
+        if zero_lines:
+            print("<{} zero bytes omitted>".format(zero_lines * width))
+            zero_lines = 0
+
+        for x in cur_line:
+            x = int(x)
+            byteline += "{:02X} ".format(x)
+            if 31 < x < 126:
+                char = chr(x)
+            else:
+                char = "."
+            asciiline += char
+
+        print("{:#08X}\t{}\t{}".format(addr, byteline, asciiline))
+        whole_buffer_empty = False
+
+    if whole_buffer_empty:
+        print("<whole buffer empty>")
+    elif zero_lines:
+        print("<zero until end>")
+
+
+class Size:
+    _KiB = 1024
+    _MiB = _KiB * 1024
+    _GiB = _MiB * 1024
+    _TiB = _GiB * 1024
+
+    def __init__(self, b: int):
+        self.bytes = b
+
+    def __int__(self):
+        return self.bytes
+
+    @classmethod
+    def from_B(cls, value):
+        return cls(value)
+
+    @classmethod
+    def from_KiB(cls, value):
+        return cls(value * cls._KiB)
+
+    @classmethod
+    def from_MiB(cls, value):
+        return cls(value * cls._MiB)
+
+    @classmethod
+    def from_GiB(cls, value):
+        return cls(value * cls._GiB)
+
+    @classmethod
+    def from_TiB(cls, value):
+        return cls(value * cls._TiB)
+
+    @property
+    def B(self):
+        return self.bytes
+
+    @property
+    def KiB(self):
+        return self.bytes / self._KiB
+
+    @property
+    def MiB(self):
+        return self.bytes / self._MiB
+
+    @property
+    def GiB(self):
+        return self.bytes / self._GiB
+
+    @property
+    def TiB(self):
+        return self.bytes / self._TiB
+
+    def __str__(self):
+        if self.bytes < self._KiB:
+            return "{} B".format(self.B)
+        elif self.bytes < self._MiB:
+            return "{} KiB".format(self.KiB)
+        elif self.bytes < self._GiB:
+            return "{} MiB".format(self.MiB)
+        elif self.bytes < self._TiB:
+            return "{} GiB".format(self.GiB)
+        else:
+            return "{} TiB".format(self.TiB)
+
+
+def print_structure(struct, indent=0):
+    print(struct)
+    for field, field_type in struct._fields_:
+        value = getattr(struct, field)
+        if hasattr(value, "_fields_"):
+            print("{}{: <20} :".format("   " * indent, field))
+            print_structure(value, indent=indent + 1)
+            continue
+
+        print("{}{: <20} : {}".format("   " * indent, field, value))
+
+
+def struct_to_dict(struct):
+    d = {}
+    for field, field_type in struct._fields_:
+        value = getattr(struct, field)
+        if hasattr(value, "_fields_"):
+            d[field] = struct_to_dict(value)
+            continue
+        d[field] = value
+
+    return d

tests/functional/pyocf/wrappers/ocf_io_wrappers.c

@@ -0,0 +1,56 @@
+/*
+ * Copyright(c) 2012-2018 Intel Corporation
+ * SPDX-License-Identifier: BSD-3-Clause-Clear
+ */
+
+#include "ocf/ocf_io.h"
+#include "ocf/ocf_core.h"
+
+struct ocf_io *ocf_core_new_io_wrapper(ocf_core_t core)
+{
+	return ocf_core_new_io(core);
+}
+
+void ocf_io_configure_wrapper(struct ocf_io *io, uint64_t addr,
+		uint32_t bytes, uint32_t dir, uint32_t class, uint64_t flags)
+{
+	ocf_io_configure(io, addr, bytes, dir, class, flags);
+}
+
+void ocf_io_set_cmpl_wrapper(struct ocf_io *io, void *context,
+		void *context2, ocf_end_io_t fn)
+{
+	ocf_io_set_cmpl(io, context, context2, fn);
+}
+
+void ocf_io_set_start_wrapper(struct ocf_io *io, ocf_start_io_t fn)
+{
+	ocf_io_set_start(io, fn);
+}
+
+void ocf_io_set_handle_wrapper(struct ocf_io *io, ocf_handle_io_t fn)
+{
+	ocf_io_set_handle(io, fn);
+}
+
+int ocf_io_set_data_wrapper(struct ocf_io *io, ctx_data_t *data,
+		uint32_t offset)
+{
+	return ocf_io_set_data(io, data, offset);
+}
+
+ctx_data_t *ocf_io_get_data_wrapper(struct ocf_io *io)
+{
+	return ocf_io_get_data(io);
+}
+
+void ocf_io_set_queue_wrapper(struct ocf_io *io, ocf_queue_t queue)
+{
+	ocf_io_set_queue(io, queue);
+}
+
+void ocf_core_submit_io_wrapper(struct ocf_io *io)
+{
+	ocf_core_submit_io(io);
+}
+

tests/functional/pyocf/wrappers/ocf_logger_wrappers.c

@@ -0,0 +1,42 @@
+
+/*
+ * Copyright(c) 2012-2018 Intel Corporation
+ * SPDX-License-Identifier: BSD-3-Clause-Clear
+ */
+
+#include <ocf/ocf_types.h>
+#include <ocf/ocf_logger.h>
+#include <stdarg.h>
+#include "ocf_env.h"
+
+#define LOG_BUFFER_SIZE 4096
+
+struct pyocf_logger_priv {
+	int (*pyocf_log)(void *pyocf_logger, ocf_logger_lvl_t lvl, char *msg);
+};
+
+int pyocf_printf_helper(ocf_logger_t logger, ocf_logger_lvl_t lvl,
+		const char *fmt, va_list args)
+{
+	char *buffer = env_zalloc(LOG_BUFFER_SIZE, ENV_MEM_NORMAL);
+	struct pyocf_logger_priv *priv = ocf_logger_get_priv(logger);
+	int ret;
+
+	if (!buffer) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	ret = vsnprintf(buffer, LOG_BUFFER_SIZE, fmt, args);
+	if (ret < 0) {
+		env_free(buffer);
+		goto out;
+	}
+
+	ret = priv->pyocf_log(logger, lvl, buffer);
+
+	env_free(buffer);
+
+out:
+	return ret;
+}

tests/functional/pyocf/wrappers/ocf_volume_wrappers.c

@@ -0,0 +1,12 @@
+
+/*
+ * Copyright(c) 2012-2018 Intel Corporation
+ * SPDX-License-Identifier: BSD-3-Clause-Clear
+ */
+
+#include "ocf/ocf_io.h"
+#include "ocf/ocf_volume.h"
+
+const char *ocf_uuid_to_str_wrapper(const struct ocf_volume_uuid *uuid) {
+	return ocf_uuid_to_str(uuid);
+}