github/ocf
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement pyocf adapter with sample OCF test

Browse Source 
PyOCF is a tool written with testing OCF functionality in mind.
It is a Python3 (3.6 version required) package which wraps OCF
by providing Python objects in place of OCF objects (volumes, queues,
etc). Thin layer of translation between OCF objects and PyOCF objects
enables using customized behaviors for OCF primitives by subclassing
PyOCF classes.

This initial version implements only WT and WI modes and single,
synchronously operating Queue.

TO DO:

  - Queues/Cleaner/MetadataUpdater implemented as Python threads
  - Loading of caches from PyOCF Volumes (fix bugs in OCF)
  - Make sure it works multi-threaded for more sophisticated tests

Co-authored-by: Jan Musial <jan.musial@intel.com>
Signed-off-by: Michal Rakowski <michal.rakowski@intel.com>
Signed-off-by: Jan Musial <jan.musial@intel.com>
This commit is contained in:
Michal Rakowski 2019-01-09 12:50:23 +01:00 committed by Jan Musial
parent 794b008127
commit e5227cef89
35 changed files with 2166 additions and 1018 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
inc/ocf_mngt.h
View File

@ -85,7 +85,17 @@ uint32_t ocf_mngt_cache_get_count(ocf_ctx_t ctx);
* @retval -OCF_ERR_INV_CACHE_ID Cache ID out of range
* @retval -OCF_ERR_CACHE_NOT_EXIST Cache with given ID is not exist
*/
int ocf_mngt_cache_get(ocf_ctx_t ctx, ocf_cache_id_t id, ocf_cache_t *cache);
int ocf_mngt_cache_get_by_id(ocf_ctx_t ctx, ocf_cache_id_t id, ocf_cache_t *cache);
/**
* @brief Increment reference counter of cache
*
* @param[in] cache OCF cache handle
*
* @retval 0 Reference counter incremented
* @retval -OCF_ERR_CACHE_NOT_AVAIL cache isn't initialised yet
*/
int ocf_mngt_cache_get(ocf_cache_t cache);
/**
* @brief Decrease reference counter in cache

10
src/mngt/ocf_mngt_common.c
View File

@ -174,7 +174,7 @@ void ocf_mngt_cache_put(ocf_cache_t cache)
}
}
int ocf_mngt_cache_get(ocf_ctx_t ocf_ctx, ocf_cache_id_t id, ocf_cache_t *cache)
int ocf_mngt_cache_get_by_id(ocf_ctx_t ocf_ctx, ocf_cache_id_t id, ocf_cache_t *cache)
{
int error = 0;
struct ocf_cache *instance = NULL;
@ -331,6 +331,14 @@ static ocf_cache_t _ocf_mngt_cache_try_get(ocf_cache_t cache)
return NULL;
}
int ocf_mngt_cache_get(ocf_cache_t cache)
{
if (!_ocf_mngt_cache_try_get(cache))
return -OCF_ERR_CACHE_NOT_AVAIL;
return 0;
}
static int _ocf_mngt_cache_get_list_cpy(ocf_ctx_t ocf_ctx, ocf_cache_t **list,
uint32_t *size)
{

2
src/ocf_logger.c
View File

@ -16,7 +16,7 @@ int ocf_log_raw(ocf_logger_t logger, ocf_logger_lvl_t lvl,
const char *fmt, ...)
{
va_list args;
int ret;
int ret = 0;
if (!logger->ops->printf)
return -ENOTSUP;

6
tests/functional/.gitignore vendored Normal file
View File

@ -0,0 +1,6 @@
__pycache__
pyocf/__pycache__
pyocf/libocf.so
*.o
pyocf/ocf/*
*.pyc

45
tests/functional/Makefile
View File

@ -5,36 +5,45 @@
PWD=$(shell pwd)
OCFDIR=$(PWD)/../../
ADAPTERDIR=$(PWD)/test_adapter
SRCDIR=$(ADAPTERDIR)/src
INCDIR=$(ADAPTERDIR)/include
ADAPTERDIR=$(PWD)/pyocf
SRCDIR=$(ADAPTERDIR)/ocf/src
INCDIR=$(ADAPTERDIR)/ocf/include
WRAPDIR=$(ADAPTERDIR)/wrappers
CC=gcc
CFLAGS=-g -Wall -I$(INCDIR) -I$(SRCDIR)
LDFLAGS=-pthread
CFLAGS=-g -Wall -I$(INCDIR) -I$(SRCDIR)/ocf/env
LDFLAGS=-pthread -lz
SRC=$(shell find $(SRCDIR) -name \*.c)
SRC=$(shell find $(SRCDIR) $(WRAPDIR) -name \*.c)
OBJS=$(patsubst %.c, %.o, $(SRC))
OCFLIB=libocf.so
OCFLIB=$(ADAPTERDIR)/libocf.so
all: sync $(OBJS) $(OCFLIB)
all: | sync
$(MAKE) $(OCFLIB)
$(OCFLIB): $(OBJS)
$(CC) -shared -o $@ $(CFLAGS) $^ -fPIC $(LDFLAGS)
@echo "Building $@"
@$(CC) -shared -o $@ $(CFLAGS) $^ -fPIC $(LDFLAGS)
%.o: %.c
$(CC) -c $(CFLAGS) -o $@ -fPIC $^ $(LDFLAGS)
@echo "Compiling $@"
@$(CC) -c $(CFLAGS) -o $@ -fPIC $^ $(LDFLAGS)
sync:
$(MAKE) -C $(OCFDIR) inc O=$(ADAPTERDIR)
$(MAKE) -C $(OCFDIR) src O=$(ADAPTERDIR)
@echo "Syncing OCF sources"
@mkdir -p $(ADAPTERDIR)/ocf
@$(MAKE) -C $(OCFDIR) inc O=$(ADAPTERDIR)/ocf
@$(MAKE) -C $(OCFDIR) src O=$(ADAPTERDIR)/ocf
@$(MAKE) -C $(OCFDIR) env O=$(ADAPTERDIR)/ocf ENV=posix
clean:
rm -rf $(OCFLIB) $(OBJS)
@rm -rf $(OCFLIB) $(OBJS)
@echo " CLEAN "
distclean:
rm -rf $(OCFLIB) $(OBJS)
rm -rf $(SRCDIR)/ocf
rm -rf $(INCDIR)/ocf
distclean: clean
@rm -rf $(OCFLIB) $(OBJS)
@rm -rf $(SRCDIR)/ocf
@rm -rf $(INCDIR)/ocf
@echo " DISTCLEAN "
.PHONY: all clean
.PHONY: all clean sync distclean

12
tests/functional/lib.py
View File

@ -1,12 +0,0 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from ctypes import *
lib = None
#TODO Consider changing lib to singleton
def LoadOcfLib():
global lib
lib = cdll.LoadLibrary('./libocf.so')

0
tests/functional/pyocf/__init__.py Normal file
View File

25
tests/functional/pyocf/ocf.py Normal file
View File

@ -0,0 +1,25 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from ctypes import *
lib = None
class OcfLib:
__lib__ = None
@classmethod
def getInstance(cls):
if cls.__lib__ is None:
lib = cdll.LoadLibrary("./pyocf/libocf.so")
lib.ocf_volume_get_uuid.restype = c_void_p
lib.ocf_volume_get_uuid.argtypes = [c_void_p]
lib.ocf_core_get_front_volume.restype = c_void_p
lib.ocf_core_get_front_volume.argtypes = [c_void_p]
cls.__lib__ = lib
return cls.__lib__

0
tests/functional/pyocf/types/__init__.py Normal file
View File

312
tests/functional/pyocf/types/cache.py Normal file
View File

@ -0,0 +1,312 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from ctypes import *
from enum import IntEnum
import logging
from datetime import timedelta
from .shared import Uuid, OcfError, CacheLineSize, CacheLines
from ..utils import Size, struct_to_dict
from .core import Core
from .stats.cache import *
from .stats.shared import *
class Backfill(Structure):
_fields_ = [("_max_queue_size", c_uint32), ("_queue_unblock_size", c_uint32)]
class CacheConfig(Structure):
_fields_ = [
("_id", c_uint16),
("_name", c_char_p),
("_cache_mode", c_uint32),
("_eviction_policy", c_uint32),
("_cache_line_size", c_uint64),
("_metadata_layout", c_uint32),
("_metadata_volatile", c_bool),
("_backfill", Backfill),
("_locked", c_bool),
("_pt_unaligned_io", c_bool),
("_use_submit_io_fast", c_bool),
]
class CacheDeviceConfig(Structure):
_fields_ = [
("_uuid", Uuid),
("_volume_type", c_uint8),
("_cache_line_size", c_uint64),
("_force", c_bool),
("_min_free_ram", c_uint64),
("_perform_test", c_bool),
("_discard_on_start", c_bool),
]
class CacheMode(IntEnum):
WT = 0
WB = 1
WA = 2
PT = 3
WI = 4
DEFAULT = WT
class EvictionPolicy(IntEnum):
LRU = 0
DEFAULT = LRU
class CleaningPolicy(IntEnum):
NOP = 0
ALRU = 1
ACP = 2
DEFAULT = ALRU
class MetadataLayout(IntEnum):
STRIPING = 0
SEQUENTIAL = 1
DEFAULT = STRIPING
class Cache:
DEFAULT_ID = 0
DEFAULT_BACKFILL_QUEUE_SIZE = 65536
DEFAULT_BACKFILL_UNBLOCK = 60000
DEFAULT_PT_UNALIGNED_IO = False
DEFAULT_USE_SUBMIT_FAST = False
def __init__(
self,
owner,
cache_id: int = DEFAULT_ID,
name: str = "",
cache_mode: CacheMode = CacheMode.DEFAULT,
eviction_policy: EvictionPolicy = EvictionPolicy.DEFAULT,
cache_line_size: CacheLineSize = CacheLineSize.DEFAULT,
metadata_layout: MetadataLayout = MetadataLayout.DEFAULT,
metadata_volatile: bool = False,
max_queue_size: int = DEFAULT_BACKFILL_QUEUE_SIZE,
queue_unblock_size: int = DEFAULT_BACKFILL_UNBLOCK,
locked: bool = True,
pt_unaligned_io: bool = DEFAULT_PT_UNALIGNED_IO,
use_submit_fast: bool = DEFAULT_USE_SUBMIT_FAST,
):
self.owner = owner
self.cache_line_size = cache_line_size
self.cfg = CacheConfig(
_id=cache_id,
_name=name.encode("ascii") if name else None,
_cache_mode=cache_mode,
_eviction_policy=eviction_policy,
_cache_line_size=cache_line_size,
_metadata_layout=metadata_layout,
_metadata_volatile=metadata_volatile,
_backfill=Backfill(
_max_queue_size=max_queue_size, _queue_unblock_size=queue_unblock_size
),
_locked=locked,
_pt_unaligned_io=pt_unaligned_io,
_use_submit_fast=use_submit_fast,
)
self.cache_handle = c_void_p()
self.queues = []
def start_cache(self):
status = self.owner.lib.ocf_mngt_cache_start(
self.owner.ctx_handle, byref(self.cache_handle), byref(self.cfg)
)
if status:
raise OcfError("Creating cache instance failed", status)
self.owner.caches += [self]
def configure_device(
self, device, force=False, perform_test=False, cache_line_size=None
):
self.device_name = device.uuid
self.dev_cfg = CacheDeviceConfig(
_uuid=Uuid(
_data=cast(
create_string_buffer(self.device_name.encode("ascii")), c_char_p
),
_size=len(self.device_name) + 1,
),
_volume_type=device.type_id,
_cache_line_size=cache_line_size
if cache_line_size
else self.cache_line_size,
_force=force,
_min_free_ram=0,
_perform_test=perform_test,
_discard_on_start=False,
)
def attach_device(
self, device, force=False, perform_test=False, cache_line_size=None,
):
self.configure_device(device, force, perform_test, cache_line_size)
status = device.owner.lib.ocf_mngt_cache_attach(
self.cache_handle, byref(self.dev_cfg)
)
if status:
raise OcfError("Attaching cache device failed", status)
def load_cache(self, device):
self.configure_device(device)
status = device.owner.lib.ocf_mngt_cache_load(
self.owner.ctx_handle,
byref(self.cache_handle),
byref(self.cfg),
byref(self.dev_cfg),
)
if status:
raise OcfError("Loading cache device failed", status)
@classmethod
def load_from_device(cls, device, name=""):
c = cls(name=name, owner=device.owner)
c.load_cache(device)
return c
@classmethod
def start_on_device(cls, device, **kwargs):
c = cls(locked=True, owner=device.owner, **kwargs)
c.start_cache()
c.attach_device(device, force=True)
return c
def _get_and_lock(self, read=True):
status = self.owner.lib.ocf_mngt_cache_get(self.cache_handle)
if status:
raise OcfError("Couldn't get cache instance", status)
if read:
status = self.owner.lib.ocf_mngt_cache_read_lock(self.cache_handle)
else:
status = self.owner.lib.ocf_mngt_cache_lock(self.cache_handle)
if status:
self.owner.lib.ocf_mngt_cache_put(self.cache_handle)
raise OcfError("Couldn't lock cache instance", status)
def _put_and_unlock(self, read=True):
if read:
self.owner.lib.ocf_mngt_cache_read_unlock(self.cache_handle)
else:
self.owner.lib.ocf_mngt_cache_unlock(self.cache_handle)
self.owner.lib.ocf_mngt_cache_put(self.cache_handle)
def get_and_read_lock(self):
self._get_and_lock(True)
def get_and_write_lock(self):
self._get_and_lock(False)
def put_and_read_unlock(self):
self._put_and_unlock(True)
def put_and_write_unlock(self):
self._put_and_unlock(False)
def add_core(self, core: Core):
self.get_and_write_lock()
status = self.owner.lib.ocf_mngt_cache_add_core(
self.cache_handle, byref(core.get_handle()), byref(core.get_cfg())
)
if status:
self.put_and_write_unlock()
raise OcfError("Failed adding core", status)
core.cache = self
self.put_and_write_unlock()
def get_stats(self):
cache_info = CacheInfo()
usage = UsageStats()
req = RequestsStats()
block = BlocksStats()
errors = ErrorsStats()
self.get_and_read_lock()
status = self.owner.lib.ocf_cache_get_info(self.cache_handle, byref(cache_info))
if status:
self.put_and_read_unlock()
raise OcfError("Failed getting cache info", status)
status = self.owner.lib.ocf_stats_collect_cache(
self.cache_handle, byref(usage), byref(req), byref(block), byref(errors)
)
if status:
self.put_and_read_unlock()
raise OcfError("Failed getting stats", status)
line_size = CacheLineSize(cache_info.cache_line_size)
self.put_and_read_unlock()
return {
"conf": {
"attached": cache_info.attached,
"volume_type": self.owner.volume_types[cache_info.volume_type],
"size": CacheLines(cache_info.size, line_size),
"inactive": {
"occupancy": CacheLines(cache_info.inactive.occupancy, line_size),
"dirty": CacheLines(cache_info.inactive.dirty, line_size),
},
"occupancy": CacheLines(cache_info.occupancy, line_size),
"dirty": CacheLines(cache_info.dirty, line_size),
"dirty_initial": CacheLines(cache_info.dirty_initial, line_size),
"dirty_for": timedelta(seconds=cache_info.dirty_for),
"cache_mode": CacheMode(cache_info.cache_mode),
"fallback_pt": {
"error_counter": cache_info.fallback_pt.error_counter,
"status": cache_info.fallback_pt.status,
},
"state": cache_info.state,
"eviction_policy": EvictionPolicy(cache_info.eviction_policy),
"cleaning_policy": CleaningPolicy(cache_info.cleaning_policy),
"cache_line_size": line_size,
"flushed": CacheLines(cache_info.flushed, line_size),
"core_count": cache_info.core_count,
"metadata_footprint": Size(cache_info.metadata_footprint),
"metadata_end_offset": Size(cache_info.metadata_end_offset),
},
"block": struct_to_dict(block),
"req": struct_to_dict(req),
"usage": struct_to_dict(usage),
"errors": struct_to_dict(errors),
}
def reset_stats(self):
self.owner.lib.ocf_core_stats_initialize_all(self.cache_handle)
def get_default_queue(self):
if not self.queues:
raise Exception("No queues added for cache")
return self.queues[0]
def stop(self):
self.get_and_write_lock()
status = self.owner.lib.ocf_mngt_cache_stop(self.cache_handle)
if status:
self.put_and_write_unlock()
raise OcfError("Failed stopping cache", status)
self.put_and_write_unlock()
self.owner.caches.remove(self)

37
tests/functional/pyocf/types/cleaner.py Normal file
View File

@ -0,0 +1,37 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from ctypes import *
from .shared import SharedOcfObject
class CleanerOps(Structure):
INIT = CFUNCTYPE(c_int, c_void_p)
STOP = CFUNCTYPE(None, c_void_p)
_fields_ = [("init", INIT), ("stop", STOP)]
class Cleaner(SharedOcfObject):
_instances_ = {}
_fields_ = [("cleaner", c_void_p)]
def __init__(self):
self._as_parameter_ = self.cleaner
super().__init__()
@classmethod
def get_ops(cls):
return CleanerOps(init=cls._init, stop=cls._stop)
@staticmethod
@CleanerOps.INIT
def _init(cleaner):
return 0
@staticmethod
@CleanerOps.STOP
def _stop(cleaner):
pass

145
tests/functional/pyocf/types/core.py Normal file
View File

@ -0,0 +1,145 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from ctypes import *
import logging
from datetime import timedelta
from ..ocf import OcfLib
from .shared import Uuid
from .volume import Volume
from .data import Data
from .io import Io, IoDir
from .stats.shared import *
from .stats.core import *
from ..utils import Size, struct_to_dict
from .queue import Queue
class UserMetadata(Structure):
_fields_ = [("data", c_void_p), ("size", c_size_t)]
class CoreConfig(Structure):
_fields_ = [
("_uuid", Uuid),
("_volume_type", c_uint8),
("_core_id", c_uint16),
("_name", c_char_p),
("_cache_id", c_uint16),
("_try_add", c_bool),
("_seq_cutoff_threshold", c_uint32),
("_user_metadata", UserMetadata),
]
class Core:
DEFAULT_ID = 4096
DEFAULT_SEQ_CUTOFF_THRESHOLD = 1024 * 1024
def __init__(
self,
device: Volume,
try_add: bool,
name: str = "",
core_id: int = DEFAULT_ID,
seq_cutoff_threshold: int = DEFAULT_SEQ_CUTOFF_THRESHOLD,
):
self.device = device
self.device_name = device.uuid
self.core_id = core_id
self.handle = c_void_p()
self.cfg = CoreConfig(
_uuid=Uuid(
_data=cast(
create_string_buffer(self.device_name.encode("ascii")), c_char_p
),
_size=len(self.device_name) + 1,
),
_core_id=self.core_id,
_name=name.encode("ascii") if name else None,
_volume_type=self.device.type_id,
_try_add=try_add,
_seq_cutoff_threshold=seq_cutoff_threshold,
_user_metadata=UserMetadata(_data=None, _size=0),
)
@classmethod
def using_device(cls, device, **kwargs):
c = cls(device=device, try_add=False, **kwargs)
return c
def get_cfg(self):
return self.cfg
def get_handle(self):
return self.handle
def new_io(self):
if not self.cache:
raise Exception("Core isn't attached to any cache")
io = OcfLib.getInstance().ocf_core_new_io_wrapper(self.handle)
return Io.from_pointer(io)
def new_core_io(self):
lib = OcfLib.getInstance()
core = lib.ocf_core_get_volume(self.handle)
io = lib.ocf_volume_new_io(core)
return Io.from_pointer(io)
def get_stats(self):
core_stats = CoreStats()
usage = UsageStats()
req = RequestsStats()
blocks = BlocksStats()
errors = ErrorsStats()
self.cache.get_and_lock(True)
status = self.cache.owner.lib.ocf_stats_collect_core(
self.handle, byref(usage), byref(req), byref(blocks), byref(errors)
)
if status:
self.cache.put_and_unlock(True)
raise OcfError("Failed collecting core stats", status)
status = self.cache.owner.lib.ocf_core_get_stats(self.handle, byref(core_stats))
if status:
self.cache.put_and_unlock(True)
raise OcfError("Failed getting core stats", status)
self.cache.put_and_unlock(True)
return {
"size": Size(core_stats.core_size_bytes),
"dirty_for": timedelta(seconds=core_stats.dirty_for),
"usage": struct_to_dict(usage),
"req": struct_to_dict(req),
"blocks": struct_to_dict(blocks),
"errors": struct_to_dict(errors),
}
def reset_stats(self):
self.cache.owner.lib.ocf_core_stats_initialize(self.handle)
def exp_obj_md5(self):
logging.getLogger("pyocf").warning(
"Reading whole exported object! This disturbs statistics values"
)
read_buffer = Data(self.device.size)
io = self.new_io()
io.configure(0, read_buffer.size, IoDir.READ, 0, 0)
io.set_data(read_buffer)
io.set_queue(self.cache.get_default_queue())
io.submit()
return read_buffer.md5()
lib = OcfLib.getInstance()
lib.ocf_core_get_volume.restype = c_void_p
lib.ocf_volume_new_io.argtypes = [c_void_p]
lib.ocf_volume_new_io.restype = c_void_p

85
tests/functional/pyocf/types/ctx.py Normal file
View File

@ -0,0 +1,85 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from ctypes import *
from enum import IntEnum
from .logger import LoggerOps, Logger
from .data import DataOps, Data
from .queue import Queue
from .cleaner import CleanerOps, Cleaner
from .metadata_updater import MetadataUpdaterOps, MetadataUpdater
from .shared import OcfError
from ..ocf import OcfLib
class OcfCtxOps(Structure):
_fields_ = [
("data", DataOps),
("cleaner", CleanerOps),
("metadata_updater", MetadataUpdaterOps),
("logger", LoggerOps),
]
class OcfCtxCfg(Structure):
_fields_ = [("name", c_char_p), ("ops", OcfCtxOps), ("logger_priv", c_void_p)]
class OcfCtx:
def __init__(self, lib, name, logger, data, mu, cleaner):
self.logger = logger
self.data = data
self.mu = mu
self.cleaner = cleaner
self.ctx_handle = c_void_p()
self.lib = lib
self.volume_types_count = 1
self.volume_types = {}
self.caches = []
self.cfg = OcfCtxCfg(
name=name,
ops=OcfCtxOps(
data=self.data.get_ops(),
cleaner=self.cleaner.get_ops(),
metadata_updater=self.mu.get_ops(),
logger=logger.get_ops(),
),
logger_priv=cast(pointer(logger.get_priv()), c_void_p),
)
result = self.lib.ocf_ctx_init(byref(self.ctx_handle), byref(self.cfg))
if result != 0:
raise OcfError("Context initialization failed", result)
def register_volume_type(self, volume_type):
self.volume_types[self.volume_types_count] = volume_type.get_props()
volume_type.type_id = self.volume_types_count
volume_type.owner = self
result = self.lib.ocf_ctx_register_volume_type(
self.ctx_handle,
self.volume_types_count,
byref(self.volume_types[self.volume_types_count]),
)
if result != 0:
raise OcfError("Data object registration failed", result)
self.volume_types_count += 1
def exit(self):
self.lib.ocf_ctx_exit(self.ctx_handle)
def get_default_ctx(logger):
return OcfCtx(
OcfLib.getInstance(),
b"PyOCF default ctx",
logger,
Data,
MetadataUpdater,
Cleaner,
)

203
tests/functional/pyocf/types/data.py Normal file
View File

@ -0,0 +1,203 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from ctypes import *
from enum import IntEnum
from hashlib import md5
from .shared import SharedOcfObject
from ..utils import print_buffer
class DataSeek(IntEnum):
BEGIN = 0
CURRENT = 1
class DataOps(Structure):
ALLOC = CFUNCTYPE(c_void_p, c_uint32)
FREE = CFUNCTYPE(None, c_void_p)
MLOCK = CFUNCTYPE(c_int, c_void_p)
MUNLOCK = CFUNCTYPE(None, c_void_p)
READ = CFUNCTYPE(c_uint32, c_void_p, c_void_p, c_uint32)
WRITE = CFUNCTYPE(c_uint32, c_void_p, c_void_p, c_uint32)
ZERO = CFUNCTYPE(c_uint32, c_void_p, c_uint32)
SEEK = CFUNCTYPE(c_uint32, c_void_p, c_uint32, c_uint32)
COPY = CFUNCTYPE(c_uint64, c_void_p, c_void_p, c_uint64, c_uint64, c_uint64)
SECURE_ERASE = CFUNCTYPE(None, c_void_p)
_fields_ = [
("_alloc", ALLOC),
("_free", FREE),
("_mlock", MLOCK),
("_munlock", MUNLOCK),
("_read", READ),
("_write", WRITE),
("_zero", ZERO),
("_seek", SEEK),
("_copy", COPY),
("_secure_erase", SECURE_ERASE),
]
class Data(SharedOcfObject):
PAGE_SIZE = 4096
_instances_ = {}
_fields_ = [("data", c_void_p)]
def __init__(self, byte_count: int):
self.size = byte_count
self.position = 0
self.buffer = create_string_buffer(int(self.size))
self.data = cast(self.buffer, c_void_p)
memset(self.data, 0, self.size)
type(self)._instances_[self.data] = self
self._as_parameter_ = self.data
super().__init__()
@classmethod
def get_ops(cls):
return DataOps(
_alloc=cls._alloc,
_free=cls._free,
_mlock=cls._mlock,
_munlock=cls._munlock,
_read=cls._read,
_write=cls._write,
_zero=cls._zero,
_seek=cls._seek,
_copy=cls._copy,
_secure_erase=cls._secure_erase,
)
@classmethod
def pages(cls, pages: int):
return cls(pages * Data.PAGE_SIZE)
@classmethod
def from_bytes(cls, source: bytes):
d = cls(len(source))
memmove(d.data, cast(source, c_void_p), len(source))
return d
@classmethod
def from_string(cls, source: str, encoding: str = "ascii"):
return cls.from_bytes(bytes(source, encoding))
def __str__(self):
char_array = cast(self.data, c_char_p)
return str(char_array.value, "ascii")
def __wstr__(self):
char_array = cast(self.data, c_wchar_p)
return str(char_array.value, "utf-8")
def set_data(self, contents):
if len(contents) > self.size:
raise Exception("Data too big to fit into allocated buffer")
memmove(self.data, cast(contents, c_void_p), len(contents))
self.position = 0
@staticmethod
@DataOps.ALLOC
def _alloc(pages):
data = Data.pages(pages)
return data.data
@staticmethod
@DataOps.FREE
def _free(data):
Data.del_object(data)
@staticmethod
@DataOps.MLOCK
def _mlock(ref):
return Data.get_instance(ref).mlock()
@staticmethod
@DataOps.MUNLOCK
def _munlock(ref):
Data.get_instance(ref).munlock()
@staticmethod
@DataOps.READ
def _read(dst, src, size):
return Data.get_instance(src).read(dst, size)
@staticmethod
@DataOps.WRITE
def _write(dst, src, size):
return Data.get_instance(dst).write(src, size)
@staticmethod
@DataOps.ZERO
def _zero(dst, size):
return Data.get_instance(dst).zero(size)
@staticmethod
@DataOps.SEEK
def _seek(dst, seek, size):
return Data.get_instance(dst).seek(DataSeek(seek), size)
@staticmethod
@DataOps.COPY
def _copy(dst, src, end, start, size):
return Data.get_instance(dst).copy(Data.get_instance(src), end, start, size)
@staticmethod
@DataOps.SECURE_ERASE
def _secure_erase(dst):
Data.get_instance(dst).secure_erase()
def read(self, dst, size):
to_read = min(self.size - self.position, size)
memmove(dst, self.data + self.position, to_read)
return to_read
def write(self, src, size):
to_write = min(self.size - self.position, size)
memmove(self.data + self.position, src, to_write)
return to_write
def mlock(self):
return 0
def munlock(self):
pass
def zero(self, size):
to_zero = min(self.size - self.position, size)
memset(self.data + self.position, 0, to_zero)
return to_zero
def seek(self, seek, size):
if seek == DataSeek.CURRENT:
to_move = min(self.size - self.position, size)
self.position += to_move
else:
to_move = min(self.size, size)
self.position = to_move
return to_move
def copy(self, src, end, start, size):
return size
def secure_erase(self):
pass
def dump(self):
print_buffer(self.buffer, self.size)
def md5(self):
m = md5()
m.update(string_at(self.data, self.size))
return m.hexdigest()

130
tests/functional/pyocf/types/io.py Normal file
View File

@ -0,0 +1,130 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from ctypes import *
from enum import IntEnum, IntFlag, auto
from ..ocf import OcfLib
from .data import Data
from .queue import Queue
class IoDir(IntEnum):
READ = 0
WRITE = 1
class IoOps(Structure):
pass
class Io(Structure):
START = CFUNCTYPE(None, c_void_p)
HANDLE = CFUNCTYPE(None, c_void_p, c_void_p)
END = CFUNCTYPE(None, c_void_p, c_int)
_instances_ = {}
_fields_ = [
("_volume", c_void_p),
("_ops", POINTER(IoOps)),
("_addr", c_uint64),
("_flags", c_uint64),
("_bytes", c_uint32),
("_class", c_uint32),
("_dir", c_uint32),
("_io_queue", c_void_p),
("_start", START),
("_handle", HANDLE),
("_end", END),
("_priv1", c_void_p),
("_priv2", c_void_p),
]
@classmethod
def from_pointer(cls, ref):
c = cls.from_address(ref)
cls._instances_[ref] = c
OcfLib.getInstance().ocf_io_set_cmpl_wrapper(byref(c), None, None, c.c_end)
return c
@classmethod
def get_instance(cls, ref):
return cls._instances_[cast(ref, c_void_p).value]
def del_object(self):
del type(self)._instances_[cast(byref(self), c_void_p).value]
def put(self):
OcfLib.getInstance().ocf_io_put(byref(self))
def get(self):
OcfLib.getInstance().ocf_io_get(byref(self))
@staticmethod
@END
def c_end(io, err):
Io.get_instance(io).end(err)
@staticmethod
@START
def c_start(io):
Io.get_instance(io).start()
@staticmethod
@HANDLE
def c_handle(io, opaque):
Io.get_instance(io).handle(opaque)
def end(self, err):
if err:
print("IO err {}".format(err))
self.put()
self.del_object()
def submit(self):
return OcfLib.getInstance().ocf_core_submit_io_wrapper(byref(self))
def configure(
self, addr: int, length: int, direction: IoDir, io_class: int, flags: int
):
OcfLib.getInstance().ocf_io_configure_wrapper(
byref(self), addr, length, direction, io_class, flags
)
def set_data(self, data: Data):
self.data = data
OcfLib.getInstance().ocf_io_set_data_wrapper(byref(self), data, 0)
def set_queue(self, queue: Queue):
self.queue = queue
OcfLib.getInstance().ocf_io_set_queue_wrapper(byref(self), queue.handle)
IoOps.SET_DATA = CFUNCTYPE(c_int, POINTER(Io), c_void_p, c_uint32)
IoOps.GET_DATA = CFUNCTYPE(c_void_p, POINTER(Io))
IoOps._fields_ = [("_set_data", IoOps.SET_DATA), ("_get_data", IoOps.GET_DATA)]
lib = OcfLib.getInstance()
lib.ocf_core_new_io_wrapper.restype = POINTER(Io)
lib.ocf_io_set_cmpl_wrapper.argtypes = [POINTER(Io), c_void_p, c_void_p, Io.END]
lib.ocf_io_configure_wrapper.argtypes = [
POINTER(Io),
c_uint64,
c_uint32,
c_uint32,
c_uint32,
c_uint64,
]
lib.ocf_io_set_queue_wrapper.argtypes = [POINTER(Io), c_uint32]
lib.ocf_core_new_io_wrapper.argtypes = [c_void_p]
lib.ocf_core_new_io_wrapper.restype = c_void_p
lib.ocf_io_set_data_wrapper.argtypes = [POINTER(Io), c_void_p, c_uint32]
lib.ocf_io_set_data_wrapper.restype = c_int
lib.ocf_io_set_queue_wrapper.argtypes = [POINTER(Io), c_void_p]

168
tests/functional/pyocf/types/logger.py Normal file
View File

@ -0,0 +1,168 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from ctypes import *
from enum import IntEnum
import logging
from io import StringIO
from ..ocf import OcfLib
logger = logging.getLogger("pyocf")
logger.setLevel(logging.DEBUG)
class LogLevel(IntEnum):
EMERG = 0
ALERT = 1
CRIT = 2
ERR = 3
WARN = 4
NOTICE = 5
INFO = 6
DEBUG = 7
LevelMapping = {
LogLevel.EMERG: logging.CRITICAL,
LogLevel.ALERT: logging.CRITICAL,
LogLevel.CRIT: logging.CRITICAL,
LogLevel.ERR: logging.ERROR,
LogLevel.WARN: logging.WARNING,
LogLevel.NOTICE: logging.INFO,
LogLevel.INFO: logging.INFO,
LogLevel.DEBUG: logging.DEBUG,
}
class LoggerOps(Structure):
OPEN = CFUNCTYPE(c_int, c_void_p)
CLOSE = CFUNCTYPE(None, c_void_p)
# PRINTF ommited - we cannot make variadic function call in ctypes
LOG = CFUNCTYPE(c_int, c_void_p, c_uint, c_char_p)
PRINTF_RL = CFUNCTYPE(c_int, c_void_p, c_char_p)
DUMP_STACK = CFUNCTYPE(c_int, c_void_p)
_fields_ = [
("_open", OPEN),
("_close", CLOSE),
("_printf", c_void_p),
("_printf_rl", PRINTF_RL),
("_dump_stack", DUMP_STACK),
]
class LoggerPriv(Structure):
_fields_ = [("_log", LoggerOps.LOG)]
class Logger(Structure):
_instances_ = {}
_fields_ = [("logger", c_void_p)]
def __init__(self):
self.ops = LoggerOps(
_open=self._open,
_printf=cast(OcfLib.getInstance().pyocf_printf_helper, c_void_p),
_close=self._close,
)
self.priv = LoggerPriv(_log=self._log)
self._as_parameter_ = cast(pointer(self.priv), c_void_p).value
self._instances_[self._as_parameter_] = self
def get_ops(self):
return self.ops
def get_priv(self):
return self.priv
@classmethod
def get_instance(cls, ctx: int):
priv = OcfLib.getInstance().ocf_logger_get_priv(ctx)
return cls._instances_[priv]
@staticmethod
@LoggerOps.LOG
def _log(ref, lvl, msg):
Logger.get_instance(ref).log(lvl, str(msg, "ascii").strip())
return 0
@staticmethod
@LoggerOps.OPEN
def _open(ref):
if hasattr(Logger.get_instance(ref), "open"):
return Logger.get_instance(ref).open()
else:
return 0
@staticmethod
@LoggerOps.CLOSE
def _close(ref):
if hasattr(Logger.get_instance(ref), "close"):
return Logger.get_instance(ref).close()
else:
return 0
class DefaultLogger(Logger):
def __init__(self, level: LogLevel = LogLevel.WARN):
super().__init__()
self.level = level
ch = logging.StreamHandler()
fmt = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
ch.setFormatter(fmt)
ch.setLevel(LevelMapping[level])
logger.addHandler(ch)
def log(self, lvl: int, msg: str):
logger.log(LevelMapping[lvl], msg)
def close(self):
logger.handlers = []
class FileLogger(Logger):
def __init__(self, f, console_level=None):
super().__init__()
fmt = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
fh = logging.FileHandler(f)
fh.setLevel(logging.DEBUG)
fh.setFormatter(fmt)
logger.addHandler(fh)
if console_level:
sh = logging.StreamHandler()
sh.setLevel(LevelMapping[console_level])
sh.setFormatter(fmt)
logger.addHandler(sh)
def log(self, lvl, msg):
logger.log(LevelMapping[lvl], msg)
def close(self):
logger.handlers = []
class BufferLogger(Logger):
def __init__(self, level: LogLevel):
super().__init__()
self.level = level
self.buffer = StringIO()
def log(self, lvl, msg):
if lvl < self.level:
self.buffer.write(msg + "\n")
def get_lines(self):
return self.buffer.getvalue().split("\n")
lib = OcfLib.getInstance()
lib.ocf_logger_get_priv.restype = c_void_p
lib.ocf_logger_get_priv.argtypes = [c_void_p]

48
tests/functional/pyocf/types/metadata_updater.py Normal file
View File

@ -0,0 +1,48 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from ctypes import *
from .shared import SharedOcfObject
class MetadataUpdaterOps(Structure):
INIT = CFUNCTYPE(c_int, c_void_p)
KICK = CFUNCTYPE(None, c_void_p)
STOP = CFUNCTYPE(None, c_void_p)
_fields_ = [("_init", INIT), ("_kick", KICK), ("_stop", STOP)]
class MetadataUpdater(SharedOcfObject):
_instances_ = {}
_fields_ = [("mu", c_void_p)]
ops = None
def __init__(self):
self._as_parameter_ = self.mu
super().__init__()
@classmethod
def get_ops(cls):
if not cls.ops:
cls.ops = MetadataUpdaterOps(
_init=cls._init, _kick=cls._kick, _stop=cls._stop
)
return cls.ops
@staticmethod
@MetadataUpdaterOps.INIT
def _init(ref):
return 0
@staticmethod
@MetadataUpdaterOps.KICK
def _kick(ref):
pass
@staticmethod
@MetadataUpdaterOps.STOP
def _stop(ref):
pass

56
tests/functional/pyocf/types/queue.py Normal file
View File

@ -0,0 +1,56 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from ctypes import *
from ..ocf import OcfLib
from .shared import SharedOcfObject, OcfError
class QueueOps(Structure):
KICK = CFUNCTYPE(None, c_void_p)
KICK_SYNC = CFUNCTYPE(None, c_void_p)
STOP = CFUNCTYPE(None, c_void_p)
_fields_ = [
("kick", KICK),
("kick_sync", KICK_SYNC),
("stop", STOP),
]
class Queue:
_instances_ = {}
def __init__(self, cache):
self.ops = QueueOps(kick_sync=type(self)._kick_sync, stop=type(self)._stop)
self.handle = c_void_p()
status = OcfLib.getInstance().ocf_queue_create(cache.cache_handle, byref(self.handle), byref(self.ops))
if status:
raise OcfError("Couldn't create queue object", status)
Queue._instances_[self.handle.value] = self
cache.queues += [self]
@classmethod
def get_instance(cls, ref):
return cls._instances_[ref]
@staticmethod
@QueueOps.KICK_SYNC
def _kick_sync(ref):
Queue.get_instance(ref).kick_sync()
@staticmethod
@QueueOps.STOP
def _stop(ref):
Queue.get_instance(ref).stop()
def kick_sync(self):
OcfLib.getInstance().ocf_queue_run(self.handle)
def stop(self):
pass

100
tests/functional/pyocf/types/shared.py Normal file
View File

@ -0,0 +1,100 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from ctypes import *
from enum import IntEnum, auto
from ..utils import Size as S
class OcfErrorCode(IntEnum):
OCF_ERR_INVAL = 1000000
OCF_ERR_INTR = auto()
OCF_ERR_NO_MEM = auto()
OCF_ERR_NO_LOCK = auto()
OCF_ERR_INVAL_VOLUME_TYPE = auto()
OCF_ERR_UNKNOWN = auto()
OCF_ERR_TOO_MANY_CACHES = auto()
OCF_ERR_NO_FREE_RAM = auto()
OCF_ERR_START_CACHE_FAIL = auto()
OCF_ERR_CACHE_IN_USE = auto()
OCF_ERR_CACHE_NOT_EXIST = auto()
OCF_ERR_CACHE_EXIST = auto()
OCF_ERR_TOO_MANY_CORES = auto()
OCF_ERR_CORE_NOT_AVAIL = auto()
OCF_ERR_NOT_OPEN_EXC = auto()
OCF_ERR_CACHE_NOT_AVAIL = auto()
OCF_ERR_IO_CLASS_NOT_EXIST = auto()
OCF_ERR_WRITE_CACHE = auto()
OCF_ERR_WRITE_CORE = auto()
OCF_ERR_DIRTY_SHUTDOWN = auto()
OCF_ERR_DIRTY_EXISTS = auto()
OCF_ERR_FLUSHING_INTERRUPTED = auto()
OCF_ERR_CANNOT_ADD_CORE_TO_POOL = auto()
OCF_ERR_CACHE_IN_INCOMPLETE_STATE = auto()
OCF_ERR_CORE_IN_INACTIVE_STATE = auto()
OCF_ERR_INVALID_CACHE_MODE = auto()
OCF_ERR_INVALID_CACHE_LINE_SIZE = auto()
class OcfError(BaseException):
def __init__(self, msg, error_code):
super().__init__(self, msg)
self.error_code = OcfErrorCode(abs(error_code))
self.msg = msg
def __str__(self):
return "{} ({})".format(self.msg, repr(self.error_code))
class SharedOcfObject(Structure):
_instances_ = {}
def __init__(self):
super().__init__()
type(self)._instances_[self._as_parameter_] = self
@classmethod
def get_instance(cls, ref: int):
try:
return cls._instances_[ref]
except:
print(
"OcfSharedObject corruption. wanted: {} instances: {}".format(
ref, cls._instances_
)
)
return None
@classmethod
def del_object(cls, ref: int):
del cls._instances_[ref]
class Uuid(Structure):
_fields_ = [("_size", c_size_t), ("_data", c_char_p)]
class CacheLineSize(IntEnum):
LINE_4KiB = S.from_KiB(4)
LINE_8KiB = S.from_KiB(8)
LINE_16KiB = S.from_KiB(16)
LINE_32KiB = S.from_KiB(32)
LINE_64KiB = S.from_KiB(64)
DEFAULT = LINE_4KiB
class CacheLines(S):
def __init__(self, count: int, line_size: CacheLineSize):
self.bytes = count * line_size
self.line_size = line_size
def __int__(self):
return int(self.bytes / self.line_size)
def __str__(self):
return "{} ({})".format(int(self), super().__str__())
__repr__ = __str__

0
tests/functional/pyocf/types/stats/__init__.py Normal file
View File

37
tests/functional/pyocf/types/stats/cache.py Normal file
View File

@ -0,0 +1,37 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from ctypes import *
class _Inactive(Structure):
_fields_ = [("occupancy", c_uint32), ("dirty", c_uint32)]
class _FallbackPt(Structure):
_fields_ = [("error_counter", c_int), ("status", c_bool)]
class CacheInfo(Structure):
_fields_ = [
("attached", c_bool),
("volume_type", c_uint8),
("size", c_uint32),
("inactive", _Inactive),
("occupancy", c_uint32),
("dirty", c_uint32),
("dirty_initial", c_uint32),
("dirty_for", c_uint32),
("cache_mode", c_uint32),
("fallback_pt", _FallbackPt),
("state", c_uint8),
("eviction_policy", c_uint32),
("cleaning_policy", c_uint32),
("cache_line_size", c_uint64),
("flushed", c_uint32),
("core_count", c_uint32),
("metadata_footprint", c_uint64),
("metadata_end_offset", c_uint32),
]

30
tests/functional/pyocf/types/stats/core.py Normal file
View File

@ -0,0 +1,30 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from ctypes import *
from .shared import *
class CoreStats(Structure):
_fields_ = [
("core_size", c_uint64),
("core_size_bytes", c_uint64),
("cache_occupancy", c_uint32),
("dirty", c_uint32),
("flushed", c_uint32),
("dirty_for", c_uint32),
("read_reqs", OcfStatsReq),
("write_reqs", OcfStatsReq),
("cache_volume", OcfStatsBlock),
("core_volume", OcfStatsBlock),
("core", OcfStatsBlock),
("cache_errors", OcfStatsError),
("core_errors", OcfStatsError),
("debug_stat", OcfStatsDebug),
("seq_cutoff_threshold", c_uint32),
("seq_cutoff_policy", c_uint32),
]

88
tests/functional/pyocf/types/stats/shared.py Normal file
View File

@ -0,0 +1,88 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from ctypes import *
class _Stat(Structure):
_fields_ = [("value", c_uint64), ("permil", c_uint64)]
class OcfStatsReq(Structure):
_fields_ = [
("partial_miss", c_uint64),
("full_miss", c_uint64),
("total", c_uint64),
("pass_through", c_uint64),
]
class OcfStatsBlock(Structure):
_fields_ = [("read", c_uint64), ("write", c_uint64)]
class OcfStatsError(Structure):
_fields_ = [("read", c_uint32), ("write", c_uint32)]
class OcfStatsDebug(Structure):
_fields_ = [
("read_size", c_uint64 * 12),
("write_size", c_uint64 * 12),
("read_align", c_uint64 * 4),
("write_align", c_uint64 * 4),
]
class UsageStats(Structure):
_fields_ = [
("occupancy", _Stat),
("free", _Stat),
("clean", _Stat),
("dirty", _Stat),
]
class RequestsStats(Structure):
_fields_ = [
("rd_hits", _Stat),
("rd_partial_misses", _Stat),
("rd_full_misses", _Stat),
("rd_total", _Stat),
("wr_hits", _Stat),
("wr_partial_misses", _Stat),
("wr_full_misses", _Stat),
("wr_total", _Stat),
("rd_pt", _Stat),
("wr_pt", _Stat),
("serviced", _Stat),
("total", _Stat),
]
class BlocksStats(Structure):
_fields_ = [
("core_volume_rd", _Stat),
("core_volume_wr", _Stat),
("core_volume_total", _Stat),
("cache_volume_rd", _Stat),
("cache_volume_wr", _Stat),
("cache_volume_total", _Stat),
("volume_rd", _Stat),
("volume_wr", _Stat),
("volume_total", _Stat),
]
class ErrorsStats(Structure):
_fields_ = [
("core_volume_rd", _Stat),
("core_volume_wr", _Stat),
("core_volume_total", _Stat),
("cache_volume_rd", _Stat),
("cache_volume_wr", _Stat),
("cache_volume_total", _Stat),
("total", _Stat),
]

260
tests/functional/pyocf/types/volume.py Normal file
View File

@ -0,0 +1,260 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from ctypes import *
from hashlib import md5
from collections import defaultdict
from .io import Io, IoOps, IoDir
from .shared import OcfError
from ..ocf import OcfLib
from ..utils import print_buffer, Size as S
from .data import Data
class VolumeCaps(Structure):
_fields_ = [("_atomic_writes", c_uint32, 1)]
class VolumeOps(Structure):
SUBMIT_IO = CFUNCTYPE(None, POINTER(Io))
SUBMIT_FLUSH = CFUNCTYPE(None, c_void_p)
SUBMIT_METADATA = CFUNCTYPE(None, c_void_p)
SUBMIT_DISCARD = CFUNCTYPE(None, c_void_p)
SUBMIT_WRITE_ZEROES = CFUNCTYPE(None, c_void_p)
OPEN = CFUNCTYPE(c_int, c_void_p)
CLOSE = CFUNCTYPE(None, c_void_p)
GET_MAX_IO_SIZE = CFUNCTYPE(c_uint, c_void_p)
GET_LENGTH = CFUNCTYPE(c_uint64, c_void_p)
_fields_ = [
("_submit_io", SUBMIT_IO),
("_submit_flush", SUBMIT_FLUSH),
("_submit_metadata", SUBMIT_METADATA),
("_submit_discard", SUBMIT_DISCARD),
("_submit_write_zeroes", SUBMIT_WRITE_ZEROES),
("_open", OPEN),
("_close", CLOSE),
("_get_max_io_size", GET_MAX_IO_SIZE),
("_get_length", GET_LENGTH),
]
class VolumeProperties(Structure):
_fields_ = [
("_name", c_char_p),
("_io_priv_size", c_uint32),
("_volume_priv_size", c_uint32),
("_caps", VolumeCaps),
("_ops", VolumeOps),
("_io_ops", IoOps),
]
class VolumeIoPriv(Structure):
_fields_ = [("_data", c_void_p)]
class Volume(Structure):
_fields_ = [("_storage", c_void_p)]
_instances_ = {}
_uuid_ = {}
def __init__(self, size: S, uuid=None):
self.size = size
if uuid:
if uuid in type(self)._uuid_:
raise Exception("Volume with uuid {} already created".format(uuid))
self.uuid = uuid
else:
self.uuid = str(id(self))
type(self)._uuid_[self.uuid] = self
self.data = create_string_buffer(int(self.size))
self._storage = cast(self.data, c_void_p)
self.reset_stats()
@classmethod
def get_props(cls):
return VolumeProperties(
_name=str(cls.__name__).encode("ascii"),
_io_priv_size=sizeof(VolumeIoPriv),
_volume_priv_size=0,
_caps=VolumeCaps(_atomic_writes=0),
_ops=VolumeOps(
_submit_io=cls._submit_io,
_submit_flush=cls._submit_flush,
_submit_metadata=cls._submit_metadata,
_submit_discard=cls._submit_discard,
_submit_write_zeroes=cls._submit_write_zeroes,
_open=cls._open,
_close=cls._close,
_get_max_io_size=cls._get_max_io_size,
_get_length=cls._get_length,
),
_io_ops=IoOps(_set_data=cls._io_set_data, _get_data=cls._io_get_data),
)
@classmethod
def get_instance(cls, ref):
return cls._instances_[ref]
@classmethod
def get_by_uuid(cls, uuid):
return cls._uuid_[uuid]
@staticmethod
@VolumeOps.SUBMIT_IO
def _submit_io(io):
io_structure = cast(io, POINTER(Io))
volume = Volume.get_instance(io_structure.contents._volume)
volume.submit_io(io_structure)
@staticmethod
@VolumeOps.SUBMIT_FLUSH
def _submit_flush(flush):
io_structure = cast(io, POINTER(Io))
volume = Volume.get_instance(io_structure.contents._volume)
volume.submit_flush(io_structure)
@staticmethod
@VolumeOps.SUBMIT_METADATA
def _submit_metadata(meta):
pass
@staticmethod
@VolumeOps.SUBMIT_DISCARD
def _submit_discard(discard):
io_structure = cast(io, POINTER(Io))
volume = Volume.get_instance(io_structure.contents._volume)
volume.submit_discard(io_structure)
@staticmethod
@VolumeOps.SUBMIT_WRITE_ZEROES
def _submit_write_zeroes(write_zeroes):
pass
@staticmethod
@CFUNCTYPE(c_int, c_void_p)
def _open(ref):
uuid_ptr = cast(OcfLib.getInstance().ocf_volume_get_uuid(ref), c_void_p)
uuid_str = cast(
OcfLib.getInstance().ocf_uuid_to_str_wrapper(uuid_ptr), c_char_p
)
uuid = str(uuid_str.value, encoding="ascii")
try:
volume = Volume.get_by_uuid(uuid)
except:
print("Tried to access unallocated volume {}".format(uuid))
print("{}".format(Volume._uuid_))
return -1
type(volume)._instances_[ref] = volume
return volume.open()
@staticmethod
@VolumeOps.CLOSE
def _close(ref):
Volume.get_instance(ref).close()
del Volume._instances_[ref]
@staticmethod
@VolumeOps.GET_MAX_IO_SIZE
def _get_max_io_size(ref):
return S.from_KiB(128)
@staticmethod
@VolumeOps.GET_LENGTH
def _get_length(ref):
return Volume.get_instance(ref).get_length()
@staticmethod
@IoOps.SET_DATA
def _io_set_data(io, data, offset):
io_priv = cast(OcfLib.getInstance().ocf_io_get_priv(io), POINTER(VolumeIoPriv))
data = Data.get_instance(data)
data.position = offset
io_priv.contents._data = cast(data, c_void_p)
return 0
@staticmethod
@IoOps.GET_DATA
def _io_get_data(io):
io_priv = cast(OcfLib.getInstance().ocf_io_get_priv(io), POINTER(VolumeIoPriv))
return io_priv.contents._data
def open(self):
return 0
def close(self):
pass
def get_length(self):
return self.size
def submit_flush(self, flush):
flush.contents._end(io, 0)
def submit_discard(self, discard):
discard.contents._end(io, 0)
def get_stats(self):
return self.stats
def reset_stats(self):
self.stats = {IoDir.WRITE: 0, IoDir.READ: 0}
def submit_io(self, io):
try:
self.stats[io.contents._dir] += 1
if io.contents._dir == IoDir.WRITE:
src_ptr = cast(io.contents._ops.contents._get_data(io), c_void_p)
src = Data.get_instance(src_ptr.value)
dst = self._storage + io.contents._addr
elif io.contents._dir == IoDir.READ:
dst_ptr = cast(io.contents._ops.contents._get_data(io), c_void_p)
dst = Data.get_instance(dst_ptr.value)
src = self._storage + io.contents._addr
memmove(dst, src, io.contents._bytes)
io.contents._end(io, 0)
except:
io.contents._end(io, -5)
def dump_contents(self, stop_after_zeros=0, offset=0, size=0):
if size == 0:
size = self.size
print_buffer(self._storage + offset, size, stop_after_zeros=stop_after_zeros)
def md5(self):
m = md5()
m.update(string_at(self._storage, self.size))
return m.hexdigest()
class ErrorDevice(Volume):
def __init__(self, size, error_sectors: set = None, uuid=None):
super().__init__(size, uuid)
self.error_sectors = error_sectors or set()
def set_mapping(self, error_sectors: set):
self.error_sectors = error_sectors
def submit_io(self, io):
if io.contents._addr in self.error_sectors:
io.contents._end(io, -5)
self.stats["errors"][io.contents._dir] += 1
else:
super().submit_io(io)
def reset_stats(self):
super().reset_stats()
self.stats["errors"] = {IoDir.WRITE: 0, IoDir.READ: 0}

139
tests/functional/pyocf/utils.py Normal file
View File

@ -0,0 +1,139 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from ctypes import *
def print_buffer(buf, length, offset=0, width=16, stop_after_zeros=0):
end = offset + length
zero_lines = 0
buf = string_at(buf, length)
whole_buffer_empty = True
stop_after_zeros = int(stop_after_zeros / width)
for addr in range(offset, end, width):
cur_line = buf[addr : min(end, addr + width)]
all_zeros = True
byteline = ""
asciiline = ""
if not any(cur_line):
if stop_after_zeros and zero_lines > stop_after_zeros:
print(
"<{} bytes of empty space encountered, stopping>".format(
stop_after_zeros * width
)
)
return
zero_lines += 1
continue
if zero_lines:
print("<{} zero bytes omitted>".format(zero_lines * width))
zero_lines = 0
for x in cur_line:
x = int(x)
byteline += "{:02X} ".format(x)
if 31 < x < 126:
char = chr(x)
else:
char = "."
asciiline += char
print("{:#08X}\t{}\t{}".format(addr, byteline, asciiline))
whole_buffer_empty = False
if whole_buffer_empty:
print("<whole buffer empty>")
elif zero_lines:
print("<zero until end>")
class Size:
_KiB = 1024
_MiB = _KiB * 1024
_GiB = _MiB * 1024
_TiB = _GiB * 1024
def __init__(self, b: int):
self.bytes = b
def __int__(self):
return self.bytes
@classmethod
def from_B(cls, value):
return cls(value)
@classmethod
def from_KiB(cls, value):
return cls(value * cls._KiB)
@classmethod
def from_MiB(cls, value):
return cls(value * cls._MiB)
@classmethod
def from_GiB(cls, value):
return cls(value * cls._GiB)
@classmethod
def from_TiB(cls, value):
return cls(value * cls._TiB)
@property
def B(self):
return self.bytes
@property
def KiB(self):
return self.bytes / self._KiB
@property
def MiB(self):
return self.bytes / self._MiB
@property
def GiB(self):
return self.bytes / self._GiB
@property
def TiB(self):
return self.bytes / self._TiB
def __str__(self):
if self.bytes < self._KiB:
return "{} B".format(self.B)
elif self.bytes < self._MiB:
return "{} KiB".format(self.KiB)
elif self.bytes < self._GiB:
return "{} MiB".format(self.MiB)
elif self.bytes < self._TiB:
return "{} GiB".format(self.GiB)
else:
return "{} TiB".format(self.TiB)
def print_structure(struct, indent=0):
print(struct)
for field, field_type in struct._fields_:
value = getattr(struct, field)
if hasattr(value, "_fields_"):
print("{}{: <20} :".format(" " * indent, field))
print_structure(value, indent=indent + 1)
continue
print("{}{: <20} : {}".format(" " * indent, field, value))
def struct_to_dict(struct):
d = {}
for field, field_type in struct._fields_:
value = getattr(struct, field)
if hasattr(value, "_fields_"):
d[field] = struct_to_dict(value)
continue
d[field] = value
return d

56
tests/functional/pyocf/wrappers/ocf_io_wrappers.c Normal file
View File

@ -0,0 +1,56 @@
/*
* Copyright(c) 2012-2018 Intel Corporation
* SPDX-License-Identifier: BSD-3-Clause-Clear
*/
#include "ocf/ocf_io.h"
#include "ocf/ocf_core.h"
struct ocf_io *ocf_core_new_io_wrapper(ocf_core_t core)
{
return ocf_core_new_io(core);
}
void ocf_io_configure_wrapper(struct ocf_io *io, uint64_t addr,
uint32_t bytes, uint32_t dir, uint32_t class, uint64_t flags)
{
ocf_io_configure(io, addr, bytes, dir, class, flags);
}
void ocf_io_set_cmpl_wrapper(struct ocf_io *io, void *context,
void *context2, ocf_end_io_t fn)
{
ocf_io_set_cmpl(io, context, context2, fn);
}
void ocf_io_set_start_wrapper(struct ocf_io *io, ocf_start_io_t fn)
{
ocf_io_set_start(io, fn);
}
void ocf_io_set_handle_wrapper(struct ocf_io *io, ocf_handle_io_t fn)
{
ocf_io_set_handle(io, fn);
}
int ocf_io_set_data_wrapper(struct ocf_io *io, ctx_data_t *data,
uint32_t offset)
{
return ocf_io_set_data(io, data, offset);
}
ctx_data_t *ocf_io_get_data_wrapper(struct ocf_io *io)
{
return ocf_io_get_data(io);
}
void ocf_io_set_queue_wrapper(struct ocf_io *io, ocf_queue_t queue)
{
ocf_io_set_queue(io, queue);
}
void ocf_core_submit_io_wrapper(struct ocf_io *io)
{
ocf_core_submit_io(io);
}

42
tests/functional/pyocf/wrappers/ocf_logger_wrappers.c Normal file
View File

@ -0,0 +1,42 @@
/*
* Copyright(c) 2012-2018 Intel Corporation
* SPDX-License-Identifier: BSD-3-Clause-Clear
*/
#include <ocf/ocf_types.h>
#include <ocf/ocf_logger.h>
#include <stdarg.h>
#include "ocf_env.h"
#define LOG_BUFFER_SIZE 4096
struct pyocf_logger_priv {
int (*pyocf_log)(void *pyocf_logger, ocf_logger_lvl_t lvl, char *msg);
};
int pyocf_printf_helper(ocf_logger_t logger, ocf_logger_lvl_t lvl,
const char *fmt, va_list args)
{
char *buffer = env_zalloc(LOG_BUFFER_SIZE, ENV_MEM_NORMAL);
struct pyocf_logger_priv *priv = ocf_logger_get_priv(logger);
int ret;
if (!buffer) {
ret = -ENOMEM;
goto out;
}
ret = vsnprintf(buffer, LOG_BUFFER_SIZE, fmt, args);
if (ret < 0) {
env_free(buffer);
goto out;
}
ret = priv->pyocf_log(logger, lvl, buffer);
env_free(buffer);
out:
return ret;
}

12
tests/functional/pyocf/wrappers/ocf_volume_wrappers.c Normal file
View File

@ -0,0 +1,12 @@
/*
* Copyright(c) 2012-2018 Intel Corporation
* SPDX-License-Identifier: BSD-3-Clause-Clear
*/
#include "ocf/ocf_io.h"
#include "ocf/ocf_volume.h"
const char *ocf_uuid_to_str_wrapper(const struct ocf_volume_uuid *uuid) {
return ocf_uuid_to_str(uuid);
}

142
tests/functional/test_adapter/src/ocf_env.c
View File

@ -1,142 +0,0 @@
/*
* Copyright(c) 2019 Intel Corporation
* SPDX-License-Identifier: BSD-3-Clause-Clear
*/
#include "ocf_env.h"
#include <sched.h>
#include <execinfo.h>
struct _env_allocator {
/*!< Memory pool ID unique name */
char *name;
/*!< Size of specific item of memory pool */
uint32_t item_size;
/*!< Number of currently allocated items in pool */
env_atomic count;
};
static inline size_t env_allocator_align(size_t size)
{
if (size <= 2)
return size;
return (1ULL << 32) >> __builtin_clz(size - 1);
}
struct _env_allocator_item {
uint32_t flags;
uint32_t cpu;
char data[];
};
void *env_allocator_new(env_allocator *allocator)
{
struct _env_allocator_item *item = NULL;
item = calloc(1, allocator->item_size);
if (item) {
item->cpu = 0;
env_atomic_inc(&allocator->count);
}
return &item->data;
}
env_allocator *env_allocator_create(uint32_t size, const char *fmt_name, ...)
{
char name[OCF_ALLOCATOR_NAME_MAX] = { '\0' };
int result, error = -1;
va_list args;
env_allocator *allocator = calloc(1, sizeof(*allocator));
if (!allocator) {
error = __LINE__;
goto err;
}
allocator->item_size = size + sizeof(struct _env_allocator_item);
/* Format allocator name */
va_start(args, fmt_name);
result = vsnprintf(name, sizeof(name), fmt_name, args);
va_end(args);
if ((result > 0) && (result < sizeof(name))) {
allocator->name = strdup(name);
if (!allocator->name) {
error = __LINE__;
goto err;
}
} else {
/* Formated string name exceed max allowed size of name */
error = __LINE__;
goto err;
}
return allocator;
err:
printf("Cannot create memory allocator, ERROR %d", error);
env_allocator_destroy(allocator);
return NULL;
}
void env_allocator_del(env_allocator *allocator, void *obj)
{
struct _env_allocator_item *item =
container_of(obj, struct _env_allocator_item, data);
env_atomic_dec(&allocator->count);
free(item);
}
void env_allocator_destroy(env_allocator *allocator)
{
if (allocator) {
if (env_atomic_read(&allocator->count)) {
printf("Not all objects deallocated\n");
ENV_WARN(true, OCF_PREFIX_SHORT" Cleanup problem\n");
}
free(allocator->name);
free(allocator);
}
}
uint32_t env_allocator_item_count(env_allocator *allocator)
{
return env_atomic_read(&allocator->count);
}
/* *** DEBUGING *** */
#define ENV_TRACE_DEPTH 16
void env_stack_trace(void)
{
void *trace[ENV_TRACE_DEPTH];
char **messages = NULL;
int i, size;
size = backtrace(trace, ENV_TRACE_DEPTH);
messages = backtrace_symbols(trace, size);
printf("[stack trace]>>>\n");
for (i = 0; i < size; ++i)
printf("%s\n", messages[i]);
printf("<<<[stack trace]\n");
free(messages);
}
/* *** CRC *** */
uint32_t env_crc32(uint32_t crc, uint8_t const *data, size_t len)
{
return crc32(crc, data, len);
}

653
tests/functional/test_adapter/src/ocf_env.h
View File

@ -1,653 +0,0 @@
/*
* Copyright(c) 2019 Intel Corporation
* SPDX-License-Identifier: BSD-3-Clause-Clear
*/
#ifndef __OCF_ENV_H__
#define __OCF_ENV_H__
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#ifndef __USE_GNU
#define __USE_GNU
#endif
#include <linux/limits.h>
#include <linux/stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdarg.h>
#include <stddef.h>
#include <string.h>
#include <pthread.h>
#include <assert.h>
#include <semaphore.h>
#include <errno.h>
#include <limits.h>
#include <unistd.h>
#include <inttypes.h>
#include <sys/time.h>
#include <sys/param.h>
#include <zlib.h>
#include "ocf_env_list.h"
#include "ocf_env_headers.h"
/* linux sector 512-bytes */
#define ENV_SECTOR_SHIFT 9
typedef uint8_t u8;
typedef uint16_t u16;
typedef uint32_t u32;
typedef uint64_t u64;
typedef uint64_t sector_t;
#define __packed __attribute__((packed))
#define __aligned(x) __attribute__((aligned(x)))
#define likely(cond) __builtin_expect(!!(cond), 1)
#define unlikely(cond) __builtin_expect(!!(cond), 0)
#define min(a,b) MIN(a,b)
#define OCF_ALLOCATOR_NAME_MAX 128
#define PAGE_SIZE 4096
/* *** MEMORY MANAGEMENT *** */
#define ENV_MEM_NORMAL 0
#define ENV_MEM_NOIO 0
#define ENV_MEM_ATOMIC 0
static inline void *env_malloc(size_t size, int flags)
{
return malloc(size);
}
static inline void *env_zalloc(size_t size, int flags)
{
void *ptr = malloc(size);
if (ptr)
memset(ptr, 0, size);
return ptr;
}
static inline void env_free(const void *ptr)
{
free((void *)ptr);
}
static inline void *env_vmalloc(size_t size)
{
return malloc(size);
}
static inline void *env_vzalloc(size_t size)
{
return env_zalloc(size, 0);
}
static inline void env_vfree(const void *ptr)
{
free((void *)ptr);
}
static inline uint64_t env_get_free_memory(void)
{
return sysconf(_SC_PAGESIZE) * sysconf(_SC_AVPHYS_PAGES);
}
/* *** ALLOCATOR *** */
typedef struct _env_allocator env_allocator;
env_allocator *env_allocator_create(uint32_t size, const char *fmt_name, ...);
void env_allocator_destroy(env_allocator *allocator);
void *env_allocator_new(env_allocator *allocator);
void env_allocator_del(env_allocator *allocator, void *item);
uint32_t env_allocator_item_count(env_allocator *allocator);
/* *** MUTEX *** */
typedef struct {
pthread_mutex_t m;
} env_mutex;
#define env_cond_resched() ({})
static inline int env_mutex_init(env_mutex *mutex)
{
if(pthread_mutex_init(&mutex->m, NULL))
return 1;
return 0;
}
static inline void env_mutex_lock(env_mutex *mutex)
{
pthread_mutex_lock(&mutex->m);
}
static inline int env_mutex_lock_interruptible(env_mutex *mutex)
{
env_mutex_lock(mutex);
return 0;
}
static inline int env_mutex_trylock(env_mutex *mutex)
{
if (pthread_mutex_trylock(&mutex->m) == 0)
return 1;
return 0;
}
static inline void env_mutex_unlock(env_mutex *mutex)
{
pthread_mutex_unlock(&mutex->m);
}
static inline int env_mutex_is_locked(env_mutex *mutex)
{
if (env_mutex_trylock(mutex)) {
env_mutex_unlock(mutex);
return 1;
}
return 0;
}
/* *** RECURSIVE MUTEX *** */
typedef env_mutex env_rmutex;
static inline int env_rmutex_init(env_rmutex *rmutex)
{
env_mutex_init(rmutex);
return 0;
}
static inline void env_rmutex_lock(env_rmutex *rmutex)
{
env_mutex_lock(rmutex);
}
static inline int env_rmutex_lock_interruptible(env_rmutex *rmutex)
{
return env_mutex_lock_interruptible(rmutex);
}
static inline int env_rmutex_trylock(env_rmutex *rmutex)
{
return env_mutex_trylock(rmutex);
}
static inline void env_rmutex_unlock(env_rmutex *rmutex)
{
env_mutex_unlock(rmutex);
}
static inline int env_rmutex_is_locked(env_rmutex *rmutex)
{
return env_mutex_is_locked(rmutex);
}
/* *** RW SEMAPHORE *** */
typedef struct {
pthread_rwlock_t lock;
} env_rwsem;
static inline int env_rwsem_init(env_rwsem *s)
{
return pthread_rwlock_init(&s->lock, NULL);
}
static inline void env_rwsem_up_read(env_rwsem *s)
{
pthread_rwlock_unlock(&s->lock);
}
static inline void env_rwsem_down_read(env_rwsem *s)
{
pthread_rwlock_rdlock(&s->lock);
}
static inline int env_rwsem_down_read_trylock(env_rwsem *s)
{
int result = pthread_rwlock_tryrdlock(&s->lock);
if (result == 0)
return 1;
else
return 0;
}
static inline void env_rwsem_up_write(env_rwsem *s)
{
pthread_rwlock_unlock(&s->lock);
}
static inline void env_rwsem_down_write(env_rwsem *s)
{
pthread_rwlock_wrlock(&s->lock);
}
static inline int env_rwsem_down_write_trylock(env_rwsem *s)
{
int result = pthread_rwlock_trywrlock(&s->lock);
if (result == 0)
return 1;
else
return 0;
}
static inline int env_rwsem_is_locked(env_rwsem *s)
{
if (env_rwsem_down_read_trylock(s)) {
env_rwsem_up_read(s);
return 0;
}
return 1;
}
static inline int env_rwsem_down_write_interruptible(env_rwsem *s)
{
env_rwsem_down_write(s);
return 0;
}
static inline int env_rwsem_down_read_interruptible(env_rwsem *s)
{
env_rwsem_down_read(s);
return 0;
}
/* *** COMPLETION *** */
struct completion {
sem_t sem;
};
typedef struct completion env_completion;
static inline void env_completion_init(env_completion *completion)
{
sem_init(&completion->sem, 0, 0);
}
static inline void env_completion_wait(env_completion *completion)
{
sem_wait(&completion->sem);
}
static inline void env_completion_complete(env_completion *completion)
{
sem_post(&completion->sem);
}
static inline void env_completion_complete_and_exit(
env_completion *completion, int ret)
{
env_completion_complete(completion); /* TODO */
}
/* *** ATOMIC VARIABLES *** */
typedef struct {
volatile int counter;
} env_atomic;
typedef struct {
volatile long counter;
} env_atomic64;
static inline int env_atomic_read(const env_atomic *a)
{
return a->counter; /* TODO */
}
static inline void env_atomic_set(env_atomic *a, int i)
{
a->counter = i; /* TODO */
}
static inline void env_atomic_add(int i, env_atomic *a)
{
__sync_add_and_fetch(&a->counter, i);
}
static inline void env_atomic_sub(int i, env_atomic *a)
{
__sync_sub_and_fetch(&a->counter, i);
}
static inline bool env_atomic_sub_and_test(int i, env_atomic *a)
{
return __sync_sub_and_fetch(&a->counter, i) == 0;
}
static inline void env_atomic_inc(env_atomic *a)
{
env_atomic_add(1, a);
}
static inline void env_atomic_dec(env_atomic *a)
{
env_atomic_sub(1, a);
}
static inline bool env_atomic_dec_and_test(env_atomic *a)
{
return __sync_sub_and_fetch(&a->counter, 1) == 0;
}
static inline bool env_atomic_inc_and_test(env_atomic *a)
{
return __sync_add_and_fetch(&a->counter, 1) == 0;
}
static inline int env_atomic_add_return(int i, env_atomic *a)
{
return __sync_add_and_fetch(&a->counter, i);
}
static inline int env_atomic_sub_return(int i, env_atomic *a)
{
return __sync_sub_and_fetch(&a->counter, i);
}
static inline int env_atomic_inc_return(env_atomic *a)
{
return env_atomic_add_return(1, a);
}
static inline int env_atomic_dec_return(env_atomic *a)
{
return env_atomic_sub_return(1, a);
}
static inline int env_atomic_cmpxchg(env_atomic *a, int old, int new_value)
{
return __sync_val_compare_and_swap(&a->counter, old, new_value);
}
static inline int env_atomic_add_unless(env_atomic *a, int i, int u)
{
int c, old;
c = env_atomic_read(a);
for (;;) {
if (unlikely(c == (u)))
break;
old = env_atomic_cmpxchg((a), c, c + (i));
if (likely(old == c))
break;
c = old;
}
return c != (u);
}
static inline long env_atomic64_read(const env_atomic64 *a)
{
return a->counter; /* TODO */
}
static inline void env_atomic64_set(env_atomic64 *a, long i)
{
a->counter = i; /* TODO */
}
static inline void env_atomic64_add(long i, env_atomic64 *a)
{
__sync_add_and_fetch(&a->counter, i);
}
static inline void env_atomic64_sub(long i, env_atomic64 *a)
{
__sync_sub_and_fetch(&a->counter, i);
}
static inline void env_atomic64_inc(env_atomic64 *a)
{
env_atomic64_add(1, a);
}
static inline void env_atomic64_dec(env_atomic64 *a)
{
env_atomic64_sub(1, a);
}
static inline long env_atomic64_cmpxchg(env_atomic64 *a, long old, long new)
{
return __sync_val_compare_and_swap(&a->counter, old, new);
}
/* *** SPIN LOCKS *** */
typedef struct {
pthread_spinlock_t lock;
} env_spinlock;
static inline void env_spinlock_init(env_spinlock *l)
{
pthread_spin_init(&l->lock, 0);
}
static inline void env_spinlock_lock(env_spinlock *l)
{
pthread_spin_lock(&l->lock);
}
static inline void env_spinlock_unlock(env_spinlock *l)
{
pthread_spin_unlock(&l->lock);
}
static inline void env_spinlock_lock_irq(env_spinlock *l)
{
env_spinlock_lock(l);
}
static inline void env_spinlock_unlock_irq(env_spinlock *l)
{
env_spinlock_unlock(l);
}
#define env_spinlock_lock_irqsave(l, flags) \
(void)flags; \
env_spinlock_lock(l)
#define env_spinlock_unlock_irqrestore(l, flags) \
(void)flags; \
env_spinlock_unlock(l)
/* *** RW LOCKS *** */
typedef struct {
pthread_rwlock_t lock;
} env_rwlock;
static inline void env_rwlock_init(env_rwlock *l)
{
pthread_rwlock_init(&l->lock, NULL);
}
static inline void env_rwlock_read_lock(env_rwlock *l)
{
pthread_rwlock_rdlock(&l->lock);
}
static inline void env_rwlock_read_unlock(env_rwlock *l)
{
pthread_rwlock_unlock(&l->lock);
}
static inline void env_rwlock_write_lock(env_rwlock *l)
{
pthread_rwlock_wrlock(&l->lock);
}
static inline void env_rwlock_write_unlock(env_rwlock *l)
{
pthread_rwlock_unlock(&l->lock);
}
/* *** WAITQUEUE *** */
typedef struct {
sem_t sem;
} env_waitqueue;
static inline void env_waitqueue_init(env_waitqueue *w)
{
sem_init(&w->sem, 0, 0);
}
static inline void env_waitqueue_wake_up(env_waitqueue *w)
{
sem_post(&w->sem);
}
#define env_waitqueue_wait(w, condition) \
({ \
int __ret = 0; \
if (!(condition)) \
sem_wait(&w.sem); \
__ret = __ret; \
})
/* *** BIT OPERATIONS *** */
static inline void env_bit_set(int nr, volatile void *addr)
{
char *byte = (char *)addr + (nr >> 3);
char mask = 1 << (nr & 7);
__sync_or_and_fetch(byte, mask);
}
static inline void env_bit_clear(int nr, volatile void *addr)
{
char *byte = (char *)addr + (nr >> 3);
char mask = 1 << (nr & 7);
mask = ~mask;
__sync_and_and_fetch(byte, mask);
}
static inline bool env_bit_test(int nr, const volatile unsigned long *addr)
{
const char *byte = (char *)addr + (nr >> 3);
char mask = 1 << (nr & 7);
return !!(*byte & mask);
}
/* *** SCHEDULING *** */
static inline void env_schedule(void)
{
sched_yield();
}
static inline int env_in_interrupt(void)
{
return 0;
}
static inline uint64_t env_get_tick_count(void)
{
struct timeval tv;
gettimeofday(&tv, NULL);
return tv.tv_sec * 1000 + tv.tv_usec / 1000;
}
static inline uint64_t env_ticks_to_msecs(uint64_t j)
{
return j;
}
static inline uint64_t env_ticks_to_secs(uint64_t j)
{
return j / 1000;
}
static inline uint64_t env_secs_to_ticks(uint64_t j)
{
return j * 1000;
}
/* *** SORTING *** */
static inline void env_sort(void *base, size_t num, size_t size,
int (*cmp_fn)(const void *, const void *),
void (*swap_fn)(void *, void *, int size))
{
qsort(base, num, size, cmp_fn);
}
/* *** STRING OPERATIONS *** */
#define env_memset(dest, dmax, val) ({ \
memset(dest, val, dmax); \
0; \
})
#define env_memcpy(dest, dmax, src, slen) ({ \
memcpy(dest, src, min(dmax, slen)); \
0; \
})
#define env_memcmp(s1, s1max, s2, s2max, diff) ({ \
*diff = memcmp(s1, s2, min(s1max, s2max)); \
0; \
})
#define env_strdup strndup
#define env_strnlen(s, smax) strnlen(s, smax)
#define env_strncmp strncmp
#define env_strncpy(dest, dmax, src, slen) ({ \
strncpy(dest, src, min(dmax, slen)); \
0; \
})
/* *** DEBUGING *** */
#define ENV_WARN(cond, fmt...) printf(fmt)
#define ENV_WARN_ON(cond) ;
#define ENV_WARN_ONCE(cond, fmt...) ENV_WARN(cond, fmt)
#define ENV_BUG() assert(0)
#define ENV_BUG_ON(cond) assert(!(cond))
#define container_of(ptr, type, member) ({ \
const typeof(((type *)0)->member)*__mptr = (ptr); \
(type *)((char *)__mptr - offsetof(type, member)); })
#define ARRAY_SIZE(x) (sizeof(x) / sizeof(*(x)))
static inline void env_msleep(uint64_t n)
{
usleep(n * 1000);
}
struct env_timeval {
uint64_t sec, usec;
};
static inline void env_gettimeofday(struct env_timeval *tv)
{
struct timeval t;
gettimeofday(&t, NULL);
tv->sec = t.tv_sec;
tv->usec = t.tv_usec;
}
uint32_t env_crc32(uint32_t crc, uint8_t const *data, size_t len);
#define ENV_PRIu64 "lu"
#endif /* __OCF_ENV_H__ */

22
tests/functional/test_adapter/src/ocf_env_headers.h
View File

@ -1,22 +0,0 @@
/*
* Copyright(c) 2019 Intel Corporation
* SPDX-License-Identifier: BSD-3-Clause-Clear
*/
#ifndef __OCF_ENV_HEADERS_H__
#define __OCF_ENV_HEADERS_H__
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
/* TODO: Move prefix printing to context logger. */
#define OCF_LOGO "Intel(R) OCF"
#define OCF_PREFIX_SHORT "[" OCF_LOGO "] "
#define OCF_PREFIX_LONG "Open CAS Framework"
#define OCF_VERSION_MAIN 1
#define OCF_VERSION_MAJOR 1
#define OCF_VERSION_MINOR 1
#endif /* __OCF_ENV_HEADERS_H__ */

168
tests/functional/test_adapter/src/ocf_env_list.h
View File

@ -1,168 +0,0 @@
/*
* Copyright(c) 2019 Intel Corporation
* SPDX-License-Identifier: BSD-3-Clause-Clear
*/
#ifndef __OCF_ENV_LIST__
#define __OCF_ENV_LIST__
#define LIST_POISON1 ((void *)0x101)
#define LIST_POISON2 ((void *)0x202)
/**
* List entry structure mimicking linux kernel based one.
*/
struct list_head {
struct list_head *next;
struct list_head *prev;
};
/**
* start an empty list
*/
#define INIT_LIST_HEAD(l) { (l)->prev = l; (l)->next = l; }
/**
* Add item to list head.
* @param it list entry to be added
* @param l1 list main node (head)
*/
static inline void list_add(struct list_head *it, struct list_head *l1)
{
it->prev = l1;
it->next = l1->next;
l1->next->prev = it;
l1->next = it;
}
/**
* Add item it to tail.
* @param it list entry to be added
* @param l1 list main node (head)
*/
static inline void list_add_tail(struct list_head *it, struct list_head *l1)
{
it->prev = l1->prev;
it->next = l1;
l1->prev->next = it;
l1->prev = it;
}
/**
* check if a list is empty (return true)
* @param l1 list main node (head)
*/
static inline int list_empty(struct list_head *l1)
{
return l1->next == l1;
}
/**
* delete an entry from a list
* @param it list entry to be deleted
*/
static inline void list_del(struct list_head *it)
{
it->next->prev = it->prev;
it->prev->next = it->next;
}
/**
* Move element to list head.
* @param it list entry to be moved
* @param l1 list main node (head)
*/
static inline void list_move(struct list_head *it, struct list_head *l1)
{
list_del(it);
list_add(it, l1);
}
/**
* Move element to list tail.
* @param it list entry to be moved
* @param l1 list main node (head)
*/
static inline void list_move_tail(struct list_head *it, struct list_head *l1)
{
list_del(it);
list_add_tail(it, l1);
}
/**
* Extract an entry.
* @param list_head_i list head item, from which entry is extracted
* @param item_type type (struct) of list entry
* @param field_name name of list_head field within item_type
*/
#define list_entry(list_head_i, item_type, field_name) \
(item_type *)(((void*)(list_head_i)) - offsetof(item_type, field_name))
#define list_first_entry(list_head_i, item_type, field_name) \
list_entry((list_head_i)->next, item_type, field_name)
/**
* @param iterator uninitialized list_head pointer, to be used as iterator
* @param plist list head (main node)
*/
#define list_for_each(iterator, plist) \
for (iterator = (plist)->next; \
(iterator)->next != (plist)->next; \
iterator = (iterator)->next)
/**
* Safe version of list_for_each which works even if entries are deleted during
* loop.
* @param iterator uninitialized list_head pointer, to be used as iterator
* @param q another uninitialized list_head, used as helper
* @param plist list head (main node)
*/
/*
* Algorithm handles situation, where q is deleted.
* consider in example 3 element list with header h:
*
* h -> 1 -> 2 -> 3 ->
*1. i q
*
*2. i q
*
*3. q i
*/
#define list_for_each_safe(iterator, q, plist) \
for (iterator = (q = (plist)->next->next)->prev; \
(q) != (plist)->next; \
iterator = (q = (q)->next)->prev)
#define _list_entry_helper(item, head, field_name) \
list_entry(head, typeof(*item), field_name)
/**
* Iterate over list entries.
* @param list pointer to list item (iterator)
* @param plist pointer to list_head item
* @param field_name name of list_head field in list entry
*/
#define list_for_each_entry(item, plist, field_name) \
for (item = _list_entry_helper(item, (plist)->next, field_name); \
_list_entry_helper(item, (item)->field_name.next, field_name) !=\
_list_entry_helper(item, (plist)->next, field_name); \
item = _list_entry_helper(item, (item)->field_name.next, field_name))
/**
* Safe version of list_for_each_entry which works even if entries are deleted
* during loop.
* @param list pointer to list item (iterator)
* @param q another pointer to list item, used as helper
* @param plist pointer to list_head item
* @param field_name name of list_head field in list entry
*/
#define list_for_each_entry_safe(item, q, plist, field_name) \
for (item = _list_entry_helper(item, (plist)->next, field_name), \
q = _list_entry_helper(item, (item)->field_name.next, field_name); \
_list_entry_helper(item, (item)->field_name.next, field_name) != \
_list_entry_helper(item, (plist)->next, field_name); \
item = q, q = _list_entry_helper(q, (q)->field_name.next, field_name))
#endif // __OCF_ENV_LIST__

0
tests/functional/tests/basic/__init__.py Normal file
View File

97
tests/functional/tests/basic/test_pyocf.py Normal file
View File

@ -0,0 +1,97 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
import pytest
from pyocf.types.cache import Cache
from pyocf.types.core import Core
from pyocf.types.volume import Volume, ErrorDevice
from pyocf.types.data import Data
from pyocf.types.io import IoDir
from pyocf.types.queue import Queue
from pyocf.utils import Size as S
from pyocf.types.shared import OcfError
def test_ctx_fixture(pyocf_ctx):
pass
def test_adding_cores(pyocf_ctx):
cache_device = Volume(S.from_MiB(200))
core1_device = Volume(S.from_MiB(400))
core2_device = Volume(S.from_MiB(400))
cache = Cache.start_on_device(cache_device)
core1 = Core.using_device(core1_device)
core2 = Core.using_device(core2_device)
cache.add_core(core1)
cache.add_core(core2)
def test_adding_core_twice(pyocf_ctx):
cache_device = Volume(S.from_MiB(200))
core_device = Volume(S.from_MiB(400))
cache = Cache.start_on_device(cache_device)
core = Core.using_device(core_device)
cache.add_core(core)
with pytest.raises(OcfError):
cache.add_core(core)
def test_simple_wt_write(pyocf_ctx):
cache_device = Volume(S.from_MiB(100))
core_device = Volume(S.from_MiB(200))
cache = Cache.start_on_device(cache_device)
core = Core.using_device(core_device)
queue = Queue(cache)
cache.add_core(core)
cache_device.reset_stats()
core_device.reset_stats()
write_data = Data.from_string("This is test data")
io = core.new_io()
io.set_data(write_data)
io.configure(20, write_data.size, IoDir.WRITE, 0, 0)
io.set_queue(queue)
io.submit()
assert cache_device.get_stats()[IoDir.WRITE] == 1
stats = cache.get_stats()
assert stats["req"]["wr_full_misses"]["value"] == 1
assert stats["usage"]["occupancy"]["value"] == 1
assert core.exp_obj_md5() == core_device.md5()
def test_start_corrupted_metadata_lba(pyocf_ctx):
cache_device = ErrorDevice(S.from_MiB(100), error_sectors=set([0]))
with pytest.raises(OcfError, match="OCF_ERR_WRITE_CACHE"):
cache = Cache.start_on_device(cache_device)
def test_load_cache_no_preexisting_data(pyocf_ctx):
cache_device = Volume(S.from_MiB(100))
with pytest.raises(OcfError, match="OCF_ERR_INVAL"):
cache = Cache.load_from_device(cache_device)
# TODO: Find out why this fails and fix
@pytest.mark.xfail
def test_load_cache(pyocf_ctx):
cache_device = Volume(S.from_MiB(100))
cache = Cache.start_on_device(cache_device)
cache.stop()
cache = Cache.load_from_device(cache_device)

42
tests/functional/tests/conftest.py Normal file
View File

@ -0,0 +1,42 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
import os
import sys
import pytest
sys.path.append(os.path.join(os.path.dirname(__file__), os.path.pardir))
from pyocf.types.logger import LogLevel, DefaultLogger, BufferLogger
from pyocf.types.volume import Volume, ErrorDevice
from pyocf.types.ctx import get_default_ctx
from pyocf.ocf import OcfLib
def pytest_configure(config):
sys.path.append(os.path.join(os.path.dirname(__file__), os.path.pardir))
@pytest.fixture()
def pyocf_ctx():
c = get_default_ctx(DefaultLogger(LogLevel.WARN))
c.register_volume_type(Volume)
c.register_volume_type(ErrorDevice)
yield c
for cache in c.caches:
cache.stop()
c.exit()
@pytest.fixture()
def pyocf_ctx_log_buffer():
logger = BufferLogger(LogLevel.DEBUG)
c = get_default_ctx(logger)
c.register_volume_type(Volume)
c.register_volume_type(ErrorDevice)
yield logger
for cache in c.caches:
cache.stop()