github/ocf
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #703 from jfckm/metadata-corruption-tests

Browse Source 
Metadata corruption tests
This commit is contained in:
Robert Baldyga
2022-07-22 16:06:41 +02:00
committed by GitHub
parent a478b67513 f1a8bac1db
commit 93b6ddb8a7
32 changed files with 802 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
tests/functional/pyocf/c/helpers/metadata_helpers.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright(c) 2022-2022 Intel Corporation
* Copyright(c) 2022 Intel Corporation
* SPDX-License-Identifier: BSD-3-Clause
*/
@@ -8,6 +8,7 @@
#include "../src/ocf/ocf_cache_priv.h"
#include "../src/ocf/metadata/metadata_raw.h"
#include "../src/ocf/metadata/metadata_internal.h"
#include "../src/ocf/metadata/metadata_superblock.h"
uint64_t ocf_get_metadata_segment_start_page(ocf_cache_t cache, int segment)
{
@@ -46,6 +47,9 @@ uint64_t ocf_get_metadata_segment_elem_size(ocf_cache_t cache, int segment)
struct ocf_metadata_ctrl *ctrl = cache->metadata.priv;
struct ocf_metadata_raw *raw = &ctrl->raw_desc[segment];
if (segment == metadata_segment_sb_config)
return offsetof(struct ocf_superblock_config, checksum);
return raw->entry_size;
}

4
tests/functional/pyocf/rio.py
View File

@@ -251,6 +251,8 @@ class Rio:
self.errors.update({thread.name: thread.errors})
self.error_count += len(thread.errors)
self.global_jobspec.target.close()
return self
def __del__(self):
@@ -275,6 +277,8 @@ class Rio:
queues = cycle(queues)
self.global_jobspec.target.open()
for job in jobs:
spec = job.merge(self.global_jobspec)
thread = Rio.RioThread(spec, next(queues))

26
tests/functional/pyocf/types/cache.py
View File

@@ -300,10 +300,13 @@ class Cache:
c.wait()
self.write_unlock()
self.device = None
if c.results["error"]:
raise OcfError("Failed to detach failover cache device", c.results["error"])
def standby_activate(self, device, open_cores=True):
self.device = device
device_cfg = self.alloc_device_config(device)
activate_cfg = CacheStandbyActivateConfig(
@@ -322,6 +325,7 @@ class Cache:
self.free_device_config(device_cfg)
if c.results["error"]:
self.device = None
raise OcfError("Failed to activate standby cache", c.results["error"])
def change_cache_mode(self, cache_mode: CacheMode):
@@ -575,7 +579,6 @@ class Cache:
disable_cleaner=False,
):
self.device = device
self.device_name = device.uuid
device_config = self.alloc_device_config(device, perform_test=perform_test)
@@ -607,7 +610,6 @@ class Cache:
def standby_attach(self, device, force=False, disable_cleaner=False):
self.device = device
self.device_name = device.uuid
device_config = self.alloc_device_config(device, perform_test=False)
@@ -639,7 +641,6 @@ class Cache:
def standby_load(self, device, perform_test=True, disable_cleaner=False):
self.device = device
self.device_name = device.uuid
device_config = self.alloc_device_config(device, perform_test=perform_test)
@@ -661,6 +662,7 @@ class Cache:
self.free_device_config(device_config)
if c.results["error"]:
self.device = None
raise OcfError("Loading standby cache device failed", c.results["error"])
def detach_device(self):
@@ -679,7 +681,6 @@ class Cache:
def load_cache(self, device, open_cores=True, disable_cleaner=False):
self.device = device
self.device_name = device.uuid
device_config = self.alloc_device_config(device)
@@ -701,8 +702,25 @@ class Cache:
self.free_device_config(device_config)
if c.results["error"]:
self.device = None
raise OcfError("Loading cache device failed", c.results["error"])
@classmethod
def load_standby_from_device(cls, device, owner=None, name="cache", cache_line_size=None):
if owner is None:
owner = OcfCtx.get_default()
c = cls(name=name, owner=owner, cache_line_size=cache_line_size)
c.start_cache()
try:
c.standby_load(device)
except: # noqa E722
c.stop()
raise
return c
@classmethod
def load_from_device(
cls, device, owner=None, name="cache", open_cores=True, disable_cleaner=False

19
tests/functional/pyocf/types/cvolume.py
View File

@@ -58,13 +58,20 @@ class CVolume(OcfInternalVolume):
def get_c_handle(self):
return self.cvol.value
def do_open(self):
ret = self.lib.ocf_volume_open(self.cvol, c_void_p())
if ret != 0:
raise OcfError("openning composite volume failed", ret)
def open(self):
ret = super().open()
if ret == 0:
ret = self.lib.ocf_volume_open(self.handle, c_void_p())
if ret:
raise OcfError("opening composite volume failed", ret)
return ret
def close(self):
self.lib.ocf_volume_close(self.handle)
super().close()
def do_close(self):
self.lib.ocf_volume_close(self.cvol)
lib = OcfLib.getInstance()

16
tests/functional/pyocf/types/logger.py
View File

@@ -1,5 +1,5 @@
#
# Copyright(c) 2019-2021 Intel Corporation
# Copyright(c) 2019-2022 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause
#
@@ -160,13 +160,19 @@ class FileLogger(Logger):
logger.handlers = []
class BufferLogger(Logger):
def __init__(self, level: LogLevel):
super().__init__()
self.level = level
class BufferLogger(DefaultLogger):
def __init__(
self,
console_level: LogLevel = LogLevel.WARN,
buffer_level: LogLevel = LogLevel.DEBUG,
name: str = ""
):
super().__init__(console_level, name)
self.level = buffer_level
self.buffer = StringIO()
def log(self, lvl, msg):
super().log(lvl, msg)
if lvl < self.level:
self.buffer.write(msg + "\n")

83
tests/functional/pyocf/types/volume.py
View File

@@ -23,6 +23,7 @@ from ctypes import (
from hashlib import md5
import weakref
from enum import IntEnum
import warnings
from .io import Io, IoOps, IoDir
from .queue import Queue
@@ -89,7 +90,7 @@ VOLUME_POISON = 0x13
class Volume:
_instances_ = weakref.WeakValueDictionary()
_instances_ = {}
_uuid_ = weakref.WeakValueDictionary()
_ops_ = {}
_props_ = {}
@@ -143,16 +144,25 @@ class Volume:
try:
volume = Volume.get_by_uuid(uuid)
except: # noqa E722 TODO:Investigate whether this really should be so broad
print("Tried to access unallocated volume {}".format(uuid))
print("{}".format(Volume._uuid_))
warnings.warn("Tried to access unallocated volume {}".format(uuid))
return -1
return Volume.s_open(ref, volume)
ret = volume.open()
if not ret:
Volume._instances_[ref] = volume
volume.handle = ref
return ret
@VolumeOps.CLOSE
def _close(ref):
volume = Volume.get_instance(ref)
Volume.s_close(volume)
del Volume._instances_[volume.handle]
volume.handle = None
volume.close()
@VolumeOps.GET_MAX_IO_SIZE
def _get_max_io_size(ref):
@@ -178,30 +188,19 @@ class Volume:
return Volume._ops_[cls]
@staticmethod
def s_open(ref, volume):
if volume.opened:
def open(self):
if self.opened:
return -OcfErrorCode.OCF_ERR_NOT_OPEN_EXC
Volume._instances_[ref] = volume
volume.handle = ref
self.opened = True
ret = volume.do_open()
if ret == 0:
volume.opened = True
return 0
return ret
@staticmethod
def s_close(volume):
if not volume.opened:
def close(self):
if not self.opened:
return
volume.do_close()
volume.opened = False
del Volume._instances_[volume.handle]
volume.handle = None
self.opened = False
@classmethod
def get_io_ops(cls):
@@ -228,11 +227,11 @@ class Volume:
@classmethod
def get_instance(cls, ref):
instance = cls._instances_[ref]
if instance is None:
print("tried to access {} but it's gone".format(ref))
if ref not in cls._instances_:
warnings.warn(f"tried to access volume ref {ref} but it's gone")
return None
return instance
return cls._instances_[ref]
@classmethod
def get_by_uuid(cls, uuid):
@@ -269,12 +268,6 @@ class Volume:
self.opened = False
self.handle = None
def do_open(self):
return 0
def do_close(self):
pass
def get_length(self):
raise NotImplementedError
@@ -451,6 +444,16 @@ class ErrorDevice(Volume):
def set_mapping(self, error_sectors: set):
self.error_sectors = error_sectors
def open(self):
ret = self.vol.open()
if ret:
return ret
return super().open()
def close(self):
super().close()
self.vol.close()
def should_forward_io(self, io):
if not self.armed:
return True
@@ -519,6 +522,10 @@ class ErrorDevice(Volume):
def get_copy(self):
return self.vol.get_copy()
def close(self):
super().close()
self.vol.close()
class TraceDevice(Volume):
class IoType(IntEnum):
@@ -531,6 +538,16 @@ class TraceDevice(Volume):
super().__init__(uuid)
self.trace_fcn = trace_fcn
def open(self):
ret = self.vol.open()
if ret:
return ret
return super().open()
def close(self):
super().close()
self.vol.close()
def _trace(self, io, io_type):
submit = True

4
tests/functional/pyocf/types/volume_cache.py
View File

@@ -13,12 +13,10 @@ from .volume import Volume
class CacheVolume(OcfInternalVolume):
def __init__(self, cache, open=False, uuid=None):
def __init__(self, cache, uuid=None):
super().__init__(cache, uuid)
self.cache = cache
self.lib = cache.owner.lib
if open:
self.open()
def get_c_handle(self):
return self.cache.get_c_front_volume()

4
tests/functional/pyocf/types/volume_core.py
View File

@@ -10,12 +10,10 @@ from .volume import Volume
class CoreVolume(OcfInternalVolume):
def __init__(self, core, open=False, uuid=None):
def __init__(self, core, uuid=None):
super().__init__(core, uuid)
self.core = core
self.lib = core.cache.owner.lib
if open:
self.open()
def get_c_handle(self):
return self.core.get_c_front_volume()

20
tests/functional/pyocf/types/volume_exp_obj.py
View File

@@ -11,7 +11,7 @@ from .volume import Volume, VOLUME_POISON
from pyocf.utils import Size
from pyocf.types.data import Data
from pyocf.types.io import IoDir, Io
from pyocf.types.shared import OcfCompletion
from pyocf.types.shared import OcfCompletion, OcfError
class OcfInternalVolume(Volume):
@@ -20,9 +20,8 @@ class OcfInternalVolume(Volume):
self.parent = parent
def __alloc_io(self, addr, _bytes, _dir, _class, _flags):
vol = self.parent.get_front_volume()
queue = self.parent.get_default_queue() # TODO multiple queues?
return vol.new_io(queue, addr, _bytes, _dir, _class, _flags)
return self.new_io(queue, addr, _bytes, _dir, _class, _flags)
def _alloc_io(self, io):
exp_obj_io = self.__alloc_io(
@@ -33,7 +32,6 @@ class OcfInternalVolume(Volume):
io.contents._flags,
)
lib = OcfLib.getInstance()
cdata = OcfLib.getInstance().ocf_io_get_data(io)
OcfLib.getInstance().ocf_io_set_data(byref(exp_obj_io), cdata, 0)
@@ -87,6 +85,7 @@ class OcfInternalVolume(Volume):
raise NotImplementedError
def _exp_obj_md5(self, read_size):
self.open()
logging.getLogger("pyocf").warning(
"Reading whole exported object! This disturbs statistics values"
)
@@ -111,14 +110,23 @@ class OcfInternalVolume(Volume):
read_buffer_all.copy(read_buffer, position, 0, read_size)
position += read_size
self.close()
return read_buffer_all.md5()
def open(self):
ret = super().open()
if ret:
return ret
handle = self.get_c_handle()
return Volume.s_open(handle, self)
self.handle = handle
return ret
def close(self):
return Volume.s_close(self)
super().close()
self.handle = None
lib = OcfLib.getInstance()

27
tests/functional/pyocf/types/volume_replicated.py
View File

@@ -5,6 +5,7 @@
from threading import Lock
from .volume import Volume, VOLUME_POISON
from .shared import OcfErrorCode
from .io import Io, IoDir
from ctypes import cast, c_void_p, CFUNCTYPE, c_int, POINTER, memmove, sizeof, pointer
@@ -15,21 +16,27 @@ class ReplicatedVolume(Volume):
self.primary = primary
self.secondary = secondary
if secondary.get_max_io_size() < primary.get_max_io_size():
raise Exception("secondary volume max io size too small")
if secondary.get_length() < primary.get_length():
raise Exception("secondary volume size too small")
def do_open(self):
ret = self.primary.do_open()
def open(self):
ret = self.primary.open()
if ret:
raise Exception(f"Couldn't open primary volume. ({ret})")
return ret
ret = self.secondary.do_open()
ret = self.secondary.open()
if ret:
self.primary.close()
return ret
raise Exception(f"Couldn't open secondary volume. ({ret})")
return ret
if self.secondary.get_max_io_size() < self.primary.get_max_io_size():
raise Exception("secondary volume max io size too small")
return -OcfErrorCode.OCF_ERR_INVAL
if self.secondary.get_length() < self.primary.get_length():
raise Exception("secondary volume size too small")
return -OcfErrorCode.OCF_ERR_INVAL
return super().open()
def close(self):
super().close()
self.primary.close()
self.secondary.close()

17
tests/functional/pyocf/utils.py
View File

@@ -61,12 +61,29 @@ class Size:
_SECTOR_SIZE = 512
_PAGE_SIZE = 4096
_unit_mapping = {
"B": 1,
"kiB": _KiB,
"MiB": _MiB,
"GiB": _GiB,
"TiB": _TiB,
}
def __init__(self, b: int, sector_aligned: bool = False):
if sector_aligned:
self.bytes = int(((b + self._SECTOR_SIZE - 1) // self._SECTOR_SIZE) * self._SECTOR_SIZE)
else:
self.bytes = int(b)
@classmethod
def from_string(cls, string):
string = string.strip()
number, unit = string.split(" ")
number = float(number)
unit = cls._unit_mapping[unit]
return cls(int(number * unit))
def __lt__(self, other):
return int(self) < int(other)