Merge pull request #695 from arutk/failover_test_5

pyocf: failover functional and power failure recovery tests

tests/functional/pyocf/c/helpers/collision_addr.c

@@ -0,0 +1,27 @@
+/*
+ * Copyright(c) 2022-2022 Intel Corporation
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include "ocf/ocf_io.h"
+#include "ocf/ocf_cache.h"
+#include "../src/ocf/ocf_cache_priv.h"
+#include "../src/ocf/metadata/metadata_raw.h"
+#include "../src/ocf/metadata/metadata_internal.h"
+
+// get collision metadata segment start and size (excluding padding)
+uint64_t ocf_get_collision_start_page_helper(ocf_cache_t cache)
+{
+	struct ocf_metadata_ctrl *ctrl = cache->metadata.priv;
+	struct ocf_metadata_raw *raw = &ctrl->raw_desc[metadata_segment_collision];
+
+	return raw->ssd_pages_offset;
+}
+
+uint64_t ocf_get_collision_page_count_helper(ocf_cache_t cache)
+{
+	struct ocf_metadata_ctrl *ctrl = cache->metadata.priv;
+	struct ocf_metadata_raw *raw = &ctrl->raw_desc[metadata_segment_collision];
+
+	return raw->ssd_pages;
+}

tests/functional/pyocf/c/helpers/collision_addr.h

@@ -0,0 +1,9 @@
+/*
+ * Copyright(c) 2022-2022 Intel Corporation
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#pragma once
+
+uint64_t ocf_get_collision_start_page_helper(ocf_cache_t cache);
+uint64_t ocf_get_collision_page_count_helper(ocf_cache_t cache);

tests/functional/pyocf/helpers.py

@@ -0,0 +1,16 @@
+#
+# Copyright(c) 2022 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause
+#
+#
+from .ocf import OcfLib
+
+
+def get_collision_segment_page_location(cache):
+    lib = OcfLib.getInstance()
+    return int(lib.ocf_get_collision_start_page_helper(cache))
+
+
+def get_collision_segment_size(cache):
+    lib = OcfLib.getInstance()
+    return int(lib.ocf_get_collision_page_count_helper(cache))

tests/functional/pyocf/rio.py

@@ -261,12 +261,12 @@ class Rio:
         self._threads = []
         self.errors = {}
 
-    def run(self, queues=None):
+    def run(self, queues):
         self.run_async(queues)
         self.wait_for_completion()
         return self
 
-    def run_async(self, queues=None):
+    def run_async(self, queues):
         self.clear()
 
         jobs = deepcopy(self.jobs)
@@ -274,8 +274,6 @@ class Rio:
         if not jobs:
             jobs = [self.global_jobspec for _ in range(self.global_jobspec.njobs)]
 
-        if not queues:
-            queues = [self.global_jobspec.target.cache.get_default_queue()]
         queues = cycle(queues)
 
         for job in jobs:

tests/functional/pyocf/types/cache.py

@@ -39,7 +39,7 @@ from .io import IoDir
 from .ioclass import IoClassesInfo, IoClassInfo
 from .stats.shared import UsageStats, RequestsStats, BlocksStats, ErrorsStats
 from .ctx import OcfCtx
-from .volume import RamVolume
+from .volume import RamVolume, Volume
 
 class Backfill(Structure):
     _fields_ = [("_max_queue_size", c_uint32), ("_queue_unblock_size", c_uint32)]
@@ -205,10 +205,7 @@ class Cache:
         self.cores = []
 
     def start_cache(
-        self,
-        default_io_queue: Queue = None,
-        mngt_queue: Queue = None,
-        locked: bool = False,
+        self, init_mngmt_queue=True, init_default_io_queue=True, locked: bool = False,
     ):
         cfg = CacheConfig(
             _name=self.name.encode("ascii"),
@@ -231,20 +228,24 @@ class Cache:
         if status:
             raise OcfError("Creating cache instance failed", status)
 
-        self.mngt_queue = mngt_queue or Queue(self, "mgmt-{}".format(self.get_name()))
+        if init_mngmt_queue:
+            self.mngt_queue = Queue(self, "mgmt-{}".format(self.get_name()))
+            status = self.owner.lib.ocf_mngt_cache_set_mngt_queue(self, self.mngt_queue)
+            if status:
+                raise OcfError("Error setting management queue", status)
 
-        if default_io_queue:
-            self.io_queues += [default_io_queue]
+        if init_default_io_queue:
+            self.io_queues = [Queue(self, "default-io-{}".format(self.get_name()))]
         else:
-            self.io_queues += [Queue(self, "default-io-{}".format(self.get_name()))]
-
-        status = self.owner.lib.ocf_mngt_cache_set_mngt_queue(self, self.mngt_queue)
-        if status:
-            raise OcfError("Error setting management queue", status)
+            self.io_queues = []
 
         self.started = True
         self.owner.caches.append(self)
 
+    def add_io_queue(self, *args, **kwargs):
+        q = Queue(self, args, **kwargs)
+        self.io_queues += [q]
+
     def standby_detach(self):
         self.write_lock()
         c = OcfCompletion([("cache", c_void_p), ("priv", c_void_p), ("error", c_int)])
@@ -560,11 +561,11 @@ class Cache:
                 c.results["error"],
             )
 
-    def standby_load(self, device):
+    def standby_load(self, device, perform_test=True):
         self.device = device
         self.device_name = device.uuid
 
-        device_config = Cache.generate_device_config(device)
+        device_config = Cache.generate_device_config(device, perform_test=perform_test)
 
         attach_cfg = CacheAttachConfig(
             _device=device_config,
@@ -755,8 +756,51 @@ class Cache:
     def get_volume(self):
         return Volume.get_instance(lib.ocf_cache_get_volume(self.cache_handle))
 
-    def get_stats(self):
+    def get_conf(self):
         cache_info = CacheInfo()
+
+        self.read_lock()
+
+        status = self.owner.lib.ocf_cache_get_info(self.cache_handle, byref(cache_info))
+
+        self.read_unlock()
+
+        if status:
+            raise OcfError("Failed getting cache info", status)
+
+        line_size = CacheLineSize(cache_info.cache_line_size)
+        cache_name = self.owner.lib.ocf_cache_get_name(self).decode("ascii")
+
+        return {
+            "attached": cache_info.attached,
+            "volume_type": self.owner.volume_types[cache_info.volume_type],
+            "size": CacheLines(cache_info.size, line_size),
+            "inactive": {
+                "occupancy": CacheLines(cache_info.inactive.occupancy.value, line_size),
+                "dirty": CacheLines(cache_info.inactive.dirty.value, line_size),
+                "clean": CacheLines(cache_info.inactive.clean.value, line_size),
+            },
+            "occupancy": CacheLines(cache_info.occupancy, line_size),
+            "dirty": CacheLines(cache_info.dirty, line_size),
+            "dirty_initial": CacheLines(cache_info.dirty_initial, line_size),
+            "dirty_for": timedelta(seconds=cache_info.dirty_for),
+            "cache_mode": CacheMode(cache_info.cache_mode),
+            "fallback_pt": {
+                "error_counter": cache_info.fallback_pt.error_counter,
+                "status": cache_info.fallback_pt.status,
+            },
+            "state": cache_info.state,
+            "cleaning_policy": CleaningPolicy(cache_info.cleaning_policy),
+            "promotion_policy": PromotionPolicy(cache_info.promotion_policy),
+            "cache_line_size": line_size,
+            "flushed": CacheLines(cache_info.flushed, line_size),
+            "core_count": cache_info.core_count,
+            "metadata_footprint": Size(cache_info.metadata_footprint),
+            "metadata_end_offset": Size(cache_info.metadata_end_offset),
+            "cache_name": cache_name,
+        }
+
+    def get_stats(self):
         usage = UsageStats()
         req = RequestsStats()
         block = BlocksStats()
@@ -764,53 +808,19 @@ class Cache:
 
         self.read_lock()
 
-        status = self.owner.lib.ocf_cache_get_info(self.cache_handle, byref(cache_info))
-        if status:
-            self.read_unlock()
-            raise OcfError("Failed getting cache info", status)
+        conf = self.get_conf()
 
         status = self.owner.lib.ocf_stats_collect_cache(
             self.cache_handle, byref(usage), byref(req), byref(block), byref(errors)
         )
-        if status:
-            self.read_unlock()
-            raise OcfError("Failed getting stats", status)
-
-        line_size = CacheLineSize(cache_info.cache_line_size)
-        cache_name = self.owner.lib.ocf_cache_get_name(self).decode("ascii")
 
         self.read_unlock()
+
+        if status:
+            raise OcfError("Failed getting stats", status)
+
         return {
-            "conf": {
-                "attached": cache_info.attached,
-                "volume_type": self.owner.volume_types[cache_info.volume_type],
-                "size": CacheLines(cache_info.size, line_size),
-                "inactive": {
-                    "occupancy": CacheLines(
-                        cache_info.inactive.occupancy.value, line_size
-                    ),
-                    "dirty": CacheLines(cache_info.inactive.dirty.value, line_size),
-                    "clean": CacheLines(cache_info.inactive.clean.value, line_size),
-                },
-                "occupancy": CacheLines(cache_info.occupancy, line_size),
-                "dirty": CacheLines(cache_info.dirty, line_size),
-                "dirty_initial": CacheLines(cache_info.dirty_initial, line_size),
-                "dirty_for": timedelta(seconds=cache_info.dirty_for),
-                "cache_mode": CacheMode(cache_info.cache_mode),
-                "fallback_pt": {
-                    "error_counter": cache_info.fallback_pt.error_counter,
-                    "status": cache_info.fallback_pt.status,
-                },
-                "state": cache_info.state,
-                "cleaning_policy": CleaningPolicy(cache_info.cleaning_policy),
-                "promotion_policy": PromotionPolicy(cache_info.promotion_policy),
-                "cache_line_size": line_size,
-                "flushed": CacheLines(cache_info.flushed, line_size),
-                "core_count": cache_info.core_count,
-                "metadata_footprint": Size(cache_info.metadata_footprint),
-                "metadata_end_offset": Size(cache_info.metadata_end_offset),
-                "cache_name": cache_name,
-            },
+            "conf": conf,
             "block": struct_to_dict(block),
             "req": struct_to_dict(req),
             "usage": struct_to_dict(usage),

tests/functional/pyocf/types/queue.py

@@ -1,5 +1,5 @@
 #
-# Copyright(c) 2019-2021 Intel Corporation
+# Copyright(c) 2019-2022 Intel Corporation
 # SPDX-License-Identifier: BSD-3-Clause
 #
 
@@ -45,6 +45,7 @@ class Queue:
     def __init__(self, cache, name):
 
         self.ops = QueueOps(kick=type(self)._kick, stop=type(self)._stop)
+        self.name = name
 
         self.handle = c_void_p()
         status = OcfLib.getInstance().ocf_queue_create(

tests/functional/pyocf/types/volume.py

@@ -405,16 +405,17 @@ class RamVolume(Volume):
         return string_at(self.data_ptr, self.size)
 
 
-class ErrorDevice(RamVolume):
+class ErrorDevice(Volume):
     def __init__(
         self,
-        size,
+        vol,
         error_sectors: set = None,
         error_seq_no: dict = None,
         armed=True,
         uuid=None,
     ):
-        super().__init__(size, uuid)
+        self.vol = vol
+        super().__init__(uuid)
         self.error_sectors = error_sectors
         self.error_seq_no = error_seq_no
         self.armed = armed
@@ -426,7 +427,7 @@ class ErrorDevice(RamVolume):
 
     def do_submit_io(self, io):
         if not self.armed:
-            super().do_submit_io(io)
+            self.vol.do_submit_io(io)
             return
 
         direction = IoDir(io.contents._dir)
@@ -451,7 +452,7 @@ class ErrorDevice(RamVolume):
             io.contents._end(io, -OcfErrorCode.OCF_ERR_IO)
             self.stats["errors"][direction] += 1
         else:
-            super().do_submit_io(io)
+            self.vol.do_submit_io(io)
 
     def arm(self):
         self.armed = True
@@ -463,9 +464,30 @@ class ErrorDevice(RamVolume):
         return self.error
 
     def reset_stats(self):
+        self.vol.reset_stats()
         super().reset_stats()
         self.stats["errors"] = {IoDir.WRITE: 0, IoDir.READ: 0}
 
+    def get_length(self):
+        return self.vol.get_length()
+
+    def get_max_io_size(self):
+        return self.vol.get_max_io_size()
+
+    def do_submit_flush(self, flush):
+        return self.vol.do_submit_flush(flush)
+
+    def do_submit_discard(self, discard):
+        return self.vol.do_submit_discard(discard)
+
+    def dump(self, offset=0, size=0, ignore=VOLUME_POISON, **kwargs):
+        return self.vol.dump(offset, size, ignore=ignore, **kwargs)
+
+    def md5(self):
+        return self.vol.md5()
+
+    def get_copy(self):
+        return self.vol.get_copy()
 
 lib = OcfLib.getInstance()
 lib.ocf_io_get_priv.restype = POINTER(VolumeIoPriv)

tests/functional/pyocf/types/volume_cache.py

@@ -22,10 +22,11 @@ class CacheVolume(ExpObjVolume):
 
     def open(self):
         return Volume.open(
-            self.lib.ocf_cache_get_front_volume(self.cache.handle),
+            self.lib.ocf_cache_get_front_volume(self.cache.cache_handle),
             self
         )
 
     def md5(self):
-        cache_line_size = int(self.cache.get_stats()['conf']['cache_line_size'])
+        out = self.cache.get_conf()
+        cache_line_size = int(out['cache_line_size'])
         return self._exp_obj_md5(cache_line_size)

tests/functional/pyocf/types/volume_exp_obj.py

@@ -49,10 +49,10 @@ class ExpObjVolume(Volume):
         return exp_obj_io
 
     def get_length(self):
-        return Size.from_B(OcfLib.getInstance().ocf_volume_get_length(self.c_vol))
+        return Size.from_B(OcfLib.getInstance().ocf_volume_get_length(self.handle))
 
     def get_max_io_size(self):
-        return Size.from_B(OcfLib.getInstance().ocf_volume_get_max_io_size(self.c_vol))
+        return Size.from_B(OcfLib.getInstance().ocf_volume_get_max_io_size(self.handle))
 
     def do_submit_io(self, io):
         io = self._alloc_io(io)

tests/functional/pyocf/utils.py

@@ -1,5 +1,5 @@
 #
-# Copyright(c) 2019-2021 Intel Corporation
+# Copyright(c) 2019-2022 Intel Corporation
 # SPDX-License-Identifier: BSD-3-Clause
 #
 
@@ -72,6 +72,7 @@ class Size:
     _GiB = _MiB * 1024
     _TiB = _GiB * 1024
     _SECTOR_SIZE = 512
+    _PAGE_SIZE = 4096
 
     def __init__(self, b: int, sector_aligned: bool = False):
         if sector_aligned:
@@ -130,6 +131,10 @@ class Size:
     def from_sector(cls, value):
         return cls(value * cls._SECTOR_SIZE)
 
+    @classmethod
+    def from_page(cls, value):
+        return cls(value * cls._PAGE_SIZE)
+
     @property
     def B(self):
         return self.bytes