Merge pull request #695 from arutk/failover_test_5

pyocf: failover functional and power failure recovery tests

env/posix/ocf_env.h

@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2019-2021 Intel Corporation
+ * Copyright(c) 2019-2022 Intel Corporation
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -67,11 +67,13 @@ typedef uint64_t sector_t;
 #define ENV_MEM_ATOMIC	0
 
 /* DEBUGING */
+void env_stack_trace(void);
+
 #define ENV_WARN(cond, fmt...)		printf(fmt)
 #define ENV_WARN_ON(cond)		;
 #define ENV_WARN_ONCE(cond, fmt...)	ENV_WARN(cond, fmt)
 
-#define ENV_BUG()			assert(0)
+#define ENV_BUG()			do {env_stack_trace(); assert(0);} while(0)
 #define ENV_BUG_ON(cond)		do { if (cond) ENV_BUG(); } while (0)
 #define ENV_BUILD_BUG_ON(cond)		_Static_assert(!(cond), "static "\
 					"assertion failure")

tests/functional/Makefile

@@ -1,5 +1,5 @@
 #
-# Copyright(c) 2019-2021 Intel Corporation
+# Copyright(c) 2019-2022 Intel Corporation
 # SPDX-License-Identifier: BSD-3-Clause
 #
 
@@ -8,13 +8,14 @@ OCFDIR=$(PWD)/../../
 ADAPTERDIR=$(PWD)/pyocf
 SRCDIR=$(ADAPTERDIR)/ocf/src
 INCDIR=$(ADAPTERDIR)/ocf/include
-WRAPDIR=$(ADAPTERDIR)/wrappers
+WRAPDIR=$(ADAPTERDIR)/c/wrappers
+HELPDIR=$(ADAPTERDIR)/c/helpers
 
 CC=gcc
 CFLAGS=-g -Wall -I$(INCDIR) -I$(SRCDIR)/ocf/env
 LDFLAGS=-pthread -lz
 
-SRC=$(shell find $(SRCDIR) $(WRAPDIR) -name \*.c)
+SRC=$(shell find $(SRCDIR) $(WRAPDIR) $(HELPDIR) -name \*.c)
 OBJS=$(patsubst %.c, %.o, $(SRC))
 OCFLIB=$(ADAPTERDIR)/libocf.so
 

tests/functional/pyocf/c/helpers/collision_addr.c

@@ -0,0 +1,27 @@
+/*
+ * Copyright(c) 2022-2022 Intel Corporation
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include "ocf/ocf_io.h"
+#include "ocf/ocf_cache.h"
+#include "../src/ocf/ocf_cache_priv.h"
+#include "../src/ocf/metadata/metadata_raw.h"
+#include "../src/ocf/metadata/metadata_internal.h"
+
+// get collision metadata segment start and size (excluding padding)
+uint64_t ocf_get_collision_start_page_helper(ocf_cache_t cache)
+{
+	struct ocf_metadata_ctrl *ctrl = cache->metadata.priv;
+	struct ocf_metadata_raw *raw = &ctrl->raw_desc[metadata_segment_collision];
+
+	return raw->ssd_pages_offset;
+}
+
+uint64_t ocf_get_collision_page_count_helper(ocf_cache_t cache)
+{
+	struct ocf_metadata_ctrl *ctrl = cache->metadata.priv;
+	struct ocf_metadata_raw *raw = &ctrl->raw_desc[metadata_segment_collision];
+
+	return raw->ssd_pages;
+}

tests/functional/pyocf/c/helpers/collision_addr.h

@@ -0,0 +1,9 @@
+/*
+ * Copyright(c) 2022-2022 Intel Corporation
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#pragma once
+
+uint64_t ocf_get_collision_start_page_helper(ocf_cache_t cache);
+uint64_t ocf_get_collision_page_count_helper(ocf_cache_t cache);

tests/functional/pyocf/helpers.py

@@ -0,0 +1,16 @@
+#
+# Copyright(c) 2022 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause
+#
+#
+from .ocf import OcfLib
+
+
+def get_collision_segment_page_location(cache):
+    lib = OcfLib.getInstance()
+    return int(lib.ocf_get_collision_start_page_helper(cache))
+
+
+def get_collision_segment_size(cache):
+    lib = OcfLib.getInstance()
+    return int(lib.ocf_get_collision_page_count_helper(cache))

tests/functional/pyocf/rio.py

@@ -261,12 +261,12 @@ class Rio:
         self._threads = []
         self.errors = {}
 
-    def run(self, queues=None):
+    def run(self, queues):
         self.run_async(queues)
         self.wait_for_completion()
         return self
 
-    def run_async(self, queues=None):
+    def run_async(self, queues):
         self.clear()
 
         jobs = deepcopy(self.jobs)
@@ -274,8 +274,6 @@ class Rio:
         if not jobs:
             jobs = [self.global_jobspec for _ in range(self.global_jobspec.njobs)]
 
-        if not queues:
-            queues = [self.global_jobspec.target.cache.get_default_queue()]
         queues = cycle(queues)
 
         for job in jobs:

tests/functional/pyocf/types/cache.py

@@ -39,7 +39,7 @@ from .io import IoDir
 from .ioclass import IoClassesInfo, IoClassInfo
 from .stats.shared import UsageStats, RequestsStats, BlocksStats, ErrorsStats
 from .ctx import OcfCtx
-from .volume import RamVolume
+from .volume import RamVolume, Volume
 
 class Backfill(Structure):
     _fields_ = [("_max_queue_size", c_uint32), ("_queue_unblock_size", c_uint32)]
@@ -205,10 +205,7 @@ class Cache:
         self.cores = []
 
     def start_cache(
-        self,
+        self, init_mngmt_queue=True, init_default_io_queue=True, locked: bool = False,
-        default_io_queue: Queue = None,
-        mngt_queue: Queue = None,
-        locked: bool = False,
     ):
         cfg = CacheConfig(
             _name=self.name.encode("ascii"),
@@ -231,20 +228,24 @@ class Cache:
         if status:
             raise OcfError("Creating cache instance failed", status)
 
-        self.mngt_queue = mngt_queue or Queue(self, "mgmt-{}".format(self.get_name()))
+        if init_mngmt_queue:
+            self.mngt_queue = Queue(self, "mgmt-{}".format(self.get_name()))
+            status = self.owner.lib.ocf_mngt_cache_set_mngt_queue(self, self.mngt_queue)
+            if status:
+                raise OcfError("Error setting management queue", status)
 
-        if default_io_queue:
+        if init_default_io_queue:
-            self.io_queues += [default_io_queue]
+            self.io_queues = [Queue(self, "default-io-{}".format(self.get_name()))]
         else:
-            self.io_queues += [Queue(self, "default-io-{}".format(self.get_name()))]
+            self.io_queues = []
-
-        status = self.owner.lib.ocf_mngt_cache_set_mngt_queue(self, self.mngt_queue)
-        if status:
-            raise OcfError("Error setting management queue", status)
 
         self.started = True
         self.owner.caches.append(self)
 
+    def add_io_queue(self, *args, **kwargs):
+        q = Queue(self, args, **kwargs)
+        self.io_queues += [q]
+
     def standby_detach(self):
         self.write_lock()
         c = OcfCompletion([("cache", c_void_p), ("priv", c_void_p), ("error", c_int)])
@@ -560,11 +561,11 @@ class Cache:
                 c.results["error"],
             )
 
-    def standby_load(self, device):
+    def standby_load(self, device, perform_test=True):
         self.device = device
         self.device_name = device.uuid
 
-        device_config = Cache.generate_device_config(device)
+        device_config = Cache.generate_device_config(device, perform_test=perform_test)
 
         attach_cfg = CacheAttachConfig(
             _device=device_config,
@@ -755,8 +756,51 @@ class Cache:
     def get_volume(self):
         return Volume.get_instance(lib.ocf_cache_get_volume(self.cache_handle))
 
-    def get_stats(self):
+    def get_conf(self):
         cache_info = CacheInfo()
+
+        self.read_lock()
+
+        status = self.owner.lib.ocf_cache_get_info(self.cache_handle, byref(cache_info))
+
+        self.read_unlock()
+
+        if status:
+            raise OcfError("Failed getting cache info", status)
+
+        line_size = CacheLineSize(cache_info.cache_line_size)
+        cache_name = self.owner.lib.ocf_cache_get_name(self).decode("ascii")
+
+        return {
+            "attached": cache_info.attached,
+            "volume_type": self.owner.volume_types[cache_info.volume_type],
+            "size": CacheLines(cache_info.size, line_size),
+            "inactive": {
+                "occupancy": CacheLines(cache_info.inactive.occupancy.value, line_size),
+                "dirty": CacheLines(cache_info.inactive.dirty.value, line_size),
+                "clean": CacheLines(cache_info.inactive.clean.value, line_size),
+            },
+            "occupancy": CacheLines(cache_info.occupancy, line_size),
+            "dirty": CacheLines(cache_info.dirty, line_size),
+            "dirty_initial": CacheLines(cache_info.dirty_initial, line_size),
+            "dirty_for": timedelta(seconds=cache_info.dirty_for),
+            "cache_mode": CacheMode(cache_info.cache_mode),
+            "fallback_pt": {
+                "error_counter": cache_info.fallback_pt.error_counter,
+                "status": cache_info.fallback_pt.status,
+            },
+            "state": cache_info.state,
+            "cleaning_policy": CleaningPolicy(cache_info.cleaning_policy),
+            "promotion_policy": PromotionPolicy(cache_info.promotion_policy),
+            "cache_line_size": line_size,
+            "flushed": CacheLines(cache_info.flushed, line_size),
+            "core_count": cache_info.core_count,
+            "metadata_footprint": Size(cache_info.metadata_footprint),
+            "metadata_end_offset": Size(cache_info.metadata_end_offset),
+            "cache_name": cache_name,
+        }
+
+    def get_stats(self):
         usage = UsageStats()
         req = RequestsStats()
         block = BlocksStats()
@@ -764,53 +808,19 @@ class Cache:
 
         self.read_lock()
 
-        status = self.owner.lib.ocf_cache_get_info(self.cache_handle, byref(cache_info))
+        conf = self.get_conf()
-        if status:
-            self.read_unlock()
-            raise OcfError("Failed getting cache info", status)
 
         status = self.owner.lib.ocf_stats_collect_cache(
             self.cache_handle, byref(usage), byref(req), byref(block), byref(errors)
         )
-        if status:
-            self.read_unlock()
-            raise OcfError("Failed getting stats", status)
-
-        line_size = CacheLineSize(cache_info.cache_line_size)
-        cache_name = self.owner.lib.ocf_cache_get_name(self).decode("ascii")
 
         self.read_unlock()
+
+        if status:
+            raise OcfError("Failed getting stats", status)
+
         return {
-            "conf": {
+            "conf": conf,
-                "attached": cache_info.attached,
-                "volume_type": self.owner.volume_types[cache_info.volume_type],
-                "size": CacheLines(cache_info.size, line_size),
-                "inactive": {
-                    "occupancy": CacheLines(
-                        cache_info.inactive.occupancy.value, line_size
-                    ),
-                    "dirty": CacheLines(cache_info.inactive.dirty.value, line_size),
-                    "clean": CacheLines(cache_info.inactive.clean.value, line_size),
-                },
-                "occupancy": CacheLines(cache_info.occupancy, line_size),
-                "dirty": CacheLines(cache_info.dirty, line_size),
-                "dirty_initial": CacheLines(cache_info.dirty_initial, line_size),
-                "dirty_for": timedelta(seconds=cache_info.dirty_for),
-                "cache_mode": CacheMode(cache_info.cache_mode),
-                "fallback_pt": {
-                    "error_counter": cache_info.fallback_pt.error_counter,
-                    "status": cache_info.fallback_pt.status,
-                },
-                "state": cache_info.state,
-                "cleaning_policy": CleaningPolicy(cache_info.cleaning_policy),
-                "promotion_policy": PromotionPolicy(cache_info.promotion_policy),
-                "cache_line_size": line_size,
-                "flushed": CacheLines(cache_info.flushed, line_size),
-                "core_count": cache_info.core_count,
-                "metadata_footprint": Size(cache_info.metadata_footprint),
-                "metadata_end_offset": Size(cache_info.metadata_end_offset),
-                "cache_name": cache_name,
-            },
             "block": struct_to_dict(block),
             "req": struct_to_dict(req),
             "usage": struct_to_dict(usage),

tests/functional/pyocf/types/queue.py

@@ -1,5 +1,5 @@
 #
-# Copyright(c) 2019-2021 Intel Corporation
+# Copyright(c) 2019-2022 Intel Corporation
 # SPDX-License-Identifier: BSD-3-Clause
 #
 
@@ -45,6 +45,7 @@ class Queue:
     def __init__(self, cache, name):
 
         self.ops = QueueOps(kick=type(self)._kick, stop=type(self)._stop)
+        self.name = name
 
         self.handle = c_void_p()
         status = OcfLib.getInstance().ocf_queue_create(

tests/functional/pyocf/types/volume.py

@@ -405,16 +405,17 @@ class RamVolume(Volume):
         return string_at(self.data_ptr, self.size)
 
 
-class ErrorDevice(RamVolume):
+class ErrorDevice(Volume):
     def __init__(
         self,
-        size,
+        vol,
         error_sectors: set = None,
         error_seq_no: dict = None,
         armed=True,
         uuid=None,
     ):
-        super().__init__(size, uuid)
+        self.vol = vol
+        super().__init__(uuid)
         self.error_sectors = error_sectors
         self.error_seq_no = error_seq_no
         self.armed = armed
@@ -426,7 +427,7 @@ class ErrorDevice(RamVolume):
 
     def do_submit_io(self, io):
         if not self.armed:
-            super().do_submit_io(io)
+            self.vol.do_submit_io(io)
             return
 
         direction = IoDir(io.contents._dir)
@@ -451,7 +452,7 @@ class ErrorDevice(RamVolume):
             io.contents._end(io, -OcfErrorCode.OCF_ERR_IO)
             self.stats["errors"][direction] += 1
         else:
-            super().do_submit_io(io)
+            self.vol.do_submit_io(io)
 
     def arm(self):
         self.armed = True
@@ -463,9 +464,30 @@ class ErrorDevice(RamVolume):
         return self.error
 
     def reset_stats(self):
+        self.vol.reset_stats()
         super().reset_stats()
         self.stats["errors"] = {IoDir.WRITE: 0, IoDir.READ: 0}
 
+    def get_length(self):
+        return self.vol.get_length()
+
+    def get_max_io_size(self):
+        return self.vol.get_max_io_size()
+
+    def do_submit_flush(self, flush):
+        return self.vol.do_submit_flush(flush)
+
+    def do_submit_discard(self, discard):
+        return self.vol.do_submit_discard(discard)
+
+    def dump(self, offset=0, size=0, ignore=VOLUME_POISON, **kwargs):
+        return self.vol.dump(offset, size, ignore=ignore, **kwargs)
+
+    def md5(self):
+        return self.vol.md5()
+
+    def get_copy(self):
+        return self.vol.get_copy()
 
 lib = OcfLib.getInstance()
 lib.ocf_io_get_priv.restype = POINTER(VolumeIoPriv)

tests/functional/pyocf/types/volume_cache.py

@@ -22,10 +22,11 @@ class CacheVolume(ExpObjVolume):
 
     def open(self):
         return Volume.open(
-            self.lib.ocf_cache_get_front_volume(self.cache.handle),
+            self.lib.ocf_cache_get_front_volume(self.cache.cache_handle),
             self
         )
 
     def md5(self):
-        cache_line_size = int(self.cache.get_stats()['conf']['cache_line_size'])
+        out = self.cache.get_conf()
+        cache_line_size = int(out['cache_line_size'])
         return self._exp_obj_md5(cache_line_size)

tests/functional/pyocf/types/volume_exp_obj.py

@@ -49,10 +49,10 @@ class ExpObjVolume(Volume):
         return exp_obj_io
 
     def get_length(self):
-        return Size.from_B(OcfLib.getInstance().ocf_volume_get_length(self.c_vol))
+        return Size.from_B(OcfLib.getInstance().ocf_volume_get_length(self.handle))
 
     def get_max_io_size(self):
-        return Size.from_B(OcfLib.getInstance().ocf_volume_get_max_io_size(self.c_vol))
+        return Size.from_B(OcfLib.getInstance().ocf_volume_get_max_io_size(self.handle))
 
     def do_submit_io(self, io):
         io = self._alloc_io(io)

tests/functional/pyocf/utils.py

@@ -1,5 +1,5 @@
 #
-# Copyright(c) 2019-2021 Intel Corporation
+# Copyright(c) 2019-2022 Intel Corporation
 # SPDX-License-Identifier: BSD-3-Clause
 #
 
@@ -72,6 +72,7 @@ class Size:
     _GiB = _MiB * 1024
     _TiB = _GiB * 1024
     _SECTOR_SIZE = 512
+    _PAGE_SIZE = 4096
 
     def __init__(self, b: int, sector_aligned: bool = False):
         if sector_aligned:
@@ -130,6 +131,10 @@ class Size:
     def from_sector(cls, value):
         return cls(value * cls._SECTOR_SIZE)
 
+    @classmethod
+    def from_page(cls, value):
+        return cls(value * cls._PAGE_SIZE)
+
     @property
     def B(self):
         return self.bytes

tests/functional/tests/basic/test_pyocf.py

@@ -47,7 +47,8 @@ def test_simple_wt_write(pyocf_ctx):
 
 
 def test_start_corrupted_metadata_lba(pyocf_ctx):
-    cache_device = ErrorDevice(S.from_MiB(50), error_sectors=set([0]))
+    ramdisk = RamVolume(S.from_MiB(50))
+    cache_device = ErrorDevice(ramdisk, error_sectors=set([0]))
 
     with pytest.raises(OcfError, match="OCF_ERR_WRITE_CACHE"):
         cache = Cache.start_on_device(cache_device)

tests/functional/tests/conftest.py

@@ -16,6 +16,7 @@ from pyocf.types.volume_core import CoreVolume
 from pyocf.types.volume_replicated import ReplicatedVolume
 from pyocf.types.ctx import OcfCtx
 
+default_registered_volumes = [RamVolume, ErrorDevice, CacheVolume, CoreVolume, ReplicatedVolume]
 
 def pytest_configure(config):
     sys.path.append(os.path.join(os.path.dirname(__file__), os.path.pardir))
@@ -24,11 +25,8 @@ def pytest_configure(config):
 @pytest.fixture()
 def pyocf_ctx():
     c = OcfCtx.with_defaults(DefaultLogger(LogLevel.WARN))
-    c.register_volume_type(RamVolume)
+    for vol_type in default_registered_volumes:
-    c.register_volume_type(ErrorDevice)
+        c.register_volume_type(vol_type)
-    c.register_volume_type(CacheVolume)
-    c.register_volume_type(CoreVolume)
-    c.register_volume_type(ReplicatedVolume)
     yield c
     c.exit()
     gc.collect()
@@ -38,11 +36,21 @@ def pyocf_ctx():
 def pyocf_ctx_log_buffer():
     logger = BufferLogger(LogLevel.DEBUG)
     c = OcfCtx.with_defaults(logger)
-    c.register_volume_type(RamVolume)
+    for vol_type in default_registered_volumes:
-    c.register_volume_type(ErrorDevice)
+        c.register_volume_type(vol_type)
-    c.register_volume_type(CacheVolume)
-    c.register_volume_type(CoreVolume)
-    c.register_volume_type(ReplicatedVolume)
     yield logger
     c.exit()
     gc.collect()
+
+
+@pytest.fixture()
+def pyocf_2_ctx():
+    c1 = OcfCtx.with_defaults(DefaultLogger(LogLevel.WARN, "Ctx1"))
+    c2 = OcfCtx.with_defaults(DefaultLogger(LogLevel.WARN, "Ctx2"))
+    for vol_type in default_registered_volumes:
+        c1.register_volume_type(vol_type)
+        c2.register_volume_type(vol_type)
+    yield [c1, c2]
+    c1.exit()
+    c2.exit()
+    gc.collect()

tests/functional/tests/management/test_failover.py

@@ -0,0 +1,591 @@
+#
+# Copyright(c) 2022-2022 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+import pytest
+import copy
+from ctypes import c_int
+
+from pyocf.types.cache import (
+    Cache,
+    CacheMode,
+    MetadataLayout,
+    CleaningPolicy,
+)
+from pyocf.types.core import Core
+from pyocf.types.data import Data
+from pyocf.types.io import Io, IoDir
+from pyocf.types.volume import RamVolume, Volume
+from pyocf.types.volume_cache import CacheVolume
+from pyocf.types.volume_core import CoreVolume
+from pyocf.types.volume_replicated import ReplicatedVolume
+from pyocf.types.shared import (
+    OcfError,
+    OcfErrorCode,
+    OcfCompletion,
+    CacheLines,
+    CacheLineSize,
+    SeqCutOffPolicy,
+)
+from pyocf.utils import Size
+from pyocf.rio import Rio, ReadWrite
+
+
+def test_standby_stop_closes_volume(pyocf_2_ctx):
+    ctx = pyocf_2_ctx[1]
+    mode = CacheMode.WB
+    cls = CacheLineSize.LINE_4KiB
+    vol = RamVolume(Size.from_MiB(150))
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    cache.standby_attach(vol, force=False)
+    cache.stop()
+    assert not vol.opened
+
+
+def test_standby_stop_detached(pyocf_2_ctx):
+    ctx = pyocf_2_ctx[1]
+    mode = CacheMode.WB
+    cls = CacheLineSize.LINE_4KiB
+    vol = RamVolume(Size.from_MiB(150))
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    cache.standby_attach(vol, force=False)
+    cache.standby_detach()
+    assert not vol.opened
+    cache.stop()
+
+
+# verify that force flag is required to attach a standby instance
+# on a volume where standby instance had previously been running
+def test_standby_attach_force_after_standby(pyocf_2_ctx):
+    ctx = pyocf_2_ctx[1]
+    mode = CacheMode.WB
+    cls = CacheLineSize.LINE_4KiB
+    vol = RamVolume(Size.from_MiB(150))
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    cache.standby_attach(vol, force=False)
+    cache.standby_detach()
+    cache.stop()
+
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    with pytest.raises(OcfError) as ex:
+        cache.standby_attach(vol, force=False)
+    assert ex.value.error_code == OcfErrorCode.OCF_ERR_METADATA_FOUND
+
+    cache.standby_attach(vol, force=True)
+
+
+def test_standby_attach_force_after_active(pyocf_2_ctx):
+    ctx = pyocf_2_ctx[1]
+    mode = CacheMode.WB
+    cls = CacheLineSize.LINE_4KiB
+    vol = RamVolume(Size.from_MiB(150))
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    cache.attach_device(vol)
+    cache.stop()
+    assert not vol.opened
+
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    with pytest.raises(OcfError) as ex:
+        cache.standby_attach(vol, force=False)
+    assert ex.value.error_code == OcfErrorCode.OCF_ERR_METADATA_FOUND
+
+    cache.standby_attach(vol, force=True)
+
+
+# standby load from standby cache instance after clean shutdown
+def test_standby_load_after_standby_clean_shutdown(pyocf_2_ctx):
+    ctx = pyocf_2_ctx[1]
+    mode = CacheMode.WB
+    cls = CacheLineSize.LINE_4KiB
+    vol = RamVolume(Size.from_MiB(150))
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    cache.standby_attach(vol, force=False)
+    cache.stop()
+
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+
+    vol.reset_stats()
+    cache.standby_load(vol, perform_test=False)
+    assert vol.get_stats()[IoDir.WRITE] == 0
+
+    cache.stop()
+
+
+# standby load from active cache instance after clean shutdown
+def test_standby_load_after_active_clean_shutdown(pyocf_2_ctx):
+    ctx = pyocf_2_ctx[1]
+    mode = CacheMode.WB
+    cls = CacheLineSize.LINE_4KiB
+    vol = RamVolume(Size.from_MiB(150))
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    cache.attach_device(vol, force=False)
+    cache.stop()
+
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+
+    vol.reset_stats()
+    cache.standby_load(vol, perform_test=False)
+    assert vol.get_stats()[IoDir.WRITE] == 0
+
+
+# standby load from active cache instance after clean shutdown
+def test_standby_load_after_active_dirty_shutdown(pyocf_2_ctx):
+    ctx = pyocf_2_ctx[1]
+    mode = CacheMode.WB
+    cls = CacheLineSize.LINE_4KiB
+    vol = RamVolume(Size.from_MiB(150))
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    cache.attach_device(vol, force=False)
+    vol.offline()
+    with pytest.raises(OcfError) as ex:
+        cache.stop()
+    assert ex.value.error_code == OcfErrorCode.OCF_ERR_WRITE_CACHE
+    vol.online()
+
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    vol.reset_stats()
+    cache.standby_load(vol, perform_test=False)
+    assert vol.get_stats()[IoDir.WRITE] == 0
+
+    cache.stop()
+
+
+def test_standby_load_after_standby_dirty_shutdown(pyocf_2_ctx):
+    ctx = pyocf_2_ctx[1]
+    mode = CacheMode.WB
+    cls = CacheLineSize.LINE_4KiB
+    vol = RamVolume(Size.from_MiB(150))
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    cache.standby_attach(vol, force=False)
+    vol.offline()
+    cache.stop()
+
+    vol.online()
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    vol.reset_stats()
+    cache.standby_load(vol, perform_test=False)
+    assert vol.get_stats()[IoDir.WRITE] == 0
+
+    cache.stop()
+
+
+def test_standby_load_after_standby_dirty_shutdown_with_vol_test(pyocf_2_ctx):
+    ctx = pyocf_2_ctx[1]
+    mode = CacheMode.WB
+    cls = CacheLineSize.LINE_4KiB
+    vol = RamVolume(Size.from_MiB(150))
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    cache.standby_attach(vol, force=False)
+    vol.offline()
+    cache.stop()
+
+    vol.online()
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    cache.standby_load(vol)
+
+    cache.stop()
+
+
+def test_standby_activate_core_size_mismatch_after_active(pyocf_2_ctx):
+    ctx = pyocf_2_ctx[1]
+    mode = CacheMode.WB
+    cls = CacheLineSize.LINE_4KiB
+    vol = RamVolume(Size.from_MiB(150))
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    cache.attach_device(vol, force=False)
+
+    # prepare and stop cache instance with standard size core volume
+    core_vol_uuid = str(id(cache))
+    core_vol_size_initial = Size.from_MiB(150)
+    core_vol = RamVolume(core_vol_size_initial, uuid=core_vol_uuid)
+    core = Core(core_vol)
+    cache.add_core(core)
+    cache.stop()
+    cache = None
+
+    # resize core volume
+    # TODO: how to avoid manually removing vol<->uuid mapping?
+    del Volume._uuid_[core_vol.uuid]
+    core_vol = None
+    core_vol = RamVolume(2 * core_vol_size_initial, uuid=core_vol_uuid)
+
+    # standby load on the volume
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    cache.standby_load(vol)
+    cache.standby_detach()
+
+    # first attempt to activate with size mismatch
+    with pytest.raises(OcfError) as ex:
+        cache.standby_activate(vol)
+    assert ex.value.error_code == OcfErrorCode.OCF_ERR_CORE_SIZE_MISMATCH
+
+    # second attempt to activate with size mismatch
+    with pytest.raises(OcfError) as ex:
+        cache.standby_activate(vol)
+    assert ex.value.error_code == OcfErrorCode.OCF_ERR_CORE_SIZE_MISMATCH
+
+    del Volume._uuid_[core_vol.uuid]
+    core_vol = RamVolume(core_vol_size_initial, uuid=core_vol_uuid)
+
+    # attempt to activate with fixed sizE
+    cache.standby_activate(vol)
+
+    cache.stop()
+
+
+def test_standby_activate_core_size_mismatch(pyocf_2_ctx):
+    ctx = pyocf_2_ctx[1]
+    mode = CacheMode.WB
+    cls = CacheLineSize.LINE_4KiB
+    vol1 = RamVolume(Size.from_MiB(150), uuid="cv1")
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    cache.attach_device(vol1, force=False)
+
+    core_vol_uuid = str(id(cache))
+    core_vol_size_initial = Size.from_MiB(150)
+    core_vol = RamVolume(core_vol_size_initial, uuid=core_vol_uuid)
+    core2_vol = RamVolume(core_vol_size_initial)
+    core = Core(core_vol)
+    core2 = Core(core2_vol, name="core2")
+    cache.add_core(core)
+    cache.add_core(core2)
+
+    data = vol1.get_bytes()
+
+    cache.stop()
+
+    vol1 = None
+
+    del Volume._uuid_[core_vol.uuid]
+    core_vol = None
+
+    vol2 = RamVolume(Size.from_MiB(150), uuid="cv2")
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    cache.standby_attach(vol2)
+    cache_vol = CacheVolume(cache, open=True)
+
+    write_vol(cache_vol, cache.get_default_queue(), data)
+
+    core_vol = RamVolume(2 * core_vol_size_initial, uuid=core_vol_uuid)
+
+    cache.standby_detach()
+
+    # first attempt to activate with size mismatch
+    with pytest.raises(OcfError) as ex:
+        cache.standby_activate(vol2)
+    assert ex.value.error_code == OcfErrorCode.OCF_ERR_CORE_SIZE_MISMATCH
+
+    # second attempt to activate with size mismatch
+    with pytest.raises(OcfError) as ex:
+        cache.standby_activate(vol2)
+    assert ex.value.error_code == OcfErrorCode.OCF_ERR_CORE_SIZE_MISMATCH
+
+    del Volume._uuid_[core_vol.uuid]
+    core_vol = None
+    core_vol = RamVolume(core_vol_size_initial, uuid=core_vol_uuid)
+
+    # attempt to activate with fixed sizE
+    cache.standby_activate(vol2)
+
+    cache.stop()
+
+
+def test_failover_passive_first(pyocf_2_ctx):
+    ctx1 = pyocf_2_ctx[0]
+    ctx2 = pyocf_2_ctx[1]
+    mode = CacheMode.WB
+    cls = CacheLineSize.LINE_4KiB
+
+    prim_cache_backend_vol = RamVolume(Size.from_MiB(150))
+    core_backend_vol = RamVolume(Size.from_MiB(1))
+    sec_cache_backend_vol = RamVolume(Size.from_MiB(150))
+
+    # passive cache with directly on ram disk
+    cache2 = Cache(owner=ctx2, cache_mode=mode, cache_line_size=cls)
+    cache2.start_cache()
+    cache2.standby_attach(sec_cache_backend_vol)
+
+    # volume replicating cache1 ramdisk writes to cache2 cache exported object
+    cache2_exp_obj_vol = CacheVolume(cache2, open=True)
+    cache1_cache_vol = ReplicatedVolume(prim_cache_backend_vol, cache2_exp_obj_vol)
+
+    # active cache
+    cache1 = Cache.start_on_device(
+        cache1_cache_vol, ctx1, cache_mode=mode, cache_line_size=cls
+    )
+    core = Core(core_backend_vol)
+    cache1.add_core(core)
+    core_vol = CoreVolume(core, open=True)
+    queue = cache1.get_default_queue()
+
+    # some I/O
+    r = (
+        Rio()
+        .target(core_vol)
+        .njobs(1)
+        .readwrite(ReadWrite.WRITE)
+        .size(Size.from_MiB(1))
+        .qd(1)
+        .run([queue])
+    )
+
+    # capture checksum before simulated active host failure
+    md5 = core_vol.md5()
+
+    # offline primary cache volume and stop primary cache to simulate active host
+    # failure
+    cache1_cache_vol.offline()
+    with pytest.raises(OcfError) as ex:
+        cache1.stop()
+    assert ex.value.error_code == OcfErrorCode.OCF_ERR_WRITE_CACHE
+
+    # failover
+    cache2.standby_detach()
+    cache2.standby_activate(sec_cache_backend_vol, open_cores=False)
+
+    # add core explicitly with "try_add" to workaround pyocf limitations
+    core = Core(core_backend_vol)
+    cache2.add_core(core, try_add=True)
+    core_vol = CoreVolume(core, open=True)
+
+    assert md5 == core_vol.md5()
+
+
+def write_vol(vol, queue, data):
+    data_size = len(data)
+    subdata_size_max = int(Size.from_MiB(32))
+    for offset in range(0, data_size, subdata_size_max):
+        subdata_size = min(data_size - offset, subdata_size_max)
+        subdata = Data.from_bytes(data, offset, subdata_size)
+        comp = OcfCompletion([("error", c_int)])
+        io = vol.new_io(queue, offset, subdata_size, IoDir.WRITE, 0, 0,)
+        io.set_data(subdata)
+        io.callback = comp.callback
+        io.submit()
+        comp.wait()
+
+
+def test_failover_active_first(pyocf_2_ctx):
+    ctx1 = pyocf_2_ctx[0]
+    ctx2 = pyocf_2_ctx[1]
+    mode = CacheMode.WB
+    cls = CacheLineSize.LINE_4KiB
+
+    prim_cache_backend_vol = RamVolume(Size.from_MiB(150))
+    core_backend_vol = RamVolume(Size.from_MiB(1))
+
+    # active cache
+    cache1 = Cache.start_on_device(
+        prim_cache_backend_vol, ctx1, cache_mode=mode, cache_line_size=cls
+    )
+    core = Core(core_backend_vol)
+    cache1.add_core(core)
+    vol = CoreVolume(core, open=True)
+    queue1 = cache1.get_default_queue()
+
+    # some I/O
+    r = (
+        Rio()
+        .target(vol)
+        .njobs(1)
+        .readwrite(ReadWrite.WRITE)
+        .size(Size.from_MiB(1))
+        .qd(1)
+        .run([queue1])
+    )
+
+    # capture checksum before simulated active host failure
+    data_md5 = vol.md5()
+
+    prim_cache_backend_vol.offline()
+
+    with pytest.raises(OcfError) as ex:
+        cache1.stop()
+    assert ex.value.error_code == OcfErrorCode.OCF_ERR_WRITE_CACHE
+
+    # capture a copy of active cache instance data
+    data = prim_cache_backend_vol.get_bytes()
+    cache_md5 = prim_cache_backend_vol.md5()
+
+    # setup standby cache
+    sec_cache_backend_vol = RamVolume(Size.from_MiB(150))
+    cache2 = Cache(owner=ctx2, cache_mode=mode, cache_line_size=cls)
+    cache2.start_cache()
+    cache2.standby_attach(sec_cache_backend_vol)
+    vol2 = CacheVolume(cache2, open=True)
+    queue = cache2.get_default_queue()
+
+    # standby cache exported object volume
+    cache2_exp_obj_vol = CacheVolume(cache2, open=True)
+
+    # just to be sure
+    assert sec_cache_backend_vol.get_bytes() != prim_cache_backend_vol.get_bytes()
+
+    # write content of active cache volume to passive cache exported obj
+    write_vol(vol2, queue, data)
+
+    assert cache_md5 == cache2_exp_obj_vol.md5()
+
+    # volumes should have the same data
+    assert sec_cache_backend_vol.get_bytes() == prim_cache_backend_vol.get_bytes()
+
+    # failover
+    cache2.standby_detach()
+    cache2.standby_activate(sec_cache_backend_vol, open_cores=False)
+    core = Core(core_backend_vol)
+    cache2.add_core(core, try_add=True)
+    vol = CoreVolume(core, open=True)
+
+    # check data consistency
+    assert data_md5 == vol.md5()
+
+
+def test_standby_load_writes_count(pyocf_ctx):
+    # Prepare a volume with valid metadata
+    device = RamVolume(Size.from_MiB(40))
+    cache = Cache.start_on_device(device, cache_mode=CacheMode.WB)
+    cache.stop()
+
+    device.reset_stats()
+
+    cache = Cache(owner=pyocf_ctx)
+    cache.start_cache()
+
+    cache.standby_load(device, perform_test=False)
+
+    assert device.get_stats()[IoDir.WRITE] == 0
+
+
+def test_failover_line_size_mismatch(pyocf_2_ctx):
+    ctx = pyocf_2_ctx[1]
+    mode = CacheMode.WB
+    cls = CacheLineSize.LINE_4KiB
+    cls2 = CacheLineSize.LINE_64KiB
+    vol1 = RamVolume(Size.from_MiB(150), uuid="cv1")
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    cache.attach_device(vol1, force=False)
+
+    core_vol = RamVolume(Size.from_MiB(150))
+    core = Core(core_vol)
+    cache.add_core(core)
+
+    data = vol1.get_bytes()
+
+    cache.stop()
+    vol1 = None
+
+    vol2 = RamVolume(Size.from_MiB(150), uuid="cv2")
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls2)
+    cache.start_cache()
+    cache.standby_attach(vol2)
+    cache_vol = CacheVolume(cache, open=True)
+
+    write_vol(cache_vol, cache.get_default_queue(), data)
+
+    cache.get_conf()["cache_line_size"] == cls2
+
+    cache.standby_detach()
+
+    # first attempt to activate with size mismatch
+    with pytest.raises(OcfError) as ex:
+        cache.standby_activate(vol2)
+    assert ex.value.error_code == OcfErrorCode.OCF_ERR_START_CACHE_FAIL
+
+    # second attempt to activate with size mismatch
+    with pytest.raises(OcfError) as ex:
+        cache.standby_activate(vol2)
+    assert ex.value.error_code == OcfErrorCode.OCF_ERR_START_CACHE_FAIL
+
+    cache.stop()
+    cache = Cache(owner=ctx, cache_mode=mode, cache_line_size=cls)
+    cache.start_cache()
+    cache.standby_load(vol2)
+    cache.standby_detach()
+    cache.standby_activate(vol2)
+
+    cache.get_conf()["cache_line_size"] == cls
+
+    cache.stop()
+
+
+def test_failover_passive_first(pyocf_2_ctx):
+    ctx1 = pyocf_2_ctx[0]
+    ctx2 = pyocf_2_ctx[1]
+    mode = CacheMode.WB
+    cls = CacheLineSize.LINE_4KiB
+
+    prim_cache_backend_vol = RamVolume(Size.from_MiB(150))
+    core_backend_vol = RamVolume(Size.from_MiB(1))
+    sec_cache_backend_vol = RamVolume(Size.from_MiB(150))
+
+    # passive cache with directly on ram disk
+    cache2 = Cache(owner=ctx2, cache_mode=mode, cache_line_size=cls)
+    cache2.start_cache()
+    cache2.standby_attach(sec_cache_backend_vol)
+
+    # volume replicating cache1 ramdisk writes to cache2 cache exported object
+    cache2_exp_obj_vol = CacheVolume(cache2, open=True)
+    cache1_cache_vol = ReplicatedVolume(prim_cache_backend_vol, cache2_exp_obj_vol)
+
+    # active cache
+    cache1 = Cache.start_on_device(
+        cache1_cache_vol, ctx1, cache_mode=mode, cache_line_size=cls
+    )
+    core = Core(core_backend_vol)
+    cache1.add_core(core)
+    core_vol = CoreVolume(core, open=True)
+    queue = cache1.get_default_queue()
+
+    # some I/O
+    r = (
+        Rio()
+        .target(core_vol)
+        .njobs(1)
+        .readwrite(ReadWrite.WRITE)
+        .size(Size.from_MiB(1))
+        .qd(1)
+        .run([queue])
+    )
+
+    # capture checksum before simulated active host failure
+    md5 = core_vol.md5()
+
+    # offline primary cache volume and stop primary cache to simulate active host
+    # failure
+    cache1_cache_vol.offline()
+    with pytest.raises(OcfError) as ex:
+        cache1.stop()
+    assert ex.value.error_code == OcfErrorCode.OCF_ERR_WRITE_CACHE
+
+    # failover
+    cache2.standby_detach()
+    cache2.standby_activate(sec_cache_backend_vol, open_cores=False)
+
+    # add core explicitly with "try_add" to workaround pyocf limitations
+    core = Core(core_backend_vol)
+    cache2.add_core(core, try_add=True)
+    core_vol = CoreVolume(core, open=True)
+
+    assert md5 == core_vol.md5()