Merge pull request #824 from mmichal10/partial-hit-tests

Partial hit tests

tests/functional/pyocf/types/io.py

@@ -22,6 +22,11 @@ from .data import Data
 from .shared import OcfCompletion
 
 
+class WriteMode(IntEnum):
+    ZERO_PAD = 0
+    READ_MODIFY_WRITE = 1
+
+
 class IoDir(IntEnum):
     READ = 0
     WRITE = 1
@@ -120,7 +125,7 @@ class Sync:
         self.io = io
 
     def sync_submit(self, submit_method):
-        if getattr(self.io, 'callback', None):
+        if getattr(self.io, "callback", None):
             raise Exception("completion callback is already set")
         cmpl = OcfCompletion([("err", c_int)])
         self.io.callback = cmpl.callback

tests/functional/pyocf/types/volume.py

@@ -1,5 +1,6 @@
 #
 # Copyright(c) 2019-2022 Intel Corporation
+# Copyright(c) 2024 Huawei Technologies
 # SPDX-License-Identifier: BSD-3-Clause
 #
 
@@ -24,16 +25,18 @@ from hashlib import md5
 import weakref
 from enum import IntEnum
 import warnings
+from typing import Union
 
-from .io import Io, IoOps, IoDir
+from .io import Io, IoOps, IoDir, WriteMode, Sync
 from .queue import Queue
 from .shared import OcfErrorCode, Uuid
 from ..ocf import OcfLib
 from ..utils import print_buffer, Size as S
-from .data import Data
+from .data import Data, DataSeek
 from .queue import Queue
 
 
+
 class IoFlags(IntEnum):
     FLUSH = 1
 
@@ -341,6 +344,66 @@ class Volume:
         )
         return Io.from_pointer(io)
 
+    def sync_io(
+        self,
+        queue,
+        address: int,
+        data: Data,
+        direction: IoDir,
+        io_class=0,
+        flags=0,
+        submit_func=Sync.submit,
+    ):
+        assert address % 512 == 0
+        assert data.size % 512 == 0
+
+        io = self.new_io(queue, address, data.size, direction, io_class, flags)
+        io.set_data(data)
+        completion = submit_func(Sync(io))
+
+        assert int(completion.results["err"]) == 0
+
+    def write_sync_4k(
+        self,
+        queue: Queue,
+        address: int,
+        data: Union[bytes, Data],
+        mode: WriteMode,
+        io_class=0,
+        flags=0,
+    ):
+        if mode not in list(WriteMode):
+            raise ValueError(f"illegal write mode: {mode}")
+
+        size = len(data)
+
+        address_4k = (address // 4096) * 4096
+
+        end_address_4k = ((address + size + 4095) // 4096) * 4096
+        size_4k = end_address_4k - address_4k
+
+        write_data = Data(size_4k)
+
+        if mode == WriteMode.ZERO_PAD:
+            write_data.zero(size_4k)
+        elif mode == WriteMode.READ_MODIFY_WRITE:
+            self.sync_io(queue, address_4k, write_data, IoDir.READ)
+
+        write_data.seek(DataSeek.BEGIN, address - address_4k)
+        write_data.write(data, size)
+
+        self.sync_io(queue, address_4k, write_data, IoDir.WRITE, io_class, flags)
+
+    def read_sync(self, queue: Queue, address: int, size: int, io_class=0, flags=0) -> bytes:
+        read_data = Data(size)
+        self.sync_io(queue, address, read_data, IoDir.READ, io_class, flags)
+
+        data = bytes(size)
+        read_data.seek(DataSeek.BEGIN, 0)
+        read_data.read(data, size)
+
+        return data
+
 
 class RamVolume(Volume):
     props = None

tests/functional/pyocf/types/volume_core.py

@@ -1,12 +1,13 @@
 #
 # Copyright(c) 2022 Intel Corporation
+# Copyright(c) 2024 Huawei Technologies
 # SPDX-License-Identifier: BSD-3-Clause
 #
 
-from .core import Core
 from .volume_exp_obj import OcfInternalVolume
-from .io import IoDir
-from .volume import Volume
+from .queue import Queue
+from .io import Sync, IoDir
+from .data import Data
 
 
 class CoreVolume(OcfInternalVolume):
@@ -20,3 +21,16 @@ class CoreVolume(OcfInternalVolume):
 
     def md5(self):
         return self._exp_obj_md5(4096)
+
+    def sync_io(
+        self,
+        queue: Queue,
+        address: int,
+        data: Data,
+        direction: IoDir,
+        io_class=0,
+        flags=0,
+        submit_func=Sync.submit,
+    ):
+        super().sync_io(queue, address, data, direction, io_class, flags, submit_func)
+        self.core.cache.settle()

tests/functional/pyocf/utils.py

@@ -1,5 +1,6 @@
 #
 # Copyright(c) 2019-2022 Intel Corporation
+# Copyright(c) 2024 Huawei Technologies
 # SPDX-License-Identifier: BSD-3-Clause
 #
 

tests/functional/tests/engine/test_partial_hit.py

@@ -0,0 +1,169 @@
+#
+# Copyright(c) 2024 Huawei Technologies
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+import pytest
+
+
+from pyocf.types.data import Data, DataSeek
+from pyocf.types.cache import Cache, CacheMode
+from pyocf.types.core import Core
+from pyocf.types.shared import CacheLineSize
+from pyocf.types.volume import RamVolume, Volume
+from pyocf.types.volume_core import CoreVolume
+from pyocf.utils import Size
+from pyocf.types.io import IoDir
+
+
+@pytest.mark.parametrize("cacheline_size", CacheLineSize)
+@pytest.mark.parametrize("cache_mode", CacheMode)
+def test_partial_hit_write(pyocf_ctx, cacheline_size, cache_mode):
+    cache_device = RamVolume(Size.from_MiB(50))
+    core_device = RamVolume(Size.from_MiB(50))
+
+    cache = Cache.start_on_device(
+        cache_device, cache_line_size=cacheline_size, cache_mode=cache_mode
+    )
+    core = Core.using_device(core_device)
+
+    queue = cache.get_default_queue()
+
+    cache.add_core(core)
+    core_volume = CoreVolume(core)
+    core_volume.open()
+
+    # Fill core with data
+    CL = cache.cache_line_size
+    data = Data(CL // 2)
+    data.seek(DataSeek.BEGIN, 0)
+    data.write(b"A\x00\x00\x00\x00", 5)
+    core_device.sync_io(queue, 0, data, IoDir.WRITE)
+    core_device.sync_io(queue, CL // 2, data, IoDir.WRITE)
+
+    # Write 0.5 CL
+    data.seek(DataSeek.BEGIN, 0)
+    data.write(b"B\x00\x00\x00\x00", 5)
+    core_volume.sync_io(queue, 0, data, IoDir.WRITE)
+
+    data1 = core_volume.read_sync(queue, 0, CL)
+
+    assert chr(data1[0]) == "B"
+    assert chr(data1[CL // 2]) == "A"
+
+
+@pytest.mark.parametrize("cacheline_size", CacheLineSize)
+@pytest.mark.parametrize("cache_mode", CacheMode)
+def test_partial_hit_read(pyocf_ctx, cacheline_size, cache_mode):
+    cache_device = RamVolume(Size.from_MiB(50))
+    core_device = RamVolume(Size.from_MiB(50))
+
+    cache = Cache.start_on_device(
+        cache_device, cache_line_size=cacheline_size, cache_mode=cache_mode
+    )
+    core = Core.using_device(core_device)
+
+    queue = cache.get_default_queue()
+
+    cache.add_core(core)
+    core_volume = CoreVolume(core)
+    core_volume.open()
+
+    # Fill core with data
+    CL = cache.cache_line_size
+    data = Data(CL // 2)
+    data.seek(DataSeek.BEGIN, 0)
+    data.write(b"A\x00\x00\x00\x00", 5)
+    core_device.sync_io(queue, 0, data, IoDir.WRITE)
+    core_device.sync_io(queue, CL // 2, data, IoDir.WRITE)
+
+    data_read = Data(CL // 2)
+    core_volume.sync_io(queue, 0, data_read, IoDir.READ)
+
+    data1 = core_volume.read_sync(queue, 0, CL)
+
+    assert chr(data1[0]) == "A"
+    assert chr(data1[CL // 2]) == "A"
+
+
+@pytest.mark.parametrize("cacheline_size", CacheLineSize)
+@pytest.mark.parametrize("cache_mode", [CacheMode.WB, CacheMode.WO])
+def test_read_partial_hit_partial_invalidate_dirty(pyocf_ctx, cacheline_size, cache_mode):
+    cache_device = RamVolume(Size.from_MiB(50))
+    core_device = RamVolume(Size.from_MiB(50))
+
+    cache = Cache.start_on_device(
+        cache_device, cache_line_size=cacheline_size, cache_mode=cache_mode
+    )
+    core = Core.using_device(core_device)
+
+    queue = cache.get_default_queue()
+
+    cache.add_core(core)
+    core_volume = CoreVolume(core)
+    core_volume.open()
+
+    CL = cache.cache_line_size
+    data = Data(CL)
+    data.seek(DataSeek.BEGIN, 0)
+    data.write(b"A" * CL, CL)
+    core_volume.sync_io(queue, 0, data, IoDir.WRITE)
+
+    data.seek(DataSeek.BEGIN, 0)
+    data.write(b"B" * 512, 512)
+    core_volume.sync_io(queue, 512, data, IoDir.WRITE)
+
+    data1 = core_volume.read_sync(queue, 0, CL)
+
+    assert chr(data1[0]) == "A"
+    assert chr(data1[512]) == "B"
+    assert chr(data1[1023]) == "B"
+    assert chr(data1[1024]) == "A"
+
+
+@pytest.mark.parametrize("cacheline_size", CacheLineSize)
+def test_partial_hit_backfill(pyocf_ctx, cacheline_size):
+    cache_device = RamVolume(Size.from_MiB(50))
+    core_device = RamVolume(Size.from_MiB(50))
+
+    cache = Cache.start_on_device(cache_device, cache_line_size=cacheline_size)
+    core = Core.using_device(core_device)
+
+    queue = cache.get_default_queue()
+
+    cache.add_core(core)
+    core_volume = CoreVolume(core)
+    core_volume.open()
+
+    cache_device.reset_stats()
+    core_device.reset_stats()
+
+    CL = cache.cache_line_size
+    # Populate core backend volume
+    invalid_sectors = list(range(0, 6 * CL, Size._SECTOR_SIZE))
+    prefill_data = Data(Size._SECTOR_SIZE)
+    prefill_data.write(b"I\x00\x00\x00\x00", 5)
+    for addr in invalid_sectors:
+        core_device.sync_io(queue, addr, prefill_data, IoDir.WRITE)
+
+    stats = cache_device.get_stats()
+    assert stats[IoDir.WRITE] == 0
+
+    # Write data to the core
+    core_data_addr = 2 * CL + CL // 2
+    core_data_size = CL
+    valid_sectors = list(range(core_data_addr, core_data_addr + core_data_size, Size._SECTOR_SIZE))
+    valid_data = Data(core_data_size)
+    for addr in range(0, CL, Size._SECTOR_SIZE):
+        valid_data.seek(DataSeek.BEGIN, addr)
+        valid_data.write(b"C\x00\x00\x00\x00", 5)
+    core_volume.sync_io(queue, core_data_addr, valid_data, IoDir.WRITE)
+
+    invalid_sectors = [s for s in invalid_sectors if s not in valid_sectors]
+
+    read_data = core_volume.read_sync(queue, 0, 6 * CL)
+    for addr in invalid_sectors:
+        assert chr(read_data[addr]) == "I", f"data offset {addr}"
+
+    for addr in valid_sectors:
+        assert chr(read_data[addr]) == "C", f"data offset {addr}"