Merge pull request #225 from katlapinka/cas-api-plug-and-cache-params

Adapt CAS API to handle disk plugging and fix for setting clean…

test/functional/api/cas/cache.py

@@ -160,15 +160,22 @@ class Cache:
 
     def set_params_acp(self, acp_params: FlushParametersAcp):
         return casadm.set_param_cleaning_acp(self.cache_id,
-                                             acp_params.wake_up_time.total_milliseconds(),
+                                             int(acp_params.wake_up_time.total_milliseconds())
-                                             acp_params.flush_max_buffers)
+                                             if acp_params.wake_up_time else None,
+                                             int(acp_params.flush_max_buffers)
+                                             if acp_params.flush_max_buffers else None)
 
     def set_params_alru(self, alru_params: FlushParametersAlru):
         return casadm.set_param_cleaning_alru(self.cache_id,
-                                              alru_params.wake_up_time.total_seconds(),
+                                              int(alru_params.wake_up_time.total_seconds())
-                                              alru_params.staleness_time.total_seconds(),
+                                              if alru_params.wake_up_time else None,
-                                              alru_params.flush_max_buffers,
+                                              int(alru_params.staleness_time.total_seconds())
-                                              alru_params.activity_threshold.total_milliseconds())
+                                              if alru_params.staleness_time else None,
+                                              alru_params.flush_max_buffers
+                                              if alru_params.flush_max_buffers else None,
+                                              int(alru_params.activity_threshold.total_seconds()
+                                                  * 1000)
+                                              if alru_params.activity_threshold else None)
 
     def get_cache_config(self):
         return CacheConfig(self.get_cache_line_size(),

test/functional/api/cas/cache_config.py

@@ -79,11 +79,15 @@ class Time(timedelta):
 
 
 class FlushParametersAlru:
-    def __init__(self):
+    def __init__(self,
-        self.activity_threshold = None
+                 activity_threshold=None,
-        self.flush_max_buffers = None
+                 flush_max_buffers=None,
-        self.staleness_time = None
+                 staleness_time=None,
-        self.wake_up_time = None
+                 wake_up_time=None):
+        self.activity_threshold = activity_threshold
+        self.flush_max_buffers = flush_max_buffers
+        self.staleness_time = staleness_time
+        self.wake_up_time = wake_up_time
 
     @staticmethod
     def default_alru_params():
@@ -96,9 +100,11 @@ class FlushParametersAlru:
 
 
 class FlushParametersAcp:
-    def __init__(self):
+    def __init__(self,
-        self.flush_max_buffers = None
+                 flush_max_buffers=None,
-        self.wake_up_time = None
+                 wake_up_time=None):
+        self.flush_max_buffers = flush_max_buffers
+        self.wake_up_time = wake_up_time
 
     @staticmethod
     def default_acp_params():
@@ -109,9 +115,9 @@ class FlushParametersAcp:
 
 
 class SeqCutOffParameters:
-    def __init__(self):
+    def __init__(self, policy=None, threshold=None):
-        self.policy = None
+        self.policy = policy
-        self.threshold = None
+        self.threshold = threshold
 
     @staticmethod
     def default_seq_cut_off_params():

test/functional/api/cas/casadm.py

@@ -2,7 +2,9 @@
 # Copyright(c) 2019 Intel Corporation
 # SPDX-License-Identifier: BSD-3-Clause-Clear
 #
+import csv
 
+from test_utils.output import CmdException
 from typing import List
 
 from api.cas.cache import Cache
@@ -134,6 +136,13 @@ def stop_all_caches():
         raise CmdException("Error while stopping caches.", output)
 
 
+def remove_all_detached_cores():
+    from api.cas import casadm_parser
+    devices = casadm_parser.get_cas_devices_dict()
+    for dev in devices["core_pool"]:
+        TestRun.executor.run(remove_detached_cmd(dev["device"]))
+
+
 def print_statistics(cache_id: int, core_id: int = None, per_io_class: bool = False,
                      io_class_id: int = None, filter: List[StatsFilter] = None,
                      output_format: OutputFormat = None, shortcut: bool = False):
@@ -256,8 +265,11 @@ def set_param_cleaning_alru(cache_id: int, wake_up: int = None, staleness_time:
                             flush_max_buffers: int = None, activity_threshold: int = None):
     output = TestRun.executor.run(
         set_param_cleaning_alru_cmd(
-            cache_id=str(cache_id), wake_up=str(wake_up), staleness_time=str(staleness_time),
+            cache_id=str(cache_id),
-            flush_max_buffers=str(flush_max_buffers), activity_threshold=str(activity_threshold)))
+            wake_up=str(wake_up) if wake_up else None,
+            staleness_time=str(staleness_time) if staleness_time else None,
+            flush_max_buffers=str(flush_max_buffers) if flush_max_buffers else None,
+            activity_threshold=str(activity_threshold) if activity_threshold else None))
     if output.exit_code != 0:
         raise CmdException("Error while setting alru cleaning policy parameters.", output)
     return output
@@ -265,8 +277,10 @@ def set_param_cleaning_alru(cache_id: int, wake_up: int = None, staleness_time:
 
 def set_param_cleaning_acp(cache_id: int, wake_up: int = None, flush_max_buffers: int = None):
     output = TestRun.executor.run(
-        set_param_cleaning_acp_cmd(cache_id=str(cache_id), wake_up=str(wake_up),
+        set_param_cleaning_acp_cmd(
-                                   flush_max_buffers=str(flush_max_buffers)))
+            cache_id=str(cache_id),
+            wake_up=str(wake_up) if wake_up else None,
+            flush_max_buffers=str(flush_max_buffers) if flush_max_buffers else None))
     if output.exit_code != 0:
         raise CmdException("Error while setting acp cleaning policy parameters.", output)
     return output

test/functional/api/cas/casadm_parser.py

@@ -2,6 +2,7 @@
 # Copyright(c) 2019 Intel Corporation
 # SPDX-License-Identifier: BSD-3-Clause-Clear
 #
+import csv
 
 import json
 import re
@@ -172,6 +173,40 @@ def get_cores(cache_id: int):
     return cores_list
 
 
+def get_cas_devices_dict():
+    device_list = list(csv.DictReader(casadm.list_caches(OutputFormat.csv).stdout.split('\n')))
+    devices = {"core_pool": [], "caches": {}, "cores": {}}
+    core_pool = False
+    prev_cache_id = -1
+
+    for device in device_list:
+        if device["type"] == "core pool":
+            core_pool = True
+            continue
+
+        if device["type"] == "cache":
+            core_pool = False
+            prev_cache_id = int(device["id"])
+            devices["caches"].update(
+                {
+                    int(device["id"]): {
+                        "device": device["disk"],
+                        "status": device["status"],
+                    }
+                }
+            )
+        elif device["type"] == "core":
+            core = {"device": device["disk"], "status": device["status"]}
+            if core_pool:
+                devices["core_pool"].append(core)
+            else:
+                core.update({"cache_id": prev_cache_id})
+                devices["cores"].update(
+                    {(prev_cache_id, int(device["id"])): core}
+                )
+    return devices
+
+
 def get_flush_parameters_alru(cache_id: int):
     casadm_output = casadm.get_param_cleaning_alru(cache_id,
                                                    casadm.OutputFormat.csv).stdout.spltlines()

test/functional/api/cas/core.py

@@ -2,13 +2,18 @@
 # Copyright(c) 2019 Intel Corporation
 # SPDX-License-Identifier: BSD-3-Clause-Clear
 #
-from typing import List
+
 
 from api.cas.cli import *
 from api.cas.casadm_parser import *
 from api.cas.cache import Device
+from test_tools import fs_utils
 from test_utils.os_utils import *
 from api.cas.statistics import CoreStats, IoClassStats
+from datetime import timedelta, datetime
+from test_utils.os_utils import wait
+
+import time
 
 
 class CoreStatus(Enum):
@@ -118,3 +123,23 @@ class Core(Device):
         return casadm.set_param_cutoff(self.cache_id, self.core_id,
                                        threshold=None,
                                        policy=policy)
+
+    def check_if_is_present_in_os(self, should_be_visible=True):
+        device_in_system_message = "CAS device exists in OS."
+        device_not_in_system_message = "CAS device does not exist in OS."
+        item = fs_utils.ls_item(f"{self.system_path}")
+        if item is not None:
+            if should_be_visible:
+                TestRun.LOGGER.info(device_in_system_message)
+            else:
+                TestRun.fail(device_in_system_message)
+        else:
+            if should_be_visible:
+                TestRun.fail(device_not_in_system_message)
+            else:
+                TestRun.LOGGER.info(device_not_in_system_message)
+
+    def wait_for_status_change(self, expected_status: CoreStatus):
+        timeout = timedelta(minutes=1)
+        if not wait(lambda: self.get_status() == expected_status, timeout, timedelta(seconds=1)):
+            TestRun.fail(f"Core status did not change after {timeout.total_seconds()}s.")

test/functional/test-framework

@@ -1 +1 @@
-Subproject commit 82a3270d6d638aa6123bc42020e362ac3c5701d5
+Subproject commit 0fb571232040ab6e39aedfc979b1fb8ef82245c3

test/functional/tests/conftest.py

@@ -3,9 +3,10 @@
 # SPDX-License-Identifier: BSD-3-Clause-Clear
 #
 
-import pytest
 import os
 import sys
+
+import pytest
 import yaml
 import traceback
 from IPy import IP
@@ -117,7 +118,10 @@ def pytest_runtest_teardown():
                     TestRun.executor.wait_for_connection()
                 Udev.enable()
                 unmount_cas_devices()
+                casadm.remove_all_detached_cores()
                 casadm.stop_all_caches()
+                from api.cas import init_config
+                init_config.create_default_init_config()
         except Exception as ex:
             TestRun.LOGGER.warning(f"Exception occured during platform cleanup.\n"
                                    f"{str(ex)}\n{traceback.format_exc()}")
@@ -182,6 +186,7 @@ def base_prepare(item):
             try:
                 unmount_cas_devices()
                 casadm.stop_all_caches()
+                casadm.remove_all_detached_cores()
             except Exception:
                 pass  # TODO: Reboot DUT if test is executed remotely