Move OCL tests from test-framework repository

Signed-off-by: Robert Baldyga <robert.baldyga@intel.com>
This commit is contained in:
Robert Baldyga
2019-10-17 17:15:38 +02:00
parent d2666b785a
commit 4fb82abeca
35 changed files with 4102 additions and 0 deletions

View File

View File

@@ -0,0 +1,142 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from api.cas.cli import *
from api.cas.casadm_parser import *
from test_utils.os_utils import *
from api.cas.cache_config import *
from storage_devices.device import Device
from core.test_run import TestRun
from api.cas.casadm_params import *
class Cache:
def __init__(self, device_system_path):
self.cache_device = Device(device_system_path)
self.cache_id = int(self.__get_cache_id())
self.__cache_line_size = None
self.__metadata_mode = None
self.__metadata_size = None
def __get_cache_id(self):
cmd = f"{list_cmd()} | grep {self.cache_device.system_path}"
output = TestRun.executor.run(cmd)
if output.exit_code == 0 and output.stdout.strip():
return output.stdout.split()[1]
else:
raise Exception(f"There is no cache started on {self.cache_device.system_path}.")
def get_core_devices(self):
return get_cores(self.cache_id)
def get_cache_line_size(self):
if self.__cache_line_size is None:
stats = self.get_cache_statistics()
stats_line_size = stats["cache line size"]
self.__cache_line_size = CacheLineSize(stats_line_size.get_value(Unit.Byte))
return self.__cache_line_size
def get_cleaning_policy(self):
stats = self.get_cache_statistics()
cp = stats["cleaning policy"]
return CleaningPolicy[cp]
def get_eviction_policy(self):
stats = self.get_cache_statistics()
ep = stats["eviction policy"]
return EvictionPolicy[ep]
def get_metadata_mode(self):
if self.__metadata_mode is None:
stats = self.get_cache_statistics()
mm = stats["metadata mode"]
self.__metadata_mode = MetadataMode[mm]
return self.__metadata_mode
def get_metadata_size(self):
if self.__metadata_size is None:
stats = self.get_cache_statistics()
self.__metadata_size = stats["metadata memory footprint"]
return self.__metadata_size
def get_occupancy(self):
return self.get_cache_statistics()["occupancy"]
def get_status(self):
status = self.get_cache_statistics()["status"].replace(' ', '_')
return CacheStatus[status]
def get_cache_mode(self):
return CacheMode[self.get_cache_statistics()["write policy"].upper()]
def get_dirty_blocks(self):
return self.get_cache_statistics()["dirty"]
def get_dirty_for(self):
return self.get_cache_statistics()["dirty for"]
def get_clean_blocks(self):
return self.get_cache_statistics()["clean"]
def get_flush_parameters_alru(self):
return get_flush_parameters_alru(self.cache_id)
def get_flush_parameters_acp(self):
return get_flush_parameters_acp(self.cache_id)
# Casadm methods:
def get_cache_statistics(self,
io_class_id: int = None,
stat_filter: List[StatsFilter] = None,
percentage_val: bool = False):
return get_statistics(self.cache_id, None, io_class_id,
stat_filter, percentage_val)
def flush_cache(self):
casadm.flush(cache_id=self.cache_id)
sync()
assert self.get_dirty_blocks().get_value(Unit.Blocks4096) == 0
def stop(self, no_data_flush: bool = False):
return casadm.stop_cache(self.cache_id, no_data_flush)
def add_core(self, core_dev, core_id: int = None):
return casadm.add_core(self, core_dev, core_id)
def remove_core(self, core_id, force: bool = False):
return casadm.remove_core(self.cache_id, core_id, force)
def reset_counters(self):
return casadm.reset_counters(self.cache_id)
def set_cache_mode(self, cache_mode: CacheMode, flush: bool = True):
return casadm.set_cache_mode(cache_mode, self.cache_id, flush)
def load_io_class(self, file_path: str):
return casadm.load_io_classes(self.cache_id, file_path)
def list_io_classes(self, output_format: OutputFormat):
return casadm.list_io_classes(self.cache_id, output_format)
def set_seq_cutoff_parameters(self, seq_cutoff_param: SeqCutOffParameters):
return casadm.set_param_cutoff(self.cache_id,
seq_cutoff_param.threshold,
seq_cutoff_param.policy)
def set_cleaning_policy(self, cleaning_policy: CleaningPolicy):
return casadm.set_param_cleaning(self.cache_id, cleaning_policy)
def set_params_acp(self, acp_params: FlushParametersAcp):
return casadm.set_param_cleaning_acp(self.cache_id,
acp_params.wake_up_time.total_milliseconds(),
acp_params.flush_max_buffers)
def set_params_alru(self, alru_params: FlushParametersAlru):
return casadm.set_param_cleaning_alru(self.cache_id,
alru_params.wake_up_time.total_seconds(),
alru_params.staleness_time.total_seconds(),
alru_params.flush_max_buffers,
alru_params.activity_threshold.total_milliseconds())

View File

@@ -0,0 +1,114 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from enum import IntEnum, Enum
from test_utils.size import Size, Unit
from datetime import timedelta
class CacheLineSize(IntEnum):
LINE_4KiB = Size(4, Unit.KibiByte)
LINE_8KiB = Size(8, Unit.KibiByte)
LINE_16KiB = Size(16, Unit.KibiByte)
LINE_32KiB = Size(32, Unit.KibiByte)
LINE_64KiB = Size(64, Unit.KibiByte)
DEFAULT = LINE_4KiB
class CacheMode(Enum):
WT = 0
WB = 1
WA = 2
PT = 3
WO = 4
DEFAULT = WT
class SeqCutOffPolicy(Enum):
full = 0
always = 1
never = 2
DEFAULT = full
class EvictionPolicy(Enum):
lru = 0
lmp = 1
nop = 2
class MetadataMode(Enum):
normal = 0
atomic = 1
class CleaningPolicy(Enum):
alru = 0
nop = 1
acp = 2
DEFAULT = alru
class CacheStatus(Enum):
not_running = 0
running = 1
stopping = 2
initializing = 3
flushing = 4
incomplete = 5
class Time(timedelta):
def total_milliseconds(self):
return self.total_seconds() * 1000
class FlushParametersAlru:
def __init__(self):
self.activity_threshold = None
self.flush_max_buffers = None
self.staleness_time = None
self.wake_up_time = None
@staticmethod
def default_alru_params():
alru_params = FlushParametersAlru()
alru_params.activity_threshold = Time(milliseconds=10000)
alru_params.flush_max_buffers = 100
alru_params.staleness_time = Time(seconds=120)
alru_params.wake_up_time = Time(seconds=20)
return alru_params
class FlushParametersAcp:
def __init__(self):
self.flush_max_buffers = None
self.wake_up_time = None
@staticmethod
def default_acp_params():
acp_params = FlushParametersAcp()
acp_params.flush_max_buffers = 128
acp_params.wake_up_time = Time(milliseconds=10)
return acp_params
class SeqCutOffParameters:
def __init__(self):
self.policy = None
self.threshold = None
@staticmethod
def default_seq_cut_off_params():
seq_cut_off_params = SeqCutOffParameters()
seq_cut_off_params.policy = SeqCutOffPolicy.full
seq_cut_off_params.threshold = Size(1024, Unit.KibiByte)
# TODO: Use case for this will be to iterate over configurations (kernel params such as
# TODO: io scheduler, metadata layout) and prepare env before starting cache
class CacheConfig:
def __init__(self):
pass

View File

@@ -0,0 +1,18 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from aenum import Enum
from config.configuration import cas_kernel_module, disk_kernel_module
from test_utils import os_utils
from test_utils.os_utils import ModuleRemoveMethod
class CasModule(Enum):
cache = cas_kernel_module
disk = disk_kernel_module
def reload_all_cas_modules():
os_utils.unload_kernel_module(CasModule.cache, ModuleRemoveMethod.modprobe)
os_utils.load_kernel_module(CasModule.cache)

View File

@@ -0,0 +1,305 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from .cli import *
from .casctl import stop as casctl_stop
from core.test_run import TestRun
from .casadm_params import *
from api.cas.cache_config import CacheLineSize, CacheMode, SeqCutOffPolicy, CleaningPolicy
from test_utils.size import Size, Unit
from typing import List
from storage_devices.device import Device
from api.cas.core import Core
from api.cas.cache import Cache
def help(shortcut: bool = False):
return TestRun.executor.run(help_cmd(shortcut))
def start_cache(cache_dev: Device, cache_mode: CacheMode = None,
cache_line_size: CacheLineSize = None, cache_id: int = None,
force: bool = False, load: bool = False, shortcut: bool = False):
_cache_line_size = None if cache_line_size is None else str(
CacheLineSize.get_value(Unit.KibiByte))
_cache_id = None if cache_id is None else str(cache_id)
_cache_mode = None if cache_mode is None else cache_mode.name.lower()
output = TestRun.executor.run(start_cmd(
cache_dev=cache_dev.system_path, cache_mode=_cache_mode, cache_line_size=_cache_line_size,
cache_id=_cache_id, force=force, load=load, shortcut=shortcut))
if output.exit_code != 0:
raise Exception(
f"Failed to start cache. stdout: {output.stdout} \n stderr :{output.stderr}")
return Cache(cache_dev.system_path)
def stop_cache(cache_id: int, no_data_flush: bool = False, shortcut: bool = False):
output = TestRun.executor.run(
stop_cmd(cache_id=str(cache_id), no_data_flush=no_data_flush, shortcut=shortcut))
if output.exit_code != 0:
raise Exception(
f"Failed to stop cache. stdout: {output.stdout} \n stderr :{output.stderr}")
return output
def add_core(cache: Cache, core_dev: Device, core_id: int = None, shortcut: bool = False):
_core_id = None if core_id is None else str(id)
output = TestRun.executor.run(
add_core_cmd(cache_id=str(cache.cache_id), core_dev=core_dev.system_path,
core_id=_core_id, shortcut=shortcut))
if output.exit_code != 0:
raise Exception(
f"Failed to add core. stdout: {output.stdout} \n stderr :{output.stderr}")
return Core(core_dev.system_path, cache.cache_id)
def remove_core(cache_id: int, core_id: int, force: bool = False, shortcut: bool = False):
output = TestRun.executor.run(
remove_core_cmd(cache_id=str(cache_id), core_id=str(core_id),
force=force, shortcut=shortcut))
if output.exit_code != 0:
raise Exception(
f"Failed to remove core. stdout: {output.stdout} \n stderr :{output.stderr}")
def remove_detached(core_device: Device, shortcut: bool = False):
output = TestRun.executor.run(
remove_detached_cmd(core_device=core_device.system_path, shortcut=shortcut))
if output.exit_code != 0:
raise Exception(
f"Failed to remove detached core. stdout: {output.stdout} \n stderr :{output.stderr}")
return output
def reset_counters(cache_id: int, core_id: int = None, shortcut: bool = False):
_core_id = None if core_id is None else str(core_id)
output = TestRun.executor.run(
reset_counters_cmd(cache_id=str(cache_id), core_id=_core_id, shortcut=shortcut))
if output.exit_code != 0:
raise Exception(
f"Failed to reset counters. stdout: {output.stdout} \n stderr :{output.stderr}")
return output
def flush(cache_id: int, core_id: int = None, shortcut: bool = False):
if core_id is None:
command = flush_cache_cmd(cache_id=str(cache_id), shortcut=shortcut)
else:
command = flush_core_cmd(cache_id=str(cache_id), core_id=str(core_id), shortcut=shortcut)
output = TestRun.executor.run(command)
if output.exit_code != 0:
raise Exception(
f"Flushing failed. stdout: {output.stdout} \n stderr :{output.stderr}")
return output
def load_cache(device: Device, shortcut: bool = False):
output = TestRun.executor.run(
load_cmd(cache_dev=device.system_path, shortcut=shortcut))
if output.exit_code != 0:
raise Exception(
f"Failed to load cache. stdout: {output.stdout} \n stderr :{output.stderr}")
return Cache(device.system_path)
def list_caches(output_format: OutputFormat = None, shortcut: bool = False):
_output_format = None if output_format is None else output_format.name
output = TestRun.executor.run(
list_cmd(output_format=_output_format, shortcut=shortcut))
if output.exit_code != 0:
raise Exception(
f"Failed to list caches. stdout: {output.stdout} \n stderr :{output.stderr}")
return output
def print_version(output_format: OutputFormat = None, shortcut: bool = False):
_output_format = None if output_format is None else output_format.name
output = TestRun.executor.run(
version_cmd(output_format=_output_format, shortcut=shortcut))
if output.exit_code != 0:
raise Exception(
f"Failed to print version. stdout: {output.stdout} \n stderr :{output.stderr}")
return output
def format_nvme(cache_dev: Device, force: bool = False, shortcut: bool = False):
output = TestRun.executor.run(
format_cmd(cache_dev=cache_dev.system_path, force=force, shortcut=shortcut))
if output.exit_code != 0:
raise Exception(
f"Format command failed. stdout: {output.stdout} \n stderr :{output.stderr}")
return output
def stop_all_caches():
if "No caches running" in list_caches().stdout:
return
TestRun.LOGGER.info("Stop all caches")
casctl_stop()
output = list_caches()
if "No caches running" not in output.stdout:
raise Exception(
f"Error while stopping caches. stdout: {output.stdout} \n stderr :{output.stderr}")
def print_statistics(cache_id: int, core_id: int = None, per_io_class: bool = False,
io_class_id: int = None, filter: List[StatsFilter] = None,
output_format: OutputFormat = None, shortcut: bool = False):
_output_format = None if output_format is None else output_format.name
_core_id = None if core_id is None else str(core_id)
_io_class_id = None if io_class_id is None else str(io_class_id)
if filter is None:
_filter = filter
else:
names = (x.name for x in filter)
_filter = ",".join(names)
output = TestRun.executor.run(
print_statistics_cmd(
cache_id=str(cache_id), core_id=_core_id,
per_io_class=per_io_class, io_class_id=_io_class_id,
filter=_filter, output_format=_output_format, shortcut=shortcut))
if output.exit_code != 0:
raise Exception(
f"Printing statistics failed. stdout: {output.stdout} \n stderr :{output.stderr}")
return output
def set_cache_mode(cache_mode: CacheMode, cache_id: int,
flush: bool = True, shortcut: bool = False):
flush_cache = None
if cache_mode in [CacheMode.WB, CacheMode.WO]:
flush_cache = "yes" if flush else "no"
output = TestRun.executor.run(
set_cache_mode_cmd(cache_mode=cache_mode.name.lower(), cache_id=str(cache_id),
flush_cache=flush_cache, shortcut=shortcut))
if output.exit_code != 0:
raise Exception(
f"Set cache mode command failed. stdout: {output.stdout} \n stderr :{output.stderr}")
return output
def load_io_classes(cache_id: int, file: str, shortcut: bool = False):
output = TestRun.executor.run(
load_io_classes_cmd(cache_id=str(cache_id), file=file, shortcut=shortcut))
if output.exit_code != 0:
raise Exception(
f"Load IO class command failed. stdout: {output.stdout} \n stderr :{output.stderr}")
return output
def list_io_classes(cache_id: int, output_format: OutputFormat, shortcut: bool = False):
_output_format = None if output_format is None else output_format.name
output = TestRun.executor.run(
list_io_classes_cmd(cache_id=str(cache_id),
output_format=_output_format, shortcut=shortcut))
if output.exit_code != 0:
raise Exception(
f"List IO class command failed. stdout: {output.stdout} \n stderr :{output.stderr}")
return output
def get_param_cutoff(cache_id: int, core_id: int,
output_format: OutputFormat = None, shortcut: bool = False):
_output_format = None if output_format is None else output_format.name
output = TestRun.executor.run(
get_param_cutoff_cmd(cache_id=str(cache_id), core_id=str(core_id),
output_format=_output_format, shortcut=shortcut))
if output.exit_code != 0:
raise Exception(
f"Getting sequential cutoff params failed."
f" stdout: {output.stdout} \n stderr :{output.stderr}")
return output
def get_param_cleaning(cache_id: int, output_format: OutputFormat = None, shortcut: bool = False):
_output_format = None if output_format is None else output_format.name
output = TestRun.executor.run(
get_param_cleaning_cmd(cache_id=str(cache_id), output_format=_output_format,
shortcut=shortcut))
if output.exit_code != 0:
raise Exception(
f"Getting cleaning policy params failed."
f" stdout: {output.stdout} \n stderr :{output.stderr}")
return output
def get_param_cleaning_alru(cache_id: int, output_format: OutputFormat = None,
shortcut: bool = False):
_output_format = None if output_format is None else output_format.name
output = TestRun.executor.run(
get_param_cleaning_alru_cmd(cache_id=str(cache_id), output_format=_output_format,
shortcut=shortcut))
if output.exit_code != 0:
raise Exception(
f"Getting alru cleaning policy params failed."
f" stdout: {output.stdout} \n stderr :{output.stderr}")
return output
def get_param_cleaning_acp(cache_id: int, output_format: OutputFormat = None,
shortcut: bool = False):
_output_format = None if output_format is None else output_format.name
output = TestRun.executor.run(
get_param_cleaning_acp_cmd(cache_id=str(cache_id), output_format=_output_format,
shortcut=shortcut))
if output.exit_code != 0:
raise Exception(
f"Getting acp cleaning policy params failed."
f" stdout: {output.stdout} \n stderr :{output.stderr}")
return output
def set_param_cutoff(cache_id: int, core_id: int = None, threshold: Size = None,
policy: SeqCutOffPolicy = None):
_threshold = None if threshold is None else threshold.get_value(Unit.KibiByte)
if core_id is None:
command = set_param_cutoff_cmd(
cache_id=str(cache_id), threshold=_threshold,
policy=policy.name)
else:
command = set_param_cutoff_cmd(
cache_id=str(cache_id), core_id=str(core_id),
threshold=_threshold, policy=policy.name)
output = TestRun.executor.run(command)
if output.exit_code != 0:
raise Exception(
f"Error while setting sequential cut-off params."
f" stdout: {output.stdout} \n stderr :{output.stderr}")
return output
def set_param_cleaning(cache_id: int, policy: CleaningPolicy):
output = TestRun.executor.run(
set_param_cleaning_cmd(cache_id=str(cache_id), policy=policy.name))
if output.exit_code != 0:
raise Exception(
f"Error while setting cleaning policy."
f" stdout: {output.stdout} \n stderr :{output.stderr}")
return output
def set_param_cleaning_alru(cache_id: int, wake_up: int = None, staleness_time: int = None,
flush_max_buffers: int = None, activity_threshold: int = None):
output = TestRun.executor.run(
set_param_cleaning_alru_cmd(
cache_id=str(cache_id), wake_up=str(wake_up), staleness_time=str(staleness_time),
flush_max_buffers=str(flush_max_buffers), activity_threshold=str(activity_threshold)))
if output.exit_code != 0:
raise Exception(
f"Error while setting alru cleaning policy parameters."
f" stdout: {output.stdout} \n stderr :{output.stderr}")
return output
def set_param_cleaning_acp(cache_id: int, wake_up: int = None, flush_max_buffers: int = None):
output = TestRun.executor.run(
set_param_cleaning_acp_cmd(cache_id=str(cache_id), wake_up=str(wake_up),
flush_max_buffers=str(flush_max_buffers)))
if output.exit_code != 0:
raise Exception(
f"Error while setting acp cleaning policy parameters."
f" stdout: {output.stdout} \n stderr :{output.stderr}")
return output

View File

@@ -0,0 +1,20 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from enum import Enum
class OutputFormat(Enum):
table = 0
csv = 1
class StatsFilter(Enum):
all = 0
conf = 1
usage = 2
req = 3
blk = 4
err = 5

View File

@@ -0,0 +1,209 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from api.cas import casadm
from test_utils.size import parse_unit
from api.cas.cache_config import *
from api.cas.casadm_params import *
from datetime import timedelta
from typing import List
from packaging import version
import re
def parse_stats_unit(unit: str):
if unit is None:
return ""
unit = re.search(r".*[^\]]", unit).group()
if unit == "s":
return "s"
elif unit == "%":
return "%"
elif unit == "Requests":
return "requests"
else:
return parse_unit(unit)
def get_filter(filter: List[casadm.StatsFilter]):
"""Prepare list of statistic sections which should be retrieved and parsed. """
if filter is None or StatsFilter.all in filter:
_filter = [
f for f in StatsFilter if (f != StatsFilter.all and f != StatsFilter.conf)
]
else:
_filter = [
f for f in filter if (f != StatsFilter.all and f != StatsFilter.conf)
]
return _filter
def get_statistics(
cache_id: int,
core_id: int = None,
io_class_id: int = None,
filter: List[casadm.StatsFilter] = None,
percentage_val: bool = False,
):
stats = {}
_filter = get_filter(filter)
per_io_class = True if io_class_id is not None else False
# No need to retrieve all stats if user specified only 'conf' flag
if filter != [StatsFilter.conf]:
csv_stats = casadm.print_statistics(
cache_id=cache_id,
core_id=core_id,
per_io_class=per_io_class,
io_class_id=io_class_id,
filter=_filter,
output_format=casadm.OutputFormat.csv,
).stdout.splitlines()
if filter is None or StatsFilter.conf in filter or StatsFilter.all in filter:
# Conf statistics have different unit or may have no unit at all. For parsing
# convenience they are gathered separately. As this is only configuration stats
# there is no risk they are divergent.
conf_stats = casadm.print_statistics(
cache_id=cache_id,
core_id=core_id,
per_io_class=per_io_class,
io_class_id=io_class_id,
filter=[StatsFilter.conf],
output_format=casadm.OutputFormat.csv,
).stdout.splitlines()
stat_keys = conf_stats[0]
stat_values = conf_stats[1]
for (name, val) in zip(stat_keys.split(","), stat_values.split(",")):
# Some of configuration stats have no unit
try:
stat_name, stat_unit = name.split(" [")
except ValueError:
stat_name = name
stat_unit = None
stat_name = stat_name.lower()
# 'dirty for' and 'cache size' stats occurs twice
if stat_name in stats:
continue
stat_unit = parse_stats_unit(stat_unit)
if isinstance(stat_unit, Unit):
stats[stat_name] = Size(float(val), stat_unit)
elif stat_unit == "s":
stats[stat_name] = timedelta(seconds=int(val))
elif stat_unit == "":
# Some of stats without unit can be a number like IDs,
# some of them can be string like device path
try:
stats[stat_name] = float(val)
except ValueError:
stats[stat_name] = val
# No need to parse all stats if user specified only 'conf' flag
if filter == [StatsFilter.conf]:
return stats
stat_keys = csv_stats[0]
stat_values = csv_stats[1]
for (name, val) in zip(stat_keys.split(","), stat_values.split(",")):
if percentage_val and " [%]" in name:
stats[name.split(" [")[0].lower()] = float(val)
elif not percentage_val and "[%]" not in name:
stat_name, stat_unit = name.split(" [")
stat_unit = parse_stats_unit(stat_unit)
stat_name = stat_name.lower()
if isinstance(stat_unit, Unit):
stats[stat_name] = Size(float(val), stat_unit)
elif stat_unit == "requests":
stats[stat_name] = float(val)
else:
raise ValueError(f"Invalid unit {stat_unit}")
return stats
def get_caches(): # This method does not return inactive or detached CAS devices
from api.cas.cache import Cache
caches_list = []
lines = casadm.list_caches(OutputFormat.csv).stdout.split('\n')
for line in lines:
args = line.split(',')
if args[0] == "cache":
current_cache = Cache(args[2])
caches_list.append(current_cache)
return caches_list
def get_cores(cache_id: int):
from api.cas.core import Core, CoreStatus
cores_list = []
lines = casadm.list_caches(OutputFormat.csv).stdout.split('\n')
is_proper_core_line = False
for line in lines:
args = line.split(',')
if args[0] == "core" and is_proper_core_line:
core_status_str = args[3].lower()
is_valid_status = CoreStatus[core_status_str].value[0] <= 1
if is_valid_status:
cores_list.append(Core(args[2], cache_id))
if args[0] == "cache":
is_proper_core_line = True if int(args[1]) == cache_id else False
return cores_list
def get_flush_parameters_alru(cache_id: int):
casadm_output = casadm.get_param_cleaning_alru(cache_id,
casadm.OutputFormat.csv).stdout.spltlines()
flush_parameters = FlushParametersAlru()
for line in casadm_output:
if 'max buffers' in line:
flush_parameters.flush_max_buffers = int(line.split(',')[1])
if 'Activity threshold' in line:
flush_parameters.activity_threshold = Time(milliseconds=int(line.split(',')[1]))
if 'Stale buffer time' in line:
flush_parameters.staneless_time = Time(seconds=int(line.split(',')[1]))
if 'Wake up time' in line:
flush_parameters.wake_up_time = Time(seconds=int(line.split(',')[1]))
return flush_parameters
def get_flush_parameters_acp(cache_id: int):
casadm_output = casadm.get_param_cleaning_acp(cache_id,
casadm.OutputFormat.csv).stdout.spltlines()
flush_parameters = FlushParametersAcp()
for line in casadm_output:
if 'max buffers' in line:
flush_parameters.flush_max_buffers = int(line.split(',')[1])
if 'Wake up time' in line:
flush_parameters.wake_up_time = Time(milliseconds=int(line.split(',')[1]))
return flush_parameters
def get_seq_cut_off_parameters(cache_id: int, core_id: int):
casadm_output = casadm.get_param_cutoff(
cache_id, core_id, casadm.OutputFormat.csv).stdout.splitlines()
seq_cut_off_params = SeqCutOffParameters()
for line in casadm_output:
if 'threshold' in line:
seq_cut_off_params.threshold = line.split(',')[1]
if 'policy' in line:
seq_cut_off_params.policy = SeqCutOffPolicy(line.split(',')[1])
def get_casadm_version():
casadm_output = casadm.print_version(OutputFormat.csv).stdout.split('\n')
version_str = casadm_output[1].split(',')[-1]
return version.parse(version_str)

View File

@@ -0,0 +1,23 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from .cli import *
from core.test_run import TestRun
def help(shortcut: bool = False):
return TestRun.executor.run(ctl_help(shortcut))
def start():
return TestRun.executor.run(ctl_start())
def stop(flush: bool = False):
return TestRun.executor.run(ctl_stop(flush))
def init(force: bool = False):
return TestRun.executor.run(ctl_init(force))

View File

@@ -0,0 +1,254 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
import logging
LOGGER = logging.getLogger(__name__)
casadm_bin = "casadm"
casctl = "casctl"
def add_core_cmd(cache_id: str, core_dev: str, core_id: str = None, shortcut: bool = False):
command = f" -A -i {cache_id} -d {core_dev}" if shortcut \
else f" --add-core --cache-id {cache_id} --core-device {core_dev}"
if core_id is not None:
command += (" -j " if shortcut else " --core-id ") + core_id
return casadm_bin + command
def remove_core_cmd(cache_id: str, core_id: str, force: bool = False, shortcut: bool = False):
command = f" -R -i {cache_id} -j {core_id}" if shortcut \
else f" --remove-core --cache-id {cache_id} --core-id {core_id}"
if force:
command += " -f" if shortcut else " --force"
return casadm_bin + command
def remove_detached_cmd(core_device: str, shortcut: bool = False):
command = " --remove-detached" + (" -d " if shortcut else " --device ") + core_device
return casadm_bin + command
def help_cmd(shortcut: bool = False):
return casadm_bin + (" -H" if shortcut else " --help")
def reset_counters_cmd(cache_id: str, core_id: str = None, shortcut: bool = False):
command = (" -Z -i " if shortcut else " --reset-counters --cache-id ") + cache_id
if core_id is not None:
command += (" -j " if shortcut else " --core-id ") + core_id
return casadm_bin + command
def flush_cache_cmd(cache_id: str, shortcut: bool = False):
command = (" -F -i " if shortcut else " --flush-cache --cache-id ") + cache_id
return casadm_bin + command
def flush_core_cmd(cache_id: str, core_id: str, shortcut: bool = False):
command = (f" -E -i {cache_id} -j {core_id}" if shortcut
else f" --flush-core --cache-id {cache_id} --core-id {core_id}")
return casadm_bin + command
def start_cmd(cache_dev: str, cache_mode: str = None, cache_line_size: str = None,
cache_id: str = None, force: bool = False,
load: bool = False, shortcut: bool = False):
command = " -S" if shortcut else " --start-cache"
command += (" -d " if shortcut else " --cache-device ") + cache_dev
if cache_mode is not None:
command += (" -c " if shortcut else " --cache-mode ") + cache_mode
if cache_line_size is not None:
command += (" -x " if shortcut else " --cache-line-size ") + cache_line_size
if cache_id is not None:
command += (" -i " if shortcut else " --cache-id ") + cache_id
if force:
command += " -f" if shortcut else " --force"
if load:
command += " -l" if shortcut else " --load"
return casadm_bin + command
def print_statistics_cmd(cache_id: str, core_id: str = None, per_io_class: bool = False,
io_class_id: str = None, filter: str = None,
output_format: str = None, shortcut: bool = False):
command = (" -P -i " if shortcut else " --stats --cache-id ") + cache_id
if core_id is not None:
command += (" -j " if shortcut else " --core-id ") + core_id
if per_io_class:
command += " -d" if shortcut else " --io-class-id"
if io_class_id is not None:
command += " " + io_class_id
elif io_class_id is not None:
raise Exception("Per io class flag not set but ID given.")
if filter is not None:
command += (" -f " if shortcut else " --filter ") + filter
if output_format is not None:
command += (" -o " if shortcut else " --output-format ") + output_format
return casadm_bin + command
def format_cmd(cache_dev: str, force: bool = False, shortcut: bool = False):
command = (" -N -F -d " if shortcut else " --nvme --format --device ") + cache_dev
if force:
command += " -f" if shortcut else " --force"
return casadm_bin + command
def stop_cmd(cache_id: str, no_data_flush: bool = False, shortcut: bool = False):
command = " -T " if shortcut else " --stop-cache"
command += (" -i " if shortcut else " --cache-id ") + cache_id
if no_data_flush:
command += " --no-data-flush"
return casadm_bin + command
def list_cmd(output_format: str = None, shortcut: bool = False):
command = " -L" if shortcut else " --list-caches"
if output_format == "table" or output_format == "csv":
command += (" -o " if shortcut else " --output-format ") + output_format
return casadm_bin + command
def load_cmd(cache_dev: str, shortcut: bool = False):
return start_cmd(cache_dev, load=True, shortcut=shortcut)
def version_cmd(output_format: str = None, shortcut: bool = False):
command = " -V" if shortcut else " --version"
if output_format == "table" or output_format == "csv":
command += (" -o " if shortcut else " --output-format ") + output_format
return casadm_bin + command
def set_cache_mode_cmd(cache_mode: str, cache_id: str,
flush_cache: str = None, shortcut: bool = False):
command = f" -Q -c {cache_mode} -i {cache_id}" if shortcut else \
f" --set-cache-mode --cache-mode {cache_mode} --cache-id {cache_id}"
if flush_cache:
command += (" -f " if shortcut else " --flush-cache ") + flush_cache
return casadm_bin + command
def load_io_classes_cmd(cache_id: str, file: str, shortcut: bool = False):
command = f" -C -C -i {cache_id} -f {file}" if shortcut else \
f" --io-class --load-config --cache-id {cache_id} --file {file}"
return casadm_bin + command
def list_io_classes_cmd(cache_id: str, output_format: str, shortcut: bool = False):
command = f" -C -L -i {cache_id} -o {output_format}" if shortcut else \
f" --io-class --list --cache-id {cache_id} --output-format {output_format}"
return casadm_bin + command
def _get_param_cmd(namespace: str, cache_id: str, output_format: str = None,
additional_params: str = None, shortcut: bool = False):
command = f" -G -n {namespace} -i {cache_id}" if shortcut else\
f" --get-param --name {namespace} --cache-id {cache_id}"
if additional_params is not None:
command += additional_params
if output_format is not None:
command += (" -o " if shortcut else " --output-format ") + output_format
return casadm_bin + command
def get_param_cutoff_cmd(cache_id: str, core_id: str,
output_format: str = None, shortcut: bool = False):
add_param = (" -j " if shortcut else " --core-id ") + core_id
return _get_param_cmd(namespace="seq-cutoff", cache_id=cache_id, output_format=output_format,
additional_params=add_param, shortcut=shortcut)
def get_param_cleaning_cmd(cache_id: str, output_format: str = None, shortcut: bool = False):
return _get_param_cmd(namespace="cleaning", cache_id=cache_id,
output_format=output_format, shortcut=shortcut)
def get_param_cleaning_alru_cmd(cache_id: str, output_format: str = None, shortcut: bool = False):
return _get_param_cmd(namespace="cleaning-alru", cache_id=cache_id,
output_format=output_format, shortcut=shortcut)
def get_param_cleaning_acp_cmd(cache_id: str, output_format: str = None, shortcut: bool = False):
return _get_param_cmd(namespace="cleaning-acp", cache_id=cache_id,
output_format=output_format, shortcut=shortcut)
def _set_param_cmd(namespace: str, cache_id: str, additional_params: str = None,
shortcut: bool = False):
command = f" -X -n {namespace} -i {cache_id}" if shortcut else\
f" --set-param --name {namespace} --cache-id {cache_id}"
command += additional_params
return casadm_bin + command
def set_param_cutoff_cmd(cache_id: str, core_id: str = None, threshold: str = None,
policy: str = None, shortcut: bool = False):
add_params = ""
if core_id is not None:
add_params += (" -j " if shortcut else " --core-id ") + core_id
if threshold is not None:
add_params += (" -t " if shortcut else " --threshold ") + threshold
if policy is not None:
add_params += (" -p " if shortcut else " --policy ") + policy
return _set_param_cmd(namespace="seq-cutoff", cache_id=cache_id,
additional_params=add_params, shortcut=shortcut)
def set_param_cleaning_cmd(cache_id: str, policy: str, shortcut: bool = False):
add_params = (" -p " if shortcut else " --policy ") + policy
return _set_param_cmd(namespace="cleaning", cache_id=cache_id,
additional_params=add_params, shortcut=shortcut)
def set_param_cleaning_alru_cmd(cache_id: str, wake_up: str, staleness_time: str,
flush_max_buffers: str, activity_threshold: str,
shortcut: bool = False):
add_param = ""
if wake_up is not None:
add_param += (" -w " if shortcut else " --wake-up ") + wake_up
if staleness_time is not None:
add_param += (" -s " if shortcut else " --staleness-time ") + staleness_time
if flush_max_buffers is not None:
add_param += (" -b " if shortcut else " --flush-max-buffers ") + flush_max_buffers
if activity_threshold is not None:
add_param += (" -t " if shortcut else " --activity-threshold ") + activity_threshold
return _set_param_cmd(namespace="cleaning-alru", cache_id=cache_id,
additional_params=add_param, shortcut=shortcut)
def set_param_cleaning_acp_cmd(cache_id: str, wake_up: str = None,
flush_max_buffers: str = None, shortcut: bool = False):
add_param = ""
if wake_up is not None:
add_param += (" -w " if shortcut else " --wake-up ") + wake_up
if flush_max_buffers is not None:
add_param += (" -b " if shortcut else " --flush-max-buffers ") + flush_max_buffers
return _set_param_cmd(namespace="cleaning-acp", cache_id=cache_id,
additional_params=add_param, shortcut=shortcut)
def ctl_help(shortcut: bool = False):
return casctl + " --help" if shortcut else " -h"
def ctl_start():
return casctl + " start"
def ctl_stop(flush: bool = False):
command = casctl + " stop"
if flush:
command += " --flush"
return command
def ctl_init(force: bool = False):
command = casctl + " init"
if force:
command += " --force"
return command

View File

@@ -0,0 +1,81 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from typing import List
from api.cas.cli import *
from api.cas.casadm_parser import *
from api.cas.cache import Device
from test_utils.os_utils import *
class CoreStatus(Enum):
empty = 0,
active = 1,
inactive = 2,
detached = 3
class Core(Device):
def __init__(self, core_device: str, cache_id: int):
self.core_device = Device(core_device)
self.system_path = None
core_info = self.__get_core_info()
self.core_id = int(core_info["core_id"])
Device.__init__(self, core_info["exp_obj"])
self.cache_id = cache_id
def __get_core_info(self):
output = TestRun.executor.run(
list_cmd(OutputFormat.csv.name))
if output.exit_code != 0:
raise Exception("Failed to execute list caches command.")
output_lines = output.stdout.splitlines()
for line in output_lines:
split_line = line.split(',')
if split_line[0] == "core" and (split_line[2] == self.core_device.system_path
or split_line[5] == self.system_path):
return {"core_id": split_line[1],
"core_device": split_line[2],
"status": split_line[3],
"exp_obj": split_line[5]}
def get_core_statistics(self,
io_class_id: int = None,
stat_filter: List[StatsFilter] = None,
percentage_val: bool = False):
return get_statistics(self.cache_id, self.core_id, io_class_id,
stat_filter, percentage_val)
def get_status(self):
return self.__get_core_info()["status"]
def get_seq_cut_off_parameters(self):
return get_seq_cut_off_parameters(self.cache_id, self.core_id)
def get_dirty_blocks(self):
return self.get_core_statistics()["dirty"]
def get_clean_blocks(self):
return self.get_core_statistics()["clean"]
def get_occupancy(self):
return self.get_core_statistics()["occupancy"]
# Casadm methods:
def remove_core(self, force: bool = False):
return casadm.remove_core(self.cache_id, self.core_id, force)
def reset_counters(self):
return casadm.reset_counters(self.cache_id, self.core_id)
def flush_core(self):
casadm.flush(self.cache_id, self.core_id)
sync()
assert self.get_dirty_blocks().get_value(Unit.Blocks4096) == 0
def set_seq_cutoff_parameters(self, seq_cutoff_param: SeqCutOffParameters):
casadm.set_param_cutoff(self.cache_id, self.core_id,
seq_cutoff_param.threshold, seq_cutoff_param.policy)

View File

@@ -0,0 +1,85 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from api.cas import casadm_parser
from api.cas.cache_config import CacheMode
from storage_devices.device import Device
from test_tools import fs_utils
opencas_conf_path = "/etc/opencas/opencas.conf"
def create_init_config_from_running_configuration(load: bool = None, extra_flags=""):
cache_lines = []
core_lines = []
for cache in casadm_parser.get_caches():
cache_lines.append(CacheConfigLine(cache.cache_id,
cache.cache_device,
cache.get_cache_mode(),
load,
extra_flags))
for core in casadm_parser.get_cores(cache.cache_id):
core_lines.append(CoreConfigLine(cache.cache_id,
core.core_id,
core.core_device))
config_lines = []
create_default_init_config()
if len(cache_lines) > 0:
config_lines.append(CacheConfigLine.header)
for c in cache_lines:
config_lines.append(str(c))
if len(core_lines) > 0:
config_lines.append(CoreConfigLine.header)
for c in core_lines:
config_lines.append(str(c))
fs_utils.write_file(opencas_conf_path, '\n'.join(config_lines), False)
def create_default_init_config():
cas_version = casadm_parser.get_casadm_version()
fs_utils.write_file(opencas_conf_path,
f"version={'.'.join(str(x) for x in cas_version.release[0:3])}")
class CacheConfigLine:
header = "[caches]"
def __init__(self, cache_id, cache_device: Device,
cache_mode: CacheMode, load=None, extra_flags=""):
self.cache_id = cache_id
self.cache_device = cache_device
self.load = load
self.cache_mode = cache_mode
self.extra_flags = extra_flags
def __str__(self):
cache_symlink = self.cache_device.get_device_link("/dev/disk/by-id")
cache_device_path = cache_symlink.full_path if cache_symlink is not None \
else self.cache_device.system_path
params = [str(self.cache_id), cache_device_path]
if self.load is not None:
params.append("yes" if self.load else "no")
params.append(self.cache_mode.name)
params.append(self.extra_flags)
return '\t'.join(params)
class CoreConfigLine:
header = "[cores]"
def __init__(self, cache_id, core_id, core_device: Device):
self.cache_id = cache_id
self.core_id = core_id
self.core_device = core_device
def __str__(self):
core_symlink = self.core_device.get_device_link("/dev/disk/by-id")
core_device_path = core_symlink.full_path if core_symlink is not None \
else self.core_device.system_path
params = [str(self.cache_id), str(self.core_id), core_device_path]
return '\t'.join(params)

View File

@@ -0,0 +1,93 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
import logging
from tests import conftest
from core.test_run import TestRun
LOGGER = logging.getLogger(__name__)
opencas_repo_name = "open-cas-linux"
def install_opencas():
LOGGER.info("Cloning Open CAS repository.")
TestRun.executor.run(f"if [ -d {opencas_repo_name} ]; "
f"then rm -rf {opencas_repo_name}; fi")
output = TestRun.executor.run(
"git clone --recursive https://github.com/Open-CAS/open-cas-linux.git")
if output.exit_code != 0:
raise Exception(f"Error while cloning repository: {output.stdout}\n{output.stderr}")
output = TestRun.executor.run(
f"cd {opencas_repo_name} && "
f"git fetch --all && "
f"git fetch --tags {conftest.get_remote()} +refs/pull/*:refs/remotes/origin/pr/*")
if output.exit_code != 0:
raise Exception(
f"Failed to fetch: "
f"{output.stdout}\n{output.stderr}")
output = TestRun.executor.run(f"cd {opencas_repo_name} && "
f"git checkout {conftest.get_branch()}")
if output.exit_code != 0:
raise Exception(
f"Failed to checkout to {conftest.get_branch()}: {output.stdout}\n{output.stderr}")
LOGGER.info("Open CAS make and make install.")
output = TestRun.executor.run(
f"cd {opencas_repo_name} && "
"git submodule update --init --recursive && "
"./configure && "
"make -j")
if output.exit_code != 0:
raise Exception(
f"Make command executed with nonzero status: {output.stdout}\n{output.stderr}")
output = TestRun.executor.run(f"cd {opencas_repo_name} && "
f"make install")
if output.exit_code != 0:
raise Exception(
f"Error while installing Open CAS: {output.stdout}\n{output.stderr}")
LOGGER.info("Check if casadm is properly installed.")
output = TestRun.executor.run("casadm -V")
if output.exit_code != 0:
raise Exception(
f"'casadm -V' command returned an error: {output.stdout}\n{output.stderr}")
else:
LOGGER.info(output.stdout)
def uninstall_opencas():
LOGGER.info("Uninstalling Open CAS.")
output = TestRun.executor.run("casadm -V")
if output.exit_code != 0:
raise Exception("Open CAS is not properly installed.")
else:
TestRun.executor.run(f"cd {opencas_repo_name} && "
f"make uninstall")
if output.exit_code != 0:
raise Exception(
f"There was an error during uninstall process: {output.stdout}\n{output.stderr}")
def reinstall_opencas():
if check_if_installed():
uninstall_opencas()
install_opencas()
def check_if_installed():
LOGGER.info("Check if Open-CAS-Linux is installed.")
output = TestRun.executor.run("which casadm")
if output.exit_code == 0:
LOGGER.info("CAS is installed")
return True
LOGGER.info("CAS not installed")
return False

View File

@@ -0,0 +1,128 @@
#
# Copyright(c) 2019 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause-Clear
#
from datetime import timedelta
from core.test_run import TestRun
default_config_file_path = "/tmp/opencas_ioclass.conf"
MAX_IO_CLASS_ID = 32
MAX_CLASSIFICATION_DELAY = timedelta(seconds=6)
def create_ioclass_config(
add_default_rule: bool = True, ioclass_config_path: str = default_config_file_path
):
TestRun.LOGGER.info(f"Creating config file {ioclass_config_path}")
output = TestRun.executor.run(
'echo "IO class id,IO class name,Eviction priority,Allocation" '
+ f"> {ioclass_config_path}"
)
if output.exit_code != 0:
raise Exception(
"Failed to create ioclass config file. "
+ f"stdout: {output.stdout} \n stderr :{output.stderr}"
)
if add_default_rule:
output = TestRun.executor.run(
f'echo "0,unclassified,22,1" >> {ioclass_config_path}'
)
if output.exit_code != 0:
raise Exception(
"Failed to create ioclass config file. "
+ f"stdout: {output.stdout} \n stderr :{output.stderr}"
)
def remove_ioclass_config(ioclass_config_path: str = default_config_file_path):
TestRun.LOGGER.info(f"Removing config file {ioclass_config_path}")
output = TestRun.executor.run(f"rm -f {ioclass_config_path}")
if output.exit_code != 0:
raise Exception(
"Failed to remove config file. "
+ f"stdout: {output.stdout} \n stderr :{output.stderr}"
)
def add_ioclass(
ioclass_id: int,
rule: str,
eviction_priority: int,
allocation: bool,
ioclass_config_path: str = default_config_file_path,
):
new_ioclass = f"{ioclass_id},{rule},{eviction_priority},{int(allocation)}"
TestRun.LOGGER.info(
f"Adding rule {new_ioclass} " + f"to config file {ioclass_config_path}"
)
output = TestRun.executor.run(
f'echo "{new_ioclass}" >> {ioclass_config_path}'
)
if output.exit_code != 0:
raise Exception(
"Failed to append ioclass to config file. "
+ f"stdout: {output.stdout} \n stderr :{output.stderr}"
)
def get_ioclass(ioclass_id: int, ioclass_config_path: str = default_config_file_path):
TestRun.LOGGER.info(
f"Retrieving rule no.{ioclass_id} " + f"from config file {ioclass_config_path}"
)
output = TestRun.executor.run(f"cat {ioclass_config_path}")
if output.exit_code != 0:
raise Exception(
"Failed to read ioclass config file. "
+ f"stdout: {output.stdout} \n stderr :{output.stderr}"
)
ioclass_config = output.stdout.splitlines()
for ioclass in ioclass_config:
if int(ioclass.split(",")[0]) == ioclass_id:
return ioclass
def remove_ioclass(
ioclass_id: int, ioclass_config_path: str = default_config_file_path
):
TestRun.LOGGER.info(
f"Removing rule no.{ioclass_id} " + f"from config file {ioclass_config_path}"
)
output = TestRun.executor.run(f"cat {ioclass_config_path}")
if output.exit_code != 0:
raise Exception(
"Failed to read ioclass config file. "
+ f"stdout: {output.stdout} \n stderr :{output.stderr}"
)
old_ioclass_config = output.stdout.splitlines()
config_header = old_ioclass_config[0]
# First line in valid config file is always a header, not a rule - it is
# already extracted above
new_ioclass_config = [
x for x in old_ioclass_config[1:] if int(x.split(",")[0]) != ioclass_id
]
new_ioclass_config.insert(0, config_header)
if len(new_ioclass_config) == len(old_ioclass_config):
raise Exception(
f"Failed to remove ioclass {ioclass_id} from config file {ioclass_config_path}"
)
new_ioclass_config_str = "\n".join(new_ioclass_config)
output = TestRun.executor.run(
f'echo "{new_ioclass_config_str}" > {ioclass_config_path}'
)
if output.exit_code != 0:
raise Exception(
"Failed to save new ioclass config. "
+ f"stdout: {output.stdout} \n stderr :{output.stderr}"
)