From 06322f61994dea3af967d2c3385e0d65c76764a8 Mon Sep 17 00:00:00 2001 From: Jan Musial Date: Thu, 9 Jul 2020 09:29:28 +0200 Subject: [PATCH] Add PerfContainer for storing performance results Implement new package which primary goal is to collect and validate performance metrics in managable way then dump them in JSON form for further processing/storage. Example usage: container = PerfContainer() container.insert_config_param("20.03.0000", ConfigParameter.CAS_VERSION) container.insert_cache_metric(20000000, IOMetric.read_IOPS) with open("perf.json", "w") as f: json.dump(container.to_serializable_dict(), f) Signed-off-by: Jan Musial --- test/functional/requirements.txt | 1 + test/functional/utils/__init__.py | 0 test/functional/utils/performance.py | 216 +++++++++++++++++++++++++++ 3 files changed, 217 insertions(+) create mode 100644 test/functional/utils/__init__.py create mode 100644 test/functional/utils/performance.py diff --git a/test/functional/requirements.txt b/test/functional/requirements.txt index 4e9f197..daaa74f 100644 --- a/test/functional/requirements.txt +++ b/test/functional/requirements.txt @@ -1 +1,2 @@ attotime>=0.2.0 +schema==0.7.2 diff --git a/test/functional/utils/__init__.py b/test/functional/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/functional/utils/performance.py b/test/functional/utils/performance.py new file mode 100644 index 0000000..44b474d --- /dev/null +++ b/test/functional/utils/performance.py @@ -0,0 +1,216 @@ +# +# Copyright(c) 2020 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause-Clear +# + +from enum import Enum +from types import MethodType +from datetime import datetime + +from schema import Schema, Use, And, SchemaError, Or + + +class ValidatableParameter(Enum): + """ + Parameter enumeration together with schema for validating this parameter + If given parameter is always valid put False as its value, otherwise use proper Schema object + """ + + def __new__(cls, schema: Schema): + if not (isinstance(schema, Schema) or not schema): + raise Exception( + f"Invalid {cls.__name__} value. Expected: Schema instance or False, got: {schema}" + ) + + # Trick for changing value which is supplied by enum + # This way we can access Schema from enumeration member instance and still have Enum + # properties maintained + obj = object.__new__(cls) + obj._value_ = obj + obj.schema = schema + obj.validate = MethodType(cls.validate, obj) + + return obj + + def validate(self, param): + if self.schema: + param = self.schema.validate(param) + + return param + + def __repr__(self): + return f"<{type(self).__name__}.{self.name}>" + + def __str__(self): + return str(self.name) + + +class PercentileMetric: + def __init__(self, value): + value = float(value) + + if not 0 < value < 100: + raise SchemaError("Invalid percentile value") + + self.value = value + + def __str__(self): + return f"p{self.value:g}".replace(".", "_") + + +class IOMetric(ValidatableParameter): + read_IOPS = Schema(Use(int)) + write_IOPS = Schema(Use(int)) + read_BW = Schema(Use(int)) + write_BW = Schema(Use(int)) + read_CLAT_AVG = Schema(Use(int)) + write_CLAT_AVG = Schema(Use(int)) + read_CLAT_PERCENTILES = Schema({Use(PercentileMetric): Use(int)}) + write_CLAT_PERCENTILES = Schema({Use(PercentileMetric): Use(int)}) + +BuildTypes = ["master", "pr", "other"] + +class ConfigParameter(ValidatableParameter): + CAS_VERSION = Schema(Use(str)) + DUT = Schema(Use(str)) + TEST_NAME = Schema(str) + BUILD_TYPE = Schema(Or(*BuildTypes)) + CACHE_CONFIG = Schema( + {"cache_mode": Use(str), "cache_line_size": Use(str), "cleaning_policy": Use(str)} + ) + CACHE_TYPE = Schema(Use(str)) + CORE_TYPE = Schema(Use(str)) + TIMESTAMP = Schema(And(datetime, Use(str))) + + +class WorkloadParameter(ValidatableParameter): + NUM_JOBS = Schema(Use(int)) + QUEUE_DEPTH = Schema(Use(int)) + + +class MetricContainer: + def __init__(self, metric_type): + self.metrics = {} + self.metric_type = metric_type + + def insert_metric(self, metric, kind): + if not isinstance(kind, self.metric_type): + raise Exception(f"Invalid metric type. Expected: {self.metric_type}, got: {type(kind)}") + + if kind.value: + metric = kind.value.validate(metric) + + self.metrics[kind] = metric + + @property + def is_empty(self): + return len(self.metrics) == 0 + + def to_serializable_dict(self): + # No easy way for json.dump to deal with custom classes (especially custom Enums) + def stringify_dict(d): + new_dict = {} + for k, v in d.items(): + k = str(k) + + if isinstance(v, dict): + v = stringify_dict(v) + elif isinstance(v, int): + pass + elif isinstance(v, float): + pass + else: + v = str(v) + + new_dict[k] = v + + return new_dict + + return stringify_dict(self.metrics) + + +class PerfContainer: + def __init__(self): + self.conf_params = MetricContainer(ConfigParameter) + + self.workload_params = MetricContainer(WorkloadParameter) + + self.cache_metrics = MetricContainer(IOMetric) + self.core_metrics = MetricContainer(IOMetric) + self.exp_obj_metrics = MetricContainer(IOMetric) + + def insert_config_param(self, param, kind: ConfigParameter): + self.conf_params.insert_metric(param, kind) + + def insert_config_from_cache(self, cache): + cache_config = { + "cache_mode": cache.get_cache_mode(), + "cache_line_size": cache.get_cache_line_size(), + "cleaning_policy": cache.get_cleaning_policy(), + } + + self.conf_params.insert_metric(cache_config, ConfigParameter.CACHE_CONFIG) + + def insert_workload_param(self, param, kind: WorkloadParameter): + self.workload_params.insert_metric(param, kind) + + @staticmethod + def _insert_metrics_from_fio(container, result): + result = result.job + + container.insert_metric(result.read.iops, IOMetric.read_IOPS) + container.insert_metric(result.write.iops, IOMetric.write_IOPS) + container.insert_metric(result.read.bw, IOMetric.read_BW) + container.insert_metric(result.write.bw, IOMetric.write_BW) + container.insert_metric(result.read.clat_ns.mean, IOMetric.read_CLAT_AVG) + container.insert_metric(result.write.clat_ns.mean, IOMetric.write_CLAT_AVG) + if hasattr(result.read.clat_ns, "percentile"): + container.insert_metric( + vars(result.read.clat_ns.percentile), IOMetric.read_CLAT_PERCENTILES + ) + if hasattr(result.write.clat_ns, "percentile"): + container.insert_metric( + vars(result.write.clat_ns.percentile), IOMetric.write_CLAT_PERCENTILES + ) + + def insert_cache_metric(self, metric, kind: IOMetric): + self.cache_metrics.insert_metric(metric, kind) + + def insert_cache_metrics_from_fio_job(self, fio_results): + self._insert_metrics_from_fio(self.cache_metrics, fio_results) + + def insert_core_metric(self, metric, kind: IOMetric): + self.core_metrics.insert_metric(metric, kind) + + def insert_core_metrics_from_fio_job(self, fio_results): + self._insert_metrics_from_fio(self.core_metrics, fio_results) + + def insert_exp_obj_metric(self, metric, kind: IOMetric): + self.exp_obj_metrics.insert_metric(metric, kind) + + def insert_exp_obj_metrics_from_fio_job(self, fio_results): + self._insert_metrics_from_fio(self.exp_obj_metrics, fio_results) + + @property + def is_empty(self): + return ( + self.conf_params.is_empty + and self.workload_params.is_empty + and self.cache_metrics.is_empty + and self.core_metrics.is_empty + and self.exp_obj_metrics.is_empty + ) + + def to_serializable_dict(self): + ret = {**self.conf_params.to_serializable_dict()} + + if not self.workload_params.is_empty: + ret["workload_params"] = self.workload_params.to_serializable_dict() + if not self.cache_metrics.is_empty: + ret["cache_io"] = self.cache_metrics.to_serializable_dict() + if not self.core_metrics.is_empty: + ret["core_io"] = self.core_metrics.to_serializable_dict() + if not self.exp_obj_metrics.is_empty: + ret["exp_obj_io"] = self.exp_obj_metrics.to_serializable_dict() + + return ret