Add new statistics API

Signed-off-by: Rafal Stefanowski <rafal.stefanowski@intel.com>

test/functional/api/cas/cache.py

@@ -10,6 +10,7 @@ from api.cas.cache_config import *
 from storage_devices.device import Device
 from core.test_run import TestRun
 from api.cas.casadm_params import *
+from api.cas.statistics import CacheStats, IoClassStats
 
 
 class Cache:
@@ -33,52 +34,52 @@ class Cache:
 
     def get_cache_line_size(self):
         if self.__cache_line_size is None:
-            stats = self.get_cache_statistics()
-            stats_line_size = stats["cache line size"]
+            stats = self.get_statistics()
+            stats_line_size = stats.config_stats.cache_line_size
             self.__cache_line_size = CacheLineSize(stats_line_size.get_value(Unit.Byte))
         return self.__cache_line_size
 
     def get_cleaning_policy(self):
-        stats = self.get_cache_statistics()
-        cp = stats["cleaning policy"]
+        stats = self.get_statistics()
+        cp = stats.config_stats.cleaning_policy
         return CleaningPolicy[cp]
 
     def get_eviction_policy(self):
-        stats = self.get_cache_statistics()
-        ep = stats["eviction policy"]
+        stats = self.get_statistics()
+        ep = stats.config_stats.eviction_policy
         return EvictionPolicy[ep]
 
     def get_metadata_mode(self):
         if self.__metadata_mode is None:
-            stats = self.get_cache_statistics()
-            mm = stats["metadata mode"]
+            stats = self.get_statistics()
+            mm = stats.config_stats.metadata_mode
             self.__metadata_mode = MetadataMode[mm]
         return self.__metadata_mode
 
     def get_metadata_size(self):
         if self.__metadata_size is None:
-            stats = self.get_cache_statistics()
-            self.__metadata_size = stats["metadata memory footprint"]
+            stats = self.get_statistics()
+            self.__metadata_size = stats.config_stats.metadata_memory_footprint
         return self.__metadata_size
 
     def get_occupancy(self):
-        return self.get_cache_statistics()["occupancy"]
+        return self.get_statistics().usage_stats.occupancy
 
     def get_status(self):
-        status = self.get_cache_statistics()["status"].replace(' ', '_')
+        status = self.get_statistics().config_stats.status.replace(' ', '_').lower()
         return CacheStatus[status]
 
     def get_cache_mode(self):
-        return CacheMode[self.get_cache_statistics()["write policy"].upper()]
+        return CacheMode[self.get_statistics().config_stats.write_policy.upper()]
 
     def get_dirty_blocks(self):
-        return self.get_cache_statistics()["dirty"]
+        return self.get_statistics().usage_stats.dirty
 
     def get_dirty_for(self):
-        return self.get_cache_statistics()["dirty for"]
+        return self.get_statistics().config_stats.dirty_for
 
     def get_clean_blocks(self):
-        return self.get_cache_statistics()["clean"]
+        return self.get_statistics().usage_stats.clean
 
     def get_flush_parameters_alru(self):
         return get_flush_parameters_alru(self.cache_id)
@@ -88,10 +89,28 @@ class Cache:
 
     # Casadm methods:
 
-    def get_cache_statistics(self,
-                             io_class_id: int = None,
-                             stat_filter: List[StatsFilter] = None,
-                             percentage_val: bool = False):
+    def get_io_class_statistics(self,
+                                io_class_id: int,
+                                stat_filter: List[StatsFilter] = None,
+                                percentage_val: bool = False):
+        stats = get_statistics(self.cache_id, None, io_class_id,
+                               stat_filter, percentage_val)
+        return IoClassStats(stats)
+
+    def get_statistics(self,
+                       stat_filter: List[StatsFilter] = None,
+                       percentage_val: bool = False):
+        stats = get_statistics(self.cache_id, None, None,
+                               stat_filter, percentage_val)
+        return CacheStats(stats)
+
+    # TODO: Get rid of this method below by tuning 'stats' and 'io_class' tests
+    # to utilize new statistics API with method above.
+
+    def get_statistics_deprecated(self,
+                                  io_class_id: int = None,
+                                  stat_filter: List[StatsFilter] = None,
+                                  percentage_val: bool = False):
         return get_statistics(self.cache_id, None, io_class_id,
                               stat_filter, percentage_val)
 

test/functional/api/cas/core.py

@@ -8,6 +8,7 @@ from api.cas.cli import *
 from api.cas.casadm_parser import *
 from api.cas.cache import Device
 from test_utils.os_utils import *
+from api.cas.statistics import CoreStats, IoClassStats
 
 
 class CoreStatus(Enum):
@@ -45,15 +46,33 @@ class Core(Device):
                         "status": split_line[3],
                         "exp_obj": split_line[5]}
 
-    def get_core_statistics(self,
-                            io_class_id: int = None,
-                            stat_filter: List[StatsFilter] = None,
-                            percentage_val: bool = False):
+    def get_io_class_statistics(self,
+                                io_class_id: int,
+                                stat_filter: List[StatsFilter] = None,
+                                percentage_val: bool = False):
+        stats = get_statistics(self.cache_id, self.core_id, io_class_id,
+                               stat_filter, percentage_val)
+        return IoClassStats(stats)
+
+    def get_statistics(self,
+                       stat_filter: List[StatsFilter] = None,
+                       percentage_val: bool = False):
+        stats = get_statistics(self.cache_id, self.core_id, None,
+                               stat_filter, percentage_val)
+        return CoreStats(stats)
+
+    # TODO: Get rid of this method below by tuning 'stats' and 'io_class' tests
+    # to utilize new statistics API with method above.
+
+    def get_statistics_deprecated(self,
+                                  io_class_id: int = None,
+                                  stat_filter: List[StatsFilter] = None,
+                                  percentage_val: bool = False):
         return get_statistics(self.cache_id, self.core_id, io_class_id,
                               stat_filter, percentage_val)
 
     def get_status(self):
-        return self.__get_core_info()["status"]
+        return CoreStatus[self.__get_core_info()["status"].lower()]
 
     def get_seq_cut_off_parameters(self):
         return get_seq_cut_off_parameters(self.cache_id, self.core_id)
@@ -65,13 +84,13 @@ class Core(Device):
         return get_seq_cut_off_parameters(self.cache_id, self.core_id).threshold
 
     def get_dirty_blocks(self):
-        return self.get_core_statistics()["dirty"]
+        return self.get_statistics().usage_stats.dirty
 
     def get_clean_blocks(self):
-        return self.get_core_statistics()["clean"]
+        return self.get_statistics().usage_stats.clean
 
     def get_occupancy(self):
-        return self.get_core_statistics()["occupancy"]
+        return self.get_statistics().usage_stats.occupancy
 
     # Casadm methods:
 

test/functional/api/cas/statistics.py

@@ -0,0 +1,631 @@
+class CacheStats:
+    stats_list = [
+        "config_stats",
+        "usage_stats",
+        "inactive_usage_stats",
+        "request_stats",
+        "block_stats",
+        "error_stats",
+    ]
+
+    def __init__(self, stats):
+        try:
+            self.config_stats = CacheConfigStats(
+                stats["cache id"],
+                stats["cache size"],
+                stats["cache device"],
+                stats["core devices"],
+                stats["inactive core devices"],
+                stats["write policy"],
+                stats["eviction policy"],
+                stats["cleaning policy"],
+                stats["promotion policy"],
+                stats["cache line size"],
+                stats["metadata memory footprint"],
+                stats["dirty for"],
+                stats["metadata mode"],
+                stats["status"],
+            )
+        except KeyError:
+            pass
+        try:
+            self.usage_stats = UsageStats(
+                stats["occupancy"], stats["free"], stats["clean"], stats["dirty"]
+            )
+        except KeyError:
+            pass
+        try:
+            self.inactive_usage_stats = InactiveUsageStats(
+                stats["inactive occupancy"],
+                stats["inactive clean"],
+                stats["inactive dirty"],
+            )
+        except KeyError:
+            pass
+        try:
+            self.request_stats = RequestStats(
+                stats["read hits"],
+                stats["read partial misses"],
+                stats["read full misses"],
+                stats["read total"],
+                stats["write hits"],
+                stats["write partial misses"],
+                stats["write full misses"],
+                stats["write total"],
+                stats["pass-through reads"],
+                stats["pass-through writes"],
+                stats["serviced requests"],
+                stats["total requests"],
+            )
+        except KeyError:
+            pass
+        try:
+            self.block_stats = BlockStats(
+                stats["reads from core(s)"],
+                stats["writes to core(s)"],
+                stats["total to/from core(s)"],
+                stats["reads from cache"],
+                stats["writes to cache"],
+                stats["total to/from cache"],
+                stats["reads from exported object(s)"],
+                stats["writes to exported object(s)"],
+                stats["total to/from exported object(s)"],
+            )
+        except KeyError:
+            pass
+        try:
+            self.error_stats = ErrorStats(
+                stats["cache read errors"],
+                stats["cache write errors"],
+                stats["cache total errors"],
+                stats["core read errors"],
+                stats["core write errors"],
+                stats["core total errors"],
+                stats["total errors"],
+            )
+        except KeyError:
+            pass
+
+    def __str__(self):
+        status = ""
+        for stats_item in self.stats_list:
+            current_stat = getattr(self, stats_item, None)
+            if current_stat:
+                status += f"--- Cache {current_stat}"
+        return status
+
+    def __eq__(self, other):
+        if not other:
+            return False
+        for stats_item in self.stats_list:
+            if getattr(self, stats_item, None) != getattr(other, stats_item, None):
+                return False
+        return True
+
+
+class CoreStats:
+    stats_list = [
+        "config_stats",
+        "usage_stats",
+        "request_stats",
+        "block_stats",
+        "error_stats",
+    ]
+
+    def __init__(self, stats):
+        try:
+            self.config_stats = CoreConfigStats(
+                stats["core id"],
+                stats["core device"],
+                stats["exported object"],
+                stats["core size"],
+                stats["dirty for"],
+                stats["status"],
+                stats["seq cutoff threshold"],
+                stats["seq cutoff policy"],
+            )
+        except KeyError:
+            pass
+        try:
+            self.usage_stats = UsageStats(
+                stats["occupancy"], stats["free"], stats["clean"], stats["dirty"]
+            )
+        except KeyError:
+            pass
+        try:
+            self.request_stats = RequestStats(
+                stats["read hits"],
+                stats["read partial misses"],
+                stats["read full misses"],
+                stats["read total"],
+                stats["write hits"],
+                stats["write partial misses"],
+                stats["write full misses"],
+                stats["write total"],
+                stats["pass-through reads"],
+                stats["pass-through writes"],
+                stats["serviced requests"],
+                stats["total requests"],
+            )
+        except KeyError:
+            pass
+        try:
+            self.block_stats = BlockStats(
+                stats["reads from core"],
+                stats["writes to core"],
+                stats["total to/from core"],
+                stats["reads from cache"],
+                stats["writes to cache"],
+                stats["total to/from cache"],
+                stats["reads from exported object"],
+                stats["writes to exported object"],
+                stats["total to/from exported object"],
+            )
+        except KeyError:
+            pass
+        try:
+            self.error_stats = ErrorStats(
+                stats["cache read errors"],
+                stats["cache write errors"],
+                stats["cache total errors"],
+                stats["core read errors"],
+                stats["core write errors"],
+                stats["core total errors"],
+                stats["total errors"],
+            )
+        except KeyError:
+            pass
+
+    def __str__(self):
+        status = ""
+        for stats_item in self.stats_list:
+            current_stat = getattr(self, stats_item, None)
+            if current_stat:
+                status += f"--- Core {current_stat}"
+        return status
+
+    def __eq__(self, other):
+        if not other:
+            return False
+        for stats_item in self.stats_list:
+            if getattr(self, stats_item, None) != getattr(other, stats_item, None):
+                return False
+        return True
+
+
+class IoClassStats:
+    stats_list = ["config_stats", "usage_stats", "request_stats", "block_stats"]
+
+    def __init__(self, stats):
+        try:
+            self.config_stats = IoClassConfigStats(
+                stats["io class id"],
+                stats["io class name"],
+                stats["eviction priority"],
+                stats["selective allocation"],
+            )
+        except KeyError:
+            pass
+        try:
+            self.usage_stats = UsageStats(
+                stats["occupancy"], stats["free"], stats["clean"], stats["dirty"]
+            )
+        except KeyError:
+            pass
+        try:
+            self.request_stats = RequestStats(
+                stats["read hits"],
+                stats["read partial misses"],
+                stats["read full misses"],
+                stats["read total"],
+                stats["write hits"],
+                stats["write partial misses"],
+                stats["write full misses"],
+                stats["write total"],
+                stats["pass-through reads"],
+                stats["pass-through writes"],
+                stats["serviced requests"],
+                stats["total requests"],
+            )
+        except KeyError:
+            pass
+        try:
+            self.block_stats = BlockStats(
+                stats["reads from core(s)"],
+                stats["writes to core(s)"],
+                stats["total to/from core(s)"],
+                stats["reads from cache"],
+                stats["writes to cache"],
+                stats["total to/from cache"],
+                stats["reads from exported object(s)"],
+                stats["writes to exported object(s)"],
+                stats["total to/from exported object(s)"],
+            )
+        except KeyError:
+            pass
+
+    def __str__(self):
+        status = ""
+        for stats_item in self.stats_list:
+            current_stat = getattr(self, stats_item, None)
+            if current_stat:
+                status += f"--- IO class {current_stat}"
+        return status
+
+    def __eq__(self, other):
+        if not other:
+            return False
+        for stats_item in self.stats_list:
+            if getattr(self, stats_item, None) != getattr(other, stats_item, None):
+                return False
+        return True
+
+
+class CacheConfigStats:
+    def __init__(
+        self,
+        cache_id,
+        cache_size,
+        cache_dev,
+        core_dev,
+        inactive_core_dev,
+        write_policy,
+        eviction_policy,
+        cleaning_policy,
+        promotion_policy,
+        cache_line_size,
+        metadata_memory_footprint,
+        dirty_for,
+        metadata_mode,
+        status,
+    ):
+        self.cache_id = cache_id
+        self.cache_size = cache_size
+        self.cache_dev = cache_dev
+        self.core_dev = core_dev
+        self.inactive_core_dev = inactive_core_dev
+        self.write_policy = write_policy
+        self.eviction_policy = eviction_policy
+        self.cleaning_policy = cleaning_policy
+        self.promotion_policy = promotion_policy
+        self.cache_line_size = cache_line_size
+        self.metadata_memory_footprint = metadata_memory_footprint
+        self.dirty_for = dirty_for
+        self.metadata_mode = metadata_mode
+        self.status = status
+
+    def __str__(self):
+        return (
+            f"Config stats:\n"
+            f"Cache ID: {self.cache_id}\n"
+            f"Cache size: {self.cache_size}\n"
+            f"Cache device: {self.cache_dev}\n"
+            f"Core devices: {self.core_dev}\n"
+            f"Inactive core devices: {self.inactive_core_dev}\n"
+            f"Write policy: {self.write_policy}\n"
+            f"Eviction policy: {self.eviction_policy}\n"
+            f"Cleaning policy: {self.cleaning_policy}\n"
+            f"Promotion policy: {self.promotion_policy}\n"
+            f"Cache line size: {self.cache_line_size}\n"
+            f"Metadata memory footprint: {self.metadata_memory_footprint}\n"
+            f"Dirty for: {self.dirty_for}\n"
+            f"Metadata mode: {self.metadata_mode}\n"
+            f"Status: {self.status}\n"
+        )
+
+    def __eq__(self, other):
+        if not other:
+            return False
+        return (
+            self.cache_id == other.cache_id
+            and self.cache_size == other.cache_size
+            and self.cache_dev == other.cache_dev
+            and self.core_dev == other.core_dev
+            and self.inactive_core_dev == other.inactive_core_dev
+            and self.write_policy == other.write_policy
+            and self.eviction_policy == other.eviction_policy
+            and self.cleaning_policy == other.cleaning_policy
+            and self.promotion_policy == other.promotion_policy
+            and self.cache_line_size == other.cache_line_size
+            and self.metadata_memory_footprint == other.metadata_memory_footprint
+            and self.dirty_for == other.dirty_for
+            and self.metadata_mode == other.metadata_mode
+            and self.status == other.status
+        )
+
+
+class CoreConfigStats:
+    def __init__(
+        self,
+        core_id,
+        core_dev,
+        exp_obj,
+        core_size,
+        dirty_for,
+        status,
+        seq_cutoff_threshold,
+        seq_cutoff_policy,
+    ):
+        self.core_id = core_id
+        self.core_dev = core_dev
+        self.exp_obj = exp_obj
+        self.core_size = core_size
+        self.dirty_for = dirty_for
+        self.status = status
+        self.seq_cutoff_threshold = seq_cutoff_threshold
+        self.seq_cutoff_policy = seq_cutoff_policy
+
+    def __str__(self):
+        return (
+            f"Config stats:\n"
+            f"Core ID: {self.core_id}\n"
+            f"Core device: {self.core_dev}\n"
+            f"Exported object: {self.exp_obj}\n"
+            f"Core size: {self.core_size}\n"
+            f"Dirty for: {self.dirty_for}\n"
+            f"Status: {self.status}\n"
+            f"Seq cutoff threshold: {self.seq_cutoff_threshold}\n"
+            f"Seq cutoff policy: {self.seq_cutoff_policy}\n"
+        )
+
+    def __eq__(self, other):
+        if not other:
+            return False
+        return (
+            self.core_id == other.core_id
+            and self.core_dev == other.core_dev
+            and self.exp_obj == other.exp_obj
+            and self.core_size == other.core_size
+            and self.dirty_for == other.dirty_for
+            and self.status == other.status
+            and self.seq_cutoff_threshold == other.seq_cutoff_threshold
+            and self.seq_cutoff_policy == other.seq_cutoff_policy
+        )
+
+
+class IoClassConfigStats:
+    def __init__(
+        self, io_class_id, io_class_name, eviction_priority, selective_allocation
+    ):
+        self.io_class_id = io_class_id
+        self.io_class_name = io_class_name
+        self.eviction_priority = eviction_priority
+        self.selective_allocation = selective_allocation
+
+    def __str__(self):
+        return (
+            f"Config stats:\n"
+            f"IO class ID: {self.io_class_id}\n"
+            f"IO class name: {self.io_class_name}\n"
+            f"Eviction priority: {self.eviction_priority}\n"
+            f"Selective allocation: {self.selective_allocation}\n"
+        )
+
+    def __eq__(self, other):
+        if not other:
+            return False
+        return (
+            self.io_class_id == other.io_class_id
+            and self.io_class_name == other.io_class_name
+            and self.eviction_priority == other.eviction_priority
+            and self.selective_allocation == other.selective_allocation
+        )
+
+
+class UsageStats:
+    def __init__(self, occupancy, free, clean, dirty):
+        self.occupancy = occupancy
+        self.free = free
+        self.clean = clean
+        self.dirty = dirty
+
+    def __str__(self):
+        return (
+            f"Usage stats:\n"
+            f"Occupancy: {self.occupancy}\n"
+            f"Free: {self.free}\n"
+            f"Clean: {self.clean}\n"
+            f"Dirty: {self.dirty}\n"
+        )
+
+    def __eq__(self, other):
+        if not other:
+            return False
+        return (
+            self.occupancy == other.occupancy
+            and self.free == other.free
+            and self.clean == other.clean
+            and self.dirty == other.dirty
+        )
+
+
+class InactiveUsageStats:
+    def __init__(self, inactive_occupancy, inactive_clean, inactive_dirty):
+        self.inactive_occupancy = inactive_occupancy
+        self.inactive_clean = inactive_clean
+        self.inactive_dirty = inactive_dirty
+
+    def __str__(self):
+        return (
+            f"Inactive usage stats:\n"
+            f"Inactive occupancy: {self.inactive_occupancy}\n"
+            f"Inactive clean: {self.inactive_clean}\n"
+            f"Inactive dirty: {self.inactive_dirty}\n"
+        )
+
+    def __eq__(self, other):
+        if not other:
+            return False
+        return (
+            self.inactive_occupancy == other.inactive_occupancy
+            and self.inactive_clean == other.inactive_clean
+            and self.inactive_dirty == other.inactive_dirty
+        )
+
+
+class RequestStats:
+    def __init__(
+        self,
+        read_hits,
+        read_part_misses,
+        read_full_misses,
+        read_total,
+        write_hits,
+        write_part_misses,
+        write_full_misses,
+        write_total,
+        pass_through_reads,
+        pass_through_writes,
+        requests_serviced,
+        requests_total,
+    ):
+        self.read = RequestStatsChunk(
+            read_hits, read_part_misses, read_full_misses, read_total
+        )
+        self.write = RequestStatsChunk(
+            write_hits, write_part_misses, write_full_misses, write_total
+        )
+        self.pass_through_reads = pass_through_reads
+        self.pass_through_writes = pass_through_writes
+        self.requests_serviced = requests_serviced
+        self.requests_total = requests_total
+
+    def __str__(self):
+        return (
+            f"Request stats:\n"
+            f"Read:\n{self.read}"
+            f"Write:\n{self.write}"
+            f"Pass-through reads: {self.pass_through_reads}\n"
+            f"Pass-through writes: {self.pass_through_writes}\n"
+            f"Serviced requests: {self.requests_serviced}\n"
+            f"Total requests: {self.requests_total}\n"
+        )
+
+    def __eq__(self, other):
+        if not other:
+            return False
+        return (
+            self.read == other.read
+            and self.write == other.write
+            and self.pass_through_reads == other.pass_through_reads
+            and self.pass_through_writes == other.pass_through_writes
+            and self.requests_serviced == other.requests_serviced
+            and self.requests_total == other.requests_total
+        )
+
+
+class RequestStatsChunk:
+    def __init__(self, hits, part_misses, full_misses, total):
+        self.hits = hits
+        self.part_misses = part_misses
+        self.full_misses = full_misses
+        self.total = total
+
+    def __str__(self):
+        return (
+            f"Hits: {self.hits}\n"
+            f"Partial misses: {self.part_misses}\n"
+            f"Full misses: {self.full_misses}\n"
+            f"Total: {self.total}\n"
+        )
+
+    def __eq__(self, other):
+        if not other:
+            return False
+        return (
+            self.hits == other.hits
+            and self.part_misses == other.part_misses
+            and self.full_misses == other.full_misses
+            and self.total == other.total
+        )
+
+
+class BlockStats:
+    def __init__(
+        self,
+        core_reads,
+        core_writes,
+        core_total,
+        cache_reads,
+        cache_writes,
+        cache_total,
+        exp_obj_reads,
+        exp_obj_writes,
+        exp_obj_total,
+    ):
+        self.core = BasicStatsChunk(core_reads, core_writes, core_total)
+        self.cache = BasicStatsChunk(cache_reads, cache_writes, cache_total)
+        self.exp_obj = BasicStatsChunk(exp_obj_reads, exp_obj_writes, exp_obj_total)
+
+    def __str__(self):
+        return (
+            f"Block stats:\n"
+            f"Core(s):\n{self.core}"
+            f"Cache:\n{self.cache}"
+            f"Exported object(s):\n{self.exp_obj}"
+        )
+
+    def __eq__(self, other):
+        if not other:
+            return False
+        return (
+            self.core == other.core
+            and self.cache == other.cache
+            and self.exp_obj == other.exp_obj
+        )
+
+
+class ErrorStats:
+    def __init__(
+        self,
+        cache_read_errors,
+        cache_write_errors,
+        cache_total_errors,
+        core_read_errors,
+        core_write_errors,
+        core_total_errors,
+        total_errors,
+    ):
+        self.cache = BasicStatsChunk(
+            cache_read_errors, cache_write_errors, cache_total_errors
+        )
+        self.core = BasicStatsChunk(
+            core_read_errors, core_write_errors, core_total_errors
+        )
+        self.total_errors = total_errors
+
+    def __str__(self):
+        return (
+            f"Error stats:\n"
+            f"Cache errors:\n{self.cache}"
+            f"Core errors:\n{self.core}"
+            f"Total errors: {self.total_errors}\n"
+        )
+
+    def __eq__(self, other):
+        if not other:
+            return False
+        return (
+            self.cache == other.cache
+            and self.core == other.core
+            and self.total_errors == other.total_errors
+        )
+
+
+class BasicStatsChunk:
+    def __init__(self, reads, writes, total):
+        self.reads = reads
+        self.writes = writes
+        self.total = total
+
+    def __str__(self):
+        return f"Reads: {self.reads}\nWrites: {self.writes}\nTotal: {self.total}\n"
+
+    def __eq__(self, other):
+        if not other:
+            return False
+        return (
+            self.reads == other.reads
+            and self.writes == other.writes
+            and self.total == other.total
+        )