From b4daac11c2ce3124c9e1a8bae763b78e3ec87f87 Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Tue, 21 Jul 2020 12:55:01 +0200 Subject: [PATCH] Track hot items on LRU list Top 50% least recently used cachelines are not promoted to list head upon access. Only after cacheline drops to bottom 50% it is considered as a candidate to promote to list head. The purpose of this change is to reduce overhead of LRU list maintanance for hot cachelines. Signed-off-by: Adam Rutkowski --- src/eviction/lru.c | 81 +++++++++++++++++++++++++++++++++++++- src/eviction/lru_structs.h | 6 ++- 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/src/eviction/lru.c b/src/eviction/lru.c index 96ef568..fa6cbba 100644 --- a/src/eviction/lru.c +++ b/src/eviction/lru.c @@ -29,6 +29,7 @@ static void add_lru_head(ocf_cache_t cache, ENV_BUG_ON(collision_index == end_marker); node = &ocf_metadata_get_eviction_policy(cache, collision_index)->lru; + node->hot = false; /* First node to be added/ */ if (!list->num_nodes) { @@ -53,6 +54,10 @@ static void add_lru_head(ocf_cache_t cache, node->next = curr_head_index; node->prev = end_marker; curr_head->prev = collision_index; + node->hot = true; + if (!curr_head->hot) + list->last_hot = collision_index; + ++list->num_hot; list->head = collision_index; @@ -76,6 +81,9 @@ static void remove_lru_list(ocf_cache_t cache, is_head = (list->head == collision_index); is_tail = (list->tail == collision_index); + if (node->hot) + --list->num_hot; + /* Set prev and next (even if not existent) */ next_lru_node = node->next; prev_lru_node = node->prev; @@ -89,6 +97,8 @@ static void remove_lru_list(ocf_cache_t cache, list->head = end_marker; list->tail = end_marker; + list->last_hot = end_marker; + ENV_BUG_ON(list->num_hot != 0); } /* Case 2: else if this collision_index is LRU head, but not tail, @@ -102,7 +112,13 @@ static void remove_lru_list(ocf_cache_t cache, next_node = &ocf_metadata_get_eviction_policy(cache, next_lru_node)->lru; + if (list->last_hot == collision_index) { + ENV_BUG_ON(list->num_hot != 0); + list->last_hot = end_marker; + } + list->head = next_lru_node; + node->next = end_marker; next_node->prev = end_marker; } @@ -139,6 +155,11 @@ static void remove_lru_list(ocf_cache_t cache, prev_node = &ocf_metadata_get_eviction_policy(cache, prev_lru_node)->lru; + if (list->last_hot == collision_index) { + ENV_BUG_ON(list->num_hot == 0); + list->last_hot = prev_lru_node; + } + /* Update prev and next nodes */ prev_node->next = node->next; next_node->prev = node->prev; @@ -151,6 +172,53 @@ static void remove_lru_list(ocf_cache_t cache, --list->num_nodes; } +/* Increase / decrease number of hot elements to achieve target count. + * Asssumes that the list has hot element clustered together at the + * head of the list. + */ +static void balance_lru_list(ocf_cache_t cache, + struct ocf_lru_list *list) +{ + unsigned target_hot_count = list->num_nodes / OCF_LRU_HOT_RATIO; + struct lru_eviction_policy_meta *node; + + if (target_hot_count == list->num_hot) + return; + + if (list->num_hot == 0) { + node = &ocf_metadata_get_eviction_policy(cache, + list->head)->lru; + list->last_hot = list->head; + list->num_hot = 1; + node->hot = 1; + return; + } + + ENV_BUG_ON(list->last_hot == end_marker); + node = &ocf_metadata_get_eviction_policy(cache, + list->last_hot)->lru; + + if (target_hot_count > list->num_hot) { + ++list->num_hot; + list->last_hot = node->next; + node = &ocf_metadata_get_eviction_policy(cache, + node->next)->lru; + node->hot = true; + } else { + if (list->last_hot == list->head) { + node->hot = false; + list->num_hot = 0; + list->last_hot = end_marker; + } else { + ENV_BUG_ON(node->prev == end_marker); + node->hot = false; + --list->num_hot; + list->last_hot = node->prev; + } + } +} + + /*-- End of LRU functions*/ void evp_lru_init_cline(ocf_cache_t cache, ocf_cache_line_t cline) @@ -159,6 +227,7 @@ void evp_lru_init_cline(ocf_cache_t cache, ocf_cache_line_t cline) node = &ocf_metadata_get_eviction_policy(cache, cline)->lru; + node->hot = false; node->prev = end_marker; node->next = end_marker; } @@ -188,6 +257,7 @@ void evp_lru_rm_cline(ocf_cache_t cache, ocf_cache_line_t cline) list = evp_get_cline_list(cache, cline); remove_lru_list(cache, list, cline); + balance_lru_list(cache, list); } static inline void lru_iter_init(struct ocf_lru_iter *iter, ocf_cache_t cache, @@ -459,8 +529,10 @@ void evp_lru_hot_cline(ocf_cache_t cache, ocf_cache_line_t cline) struct ocf_lru_list *list; node = &ocf_metadata_get_eviction_policy(cache, cline)->lru; - list = evp_get_cline_list(cache, cline); + if (node->hot) + return; + list = evp_get_cline_list(cache, cline); if (node->next != end_marker || node->prev != end_marker || list->head == cline || list->tail == cline) { @@ -469,6 +541,7 @@ void evp_lru_hot_cline(ocf_cache_t cache, ocf_cache_line_t cline) /* Update LRU */ add_lru_head(cache, list, cline); + balance_lru_list(cache, list); } static inline void _lru_init(struct ocf_lru_list *list) @@ -476,6 +549,8 @@ static inline void _lru_init(struct ocf_lru_list *list) list->num_nodes = 0; list->head = end_marker; list->tail = end_marker; + list->num_hot = 0; + list->last_hot = end_marker; } void evp_lru_init_evp(ocf_cache_t cache, struct ocf_user_part *part) @@ -505,7 +580,9 @@ void evp_lru_clean_cline(ocf_cache_t cache, struct ocf_user_part *part, OCF_METADATA_EVICTION_LOCK(cline); remove_lru_list(cache, dirty_list, cline); + balance_lru_list(cache, dirty_list); add_lru_head(cache, clean_list, cline); + balance_lru_list(cache, clean_list); OCF_METADATA_EVICTION_UNLOCK(cline); } @@ -521,7 +598,9 @@ void evp_lru_dirty_cline(ocf_cache_t cache, struct ocf_user_part *part, OCF_METADATA_EVICTION_LOCK(cline); remove_lru_list(cache, clean_list, cline); + balance_lru_list(cache, clean_list); add_lru_head(cache, dirty_list, cline); + balance_lru_list(cache, dirty_list); OCF_METADATA_EVICTION_UNLOCK(cline); } diff --git a/src/eviction/lru_structs.h b/src/eviction/lru_structs.h index 9c2b529..22aaedb 100644 --- a/src/eviction/lru_structs.h +++ b/src/eviction/lru_structs.h @@ -7,15 +7,17 @@ #define __EVICTION_LRU_STRUCTS_H__ struct lru_eviction_policy_meta { - /* LRU pointers 2*4=8 bytes */ uint32_t prev; uint32_t next; + uint8_t hot; } __attribute__((packed)); struct ocf_lru_list { uint32_t num_nodes; uint32_t head; uint32_t tail; + uint32_t num_hot; + uint32_t last_hot; }; struct lru_eviction_policy { @@ -23,4 +25,6 @@ struct lru_eviction_policy { struct ocf_lru_list dirty; }; +#define OCF_LRU_HOT_RATIO 2 + #endif