Track hot items on LRU list

Top 50% least recently used cachelines are not promoted
to list head upon access. Only after cacheline drops to
bottom 50% it is considered as a candidate to promote
to list head.

The purpose of this change is to reduce overhead of
LRU list maintanance for hot cachelines.

Signed-off-by: Adam Rutkowski <adam.j.rutkowski@intel.com>

src/eviction/lru.c

@@ -29,6 +29,7 @@ static void add_lru_head(ocf_cache_t cache,
 	ENV_BUG_ON(collision_index == end_marker);
 
 	node = &ocf_metadata_get_eviction_policy(cache, collision_index)->lru;
+	node->hot = false;
 
 	/* First node to be added/ */
 	if (!list->num_nodes)  {
@@ -53,6 +54,10 @@ static void add_lru_head(ocf_cache_t cache,
 		node->next = curr_head_index;
 		node->prev = end_marker;
 		curr_head->prev = collision_index;
+		node->hot = true;
+		if (!curr_head->hot)
+			list->last_hot = collision_index;
+		++list->num_hot;
 
 		list->head = collision_index;
 
@@ -76,6 +81,9 @@ static void remove_lru_list(ocf_cache_t cache,
 	is_head = (list->head == collision_index);
 	is_tail = (list->tail == collision_index);
 
+	if (node->hot)
+		--list->num_hot;
+
 	/* Set prev and next (even if not existent) */
 	next_lru_node = node->next;
 	prev_lru_node = node->prev;
@@ -89,6 +97,8 @@ static void remove_lru_list(ocf_cache_t cache,
 
 		list->head = end_marker;
 		list->tail = end_marker;
+		list->last_hot = end_marker;
+		ENV_BUG_ON(list->num_hot != 0);
 	}
 
 	/* Case 2: else if this collision_index is LRU head, but not tail,
@@ -102,7 +112,13 @@ static void remove_lru_list(ocf_cache_t cache,
 		next_node = &ocf_metadata_get_eviction_policy(cache,
 				next_lru_node)->lru;
 
+		if (list->last_hot == collision_index) {
+			ENV_BUG_ON(list->num_hot != 0);
+			list->last_hot = end_marker;
+		}
+
 		list->head = next_lru_node;
+
 		node->next = end_marker;
 		next_node->prev = end_marker;
 	}
@@ -139,6 +155,11 @@ static void remove_lru_list(ocf_cache_t cache,
 		prev_node = &ocf_metadata_get_eviction_policy(cache,
 				prev_lru_node)->lru;
 
+		if (list->last_hot == collision_index) {
+			ENV_BUG_ON(list->num_hot == 0);
+			list->last_hot = prev_lru_node;
+		}
+
 		/* Update prev and next nodes */
 		prev_node->next = node->next;
 		next_node->prev = node->prev;
@@ -151,6 +172,53 @@ static void remove_lru_list(ocf_cache_t cache,
 	--list->num_nodes;
 }
 
+/* Increase / decrease number of hot elements to achieve target count.
+ * Asssumes that the list has hot element clustered together at the
+ * head of the list.
+ */
+static void balance_lru_list(ocf_cache_t cache,
+		struct ocf_lru_list *list)
+{
+	unsigned target_hot_count = list->num_nodes / OCF_LRU_HOT_RATIO;
+	struct lru_eviction_policy_meta *node;
+
+	if (target_hot_count == list->num_hot)
+		return;
+
+	if (list->num_hot == 0) {
+		node = &ocf_metadata_get_eviction_policy(cache,
+				list->head)->lru;
+		list->last_hot = list->head;
+		list->num_hot = 1;
+		node->hot = 1;
+		return;
+	}
+
+	ENV_BUG_ON(list->last_hot == end_marker);
+	node = &ocf_metadata_get_eviction_policy(cache,
+			list->last_hot)->lru;
+
+	if (target_hot_count > list->num_hot) {
+		++list->num_hot;
+		list->last_hot = node->next;
+		node = &ocf_metadata_get_eviction_policy(cache,
+				node->next)->lru;
+		node->hot = true;
+	} else {
+		if (list->last_hot == list->head) {
+			node->hot = false;
+			list->num_hot = 0;
+			list->last_hot = end_marker;
+		} else {
+			ENV_BUG_ON(node->prev == end_marker);
+			node->hot = false;
+			--list->num_hot;
+			list->last_hot = node->prev;
+		}
+	}
+}
+
+
 /*-- End of LRU functions*/
 
 void evp_lru_init_cline(ocf_cache_t cache, ocf_cache_line_t cline)
@@ -159,6 +227,7 @@ void evp_lru_init_cline(ocf_cache_t cache, ocf_cache_line_t cline)
 
 	node = &ocf_metadata_get_eviction_policy(cache, cline)->lru;
 
+	node->hot = false;
 	node->prev = end_marker;
 	node->next = end_marker;
 }
@@ -188,6 +257,7 @@ void evp_lru_rm_cline(ocf_cache_t cache, ocf_cache_line_t cline)
 
 	list = evp_get_cline_list(cache, cline);
 	remove_lru_list(cache, list, cline);
+	balance_lru_list(cache, list);
 }
 
 static inline void lru_iter_init(struct ocf_lru_iter *iter, ocf_cache_t cache,
@@ -459,8 +529,10 @@ void evp_lru_hot_cline(ocf_cache_t cache, ocf_cache_line_t cline)
 	struct ocf_lru_list *list;
 
 	node = &ocf_metadata_get_eviction_policy(cache, cline)->lru;
-	list = evp_get_cline_list(cache, cline);
+	if (node->hot)
+		return;
 
+	list = evp_get_cline_list(cache, cline);
 	if (node->next != end_marker ||
 			node->prev != end_marker ||
 			list->head == cline || list->tail == cline) {
@@ -469,6 +541,7 @@ void evp_lru_hot_cline(ocf_cache_t cache, ocf_cache_line_t cline)
 
 	/* Update LRU */
 	add_lru_head(cache, list, cline);
+	balance_lru_list(cache, list);
 }
 
 static inline void _lru_init(struct ocf_lru_list *list)
@@ -476,6 +549,8 @@ static inline void _lru_init(struct ocf_lru_list *list)
 	list->num_nodes = 0;
 	list->head = end_marker;
 	list->tail = end_marker;
+	list->num_hot = 0;
+	list->last_hot = end_marker;
 }
 
 void evp_lru_init_evp(ocf_cache_t cache, struct ocf_user_part *part)
@@ -505,7 +580,9 @@ void evp_lru_clean_cline(ocf_cache_t cache, struct ocf_user_part *part,
 
 	OCF_METADATA_EVICTION_LOCK(cline);
 	remove_lru_list(cache, dirty_list, cline);
+	balance_lru_list(cache, dirty_list);
 	add_lru_head(cache, clean_list, cline);
+	balance_lru_list(cache, clean_list);
 	OCF_METADATA_EVICTION_UNLOCK(cline);
 }
 
@@ -521,7 +598,9 @@ void evp_lru_dirty_cline(ocf_cache_t cache, struct ocf_user_part *part,
 
 	OCF_METADATA_EVICTION_LOCK(cline);
 	remove_lru_list(cache, clean_list, cline);
+	balance_lru_list(cache, clean_list);
 	add_lru_head(cache, dirty_list, cline);
+	balance_lru_list(cache, dirty_list);
 	OCF_METADATA_EVICTION_UNLOCK(cline);
 }
 

src/eviction/lru_structs.h

@@ -7,15 +7,17 @@
 #define __EVICTION_LRU_STRUCTS_H__
 
 struct lru_eviction_policy_meta {
-	/* LRU pointers 2*4=8 bytes */
 	uint32_t prev;
 	uint32_t next;
+	uint8_t hot;
 } __attribute__((packed));
 
 struct ocf_lru_list {
 	uint32_t num_nodes;
 	uint32_t head;
 	uint32_t tail;
+	uint32_t num_hot;
+	uint32_t last_hot;
 };
 
 struct lru_eviction_policy {
@@ -23,4 +25,6 @@ struct lru_eviction_policy {
 	struct ocf_lru_list dirty;
 };
 
+#define OCF_LRU_HOT_RATIO 2
+
 #endif