Merge pull request #396 from arutk/lru_refactor

Simplify and modularize LRU list code

src/eviction/lru.c

@@ -15,176 +15,115 @@
 
 #define OCF_EVICTION_MAX_SCAN 1024
 
-/* -- Start of LRU functions --*/
+static const ocf_cache_line_t end_marker = (ocf_cache_line_t)-1;
-
-/* Returns 1 if the given collision_index is the _head_ of
- * the LRU list, 0 otherwise.
- */
-/* static inline int is_lru_head(unsigned collision_index) {
- *	return collision_index == lru_list.lru_head;
- * }
- */
-
-#define is_lru_head(x) (x == collision_table_entries)
-#define is_lru_tail(x) (x == collision_table_entries)
-
-/* Sets the given collision_index as the new _head_ of the LRU list. */
-static inline void update_lru_head(ocf_cache_t cache,
-		int partition_id, unsigned int collision_index,
-		int cline_dirty)
-{
-	struct ocf_user_part *part = &cache->user_parts[partition_id];
-
-
-	if (cline_dirty)
-		part->runtime->eviction.policy.lru.dirty_head = collision_index;
-	else
-		part->runtime->eviction.policy.lru.clean_head = collision_index;
-}
-
-/* Sets the given collision_index as the new _tail_ of the LRU list. */
-static inline void update_lru_tail(ocf_cache_t cache,
-		int partition_id, unsigned int collision_index,
-		int cline_dirty)
-{
-	struct ocf_user_part *part = &cache->user_parts[partition_id];
-
-	if (cline_dirty)
-		part->runtime->eviction.policy.lru.dirty_tail = collision_index;
-	else
-		part->runtime->eviction.policy.lru.clean_tail = collision_index;
-}
-
-/* Sets the given collision_index as the new _head_ and _tail_ of
- * the LRU list.
- */
-static inline void update_lru_head_tail(ocf_cache_t cache,
-		int partition_id, unsigned int collision_index, int cline_dirty)
-{
-	update_lru_head(cache, partition_id, collision_index, cline_dirty);
-	update_lru_tail(cache, partition_id, collision_index, cline_dirty);
-}
 
 /* Adds the given collision_index to the _head_ of the LRU list */
-static void add_lru_head(ocf_cache_t cache, int partition_id,
+static void add_lru_head(ocf_cache_t cache,
-		unsigned int collision_index, int cline_dirty)
+		struct ocf_lru_list *list,
-{
+		unsigned int collision_index)
-	unsigned int curr_head_index;
-	unsigned int collision_table_entries =
-			cache->device->collision_table_entries;
-	struct ocf_user_part *part = &cache->user_parts[partition_id];
-	union eviction_policy_meta eviction;
 
-	ENV_BUG_ON(!(collision_index < collision_table_entries));
+{
+	union eviction_policy_meta eviction;
+	struct lru_eviction_policy_meta *node;
+	unsigned int curr_head_index;
+
+	ENV_BUG_ON(collision_index == end_marker);
 
 	ocf_metadata_get_evicition_policy(cache, collision_index, &eviction);
+	node = &eviction.lru;
 
 	/* First node to be added/ */
-	if ((cline_dirty && !part->runtime->eviction.policy.lru.has_dirty_nodes) ||
+	if (!list->num_nodes)  {
-	    (!cline_dirty && !part->runtime->eviction.policy.lru.has_clean_nodes)) {
+		list->head = collision_index;
-		update_lru_head_tail(cache, partition_id, collision_index, cline_dirty);
+		list->tail = collision_index;
 
-		eviction.lru.next = collision_table_entries;
+		node->next = end_marker;
-		eviction.lru.prev = collision_table_entries;
+		node->prev = end_marker;
 
-		if (cline_dirty)
+		list->num_nodes = 1;
-			part->runtime->eviction.policy.lru.has_dirty_nodes = 1;
-		else
-			part->runtime->eviction.policy.lru.has_clean_nodes = 1;
 
 		ocf_metadata_set_evicition_policy(cache, collision_index,
 				&eviction);
 	} else {
-		union eviction_policy_meta eviction_curr;
+		union eviction_policy_meta eviction_curr_head;
+		struct lru_eviction_policy_meta *curr_head;
 
 		/* Not the first node to be added. */
-		curr_head_index = cline_dirty ?
+		curr_head_index = list->head;
-				part->runtime->eviction.policy.lru.dirty_head :
-				part->runtime->eviction.policy.lru.clean_head;
 
-		ENV_BUG_ON(!(curr_head_index < collision_table_entries));
+		ENV_BUG_ON(curr_head_index == end_marker);
 
 		ocf_metadata_get_evicition_policy(cache, curr_head_index,
-						&eviction_curr);
+						&eviction_curr_head);
+		curr_head = &eviction_curr_head.lru;
 
-		eviction.lru.next = curr_head_index;
+		node->next = curr_head_index;
-		eviction.lru.prev = collision_table_entries;
+		node->prev = end_marker;
-		eviction_curr.lru.prev = collision_index;
+		curr_head->prev = collision_index;
 
-		update_lru_head(cache, partition_id, collision_index, cline_dirty);
+		list->head = collision_index;
+
+		++list->num_nodes;
 
 		ocf_metadata_set_evicition_policy(cache, curr_head_index,
-				&eviction_curr);
+				&eviction_curr_head);
 		ocf_metadata_set_evicition_policy(cache, collision_index,
 				&eviction);
 	}
 }
 
 /* Deletes the node with the given collision_index from the lru list */
-static void remove_lru_list(ocf_cache_t cache, int partition_id,
+static void remove_lru_list(ocf_cache_t cache,
-		unsigned int collision_index, int cline_dirty)
+		struct ocf_lru_list *list,
+		unsigned int collision_index)
 {
-	int is_clean_head = 0, is_clean_tail = 0, is_dirty_head = 0, is_dirty_tail = 0;
+	int is_head = 0, is_tail = 0;
 	uint32_t prev_lru_node, next_lru_node;
-	uint32_t collision_table_entries = cache->device->collision_table_entries;
-	struct ocf_user_part *part = &cache->user_parts[partition_id];
 	union eviction_policy_meta eviction;
+	struct lru_eviction_policy_meta *node;
 
-	ENV_BUG_ON(!(collision_index < collision_table_entries));
+	ENV_BUG_ON(collision_index == end_marker);
 
 	ocf_metadata_get_evicition_policy(cache, collision_index, &eviction);
+	node = &eviction.lru;
 
-	/* Find out if this node is LRU _head_ or LRU _tail_ */
+	is_head = (list->head == collision_index);
-	if (part->runtime->eviction.policy.lru.clean_head == collision_index)
+	is_tail = (list->tail == collision_index);
-		is_clean_head = 1;
-	if (part->runtime->eviction.policy.lru.dirty_head == collision_index)
-		is_dirty_head = 1;
-	if (part->runtime->eviction.policy.lru.clean_tail == collision_index)
-		is_clean_tail = 1;
-	if (part->runtime->eviction.policy.lru.dirty_tail == collision_index)
-		is_dirty_tail = 1;
-	ENV_BUG_ON((is_clean_tail || is_clean_head) && (is_dirty_tail || is_dirty_head));
 
 	/* Set prev and next (even if not existent) */
-	next_lru_node = eviction.lru.next;
+	next_lru_node = node->next;
-	prev_lru_node = eviction.lru.prev;
+	prev_lru_node = node->prev;
 
 	/* Case 1: If we are head AND tail, there is only one node.
 	 * So unlink node and set that there is no node left in the list.
 	 */
-	if ((is_clean_head && is_clean_tail) || (is_dirty_head && is_dirty_tail)) {
+	if (is_head && is_tail) {
-		eviction.lru.next = collision_table_entries;
+		node->next = end_marker;
-		eviction.lru.prev = collision_table_entries;
+		node->prev = end_marker;
-
-		update_lru_head_tail(cache, partition_id, collision_table_entries, cline_dirty);
-
-		if (cline_dirty)
-			part->runtime->eviction.policy.lru.has_dirty_nodes = 0;
-		else
-			 part->runtime->eviction.policy.lru.has_clean_nodes = 0;
 
 		ocf_metadata_set_evicition_policy(cache, collision_index,
 				&eviction);
 
-		update_lru_head_tail(cache, partition_id,
+		list->head = end_marker;
-				collision_table_entries, cline_dirty);
+		list->tail = end_marker;
 	}
 
 	/* Case 2: else if this collision_index is LRU head, but not tail,
 	 * update head and return
 	 */
-	else if ((!is_clean_tail && is_clean_head) || (!is_dirty_tail && is_dirty_head)) {
+	else if (is_head) {
 		union eviction_policy_meta eviction_next;
+		struct lru_eviction_policy_meta *next_node;
 
-		ENV_BUG_ON(!(next_lru_node < collision_table_entries));
+		ENV_BUG_ON(next_lru_node == end_marker);
 
 		ocf_metadata_get_evicition_policy(cache, next_lru_node,
 				&eviction_next);
+		next_node = &eviction_next.lru;
 
-		update_lru_head(cache, partition_id, next_lru_node, cline_dirty);
+		list->head = next_lru_node;
-
+		node->next = end_marker;
-		eviction.lru.next = collision_table_entries;
+		next_node->prev = end_marker;
-		eviction_next.lru.prev = collision_table_entries;
 
 		ocf_metadata_set_evicition_policy(cache, collision_index,
 				&eviction);
@@ -196,18 +135,20 @@ static void remove_lru_list(ocf_cache_t cache, int partition_id,
 	/* Case 3: else if this collision_index is LRU tail, but not head,
 	 * update tail and return
 	 */
-	else if ((is_clean_tail && !is_clean_head) || (is_dirty_tail && !is_dirty_head)) {
+	else if (is_tail) {
 		union eviction_policy_meta eviction_prev;
+		struct lru_eviction_policy_meta *prev_node;
 
-		ENV_BUG_ON(!(prev_lru_node < collision_table_entries));
+		ENV_BUG_ON(prev_lru_node == end_marker);
 
-		update_lru_tail(cache, partition_id, prev_lru_node, cline_dirty);
+		list->tail = prev_lru_node;
 
 		ocf_metadata_get_evicition_policy(cache, prev_lru_node,
 				&eviction_prev);
+		prev_node = &eviction_prev.lru;
 
-		eviction.lru.prev = collision_table_entries;
+		node->prev = end_marker;
-		eviction_prev.lru.next = collision_table_entries;
+		prev_node->next = end_marker;
 
 		ocf_metadata_set_evicition_policy(cache, collision_index,
 				&eviction);
@@ -222,22 +163,26 @@ static void remove_lru_list(ocf_cache_t cache, int partition_id,
 	else {
 		union eviction_policy_meta eviction_prev;
 		union eviction_policy_meta eviction_next;
+		struct lru_eviction_policy_meta *prev_node;
+		struct lru_eviction_policy_meta *next_node;
 
-		ENV_BUG_ON(!(next_lru_node < collision_table_entries));
+		ENV_BUG_ON(next_lru_node == end_marker);
-		ENV_BUG_ON(!(prev_lru_node < collision_table_entries));
+		ENV_BUG_ON(prev_lru_node == end_marker);
 
 		ocf_metadata_get_evicition_policy(cache, next_lru_node,
 				&eviction_next);
+		next_node = &eviction_next.lru;
 		ocf_metadata_get_evicition_policy(cache, prev_lru_node,
 				&eviction_prev);
+		prev_node = &eviction_prev.lru;
 
 		/* Update prev and next nodes */
-		eviction_prev.lru.next = eviction.lru.next;
+		prev_node->next = node->next;
-		eviction_next.lru.prev = eviction.lru.prev;
+		next_node->prev = node->prev;
 
 		/* Update the given node */
-		eviction.lru.next = collision_table_entries;
+		node->next = end_marker;
-		eviction.lru.prev = collision_table_entries;
+		node->prev = end_marker;
 
 		ocf_metadata_set_evicition_policy(cache, collision_index,
 				&eviction);
@@ -246,6 +191,8 @@ static void remove_lru_list(ocf_cache_t cache, int partition_id,
 		ocf_metadata_set_evicition_policy(cache, prev_lru_node,
 				&eviction_prev);
 	}
+
+	--list->num_nodes;
 }
 
 /*-- End of LRU functions*/
@@ -253,11 +200,13 @@ static void remove_lru_list(ocf_cache_t cache, int partition_id,
 void evp_lru_init_cline(ocf_cache_t cache, ocf_cache_line_t cline)
 {
 	union eviction_policy_meta eviction;
+	struct lru_eviction_policy_meta *node;
 
 	ocf_metadata_get_evicition_policy(cache, cline, &eviction);
+	node = &eviction.lru;
 
-	eviction.lru.prev = cache->device->collision_table_entries;
+	node->prev = end_marker;
-	eviction.lru.next = cache->device->collision_table_entries;
+	node->next = end_marker;
 
 	ocf_metadata_set_evicition_policy(cache, cline, &eviction);
 }
@@ -267,8 +216,14 @@ void evp_lru_init_cline(ocf_cache_t cache, ocf_cache_line_t cline)
 void evp_lru_rm_cline(ocf_cache_t cache, ocf_cache_line_t cline)
 {
 	ocf_part_id_t part_id = ocf_metadata_get_partition_id(cache, cline);
+	struct ocf_user_part *part = &cache->user_parts[part_id];
+	struct ocf_lru_list *list;
 
-	remove_lru_list(cache, part_id, cline, metadata_test_dirty(cache, cline));
+	list = metadata_test_dirty(cache, cline) ?
+		&part->runtime->eviction.policy.lru.dirty :
+		&part->runtime->eviction.policy.lru.clean;
+
+	remove_lru_list(cache, list, cline);
 }
 
 static void evp_lru_clean_end(void *private_data, int error)
@@ -285,7 +240,7 @@ static int evp_lru_clean_getter(ocf_cache_t cache,
 	struct ocf_cleaner_attribs *attribs = getter_context;
 	ocf_cache_line_t prev_cline, curr_cline = attribs->getter_item;
 
-	while (curr_cline < cache->device->collision_table_entries) {
+	while (curr_cline != end_marker) {
 		ocf_metadata_get_evicition_policy(cache, curr_cline,
 				&eviction);
 		prev_cline = eviction.lru.prev;
@@ -320,7 +275,7 @@ static void evp_lru_clean(ocf_cache_t cache, ocf_queue_t io_queue,
 
 		.getter = evp_lru_clean_getter,
 		.getter_context = &attribs,
-		.getter_item = part->runtime->eviction.policy.lru.dirty_tail,
+		.getter_item = part->runtime->eviction.policy.lru.dirty.tail,
 
 		.count = count > 32 ? 32 : count,
 
@@ -402,15 +357,13 @@ uint32_t evp_lru_req_clines(ocf_cache_t cache, ocf_queue_t io_queue,
 		return 0;
 
 	i =  0;
-	curr_cline = part->runtime->eviction.policy.lru.clean_tail;
+	curr_cline = part->runtime->eviction.policy.lru.clean.tail;
 	/* Find cachelines to be evicted. */
 	while (i < cline_no) {
-		ENV_BUG_ON(curr_cline > cache->device->collision_table_entries);
-
 		if (!evp_lru_can_evict(cache))
 			break;
 
-		if (curr_cline == cache->device->collision_table_entries)
+		if (curr_cline == end_marker)
 			break;
 
 		ocf_metadata_get_evicition_policy(cache, curr_cline,
@@ -447,8 +400,8 @@ uint32_t evp_lru_req_clines(ocf_cache_t cache, ocf_queue_t io_queue,
 		curr_cline = prev_cline;
 	}
 
-	if (i < cline_no && part->runtime->eviction.policy.lru.dirty_tail !=
+	if (i < cline_no && part->runtime->eviction.policy.lru.dirty.tail !=
-			cache->device->collision_table_entries) {
+			end_marker) {
 		evp_lru_clean(cache, io_queue, part_id, cline_no - i);
 	}
 
@@ -461,62 +414,78 @@ void evp_lru_hot_cline(ocf_cache_t cache, ocf_cache_line_t cline)
 {
 	ocf_part_id_t part_id = ocf_metadata_get_partition_id(cache, cline);
 	struct ocf_user_part *part = &cache->user_parts[part_id];
-
-	uint32_t prev_lru_node, next_lru_node;
-	uint32_t collision_table_entries = cache->device->collision_table_entries;
 	union eviction_policy_meta eviction;
-
+	struct lru_eviction_policy_meta *node;
 	int cline_dirty;
+	struct ocf_lru_list *list;
 
 	ocf_metadata_get_evicition_policy(cache, cline, &eviction);
-
+	node = &eviction.lru;
-	next_lru_node = eviction.lru.next;
-	prev_lru_node = eviction.lru.prev;
 
 	cline_dirty = metadata_test_dirty(cache, cline);
+	list = cline_dirty ?
+		&part->runtime->eviction.policy.lru.dirty :
+		&part->runtime->eviction.policy.lru.clean;
 
-	if ((next_lru_node != collision_table_entries) ||
+	if (node->next != end_marker ||
-	    (prev_lru_node != collision_table_entries) ||
+			node->prev != end_marker ||
-	    ((part->runtime->eviction.policy.lru.clean_head == cline) &&
+			list->head == cline || list->tail == cline) {
-	     (part->runtime->eviction.policy.lru.clean_tail == cline)) ||
+		remove_lru_list(cache, list, cline);
-	    ((part->runtime->eviction.policy.lru.dirty_head == cline) &&
-	     (part->runtime->eviction.policy.lru.dirty_tail == cline))) {
-		remove_lru_list(cache, part_id, cline, cline_dirty);
 	}
 
 	/* Update LRU */
-	add_lru_head(cache, part_id, cline, cline_dirty);
+	add_lru_head(cache, list, cline);
+}
+
+static inline void _lru_init(struct ocf_lru_list *list)
+{
+	list->num_nodes = 0;
+	list->head = end_marker;
+	list->tail = end_marker;
 }
 
 void evp_lru_init_evp(ocf_cache_t cache, ocf_part_id_t part_id)
 {
-	unsigned int collision_table_entries =
-			cache->device->collision_table_entries;
 	struct ocf_user_part *part = &cache->user_parts[part_id];
+	struct ocf_lru_list *clean_list;
+	struct ocf_lru_list *dirty_list;
 
-	part->runtime->eviction.policy.lru.has_clean_nodes = 0;
+	clean_list = &part->runtime->eviction.policy.lru.clean;
-	part->runtime->eviction.policy.lru.has_dirty_nodes = 0;
+	dirty_list = &part->runtime->eviction.policy.lru.dirty;
-	part->runtime->eviction.policy.lru.clean_head = collision_table_entries;
+
-	part->runtime->eviction.policy.lru.clean_tail = collision_table_entries;
+	_lru_init(clean_list);
-	part->runtime->eviction.policy.lru.dirty_head = collision_table_entries;
+	_lru_init(dirty_list);
-	part->runtime->eviction.policy.lru.dirty_tail = collision_table_entries;
 }
 
 void evp_lru_clean_cline(ocf_cache_t cache, ocf_part_id_t part_id,
 		uint32_t cline)
 {
+	struct ocf_user_part *part = &cache->user_parts[part_id];
+	struct ocf_lru_list *clean_list;
+	struct ocf_lru_list *dirty_list;
+
+	clean_list = &part->runtime->eviction.policy.lru.clean;
+	dirty_list = &part->runtime->eviction.policy.lru.dirty;
+
 	OCF_METADATA_EVICTION_LOCK();
-	remove_lru_list(cache, part_id, cline, 1);
+	remove_lru_list(cache, dirty_list, cline);
-	add_lru_head(cache, part_id, cline, 0);
+	add_lru_head(cache, clean_list, cline);
 	OCF_METADATA_EVICTION_UNLOCK();
 }
 
 void evp_lru_dirty_cline(ocf_cache_t cache, ocf_part_id_t part_id,
 		uint32_t cline)
 {
+	struct ocf_user_part *part = &cache->user_parts[part_id];
+	struct ocf_lru_list *clean_list;
+	struct ocf_lru_list *dirty_list;
+
+	clean_list = &part->runtime->eviction.policy.lru.clean;
+	dirty_list = &part->runtime->eviction.policy.lru.dirty;
+
 	OCF_METADATA_EVICTION_LOCK();
-	remove_lru_list(cache, part_id, cline, 0);
+	remove_lru_list(cache, clean_list, cline);
-	add_lru_head(cache, part_id, cline, 1);
+	add_lru_head(cache, dirty_list, cline);
 	OCF_METADATA_EVICTION_UNLOCK();
 }
 

src/eviction/lru_structs.h

@@ -12,13 +12,15 @@ struct lru_eviction_policy_meta {
 	uint32_t next;
 } __attribute__((packed));
 
+struct ocf_lru_list {
+	uint32_t num_nodes;
+	uint32_t head;
+	uint32_t tail;
+};
+
 struct lru_eviction_policy {
-	int has_clean_nodes;
+	struct ocf_lru_list clean;
-	int has_dirty_nodes;
+	struct ocf_lru_list dirty;
-	uint32_t dirty_head;
-	uint32_t dirty_tail;
-	uint32_t clean_head;
-	uint32_t clean_tail;
 };
 
 #endif