Merge pull request #456 from arutk/aalru

Relax LRU list ordering to minimize list updates

src/concurrency/ocf_metadata_concurrency.c

@@ -14,11 +14,8 @@ int ocf_metadata_concurrency_init(struct ocf_metadata_lock *metadata_lock)
 	unsigned part_iter;
 	unsigned global_iter;
 
-	for (evp_iter = 0; evp_iter < OCF_NUM_EVICTION_LISTS; evp_iter++) {
+	for (evp_iter = 0; evp_iter < OCF_NUM_EVICTION_LISTS; evp_iter++)
-		err = env_spinlock_init(&metadata_lock->eviction[evp_iter]);
+		env_rwlock_init(&metadata_lock->eviction[evp_iter]);
-		if (err)
-			goto eviction_err;
-	}
 
 	env_rwlock_init(&metadata_lock->status);
 
@@ -47,9 +44,8 @@ global_err:
 
 	env_rwlock_destroy(&metadata_lock->status);
 
-eviction_err:
 	while (evp_iter--)
-		env_spinlock_destroy(&metadata_lock->eviction[evp_iter]);
+		env_rwlock_destroy(&metadata_lock->eviction[evp_iter]);
 
 	return err;
 }
@@ -62,7 +58,7 @@ void ocf_metadata_concurrency_deinit(struct ocf_metadata_lock *metadata_lock)
 		env_spinlock_destroy(&metadata_lock->partition[i]);
 
 	for (i = 0; i < OCF_NUM_EVICTION_LISTS; i++)
-		env_spinlock_destroy(&metadata_lock->eviction[i]);
+		env_rwlock_destroy(&metadata_lock->eviction[i]);
 
 	for (i = 0; i < OCF_NUM_GLOBAL_META_LOCKS; i++)
 		env_rwsem_destroy(&metadata_lock->global[i].sem);

src/concurrency/ocf_metadata_concurrency.h

@@ -28,49 +28,69 @@ int ocf_metadata_concurrency_attached_init(
 void ocf_metadata_concurrency_attached_deinit(
 		struct ocf_metadata_lock *metadata_lock);
 
-static inline void ocf_metadata_eviction_lock(
+static inline void ocf_metadata_eviction_wr_lock(
 		struct ocf_metadata_lock *metadata_lock, unsigned ev_list)
 {
-	env_spinlock_lock(&metadata_lock->eviction[ev_list]);
+	env_rwlock_write_lock(&metadata_lock->eviction[ev_list]);
 }
 
-static inline void ocf_metadata_eviction_unlock(
+static inline void ocf_metadata_eviction_wr_unlock(
 		struct ocf_metadata_lock *metadata_lock, unsigned ev_list)
 {
-	env_spinlock_unlock(&metadata_lock->eviction[ev_list]);
+	env_rwlock_write_unlock(&metadata_lock->eviction[ev_list]);
 }
 
-static inline void ocf_metadata_eviction_lock_all(
+static inline void ocf_metadata_eviction_rd_lock(
+		struct ocf_metadata_lock *metadata_lock, unsigned ev_list)
+{
+	env_rwlock_read_lock(&metadata_lock->eviction[ev_list]);
+}
+
+static inline void ocf_metadata_eviction_rd_unlock(
+		struct ocf_metadata_lock *metadata_lock, unsigned ev_list)
+{
+	env_rwlock_read_unlock(&metadata_lock->eviction[ev_list]);
+}
+static inline void ocf_metadata_eviction_wr_lock_all(
 		struct ocf_metadata_lock *metadata_lock)
 {
 	uint32_t i;
 
 	for (i = 0; i < OCF_NUM_EVICTION_LISTS; i++)
-		ocf_metadata_eviction_lock(metadata_lock, i);
+		ocf_metadata_eviction_wr_lock(metadata_lock, i);
 }
 
-static inline void ocf_metadata_eviction_unlock_all(
+static inline void ocf_metadata_eviction_wr_unlock_all(
 		struct ocf_metadata_lock *metadata_lock)
 {
 	uint32_t i;
 
 	for (i = 0; i < OCF_NUM_EVICTION_LISTS; i++)
-		ocf_metadata_eviction_unlock(metadata_lock, i);
+		ocf_metadata_eviction_wr_unlock(metadata_lock, i);
 }
 
-#define OCF_METADATA_EVICTION_LOCK(cline) \
+#define OCF_METADATA_EVICTION_WR_LOCK(cline) \
-		ocf_metadata_eviction_lock(&cache->metadata.lock, \
+		ocf_metadata_eviction_wr_lock(&cache->metadata.lock, \
 				cline % OCF_NUM_EVICTION_LISTS)
 
-#define OCF_METADATA_EVICTION_UNLOCK(cline) \
+#define OCF_METADATA_EVICTION_WR_UNLOCK(cline) \
-		ocf_metadata_eviction_unlock(&cache->metadata.lock, \
+		ocf_metadata_eviction_wr_unlock(&cache->metadata.lock, \
 				cline % OCF_NUM_EVICTION_LISTS)
 
-#define OCF_METADATA_EVICTION_LOCK_ALL() \
+#define OCF_METADATA_EVICTION_RD_LOCK(cline) \
-	ocf_metadata_eviction_lock_all(&cache->metadata.lock)
+		ocf_metadata_eviction_rd_lock(&cache->metadata.lock, \
+				cline % OCF_NUM_EVICTION_LISTS)
 
-#define OCF_METADATA_EVICTION_UNLOCK_ALL() \
+#define OCF_METADATA_EVICTION_RD_UNLOCK(cline) \
-	ocf_metadata_eviction_unlock_all(&cache->metadata.lock)
+		ocf_metadata_eviction_rd_unlock(&cache->metadata.lock, \
+				cline % OCF_NUM_EVICTION_LISTS)
+
+
+#define OCF_METADATA_EVICTION_WR_LOCK_ALL() \
+	ocf_metadata_eviction_wr_lock_all(&cache->metadata.lock)
+
+#define OCF_METADATA_EVICTION_WR_UNLOCK_ALL() \
+	ocf_metadata_eviction_wr_unlock_all(&cache->metadata.lock)
 
 static inline void ocf_metadata_partition_lock(
 		struct ocf_metadata_lock *metadata_lock,

src/eviction/lru.c

@@ -29,6 +29,7 @@ static void add_lru_head(ocf_cache_t cache,
 	ENV_BUG_ON(collision_index == end_marker);
 
 	node = &ocf_metadata_get_eviction_policy(cache, collision_index)->lru;
+	node->hot = false;
 
 	/* First node to be added/ */
 	if (!list->num_nodes)  {
@@ -53,6 +54,10 @@ static void add_lru_head(ocf_cache_t cache,
 		node->next = curr_head_index;
 		node->prev = end_marker;
 		curr_head->prev = collision_index;
+		node->hot = true;
+		if (!curr_head->hot)
+			list->last_hot = collision_index;
+		++list->num_hot;
 
 		list->head = collision_index;
 
@@ -76,6 +81,9 @@ static void remove_lru_list(ocf_cache_t cache,
 	is_head = (list->head == collision_index);
 	is_tail = (list->tail == collision_index);
 
+	if (node->hot)
+		--list->num_hot;
+
 	/* Set prev and next (even if not existent) */
 	next_lru_node = node->next;
 	prev_lru_node = node->prev;
@@ -89,6 +97,8 @@ static void remove_lru_list(ocf_cache_t cache,
 
 		list->head = end_marker;
 		list->tail = end_marker;
+		list->last_hot = end_marker;
+		ENV_BUG_ON(list->num_hot != 0);
 	}
 
 	/* Case 2: else if this collision_index is LRU head, but not tail,
@@ -102,7 +112,13 @@ static void remove_lru_list(ocf_cache_t cache,
 		next_node = &ocf_metadata_get_eviction_policy(cache,
 				next_lru_node)->lru;
 
+		if (list->last_hot == collision_index) {
+			ENV_BUG_ON(list->num_hot != 0);
+			list->last_hot = end_marker;
+		}
+
 		list->head = next_lru_node;
+
 		node->next = end_marker;
 		next_node->prev = end_marker;
 	}
@@ -139,6 +155,11 @@ static void remove_lru_list(ocf_cache_t cache,
 		prev_node = &ocf_metadata_get_eviction_policy(cache,
 				prev_lru_node)->lru;
 
+		if (list->last_hot == collision_index) {
+			ENV_BUG_ON(list->num_hot == 0);
+			list->last_hot = prev_lru_node;
+		}
+
 		/* Update prev and next nodes */
 		prev_node->next = node->next;
 		next_node->prev = node->prev;
@@ -151,6 +172,53 @@ static void remove_lru_list(ocf_cache_t cache,
 	--list->num_nodes;
 }
 
+/* Increase / decrease number of hot elements to achieve target count.
+ * Asssumes that the list has hot element clustered together at the
+ * head of the list.
+ */
+static void balance_lru_list(ocf_cache_t cache,
+		struct ocf_lru_list *list)
+{
+	unsigned target_hot_count = list->num_nodes / OCF_LRU_HOT_RATIO;
+	struct lru_eviction_policy_meta *node;
+
+	if (target_hot_count == list->num_hot)
+		return;
+
+	if (list->num_hot == 0) {
+		node = &ocf_metadata_get_eviction_policy(cache,
+				list->head)->lru;
+		list->last_hot = list->head;
+		list->num_hot = 1;
+		node->hot = 1;
+		return;
+	}
+
+	ENV_BUG_ON(list->last_hot == end_marker);
+	node = &ocf_metadata_get_eviction_policy(cache,
+			list->last_hot)->lru;
+
+	if (target_hot_count > list->num_hot) {
+		++list->num_hot;
+		list->last_hot = node->next;
+		node = &ocf_metadata_get_eviction_policy(cache,
+				node->next)->lru;
+		node->hot = true;
+	} else {
+		if (list->last_hot == list->head) {
+			node->hot = false;
+			list->num_hot = 0;
+			list->last_hot = end_marker;
+		} else {
+			ENV_BUG_ON(node->prev == end_marker);
+			node->hot = false;
+			--list->num_hot;
+			list->last_hot = node->prev;
+		}
+	}
+}
+
+
 /*-- End of LRU functions*/
 
 void evp_lru_init_cline(ocf_cache_t cache, ocf_cache_line_t cline)
@@ -159,6 +227,7 @@ void evp_lru_init_cline(ocf_cache_t cache, ocf_cache_line_t cline)
 
 	node = &ocf_metadata_get_eviction_policy(cache, cline)->lru;
 
+	node->hot = false;
 	node->prev = end_marker;
 	node->next = end_marker;
 }
@@ -188,6 +257,7 @@ void evp_lru_rm_cline(ocf_cache_t cache, ocf_cache_line_t cline)
 
 	list = evp_get_cline_list(cache, cline);
 	remove_lru_list(cache, list, cline);
+	balance_lru_list(cache, list);
 }
 
 static inline void lru_iter_init(struct ocf_lru_iter *iter, ocf_cache_t cache,
@@ -457,10 +527,21 @@ void evp_lru_hot_cline(ocf_cache_t cache, ocf_cache_line_t cline)
 {
 	struct lru_eviction_policy_meta *node;
 	struct ocf_lru_list *list;
+	bool hot;
 
 	node = &ocf_metadata_get_eviction_policy(cache, cline)->lru;
+
+	OCF_METADATA_EVICTION_RD_LOCK(cline);
+	hot = node->hot;
+	OCF_METADATA_EVICTION_RD_UNLOCK(cline);
+
+	if (hot)
+		return;
+
 	list = evp_get_cline_list(cache, cline);
 
+	OCF_METADATA_EVICTION_WR_LOCK(cline);
+
 	if (node->next != end_marker ||
 			node->prev != end_marker ||
 			list->head == cline || list->tail == cline) {
@@ -469,6 +550,9 @@ void evp_lru_hot_cline(ocf_cache_t cache, ocf_cache_line_t cline)
 
 	/* Update LRU */
 	add_lru_head(cache, list, cline);
+	balance_lru_list(cache, list);
+
+	OCF_METADATA_EVICTION_WR_UNLOCK(cline);
 }
 
 static inline void _lru_init(struct ocf_lru_list *list)
@@ -476,6 +560,8 @@ static inline void _lru_init(struct ocf_lru_list *list)
 	list->num_nodes = 0;
 	list->head = end_marker;
 	list->tail = end_marker;
+	list->num_hot = 0;
+	list->last_hot = end_marker;
 }
 
 void evp_lru_init_evp(ocf_cache_t cache, struct ocf_user_part *part)
@@ -503,10 +589,12 @@ void evp_lru_clean_cline(ocf_cache_t cache, struct ocf_user_part *part,
 	clean_list = evp_lru_get_list(part, ev_list, true);
 	dirty_list = evp_lru_get_list(part, ev_list, false);
 
-	OCF_METADATA_EVICTION_LOCK(cline);
+	OCF_METADATA_EVICTION_WR_LOCK(cline);
 	remove_lru_list(cache, dirty_list, cline);
+	balance_lru_list(cache, dirty_list);
 	add_lru_head(cache, clean_list, cline);
-	OCF_METADATA_EVICTION_UNLOCK(cline);
+	balance_lru_list(cache, clean_list);
+	OCF_METADATA_EVICTION_WR_UNLOCK(cline);
 }
 
 void evp_lru_dirty_cline(ocf_cache_t cache, struct ocf_user_part *part,
@@ -519,9 +607,11 @@ void evp_lru_dirty_cline(ocf_cache_t cache, struct ocf_user_part *part,
 	clean_list = evp_lru_get_list(part, ev_list, true);
 	dirty_list = evp_lru_get_list(part, ev_list, false);
 
-	OCF_METADATA_EVICTION_LOCK(cline);
+	OCF_METADATA_EVICTION_WR_LOCK(cline);
 	remove_lru_list(cache, clean_list, cline);
+	balance_lru_list(cache, clean_list);
 	add_lru_head(cache, dirty_list, cline);
-	OCF_METADATA_EVICTION_UNLOCK(cline);
+	balance_lru_list(cache, dirty_list);
+	OCF_METADATA_EVICTION_WR_UNLOCK(cline);
 }
 

src/eviction/lru_structs.h

@@ -7,15 +7,17 @@
 #define __EVICTION_LRU_STRUCTS_H__
 
 struct lru_eviction_policy_meta {
-	/* LRU pointers 2*4=8 bytes */
 	uint32_t prev;
 	uint32_t next;
+	uint8_t hot;
 } __attribute__((packed));
 
 struct ocf_lru_list {
 	uint32_t num_nodes;
 	uint32_t head;
 	uint32_t tail;
+	uint32_t num_hot;
+	uint32_t last_hot;
 };
 
 struct lru_eviction_policy {
@@ -23,4 +25,6 @@ struct lru_eviction_policy {
 	struct ocf_lru_list dirty;
 };
 
+#define OCF_LRU_HOT_RATIO 2
+
 #endif

src/eviction/ops.h

@@ -36,9 +36,9 @@ static inline void ocf_eviction_purge_cache_line(
 	ENV_BUG_ON(type >= ocf_eviction_max);
 
 	if (likely(evict_policy_ops[type].rm_cline)) {
-		OCF_METADATA_EVICTION_LOCK(line);
+		OCF_METADATA_EVICTION_WR_LOCK(line);
 		evict_policy_ops[type].rm_cline(cache, line);
-		OCF_METADATA_EVICTION_UNLOCK(line);
+		OCF_METADATA_EVICTION_WR_UNLOCK(line);
 	}
 }
 
@@ -83,9 +83,7 @@ static inline void ocf_eviction_set_hot_cache_line(
 	ENV_BUG_ON(type >= ocf_eviction_max);
 
 	if (likely(evict_policy_ops[type].hot_cline)) {
-		OCF_METADATA_EVICTION_LOCK(line);
 		evict_policy_ops[type].hot_cline(cache, line);
-		OCF_METADATA_EVICTION_UNLOCK(line);
 	}
 }
 
@@ -97,9 +95,9 @@ static inline void ocf_eviction_initialize(struct ocf_cache *cache,
 	ENV_BUG_ON(type >= ocf_eviction_max);
 
 	if (likely(evict_policy_ops[type].init_evp)) {
-		OCF_METADATA_EVICTION_LOCK_ALL();
+		OCF_METADATA_EVICTION_WR_LOCK_ALL();
 		evict_policy_ops[type].init_evp(cache, part);
-		OCF_METADATA_EVICTION_UNLOCK_ALL();
+		OCF_METADATA_EVICTION_WR_UNLOCK_ALL();
 	}
 }
 

src/metadata/metadata_structs.h

@@ -56,7 +56,7 @@ struct ocf_metadata_lock
 	struct ocf_metadata_global_lock global[OCF_NUM_GLOBAL_META_LOCKS];
 			/*!< global metadata lock (GML) */
 	env_rwlock status; /*!< Fast lock for status bits */
-	env_spinlock eviction[OCF_NUM_EVICTION_LISTS]; /*!< Fast lock for eviction policy */
+	env_rwlock eviction[OCF_NUM_EVICTION_LISTS]; /*!< Fast lock for eviction policy */
 	env_rwsem *hash; /*!< Hash bucket locks */
 	env_rwsem *collision_pages; /*!< Collision table page locks */
 	env_spinlock partition[OCF_IO_CLASS_MAX]; /* partition lock */

tests/unit/tests/eviction/lru.c/lru.c

@@ -0,0 +1,323 @@
+/*
+ * <tested_file_path>src/eviction/lru.c</tested_file_path>
+ * <tested_function>_lru_init</tested_function>
+ * <functions_to_leave>
+ * 	update_lru_head
+ * 	update_lru_tail
+ * 	update_lru_head_tail
+ *      _lru_init
+ * 	add_lru_head
+ * 	remove_lru_list
+ * 	balance_lru_list
+ * </functions_to_leave>
+ */
+
+#undef static
+
+#undef inline
+
+
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+#include "print_desc.h"
+
+#include "eviction.h"
+#include "lru.h"
+#include "ops.h"
+#include "../utils/utils_cleaner.h"
+#include "../utils/utils_cache_line.h"
+#include "../concurrency/ocf_concurrency.h"
+#include "../mngt/ocf_mngt_common.h"
+#include "../engine/engine_zero.h"
+#include "../ocf_request.h"
+
+#include "eviction/lru.c/lru_generated_wraps.c"
+
+#define META_COUNT 128
+
+static union eviction_policy_meta meta[META_COUNT];
+
+union eviction_policy_meta*
+__wrap_ocf_metadata_get_eviction_policy(ocf_cache_t cache, ocf_cache_line_t line)
+{
+	assert (line < META_COUNT);
+	return &meta[line];
+}
+
+static const unsigned end_marker = -1;
+
+static void _lru_init_test01(void **state)
+{
+	struct ocf_lru_list l;
+
+	print_test_description("test init\n");
+
+	_lru_init(&l, end_marker);
+
+	assert_int_equal(l.num_hot, 0);
+	assert_int_equal(l.num_nodes, 0);
+	assert_int_equal(l.head, end_marker);
+	assert_int_equal(l.tail, end_marker);
+	assert_int_equal(l.last_hot, end_marker);
+
+	assert_int_equal(1,1);
+}
+
+static void check_hot_elems(struct ocf_lru_list *l)
+{
+	unsigned i;
+	unsigned curr = l->head;
+
+	for (i = 0; i < l->num_hot; i++) {
+		assert_int_equal(meta[curr].lru.hot, 1);
+		curr = meta[curr].lru.next;
+	}
+	for (i = l->num_hot; i < l->num_nodes; i++) {
+		assert_int_equal(meta[curr].lru.hot, 0);
+		curr = meta[curr].lru.next;
+	}
+}
+
+static void _lru_init_test02(void **state)
+{
+	struct ocf_lru_list l;
+	unsigned i;
+
+	memset(meta, 0, sizeof(meta));
+
+	print_test_description("test add\n");
+
+	_lru_init(&l, end_marker);
+
+	for (i = 1; i <= 8; i++)
+	{
+		add_lru_head(NULL, &l, i, end_marker);
+		balance_lru_list(NULL, &l, end_marker);
+		assert_int_equal(l.num_hot, i / 2);
+		assert_int_equal(l.num_nodes, i);
+		assert_int_equal(l.head, i);
+		assert_int_equal(l.tail, 1);
+		assert_int_equal(l.last_hot, i < 2 ? end_marker :
+				i - i / 2 + 1);
+		check_hot_elems(&l);
+	}
+}
+
+static void _lru_init_test03(void **state)
+{
+	struct ocf_lru_list l;
+	unsigned i;
+
+	memset(meta, 0, sizeof(meta));
+
+	print_test_description("remove head\n");
+
+	_lru_init(&l, end_marker);
+
+	for (i = 1; i <= 8; i++) {
+		add_lru_head(NULL, &l, i, end_marker);
+		balance_lru_list(NULL, &l, end_marker);
+	}
+
+	for (i = 8; i >= 1; i--) {
+		assert_int_equal(l.num_hot, i / 2);
+		assert_int_equal(l.num_nodes, i);
+		assert_int_equal(l.head, i);
+		assert_int_equal(l.tail, 1);
+		assert_int_equal(l.last_hot, i < 2 ? end_marker :
+				i - i / 2 + 1);
+		check_hot_elems(&l);
+
+		remove_lru_list(NULL, &l, i, end_marker);
+		balance_lru_list(NULL, &l, end_marker);
+	}
+
+	assert_int_equal(l.num_hot, 0);
+	assert_int_equal(l.num_nodes, 0);
+	assert_int_equal(l.head, end_marker);
+	assert_int_equal(l.tail, end_marker);
+	assert_int_equal(l.last_hot, end_marker);
+}
+
+static void _lru_init_test04(void **state)
+{
+	struct ocf_lru_list l;
+	unsigned i;
+
+	memset(meta, 0, sizeof(meta));
+
+	print_test_description("remove tail\n");
+
+	_lru_init(&l, end_marker);
+
+	for (i = 1; i <= 8; i++) {
+		add_lru_head(NULL, &l, i, end_marker);
+		balance_lru_list(NULL, &l, end_marker);
+	}
+
+	for (i = 8; i >= 1; i--) {
+		assert_int_equal(l.num_hot, i / 2);
+		assert_int_equal(l.num_nodes, i);
+		assert_int_equal(l.head, 8);
+		assert_int_equal(l.tail, 9 - i);
+		assert_int_equal(l.last_hot, i < 2 ? end_marker :
+				8 - i / 2 + 1);
+		check_hot_elems(&l);
+
+		remove_lru_list(NULL, &l, 9 - i, end_marker);
+		balance_lru_list(NULL, &l, end_marker);
+	}
+
+	assert_int_equal(l.num_hot, 0);
+	assert_int_equal(l.num_nodes, 0);
+	assert_int_equal(l.head, end_marker);
+	assert_int_equal(l.tail, end_marker);
+	assert_int_equal(l.last_hot, end_marker);
+}
+
+static void _lru_init_test05(void **state)
+{
+	struct ocf_lru_list l;
+	unsigned i, j;
+	bool present[9];
+	unsigned count;
+
+	memset(meta, 0, sizeof(meta));
+
+	print_test_description("remove last hot\n");
+
+	_lru_init(&l, end_marker);
+
+	for (i = 1; i <= 8; i++) {
+		add_lru_head(NULL, &l, i, end_marker);
+		balance_lru_list(NULL, &l, end_marker);
+		present[i] = true;
+	}
+
+	for (i = 8; i >= 3; i--) {
+		assert_int_equal(l.num_hot, i / 2);
+		assert_int_equal(l.num_nodes, i);
+		assert_int_equal(l.head, 8);
+		assert_int_equal(l.tail, 1);
+
+		count = 0;
+		j = 8;
+		while (count < i / 2) {
+			if (present[j])
+				++count;
+			--j;
+		}
+
+		assert_int_equal(l.last_hot, j + 1);
+		check_hot_elems(&l);
+
+		present[l.last_hot] = false;
+		remove_lru_list(NULL, &l, l.last_hot, end_marker);
+		balance_lru_list(NULL, &l, end_marker);
+	}
+
+	assert_int_equal(l.num_hot, 1);
+	assert_int_equal(l.num_nodes, 2);
+	assert_int_equal(l.head, 2);
+	assert_int_equal(l.tail, 1);
+	assert_int_equal(l.last_hot, 2);
+}
+
+static void _lru_init_test06(void **state)
+{
+	struct ocf_lru_list l;
+	unsigned i;
+	unsigned count;
+
+	memset(meta, 0, sizeof(meta));
+
+	print_test_description("remove middle hot\n");
+
+	_lru_init(&l, end_marker);
+
+	for (i = 1; i <= 8; i++) {
+		add_lru_head(NULL, &l, i, end_marker);
+		balance_lru_list(NULL, &l, end_marker);
+	}
+
+	count = 8;
+
+	remove_lru_list(NULL, &l, 7, end_marker);
+	balance_lru_list(NULL, &l, end_marker);
+	--count;
+	assert_int_equal(l.num_hot, count / 2);
+	assert_int_equal(l.num_nodes, count);
+	assert_int_equal(l.head, 8);
+	assert_int_equal(l.tail, 1);
+	assert_int_equal(l.last_hot, 5);
+
+	remove_lru_list(NULL, &l, 6, end_marker);
+	balance_lru_list(NULL, &l, end_marker);
+	--count;
+	assert_int_equal(l.num_hot, count / 2);
+	assert_int_equal(l.num_nodes, count);
+	assert_int_equal(l.head, 8);
+	assert_int_equal(l.tail, 1);
+	assert_int_equal(l.last_hot, 4);
+
+	remove_lru_list(NULL, &l, 5, end_marker);
+	balance_lru_list(NULL, &l, end_marker);
+	--count;
+	assert_int_equal(l.num_hot, count / 2);
+	assert_int_equal(l.num_nodes, count);
+	assert_int_equal(l.head, 8);
+	assert_int_equal(l.tail, 1);
+	assert_int_equal(l.last_hot, 4);
+
+	remove_lru_list(NULL, &l, 4, end_marker);
+	balance_lru_list(NULL, &l, end_marker);
+	--count;
+	assert_int_equal(l.num_hot, count / 2);
+	assert_int_equal(l.num_nodes, count);
+	assert_int_equal(l.head, 8);
+	assert_int_equal(l.tail, 1);
+	assert_int_equal(l.last_hot, 3);
+
+	remove_lru_list(NULL, &l, 3, end_marker);
+	balance_lru_list(NULL, &l, end_marker);
+	--count;
+	assert_int_equal(l.num_hot, count / 2);
+	assert_int_equal(l.num_nodes, count);
+	assert_int_equal(l.head, 8);
+	assert_int_equal(l.tail, 1);
+	assert_int_equal(l.last_hot, 8);
+
+	remove_lru_list(NULL, &l, 8, end_marker);
+	balance_lru_list(NULL, &l, end_marker);
+	--count;
+	assert_int_equal(l.num_hot, count / 2);
+	assert_int_equal(l.num_nodes, count);
+	assert_int_equal(l.head, 2);
+	assert_int_equal(l.tail, 1);
+	assert_int_equal(l.last_hot, 2);
+
+	remove_lru_list(NULL, &l, 2, end_marker);
+	balance_lru_list(NULL, &l, end_marker);
+	--count;
+	assert_int_equal(l.num_hot, count / 2);
+	assert_int_equal(l.num_nodes, count);
+	assert_int_equal(l.head, 1);
+	assert_int_equal(l.tail, 1);
+	assert_int_equal(l.last_hot, end_marker);
+}
+
+int main(void)
+{
+	const struct CMUnitTest tests[] = {
+		cmocka_unit_test(_lru_init_test01),
+		cmocka_unit_test(_lru_init_test02),
+		cmocka_unit_test(_lru_init_test03),
+		cmocka_unit_test(_lru_init_test04),
+		cmocka_unit_test(_lru_init_test05),
+		cmocka_unit_test(_lru_init_test06)
+	};
+
+	return cmocka_run_group_tests(tests, NULL, NULL);
+}