github/ocf
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #456 from arutk/aalru

Browse Source 
Relax LRU list ordering to minimize list updates
This commit is contained in:
Robert Baldyga
2021-02-18 13:48:54 +01:00
committed by GitHub
parent 91cbeed611 9e98eec361
commit 75baec5aa5
7 changed files with 467 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
src/eviction/lru.c
View File

@@ -29,6 +29,7 @@ static void add_lru_head(ocf_cache_t cache,
ENV_BUG_ON(collision_index == end_marker);
node = &ocf_metadata_get_eviction_policy(cache, collision_index)->lru;
node->hot = false;
/* First node to be added/ */
if (!list->num_nodes) {
@@ -53,6 +54,10 @@ static void add_lru_head(ocf_cache_t cache,
node->next = curr_head_index;
node->prev = end_marker;
curr_head->prev = collision_index;
node->hot = true;
if (!curr_head->hot)
list->last_hot = collision_index;
++list->num_hot;
list->head = collision_index;
@@ -76,6 +81,9 @@ static void remove_lru_list(ocf_cache_t cache,
is_head = (list->head == collision_index);
is_tail = (list->tail == collision_index);
if (node->hot)
--list->num_hot;
/* Set prev and next (even if not existent) */
next_lru_node = node->next;
prev_lru_node = node->prev;
@@ -89,6 +97,8 @@ static void remove_lru_list(ocf_cache_t cache,
list->head = end_marker;
list->tail = end_marker;
list->last_hot = end_marker;
ENV_BUG_ON(list->num_hot != 0);
}
/* Case 2: else if this collision_index is LRU head, but not tail,
@@ -102,7 +112,13 @@ static void remove_lru_list(ocf_cache_t cache,
next_node = &ocf_metadata_get_eviction_policy(cache,
next_lru_node)->lru;
if (list->last_hot == collision_index) {
ENV_BUG_ON(list->num_hot != 0);
list->last_hot = end_marker;
}
list->head = next_lru_node;
node->next = end_marker;
next_node->prev = end_marker;
}
@@ -139,6 +155,11 @@ static void remove_lru_list(ocf_cache_t cache,
prev_node = &ocf_metadata_get_eviction_policy(cache,
prev_lru_node)->lru;
if (list->last_hot == collision_index) {
ENV_BUG_ON(list->num_hot == 0);
list->last_hot = prev_lru_node;
}
/* Update prev and next nodes */
prev_node->next = node->next;
next_node->prev = node->prev;
@@ -151,6 +172,53 @@ static void remove_lru_list(ocf_cache_t cache,
--list->num_nodes;
}
/* Increase / decrease number of hot elements to achieve target count.
* Asssumes that the list has hot element clustered together at the
* head of the list.
*/
static void balance_lru_list(ocf_cache_t cache,
struct ocf_lru_list *list)
{
unsigned target_hot_count = list->num_nodes / OCF_LRU_HOT_RATIO;
struct lru_eviction_policy_meta *node;
if (target_hot_count == list->num_hot)
return;
if (list->num_hot == 0) {
node = &ocf_metadata_get_eviction_policy(cache,
list->head)->lru;
list->last_hot = list->head;
list->num_hot = 1;
node->hot = 1;
return;
}
ENV_BUG_ON(list->last_hot == end_marker);
node = &ocf_metadata_get_eviction_policy(cache,
list->last_hot)->lru;
if (target_hot_count > list->num_hot) {
++list->num_hot;
list->last_hot = node->next;
node = &ocf_metadata_get_eviction_policy(cache,
node->next)->lru;
node->hot = true;
} else {
if (list->last_hot == list->head) {
node->hot = false;
list->num_hot = 0;
list->last_hot = end_marker;
} else {
ENV_BUG_ON(node->prev == end_marker);
node->hot = false;
--list->num_hot;
list->last_hot = node->prev;
}
}
}
/*-- End of LRU functions*/
void evp_lru_init_cline(ocf_cache_t cache, ocf_cache_line_t cline)
@@ -159,6 +227,7 @@ void evp_lru_init_cline(ocf_cache_t cache, ocf_cache_line_t cline)
node = &ocf_metadata_get_eviction_policy(cache, cline)->lru;
node->hot = false;
node->prev = end_marker;
node->next = end_marker;
}
@@ -188,6 +257,7 @@ void evp_lru_rm_cline(ocf_cache_t cache, ocf_cache_line_t cline)
list = evp_get_cline_list(cache, cline);
remove_lru_list(cache, list, cline);
balance_lru_list(cache, list);
}
static inline void lru_iter_init(struct ocf_lru_iter *iter, ocf_cache_t cache,
@@ -457,10 +527,21 @@ void evp_lru_hot_cline(ocf_cache_t cache, ocf_cache_line_t cline)
{
struct lru_eviction_policy_meta *node;
struct ocf_lru_list *list;
bool hot;
node = &ocf_metadata_get_eviction_policy(cache, cline)->lru;
OCF_METADATA_EVICTION_RD_LOCK(cline);
hot = node->hot;
OCF_METADATA_EVICTION_RD_UNLOCK(cline);
if (hot)
return;
list = evp_get_cline_list(cache, cline);
OCF_METADATA_EVICTION_WR_LOCK(cline);
if (node->next != end_marker ||
node->prev != end_marker ||
list->head == cline || list->tail == cline) {
@@ -469,6 +550,9 @@ void evp_lru_hot_cline(ocf_cache_t cache, ocf_cache_line_t cline)
/* Update LRU */
add_lru_head(cache, list, cline);
balance_lru_list(cache, list);
OCF_METADATA_EVICTION_WR_UNLOCK(cline);
}
static inline void _lru_init(struct ocf_lru_list *list)
@@ -476,6 +560,8 @@ static inline void _lru_init(struct ocf_lru_list *list)
list->num_nodes = 0;
list->head = end_marker;
list->tail = end_marker;
list->num_hot = 0;
list->last_hot = end_marker;
}
void evp_lru_init_evp(ocf_cache_t cache, struct ocf_user_part *part)
@@ -503,10 +589,12 @@ void evp_lru_clean_cline(ocf_cache_t cache, struct ocf_user_part *part,
clean_list = evp_lru_get_list(part, ev_list, true);
dirty_list = evp_lru_get_list(part, ev_list, false);
OCF_METADATA_EVICTION_LOCK(cline);
OCF_METADATA_EVICTION_WR_LOCK(cline);
remove_lru_list(cache, dirty_list, cline);
balance_lru_list(cache, dirty_list);
add_lru_head(cache, clean_list, cline);
OCF_METADATA_EVICTION_UNLOCK(cline);
balance_lru_list(cache, clean_list);
OCF_METADATA_EVICTION_WR_UNLOCK(cline);
}
void evp_lru_dirty_cline(ocf_cache_t cache, struct ocf_user_part *part,
@@ -519,9 +607,11 @@ void evp_lru_dirty_cline(ocf_cache_t cache, struct ocf_user_part *part,
clean_list = evp_lru_get_list(part, ev_list, true);
dirty_list = evp_lru_get_list(part, ev_list, false);
OCF_METADATA_EVICTION_LOCK(cline);
OCF_METADATA_EVICTION_WR_LOCK(cline);
remove_lru_list(cache, clean_list, cline);
balance_lru_list(cache, clean_list);
add_lru_head(cache, dirty_list, cline);
OCF_METADATA_EVICTION_UNLOCK(cline);
balance_lru_list(cache, dirty_list);
OCF_METADATA_EVICTION_WR_UNLOCK(cline);
}

6
src/eviction/lru_structs.h
View File

@@ -7,15 +7,17 @@
#define __EVICTION_LRU_STRUCTS_H__
struct lru_eviction_policy_meta {
/* LRU pointers 2*4=8 bytes */
uint32_t prev;
uint32_t next;
uint8_t hot;
} __attribute__((packed));
struct ocf_lru_list {
uint32_t num_nodes;
uint32_t head;
uint32_t tail;
uint32_t num_hot;
uint32_t last_hot;
};
struct lru_eviction_policy {
@@ -23,4 +25,6 @@ struct lru_eviction_policy {
struct ocf_lru_list dirty;
};
#define OCF_LRU_HOT_RATIO 2
#endif

10
src/eviction/ops.h
View File

@@ -36,9 +36,9 @@ static inline void ocf_eviction_purge_cache_line(
ENV_BUG_ON(type >= ocf_eviction_max);
if (likely(evict_policy_ops[type].rm_cline)) {
OCF_METADATA_EVICTION_LOCK(line);
OCF_METADATA_EVICTION_WR_LOCK(line);
evict_policy_ops[type].rm_cline(cache, line);
OCF_METADATA_EVICTION_UNLOCK(line);
OCF_METADATA_EVICTION_WR_UNLOCK(line);
}
}
@@ -83,9 +83,7 @@ static inline void ocf_eviction_set_hot_cache_line(
ENV_BUG_ON(type >= ocf_eviction_max);
if (likely(evict_policy_ops[type].hot_cline)) {
OCF_METADATA_EVICTION_LOCK(line);
evict_policy_ops[type].hot_cline(cache, line);
OCF_METADATA_EVICTION_UNLOCK(line);
}
}
@@ -97,9 +95,9 @@ static inline void ocf_eviction_initialize(struct ocf_cache *cache,
ENV_BUG_ON(type >= ocf_eviction_max);
if (likely(evict_policy_ops[type].init_evp)) {
OCF_METADATA_EVICTION_LOCK_ALL();
OCF_METADATA_EVICTION_WR_LOCK_ALL();
evict_policy_ops[type].init_evp(cache, part);
OCF_METADATA_EVICTION_UNLOCK_ALL();
OCF_METADATA_EVICTION_WR_UNLOCK_ALL();
}
}