From b4daac11c2ce3124c9e1a8bae763b78e3ec87f87 Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Tue, 21 Jul 2020 12:55:01 +0200 Subject: [PATCH 1/5] Track hot items on LRU list Top 50% least recently used cachelines are not promoted to list head upon access. Only after cacheline drops to bottom 50% it is considered as a candidate to promote to list head. The purpose of this change is to reduce overhead of LRU list maintanance for hot cachelines. Signed-off-by: Adam Rutkowski --- src/eviction/lru.c | 81 +++++++++++++++++++++++++++++++++++++- src/eviction/lru_structs.h | 6 ++- 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/src/eviction/lru.c b/src/eviction/lru.c index 96ef568..fa6cbba 100644 --- a/src/eviction/lru.c +++ b/src/eviction/lru.c @@ -29,6 +29,7 @@ static void add_lru_head(ocf_cache_t cache, ENV_BUG_ON(collision_index == end_marker); node = &ocf_metadata_get_eviction_policy(cache, collision_index)->lru; + node->hot = false; /* First node to be added/ */ if (!list->num_nodes) { @@ -53,6 +54,10 @@ static void add_lru_head(ocf_cache_t cache, node->next = curr_head_index; node->prev = end_marker; curr_head->prev = collision_index; + node->hot = true; + if (!curr_head->hot) + list->last_hot = collision_index; + ++list->num_hot; list->head = collision_index; @@ -76,6 +81,9 @@ static void remove_lru_list(ocf_cache_t cache, is_head = (list->head == collision_index); is_tail = (list->tail == collision_index); + if (node->hot) + --list->num_hot; + /* Set prev and next (even if not existent) */ next_lru_node = node->next; prev_lru_node = node->prev; @@ -89,6 +97,8 @@ static void remove_lru_list(ocf_cache_t cache, list->head = end_marker; list->tail = end_marker; + list->last_hot = end_marker; + ENV_BUG_ON(list->num_hot != 0); } /* Case 2: else if this collision_index is LRU head, but not tail, @@ -102,7 +112,13 @@ static void remove_lru_list(ocf_cache_t cache, next_node = &ocf_metadata_get_eviction_policy(cache, next_lru_node)->lru; + if (list->last_hot == collision_index) { + ENV_BUG_ON(list->num_hot != 0); + list->last_hot = end_marker; + } + list->head = next_lru_node; + node->next = end_marker; next_node->prev = end_marker; } @@ -139,6 +155,11 @@ static void remove_lru_list(ocf_cache_t cache, prev_node = &ocf_metadata_get_eviction_policy(cache, prev_lru_node)->lru; + if (list->last_hot == collision_index) { + ENV_BUG_ON(list->num_hot == 0); + list->last_hot = prev_lru_node; + } + /* Update prev and next nodes */ prev_node->next = node->next; next_node->prev = node->prev; @@ -151,6 +172,53 @@ static void remove_lru_list(ocf_cache_t cache, --list->num_nodes; } +/* Increase / decrease number of hot elements to achieve target count. + * Asssumes that the list has hot element clustered together at the + * head of the list. + */ +static void balance_lru_list(ocf_cache_t cache, + struct ocf_lru_list *list) +{ + unsigned target_hot_count = list->num_nodes / OCF_LRU_HOT_RATIO; + struct lru_eviction_policy_meta *node; + + if (target_hot_count == list->num_hot) + return; + + if (list->num_hot == 0) { + node = &ocf_metadata_get_eviction_policy(cache, + list->head)->lru; + list->last_hot = list->head; + list->num_hot = 1; + node->hot = 1; + return; + } + + ENV_BUG_ON(list->last_hot == end_marker); + node = &ocf_metadata_get_eviction_policy(cache, + list->last_hot)->lru; + + if (target_hot_count > list->num_hot) { + ++list->num_hot; + list->last_hot = node->next; + node = &ocf_metadata_get_eviction_policy(cache, + node->next)->lru; + node->hot = true; + } else { + if (list->last_hot == list->head) { + node->hot = false; + list->num_hot = 0; + list->last_hot = end_marker; + } else { + ENV_BUG_ON(node->prev == end_marker); + node->hot = false; + --list->num_hot; + list->last_hot = node->prev; + } + } +} + + /*-- End of LRU functions*/ void evp_lru_init_cline(ocf_cache_t cache, ocf_cache_line_t cline) @@ -159,6 +227,7 @@ void evp_lru_init_cline(ocf_cache_t cache, ocf_cache_line_t cline) node = &ocf_metadata_get_eviction_policy(cache, cline)->lru; + node->hot = false; node->prev = end_marker; node->next = end_marker; } @@ -188,6 +257,7 @@ void evp_lru_rm_cline(ocf_cache_t cache, ocf_cache_line_t cline) list = evp_get_cline_list(cache, cline); remove_lru_list(cache, list, cline); + balance_lru_list(cache, list); } static inline void lru_iter_init(struct ocf_lru_iter *iter, ocf_cache_t cache, @@ -459,8 +529,10 @@ void evp_lru_hot_cline(ocf_cache_t cache, ocf_cache_line_t cline) struct ocf_lru_list *list; node = &ocf_metadata_get_eviction_policy(cache, cline)->lru; - list = evp_get_cline_list(cache, cline); + if (node->hot) + return; + list = evp_get_cline_list(cache, cline); if (node->next != end_marker || node->prev != end_marker || list->head == cline || list->tail == cline) { @@ -469,6 +541,7 @@ void evp_lru_hot_cline(ocf_cache_t cache, ocf_cache_line_t cline) /* Update LRU */ add_lru_head(cache, list, cline); + balance_lru_list(cache, list); } static inline void _lru_init(struct ocf_lru_list *list) @@ -476,6 +549,8 @@ static inline void _lru_init(struct ocf_lru_list *list) list->num_nodes = 0; list->head = end_marker; list->tail = end_marker; + list->num_hot = 0; + list->last_hot = end_marker; } void evp_lru_init_evp(ocf_cache_t cache, struct ocf_user_part *part) @@ -505,7 +580,9 @@ void evp_lru_clean_cline(ocf_cache_t cache, struct ocf_user_part *part, OCF_METADATA_EVICTION_LOCK(cline); remove_lru_list(cache, dirty_list, cline); + balance_lru_list(cache, dirty_list); add_lru_head(cache, clean_list, cline); + balance_lru_list(cache, clean_list); OCF_METADATA_EVICTION_UNLOCK(cline); } @@ -521,7 +598,9 @@ void evp_lru_dirty_cline(ocf_cache_t cache, struct ocf_user_part *part, OCF_METADATA_EVICTION_LOCK(cline); remove_lru_list(cache, clean_list, cline); + balance_lru_list(cache, clean_list); add_lru_head(cache, dirty_list, cline); + balance_lru_list(cache, dirty_list); OCF_METADATA_EVICTION_UNLOCK(cline); } diff --git a/src/eviction/lru_structs.h b/src/eviction/lru_structs.h index 9c2b529..22aaedb 100644 --- a/src/eviction/lru_structs.h +++ b/src/eviction/lru_structs.h @@ -7,15 +7,17 @@ #define __EVICTION_LRU_STRUCTS_H__ struct lru_eviction_policy_meta { - /* LRU pointers 2*4=8 bytes */ uint32_t prev; uint32_t next; + uint8_t hot; } __attribute__((packed)); struct ocf_lru_list { uint32_t num_nodes; uint32_t head; uint32_t tail; + uint32_t num_hot; + uint32_t last_hot; }; struct lru_eviction_policy { @@ -23,4 +25,6 @@ struct lru_eviction_policy { struct ocf_lru_list dirty; }; +#define OCF_LRU_HOT_RATIO 2 + #endif From 1908707a3d7fc308def4b5747bffb1db51b7a2a1 Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Tue, 21 Jul 2020 13:00:11 +0200 Subject: [PATCH 2/5] LRU list unit tests Signed-off-by: Adam Rutkowski --- tests/unit/tests/eviction/lru.c/lru.c | 323 ++++++++++++++++++++++++++ 1 file changed, 323 insertions(+) create mode 100644 tests/unit/tests/eviction/lru.c/lru.c diff --git a/tests/unit/tests/eviction/lru.c/lru.c b/tests/unit/tests/eviction/lru.c/lru.c new file mode 100644 index 0000000..f3d43ec --- /dev/null +++ b/tests/unit/tests/eviction/lru.c/lru.c @@ -0,0 +1,323 @@ +/* + * src/eviction/lru.c + * _lru_init + * + * update_lru_head + * update_lru_tail + * update_lru_head_tail + * _lru_init + * add_lru_head + * remove_lru_list + * balance_lru_list + * + */ + +#undef static + +#undef inline + + +#include +#include +#include +#include +#include "print_desc.h" + +#include "eviction.h" +#include "lru.h" +#include "ops.h" +#include "../utils/utils_cleaner.h" +#include "../utils/utils_cache_line.h" +#include "../concurrency/ocf_concurrency.h" +#include "../mngt/ocf_mngt_common.h" +#include "../engine/engine_zero.h" +#include "../ocf_request.h" + +#include "eviction/lru.c/lru_generated_wraps.c" + +#define META_COUNT 128 + +static union eviction_policy_meta meta[META_COUNT]; + +union eviction_policy_meta* +__wrap_ocf_metadata_get_eviction_policy(ocf_cache_t cache, ocf_cache_line_t line) +{ + assert (line < META_COUNT); + return &meta[line]; +} + +static const unsigned end_marker = -1; + +static void _lru_init_test01(void **state) +{ + struct ocf_lru_list l; + + print_test_description("test init\n"); + + _lru_init(&l, end_marker); + + assert_int_equal(l.num_hot, 0); + assert_int_equal(l.num_nodes, 0); + assert_int_equal(l.head, end_marker); + assert_int_equal(l.tail, end_marker); + assert_int_equal(l.last_hot, end_marker); + + assert_int_equal(1,1); +} + +static void check_hot_elems(struct ocf_lru_list *l) +{ + unsigned i; + unsigned curr = l->head; + + for (i = 0; i < l->num_hot; i++) { + assert_int_equal(meta[curr].lru.hot, 1); + curr = meta[curr].lru.next; + } + for (i = l->num_hot; i < l->num_nodes; i++) { + assert_int_equal(meta[curr].lru.hot, 0); + curr = meta[curr].lru.next; + } +} + +static void _lru_init_test02(void **state) +{ + struct ocf_lru_list l; + unsigned i; + + memset(meta, 0, sizeof(meta)); + + print_test_description("test add\n"); + + _lru_init(&l, end_marker); + + for (i = 1; i <= 8; i++) + { + add_lru_head(NULL, &l, i, end_marker); + balance_lru_list(NULL, &l, end_marker); + assert_int_equal(l.num_hot, i / 2); + assert_int_equal(l.num_nodes, i); + assert_int_equal(l.head, i); + assert_int_equal(l.tail, 1); + assert_int_equal(l.last_hot, i < 2 ? end_marker : + i - i / 2 + 1); + check_hot_elems(&l); + } +} + +static void _lru_init_test03(void **state) +{ + struct ocf_lru_list l; + unsigned i; + + memset(meta, 0, sizeof(meta)); + + print_test_description("remove head\n"); + + _lru_init(&l, end_marker); + + for (i = 1; i <= 8; i++) { + add_lru_head(NULL, &l, i, end_marker); + balance_lru_list(NULL, &l, end_marker); + } + + for (i = 8; i >= 1; i--) { + assert_int_equal(l.num_hot, i / 2); + assert_int_equal(l.num_nodes, i); + assert_int_equal(l.head, i); + assert_int_equal(l.tail, 1); + assert_int_equal(l.last_hot, i < 2 ? end_marker : + i - i / 2 + 1); + check_hot_elems(&l); + + remove_lru_list(NULL, &l, i, end_marker); + balance_lru_list(NULL, &l, end_marker); + } + + assert_int_equal(l.num_hot, 0); + assert_int_equal(l.num_nodes, 0); + assert_int_equal(l.head, end_marker); + assert_int_equal(l.tail, end_marker); + assert_int_equal(l.last_hot, end_marker); +} + +static void _lru_init_test04(void **state) +{ + struct ocf_lru_list l; + unsigned i; + + memset(meta, 0, sizeof(meta)); + + print_test_description("remove tail\n"); + + _lru_init(&l, end_marker); + + for (i = 1; i <= 8; i++) { + add_lru_head(NULL, &l, i, end_marker); + balance_lru_list(NULL, &l, end_marker); + } + + for (i = 8; i >= 1; i--) { + assert_int_equal(l.num_hot, i / 2); + assert_int_equal(l.num_nodes, i); + assert_int_equal(l.head, 8); + assert_int_equal(l.tail, 9 - i); + assert_int_equal(l.last_hot, i < 2 ? end_marker : + 8 - i / 2 + 1); + check_hot_elems(&l); + + remove_lru_list(NULL, &l, 9 - i, end_marker); + balance_lru_list(NULL, &l, end_marker); + } + + assert_int_equal(l.num_hot, 0); + assert_int_equal(l.num_nodes, 0); + assert_int_equal(l.head, end_marker); + assert_int_equal(l.tail, end_marker); + assert_int_equal(l.last_hot, end_marker); +} + +static void _lru_init_test05(void **state) +{ + struct ocf_lru_list l; + unsigned i, j; + bool present[9]; + unsigned count; + + memset(meta, 0, sizeof(meta)); + + print_test_description("remove last hot\n"); + + _lru_init(&l, end_marker); + + for (i = 1; i <= 8; i++) { + add_lru_head(NULL, &l, i, end_marker); + balance_lru_list(NULL, &l, end_marker); + present[i] = true; + } + + for (i = 8; i >= 3; i--) { + assert_int_equal(l.num_hot, i / 2); + assert_int_equal(l.num_nodes, i); + assert_int_equal(l.head, 8); + assert_int_equal(l.tail, 1); + + count = 0; + j = 8; + while (count < i / 2) { + if (present[j]) + ++count; + --j; + } + + assert_int_equal(l.last_hot, j + 1); + check_hot_elems(&l); + + present[l.last_hot] = false; + remove_lru_list(NULL, &l, l.last_hot, end_marker); + balance_lru_list(NULL, &l, end_marker); + } + + assert_int_equal(l.num_hot, 1); + assert_int_equal(l.num_nodes, 2); + assert_int_equal(l.head, 2); + assert_int_equal(l.tail, 1); + assert_int_equal(l.last_hot, 2); +} + +static void _lru_init_test06(void **state) +{ + struct ocf_lru_list l; + unsigned i; + unsigned count; + + memset(meta, 0, sizeof(meta)); + + print_test_description("remove middle hot\n"); + + _lru_init(&l, end_marker); + + for (i = 1; i <= 8; i++) { + add_lru_head(NULL, &l, i, end_marker); + balance_lru_list(NULL, &l, end_marker); + } + + count = 8; + + remove_lru_list(NULL, &l, 7, end_marker); + balance_lru_list(NULL, &l, end_marker); + --count; + assert_int_equal(l.num_hot, count / 2); + assert_int_equal(l.num_nodes, count); + assert_int_equal(l.head, 8); + assert_int_equal(l.tail, 1); + assert_int_equal(l.last_hot, 5); + + remove_lru_list(NULL, &l, 6, end_marker); + balance_lru_list(NULL, &l, end_marker); + --count; + assert_int_equal(l.num_hot, count / 2); + assert_int_equal(l.num_nodes, count); + assert_int_equal(l.head, 8); + assert_int_equal(l.tail, 1); + assert_int_equal(l.last_hot, 4); + + remove_lru_list(NULL, &l, 5, end_marker); + balance_lru_list(NULL, &l, end_marker); + --count; + assert_int_equal(l.num_hot, count / 2); + assert_int_equal(l.num_nodes, count); + assert_int_equal(l.head, 8); + assert_int_equal(l.tail, 1); + assert_int_equal(l.last_hot, 4); + + remove_lru_list(NULL, &l, 4, end_marker); + balance_lru_list(NULL, &l, end_marker); + --count; + assert_int_equal(l.num_hot, count / 2); + assert_int_equal(l.num_nodes, count); + assert_int_equal(l.head, 8); + assert_int_equal(l.tail, 1); + assert_int_equal(l.last_hot, 3); + + remove_lru_list(NULL, &l, 3, end_marker); + balance_lru_list(NULL, &l, end_marker); + --count; + assert_int_equal(l.num_hot, count / 2); + assert_int_equal(l.num_nodes, count); + assert_int_equal(l.head, 8); + assert_int_equal(l.tail, 1); + assert_int_equal(l.last_hot, 8); + + remove_lru_list(NULL, &l, 8, end_marker); + balance_lru_list(NULL, &l, end_marker); + --count; + assert_int_equal(l.num_hot, count / 2); + assert_int_equal(l.num_nodes, count); + assert_int_equal(l.head, 2); + assert_int_equal(l.tail, 1); + assert_int_equal(l.last_hot, 2); + + remove_lru_list(NULL, &l, 2, end_marker); + balance_lru_list(NULL, &l, end_marker); + --count; + assert_int_equal(l.num_hot, count / 2); + assert_int_equal(l.num_nodes, count); + assert_int_equal(l.head, 1); + assert_int_equal(l.tail, 1); + assert_int_equal(l.last_hot, end_marker); +} + +int main(void) +{ + const struct CMUnitTest tests[] = { + cmocka_unit_test(_lru_init_test01), + cmocka_unit_test(_lru_init_test02), + cmocka_unit_test(_lru_init_test03), + cmocka_unit_test(_lru_init_test04), + cmocka_unit_test(_lru_init_test05), + cmocka_unit_test(_lru_init_test06) + }; + + return cmocka_run_group_tests(tests, NULL, NULL); +} From 9690f13bef8faae287062daa860fb4c191690457 Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Fri, 5 Feb 2021 16:05:20 -0600 Subject: [PATCH 3/5] Change eviction spin lock to RW lock Signed-off-by: Adam Rutkowski --- src/concurrency/ocf_metadata_concurrency.c | 12 +++----- src/concurrency/ocf_metadata_concurrency.h | 32 +++++++++++----------- src/eviction/lru.c | 8 +++--- src/eviction/ops.h | 12 ++++---- src/metadata/metadata_structs.h | 2 +- 5 files changed, 31 insertions(+), 35 deletions(-) diff --git a/src/concurrency/ocf_metadata_concurrency.c b/src/concurrency/ocf_metadata_concurrency.c index ef0a737..f2d7b90 100644 --- a/src/concurrency/ocf_metadata_concurrency.c +++ b/src/concurrency/ocf_metadata_concurrency.c @@ -12,11 +12,8 @@ int ocf_metadata_concurrency_init(struct ocf_metadata_lock *metadata_lock) unsigned evp_iter; unsigned part_iter; - for (evp_iter = 0; evp_iter < OCF_NUM_EVICTION_LISTS; evp_iter++) { - err = env_spinlock_init(&metadata_lock->eviction[evp_iter]); - if (err) - goto eviction_err; - } + for (evp_iter = 0; evp_iter < OCF_NUM_EVICTION_LISTS; evp_iter++) + env_rwlock_init(&metadata_lock->eviction[evp_iter]); env_rwlock_init(&metadata_lock->status); @@ -38,9 +35,8 @@ spinlocks_err: rwsem_err: env_rwlock_destroy(&metadata_lock->status); -eviction_err: while (evp_iter--) - env_spinlock_destroy(&metadata_lock->eviction[evp_iter]); + env_rwlock_destroy(&metadata_lock->eviction[evp_iter]); return err; } @@ -53,7 +49,7 @@ void ocf_metadata_concurrency_deinit(struct ocf_metadata_lock *metadata_lock) env_spinlock_destroy(&metadata_lock->partition[i]); for (i = 0; i < OCF_NUM_EVICTION_LISTS; i++) - env_spinlock_destroy(&metadata_lock->eviction[i]); + env_rwlock_destroy(&metadata_lock->eviction[i]); env_rwlock_destroy(&metadata_lock->status); env_rwsem_destroy(&metadata_lock->global); diff --git a/src/concurrency/ocf_metadata_concurrency.h b/src/concurrency/ocf_metadata_concurrency.h index b981bca..172e591 100644 --- a/src/concurrency/ocf_metadata_concurrency.h +++ b/src/concurrency/ocf_metadata_concurrency.h @@ -22,49 +22,49 @@ int ocf_metadata_concurrency_attached_init( void ocf_metadata_concurrency_attached_deinit( struct ocf_metadata_lock *metadata_lock); -static inline void ocf_metadata_eviction_lock( +static inline void ocf_metadata_eviction_wr_lock( struct ocf_metadata_lock *metadata_lock, unsigned ev_list) { - env_spinlock_lock(&metadata_lock->eviction[ev_list]); + env_rwlock_write_lock(&metadata_lock->eviction[ev_list]); } -static inline void ocf_metadata_eviction_unlock( +static inline void ocf_metadata_eviction_wr_unlock( struct ocf_metadata_lock *metadata_lock, unsigned ev_list) { - env_spinlock_unlock(&metadata_lock->eviction[ev_list]); + env_rwlock_write_unlock(&metadata_lock->eviction[ev_list]); } -static inline void ocf_metadata_eviction_lock_all( +static inline void ocf_metadata_eviction_wr_lock_all( struct ocf_metadata_lock *metadata_lock) { uint32_t i; for (i = 0; i < OCF_NUM_EVICTION_LISTS; i++) - ocf_metadata_eviction_lock(metadata_lock, i); + ocf_metadata_eviction_wr_lock(metadata_lock, i); } -static inline void ocf_metadata_eviction_unlock_all( +static inline void ocf_metadata_eviction_wr_unlock_all( struct ocf_metadata_lock *metadata_lock) { uint32_t i; for (i = 0; i < OCF_NUM_EVICTION_LISTS; i++) - ocf_metadata_eviction_unlock(metadata_lock, i); + ocf_metadata_eviction_wr_unlock(metadata_lock, i); } -#define OCF_METADATA_EVICTION_LOCK(cline) \ - ocf_metadata_eviction_lock(&cache->metadata.lock, \ +#define OCF_METADATA_EVICTION_WR_LOCK(cline) \ + ocf_metadata_eviction_wr_lock(&cache->metadata.lock, \ cline % OCF_NUM_EVICTION_LISTS) -#define OCF_METADATA_EVICTION_UNLOCK(cline) \ - ocf_metadata_eviction_unlock(&cache->metadata.lock, \ +#define OCF_METADATA_EVICTION_WR_UNLOCK(cline) \ + ocf_metadata_eviction_wr_unlock(&cache->metadata.lock, \ cline % OCF_NUM_EVICTION_LISTS) -#define OCF_METADATA_EVICTION_LOCK_ALL() \ - ocf_metadata_eviction_lock_all(&cache->metadata.lock) +#define OCF_METADATA_EVICTION_WR_LOCK_ALL() \ + ocf_metadata_eviction_wr_lock_all(&cache->metadata.lock) -#define OCF_METADATA_EVICTION_UNLOCK_ALL() \ - ocf_metadata_eviction_unlock_all(&cache->metadata.lock) +#define OCF_METADATA_EVICTION_WR_UNLOCK_ALL() \ + ocf_metadata_eviction_wr_unlock_all(&cache->metadata.lock) static inline void ocf_metadata_partition_lock( struct ocf_metadata_lock *metadata_lock, diff --git a/src/eviction/lru.c b/src/eviction/lru.c index fa6cbba..a8f938b 100644 --- a/src/eviction/lru.c +++ b/src/eviction/lru.c @@ -578,12 +578,12 @@ void evp_lru_clean_cline(ocf_cache_t cache, struct ocf_user_part *part, clean_list = evp_lru_get_list(part, ev_list, true); dirty_list = evp_lru_get_list(part, ev_list, false); - OCF_METADATA_EVICTION_LOCK(cline); + OCF_METADATA_EVICTION_WR_LOCK(cline); remove_lru_list(cache, dirty_list, cline); balance_lru_list(cache, dirty_list); add_lru_head(cache, clean_list, cline); balance_lru_list(cache, clean_list); - OCF_METADATA_EVICTION_UNLOCK(cline); + OCF_METADATA_EVICTION_WR_UNLOCK(cline); } void evp_lru_dirty_cline(ocf_cache_t cache, struct ocf_user_part *part, @@ -596,11 +596,11 @@ void evp_lru_dirty_cline(ocf_cache_t cache, struct ocf_user_part *part, clean_list = evp_lru_get_list(part, ev_list, true); dirty_list = evp_lru_get_list(part, ev_list, false); - OCF_METADATA_EVICTION_LOCK(cline); + OCF_METADATA_EVICTION_WR_LOCK(cline); remove_lru_list(cache, clean_list, cline); balance_lru_list(cache, clean_list); add_lru_head(cache, dirty_list, cline); balance_lru_list(cache, dirty_list); - OCF_METADATA_EVICTION_UNLOCK(cline); + OCF_METADATA_EVICTION_WR_UNLOCK(cline); } diff --git a/src/eviction/ops.h b/src/eviction/ops.h index 2952171..56475b1 100644 --- a/src/eviction/ops.h +++ b/src/eviction/ops.h @@ -36,9 +36,9 @@ static inline void ocf_eviction_purge_cache_line( ENV_BUG_ON(type >= ocf_eviction_max); if (likely(evict_policy_ops[type].rm_cline)) { - OCF_METADATA_EVICTION_LOCK(line); + OCF_METADATA_EVICTION_WR_LOCK(line); evict_policy_ops[type].rm_cline(cache, line); - OCF_METADATA_EVICTION_UNLOCK(line); + OCF_METADATA_EVICTION_WR_UNLOCK(line); } } @@ -83,9 +83,9 @@ static inline void ocf_eviction_set_hot_cache_line( ENV_BUG_ON(type >= ocf_eviction_max); if (likely(evict_policy_ops[type].hot_cline)) { - OCF_METADATA_EVICTION_LOCK(line); + OCF_METADATA_EVICTION_WR_LOCK(line); evict_policy_ops[type].hot_cline(cache, line); - OCF_METADATA_EVICTION_UNLOCK(line); + OCF_METADATA_EVICTION_WR_UNLOCK(line); } } @@ -97,9 +97,9 @@ static inline void ocf_eviction_initialize(struct ocf_cache *cache, ENV_BUG_ON(type >= ocf_eviction_max); if (likely(evict_policy_ops[type].init_evp)) { - OCF_METADATA_EVICTION_LOCK_ALL(); + OCF_METADATA_EVICTION_WR_LOCK_ALL(); evict_policy_ops[type].init_evp(cache, part); - OCF_METADATA_EVICTION_UNLOCK_ALL(); + OCF_METADATA_EVICTION_WR_UNLOCK_ALL(); } } diff --git a/src/metadata/metadata_structs.h b/src/metadata/metadata_structs.h index 2efdc92..c6c5577 100644 --- a/src/metadata/metadata_structs.h +++ b/src/metadata/metadata_structs.h @@ -47,7 +47,7 @@ struct ocf_metadata_lock { env_rwsem global; /*!< global metadata lock (GML) */ env_rwlock status; /*!< Fast lock for status bits */ - env_spinlock eviction[OCF_NUM_EVICTION_LISTS]; /*!< Fast lock for eviction policy */ + env_rwlock eviction[OCF_NUM_EVICTION_LISTS]; /*!< Fast lock for eviction policy */ env_rwsem *hash; /*!< Hash bucket locks */ env_rwsem *collision_pages; /*!< Collision table page locks */ env_spinlock partition[OCF_IO_CLASS_MAX]; /* partition lock */ From c04bfa3962c132cac3a21e000eac3f394e60c628 Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Fri, 5 Feb 2021 16:06:38 -0600 Subject: [PATCH 4/5] Add macros to read lock eviction list Signed-off-by: Adam Rutkowski --- src/concurrency/ocf_metadata_concurrency.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/concurrency/ocf_metadata_concurrency.h b/src/concurrency/ocf_metadata_concurrency.h index 172e591..ab6085c 100644 --- a/src/concurrency/ocf_metadata_concurrency.h +++ b/src/concurrency/ocf_metadata_concurrency.h @@ -34,6 +34,17 @@ static inline void ocf_metadata_eviction_wr_unlock( env_rwlock_write_unlock(&metadata_lock->eviction[ev_list]); } +static inline void ocf_metadata_eviction_rd_lock( + struct ocf_metadata_lock *metadata_lock, unsigned ev_list) +{ + env_rwlock_read_lock(&metadata_lock->eviction[ev_list]); +} + +static inline void ocf_metadata_eviction_rd_unlock( + struct ocf_metadata_lock *metadata_lock, unsigned ev_list) +{ + env_rwlock_read_unlock(&metadata_lock->eviction[ev_list]); +} static inline void ocf_metadata_eviction_wr_lock_all( struct ocf_metadata_lock *metadata_lock) { @@ -60,6 +71,15 @@ static inline void ocf_metadata_eviction_wr_unlock_all( ocf_metadata_eviction_wr_unlock(&cache->metadata.lock, \ cline % OCF_NUM_EVICTION_LISTS) +#define OCF_METADATA_EVICTION_RD_LOCK(cline) \ + ocf_metadata_eviction_rd_lock(&cache->metadata.lock, \ + cline % OCF_NUM_EVICTION_LISTS) + +#define OCF_METADATA_EVICTION_RD_UNLOCK(cline) \ + ocf_metadata_eviction_rd_unlock(&cache->metadata.lock, \ + cline % OCF_NUM_EVICTION_LISTS) + + #define OCF_METADATA_EVICTION_WR_LOCK_ALL() \ ocf_metadata_eviction_wr_lock_all(&cache->metadata.lock) From 9e98eec36152228148484dd74bb4b0413e17db41 Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Fri, 5 Feb 2021 16:07:00 -0600 Subject: [PATCH 5/5] Only acquire read lock to verify lru elem hotness Signed-off-by: Adam Rutkowski --- src/eviction/lru.c | 13 ++++++++++++- src/eviction/ops.h | 2 -- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/eviction/lru.c b/src/eviction/lru.c index a8f938b..740b1f7 100644 --- a/src/eviction/lru.c +++ b/src/eviction/lru.c @@ -527,12 +527,21 @@ void evp_lru_hot_cline(ocf_cache_t cache, ocf_cache_line_t cline) { struct lru_eviction_policy_meta *node; struct ocf_lru_list *list; + bool hot; node = &ocf_metadata_get_eviction_policy(cache, cline)->lru; - if (node->hot) + + OCF_METADATA_EVICTION_RD_LOCK(cline); + hot = node->hot; + OCF_METADATA_EVICTION_RD_UNLOCK(cline); + + if (hot) return; list = evp_get_cline_list(cache, cline); + + OCF_METADATA_EVICTION_WR_LOCK(cline); + if (node->next != end_marker || node->prev != end_marker || list->head == cline || list->tail == cline) { @@ -542,6 +551,8 @@ void evp_lru_hot_cline(ocf_cache_t cache, ocf_cache_line_t cline) /* Update LRU */ add_lru_head(cache, list, cline); balance_lru_list(cache, list); + + OCF_METADATA_EVICTION_WR_UNLOCK(cline); } static inline void _lru_init(struct ocf_lru_list *list) diff --git a/src/eviction/ops.h b/src/eviction/ops.h index 56475b1..5d90c64 100644 --- a/src/eviction/ops.h +++ b/src/eviction/ops.h @@ -83,9 +83,7 @@ static inline void ocf_eviction_set_hot_cache_line( ENV_BUG_ON(type >= ocf_eviction_max); if (likely(evict_policy_ops[type].hot_cline)) { - OCF_METADATA_EVICTION_WR_LOCK(line); evict_policy_ops[type].hot_cline(cache, line); - OCF_METADATA_EVICTION_WR_UNLOCK(line); } }