Merge pull request #456 from arutk/aalru

Relax LRU list ordering to minimize list updates
This commit is contained in:
Robert Baldyga 2021-02-18 13:48:54 +01:00 committed by GitHub
commit 75baec5aa5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 467 additions and 36 deletions

View File

@ -14,11 +14,8 @@ int ocf_metadata_concurrency_init(struct ocf_metadata_lock *metadata_lock)
unsigned part_iter;
unsigned global_iter;
for (evp_iter = 0; evp_iter < OCF_NUM_EVICTION_LISTS; evp_iter++) {
err = env_spinlock_init(&metadata_lock->eviction[evp_iter]);
if (err)
goto eviction_err;
}
for (evp_iter = 0; evp_iter < OCF_NUM_EVICTION_LISTS; evp_iter++)
env_rwlock_init(&metadata_lock->eviction[evp_iter]);
env_rwlock_init(&metadata_lock->status);
@ -47,9 +44,8 @@ global_err:
env_rwlock_destroy(&metadata_lock->status);
eviction_err:
while (evp_iter--)
env_spinlock_destroy(&metadata_lock->eviction[evp_iter]);
env_rwlock_destroy(&metadata_lock->eviction[evp_iter]);
return err;
}
@ -62,7 +58,7 @@ void ocf_metadata_concurrency_deinit(struct ocf_metadata_lock *metadata_lock)
env_spinlock_destroy(&metadata_lock->partition[i]);
for (i = 0; i < OCF_NUM_EVICTION_LISTS; i++)
env_spinlock_destroy(&metadata_lock->eviction[i]);
env_rwlock_destroy(&metadata_lock->eviction[i]);
for (i = 0; i < OCF_NUM_GLOBAL_META_LOCKS; i++)
env_rwsem_destroy(&metadata_lock->global[i].sem);

View File

@ -28,49 +28,69 @@ int ocf_metadata_concurrency_attached_init(
void ocf_metadata_concurrency_attached_deinit(
struct ocf_metadata_lock *metadata_lock);
static inline void ocf_metadata_eviction_lock(
static inline void ocf_metadata_eviction_wr_lock(
struct ocf_metadata_lock *metadata_lock, unsigned ev_list)
{
env_spinlock_lock(&metadata_lock->eviction[ev_list]);
env_rwlock_write_lock(&metadata_lock->eviction[ev_list]);
}
static inline void ocf_metadata_eviction_unlock(
static inline void ocf_metadata_eviction_wr_unlock(
struct ocf_metadata_lock *metadata_lock, unsigned ev_list)
{
env_spinlock_unlock(&metadata_lock->eviction[ev_list]);
env_rwlock_write_unlock(&metadata_lock->eviction[ev_list]);
}
static inline void ocf_metadata_eviction_lock_all(
static inline void ocf_metadata_eviction_rd_lock(
struct ocf_metadata_lock *metadata_lock, unsigned ev_list)
{
env_rwlock_read_lock(&metadata_lock->eviction[ev_list]);
}
static inline void ocf_metadata_eviction_rd_unlock(
struct ocf_metadata_lock *metadata_lock, unsigned ev_list)
{
env_rwlock_read_unlock(&metadata_lock->eviction[ev_list]);
}
static inline void ocf_metadata_eviction_wr_lock_all(
struct ocf_metadata_lock *metadata_lock)
{
uint32_t i;
for (i = 0; i < OCF_NUM_EVICTION_LISTS; i++)
ocf_metadata_eviction_lock(metadata_lock, i);
ocf_metadata_eviction_wr_lock(metadata_lock, i);
}
static inline void ocf_metadata_eviction_unlock_all(
static inline void ocf_metadata_eviction_wr_unlock_all(
struct ocf_metadata_lock *metadata_lock)
{
uint32_t i;
for (i = 0; i < OCF_NUM_EVICTION_LISTS; i++)
ocf_metadata_eviction_unlock(metadata_lock, i);
ocf_metadata_eviction_wr_unlock(metadata_lock, i);
}
#define OCF_METADATA_EVICTION_LOCK(cline) \
ocf_metadata_eviction_lock(&cache->metadata.lock, \
#define OCF_METADATA_EVICTION_WR_LOCK(cline) \
ocf_metadata_eviction_wr_lock(&cache->metadata.lock, \
cline % OCF_NUM_EVICTION_LISTS)
#define OCF_METADATA_EVICTION_UNLOCK(cline) \
ocf_metadata_eviction_unlock(&cache->metadata.lock, \
#define OCF_METADATA_EVICTION_WR_UNLOCK(cline) \
ocf_metadata_eviction_wr_unlock(&cache->metadata.lock, \
cline % OCF_NUM_EVICTION_LISTS)
#define OCF_METADATA_EVICTION_LOCK_ALL() \
ocf_metadata_eviction_lock_all(&cache->metadata.lock)
#define OCF_METADATA_EVICTION_RD_LOCK(cline) \
ocf_metadata_eviction_rd_lock(&cache->metadata.lock, \
cline % OCF_NUM_EVICTION_LISTS)
#define OCF_METADATA_EVICTION_UNLOCK_ALL() \
ocf_metadata_eviction_unlock_all(&cache->metadata.lock)
#define OCF_METADATA_EVICTION_RD_UNLOCK(cline) \
ocf_metadata_eviction_rd_unlock(&cache->metadata.lock, \
cline % OCF_NUM_EVICTION_LISTS)
#define OCF_METADATA_EVICTION_WR_LOCK_ALL() \
ocf_metadata_eviction_wr_lock_all(&cache->metadata.lock)
#define OCF_METADATA_EVICTION_WR_UNLOCK_ALL() \
ocf_metadata_eviction_wr_unlock_all(&cache->metadata.lock)
static inline void ocf_metadata_partition_lock(
struct ocf_metadata_lock *metadata_lock,

View File

@ -29,6 +29,7 @@ static void add_lru_head(ocf_cache_t cache,
ENV_BUG_ON(collision_index == end_marker);
node = &ocf_metadata_get_eviction_policy(cache, collision_index)->lru;
node->hot = false;
/* First node to be added/ */
if (!list->num_nodes) {
@ -53,6 +54,10 @@ static void add_lru_head(ocf_cache_t cache,
node->next = curr_head_index;
node->prev = end_marker;
curr_head->prev = collision_index;
node->hot = true;
if (!curr_head->hot)
list->last_hot = collision_index;
++list->num_hot;
list->head = collision_index;
@ -76,6 +81,9 @@ static void remove_lru_list(ocf_cache_t cache,
is_head = (list->head == collision_index);
is_tail = (list->tail == collision_index);
if (node->hot)
--list->num_hot;
/* Set prev and next (even if not existent) */
next_lru_node = node->next;
prev_lru_node = node->prev;
@ -89,6 +97,8 @@ static void remove_lru_list(ocf_cache_t cache,
list->head = end_marker;
list->tail = end_marker;
list->last_hot = end_marker;
ENV_BUG_ON(list->num_hot != 0);
}
/* Case 2: else if this collision_index is LRU head, but not tail,
@ -102,7 +112,13 @@ static void remove_lru_list(ocf_cache_t cache,
next_node = &ocf_metadata_get_eviction_policy(cache,
next_lru_node)->lru;
if (list->last_hot == collision_index) {
ENV_BUG_ON(list->num_hot != 0);
list->last_hot = end_marker;
}
list->head = next_lru_node;
node->next = end_marker;
next_node->prev = end_marker;
}
@ -139,6 +155,11 @@ static void remove_lru_list(ocf_cache_t cache,
prev_node = &ocf_metadata_get_eviction_policy(cache,
prev_lru_node)->lru;
if (list->last_hot == collision_index) {
ENV_BUG_ON(list->num_hot == 0);
list->last_hot = prev_lru_node;
}
/* Update prev and next nodes */
prev_node->next = node->next;
next_node->prev = node->prev;
@ -151,6 +172,53 @@ static void remove_lru_list(ocf_cache_t cache,
--list->num_nodes;
}
/* Increase / decrease number of hot elements to achieve target count.
* Asssumes that the list has hot element clustered together at the
* head of the list.
*/
static void balance_lru_list(ocf_cache_t cache,
struct ocf_lru_list *list)
{
unsigned target_hot_count = list->num_nodes / OCF_LRU_HOT_RATIO;
struct lru_eviction_policy_meta *node;
if (target_hot_count == list->num_hot)
return;
if (list->num_hot == 0) {
node = &ocf_metadata_get_eviction_policy(cache,
list->head)->lru;
list->last_hot = list->head;
list->num_hot = 1;
node->hot = 1;
return;
}
ENV_BUG_ON(list->last_hot == end_marker);
node = &ocf_metadata_get_eviction_policy(cache,
list->last_hot)->lru;
if (target_hot_count > list->num_hot) {
++list->num_hot;
list->last_hot = node->next;
node = &ocf_metadata_get_eviction_policy(cache,
node->next)->lru;
node->hot = true;
} else {
if (list->last_hot == list->head) {
node->hot = false;
list->num_hot = 0;
list->last_hot = end_marker;
} else {
ENV_BUG_ON(node->prev == end_marker);
node->hot = false;
--list->num_hot;
list->last_hot = node->prev;
}
}
}
/*-- End of LRU functions*/
void evp_lru_init_cline(ocf_cache_t cache, ocf_cache_line_t cline)
@ -159,6 +227,7 @@ void evp_lru_init_cline(ocf_cache_t cache, ocf_cache_line_t cline)
node = &ocf_metadata_get_eviction_policy(cache, cline)->lru;
node->hot = false;
node->prev = end_marker;
node->next = end_marker;
}
@ -188,6 +257,7 @@ void evp_lru_rm_cline(ocf_cache_t cache, ocf_cache_line_t cline)
list = evp_get_cline_list(cache, cline);
remove_lru_list(cache, list, cline);
balance_lru_list(cache, list);
}
static inline void lru_iter_init(struct ocf_lru_iter *iter, ocf_cache_t cache,
@ -457,10 +527,21 @@ void evp_lru_hot_cline(ocf_cache_t cache, ocf_cache_line_t cline)
{
struct lru_eviction_policy_meta *node;
struct ocf_lru_list *list;
bool hot;
node = &ocf_metadata_get_eviction_policy(cache, cline)->lru;
OCF_METADATA_EVICTION_RD_LOCK(cline);
hot = node->hot;
OCF_METADATA_EVICTION_RD_UNLOCK(cline);
if (hot)
return;
list = evp_get_cline_list(cache, cline);
OCF_METADATA_EVICTION_WR_LOCK(cline);
if (node->next != end_marker ||
node->prev != end_marker ||
list->head == cline || list->tail == cline) {
@ -469,6 +550,9 @@ void evp_lru_hot_cline(ocf_cache_t cache, ocf_cache_line_t cline)
/* Update LRU */
add_lru_head(cache, list, cline);
balance_lru_list(cache, list);
OCF_METADATA_EVICTION_WR_UNLOCK(cline);
}
static inline void _lru_init(struct ocf_lru_list *list)
@ -476,6 +560,8 @@ static inline void _lru_init(struct ocf_lru_list *list)
list->num_nodes = 0;
list->head = end_marker;
list->tail = end_marker;
list->num_hot = 0;
list->last_hot = end_marker;
}
void evp_lru_init_evp(ocf_cache_t cache, struct ocf_user_part *part)
@ -503,10 +589,12 @@ void evp_lru_clean_cline(ocf_cache_t cache, struct ocf_user_part *part,
clean_list = evp_lru_get_list(part, ev_list, true);
dirty_list = evp_lru_get_list(part, ev_list, false);
OCF_METADATA_EVICTION_LOCK(cline);
OCF_METADATA_EVICTION_WR_LOCK(cline);
remove_lru_list(cache, dirty_list, cline);
balance_lru_list(cache, dirty_list);
add_lru_head(cache, clean_list, cline);
OCF_METADATA_EVICTION_UNLOCK(cline);
balance_lru_list(cache, clean_list);
OCF_METADATA_EVICTION_WR_UNLOCK(cline);
}
void evp_lru_dirty_cline(ocf_cache_t cache, struct ocf_user_part *part,
@ -519,9 +607,11 @@ void evp_lru_dirty_cline(ocf_cache_t cache, struct ocf_user_part *part,
clean_list = evp_lru_get_list(part, ev_list, true);
dirty_list = evp_lru_get_list(part, ev_list, false);
OCF_METADATA_EVICTION_LOCK(cline);
OCF_METADATA_EVICTION_WR_LOCK(cline);
remove_lru_list(cache, clean_list, cline);
balance_lru_list(cache, clean_list);
add_lru_head(cache, dirty_list, cline);
OCF_METADATA_EVICTION_UNLOCK(cline);
balance_lru_list(cache, dirty_list);
OCF_METADATA_EVICTION_WR_UNLOCK(cline);
}

View File

@ -7,15 +7,17 @@
#define __EVICTION_LRU_STRUCTS_H__
struct lru_eviction_policy_meta {
/* LRU pointers 2*4=8 bytes */
uint32_t prev;
uint32_t next;
uint8_t hot;
} __attribute__((packed));
struct ocf_lru_list {
uint32_t num_nodes;
uint32_t head;
uint32_t tail;
uint32_t num_hot;
uint32_t last_hot;
};
struct lru_eviction_policy {
@ -23,4 +25,6 @@ struct lru_eviction_policy {
struct ocf_lru_list dirty;
};
#define OCF_LRU_HOT_RATIO 2
#endif

View File

@ -36,9 +36,9 @@ static inline void ocf_eviction_purge_cache_line(
ENV_BUG_ON(type >= ocf_eviction_max);
if (likely(evict_policy_ops[type].rm_cline)) {
OCF_METADATA_EVICTION_LOCK(line);
OCF_METADATA_EVICTION_WR_LOCK(line);
evict_policy_ops[type].rm_cline(cache, line);
OCF_METADATA_EVICTION_UNLOCK(line);
OCF_METADATA_EVICTION_WR_UNLOCK(line);
}
}
@ -83,9 +83,7 @@ static inline void ocf_eviction_set_hot_cache_line(
ENV_BUG_ON(type >= ocf_eviction_max);
if (likely(evict_policy_ops[type].hot_cline)) {
OCF_METADATA_EVICTION_LOCK(line);
evict_policy_ops[type].hot_cline(cache, line);
OCF_METADATA_EVICTION_UNLOCK(line);
}
}
@ -97,9 +95,9 @@ static inline void ocf_eviction_initialize(struct ocf_cache *cache,
ENV_BUG_ON(type >= ocf_eviction_max);
if (likely(evict_policy_ops[type].init_evp)) {
OCF_METADATA_EVICTION_LOCK_ALL();
OCF_METADATA_EVICTION_WR_LOCK_ALL();
evict_policy_ops[type].init_evp(cache, part);
OCF_METADATA_EVICTION_UNLOCK_ALL();
OCF_METADATA_EVICTION_WR_UNLOCK_ALL();
}
}

View File

@ -56,7 +56,7 @@ struct ocf_metadata_lock
struct ocf_metadata_global_lock global[OCF_NUM_GLOBAL_META_LOCKS];
/*!< global metadata lock (GML) */
env_rwlock status; /*!< Fast lock for status bits */
env_spinlock eviction[OCF_NUM_EVICTION_LISTS]; /*!< Fast lock for eviction policy */
env_rwlock eviction[OCF_NUM_EVICTION_LISTS]; /*!< Fast lock for eviction policy */
env_rwsem *hash; /*!< Hash bucket locks */
env_rwsem *collision_pages; /*!< Collision table page locks */
env_spinlock partition[OCF_IO_CLASS_MAX]; /* partition lock */

View File

@ -0,0 +1,323 @@
/*
* <tested_file_path>src/eviction/lru.c</tested_file_path>
* <tested_function>_lru_init</tested_function>
* <functions_to_leave>
* update_lru_head
* update_lru_tail
* update_lru_head_tail
* _lru_init
* add_lru_head
* remove_lru_list
* balance_lru_list
* </functions_to_leave>
*/
#undef static
#undef inline
#include <stdarg.h>
#include <stddef.h>
#include <setjmp.h>
#include <cmocka.h>
#include "print_desc.h"
#include "eviction.h"
#include "lru.h"
#include "ops.h"
#include "../utils/utils_cleaner.h"
#include "../utils/utils_cache_line.h"
#include "../concurrency/ocf_concurrency.h"
#include "../mngt/ocf_mngt_common.h"
#include "../engine/engine_zero.h"
#include "../ocf_request.h"
#include "eviction/lru.c/lru_generated_wraps.c"
#define META_COUNT 128
static union eviction_policy_meta meta[META_COUNT];
union eviction_policy_meta*
__wrap_ocf_metadata_get_eviction_policy(ocf_cache_t cache, ocf_cache_line_t line)
{
assert (line < META_COUNT);
return &meta[line];
}
static const unsigned end_marker = -1;
static void _lru_init_test01(void **state)
{
struct ocf_lru_list l;
print_test_description("test init\n");
_lru_init(&l, end_marker);
assert_int_equal(l.num_hot, 0);
assert_int_equal(l.num_nodes, 0);
assert_int_equal(l.head, end_marker);
assert_int_equal(l.tail, end_marker);
assert_int_equal(l.last_hot, end_marker);
assert_int_equal(1,1);
}
static void check_hot_elems(struct ocf_lru_list *l)
{
unsigned i;
unsigned curr = l->head;
for (i = 0; i < l->num_hot; i++) {
assert_int_equal(meta[curr].lru.hot, 1);
curr = meta[curr].lru.next;
}
for (i = l->num_hot; i < l->num_nodes; i++) {
assert_int_equal(meta[curr].lru.hot, 0);
curr = meta[curr].lru.next;
}
}
static void _lru_init_test02(void **state)
{
struct ocf_lru_list l;
unsigned i;
memset(meta, 0, sizeof(meta));
print_test_description("test add\n");
_lru_init(&l, end_marker);
for (i = 1; i <= 8; i++)
{
add_lru_head(NULL, &l, i, end_marker);
balance_lru_list(NULL, &l, end_marker);
assert_int_equal(l.num_hot, i / 2);
assert_int_equal(l.num_nodes, i);
assert_int_equal(l.head, i);
assert_int_equal(l.tail, 1);
assert_int_equal(l.last_hot, i < 2 ? end_marker :
i - i / 2 + 1);
check_hot_elems(&l);
}
}
static void _lru_init_test03(void **state)
{
struct ocf_lru_list l;
unsigned i;
memset(meta, 0, sizeof(meta));
print_test_description("remove head\n");
_lru_init(&l, end_marker);
for (i = 1; i <= 8; i++) {
add_lru_head(NULL, &l, i, end_marker);
balance_lru_list(NULL, &l, end_marker);
}
for (i = 8; i >= 1; i--) {
assert_int_equal(l.num_hot, i / 2);
assert_int_equal(l.num_nodes, i);
assert_int_equal(l.head, i);
assert_int_equal(l.tail, 1);
assert_int_equal(l.last_hot, i < 2 ? end_marker :
i - i / 2 + 1);
check_hot_elems(&l);
remove_lru_list(NULL, &l, i, end_marker);
balance_lru_list(NULL, &l, end_marker);
}
assert_int_equal(l.num_hot, 0);
assert_int_equal(l.num_nodes, 0);
assert_int_equal(l.head, end_marker);
assert_int_equal(l.tail, end_marker);
assert_int_equal(l.last_hot, end_marker);
}
static void _lru_init_test04(void **state)
{
struct ocf_lru_list l;
unsigned i;
memset(meta, 0, sizeof(meta));
print_test_description("remove tail\n");
_lru_init(&l, end_marker);
for (i = 1; i <= 8; i++) {
add_lru_head(NULL, &l, i, end_marker);
balance_lru_list(NULL, &l, end_marker);
}
for (i = 8; i >= 1; i--) {
assert_int_equal(l.num_hot, i / 2);
assert_int_equal(l.num_nodes, i);
assert_int_equal(l.head, 8);
assert_int_equal(l.tail, 9 - i);
assert_int_equal(l.last_hot, i < 2 ? end_marker :
8 - i / 2 + 1);
check_hot_elems(&l);
remove_lru_list(NULL, &l, 9 - i, end_marker);
balance_lru_list(NULL, &l, end_marker);
}
assert_int_equal(l.num_hot, 0);
assert_int_equal(l.num_nodes, 0);
assert_int_equal(l.head, end_marker);
assert_int_equal(l.tail, end_marker);
assert_int_equal(l.last_hot, end_marker);
}
static void _lru_init_test05(void **state)
{
struct ocf_lru_list l;
unsigned i, j;
bool present[9];
unsigned count;
memset(meta, 0, sizeof(meta));
print_test_description("remove last hot\n");
_lru_init(&l, end_marker);
for (i = 1; i <= 8; i++) {
add_lru_head(NULL, &l, i, end_marker);
balance_lru_list(NULL, &l, end_marker);
present[i] = true;
}
for (i = 8; i >= 3; i--) {
assert_int_equal(l.num_hot, i / 2);
assert_int_equal(l.num_nodes, i);
assert_int_equal(l.head, 8);
assert_int_equal(l.tail, 1);
count = 0;
j = 8;
while (count < i / 2) {
if (present[j])
++count;
--j;
}
assert_int_equal(l.last_hot, j + 1);
check_hot_elems(&l);
present[l.last_hot] = false;
remove_lru_list(NULL, &l, l.last_hot, end_marker);
balance_lru_list(NULL, &l, end_marker);
}
assert_int_equal(l.num_hot, 1);
assert_int_equal(l.num_nodes, 2);
assert_int_equal(l.head, 2);
assert_int_equal(l.tail, 1);
assert_int_equal(l.last_hot, 2);
}
static void _lru_init_test06(void **state)
{
struct ocf_lru_list l;
unsigned i;
unsigned count;
memset(meta, 0, sizeof(meta));
print_test_description("remove middle hot\n");
_lru_init(&l, end_marker);
for (i = 1; i <= 8; i++) {
add_lru_head(NULL, &l, i, end_marker);
balance_lru_list(NULL, &l, end_marker);
}
count = 8;
remove_lru_list(NULL, &l, 7, end_marker);
balance_lru_list(NULL, &l, end_marker);
--count;
assert_int_equal(l.num_hot, count / 2);
assert_int_equal(l.num_nodes, count);
assert_int_equal(l.head, 8);
assert_int_equal(l.tail, 1);
assert_int_equal(l.last_hot, 5);
remove_lru_list(NULL, &l, 6, end_marker);
balance_lru_list(NULL, &l, end_marker);
--count;
assert_int_equal(l.num_hot, count / 2);
assert_int_equal(l.num_nodes, count);
assert_int_equal(l.head, 8);
assert_int_equal(l.tail, 1);
assert_int_equal(l.last_hot, 4);
remove_lru_list(NULL, &l, 5, end_marker);
balance_lru_list(NULL, &l, end_marker);
--count;
assert_int_equal(l.num_hot, count / 2);
assert_int_equal(l.num_nodes, count);
assert_int_equal(l.head, 8);
assert_int_equal(l.tail, 1);
assert_int_equal(l.last_hot, 4);
remove_lru_list(NULL, &l, 4, end_marker);
balance_lru_list(NULL, &l, end_marker);
--count;
assert_int_equal(l.num_hot, count / 2);
assert_int_equal(l.num_nodes, count);
assert_int_equal(l.head, 8);
assert_int_equal(l.tail, 1);
assert_int_equal(l.last_hot, 3);
remove_lru_list(NULL, &l, 3, end_marker);
balance_lru_list(NULL, &l, end_marker);
--count;
assert_int_equal(l.num_hot, count / 2);
assert_int_equal(l.num_nodes, count);
assert_int_equal(l.head, 8);
assert_int_equal(l.tail, 1);
assert_int_equal(l.last_hot, 8);
remove_lru_list(NULL, &l, 8, end_marker);
balance_lru_list(NULL, &l, end_marker);
--count;
assert_int_equal(l.num_hot, count / 2);
assert_int_equal(l.num_nodes, count);
assert_int_equal(l.head, 2);
assert_int_equal(l.tail, 1);
assert_int_equal(l.last_hot, 2);
remove_lru_list(NULL, &l, 2, end_marker);
balance_lru_list(NULL, &l, end_marker);
--count;
assert_int_equal(l.num_hot, count / 2);
assert_int_equal(l.num_nodes, count);
assert_int_equal(l.head, 1);
assert_int_equal(l.tail, 1);
assert_int_equal(l.last_hot, end_marker);
}
int main(void)
{
const struct CMUnitTest tests[] = {
cmocka_unit_test(_lru_init_test01),
cmocka_unit_test(_lru_init_test02),
cmocka_unit_test(_lru_init_test03),
cmocka_unit_test(_lru_init_test04),
cmocka_unit_test(_lru_init_test05),
cmocka_unit_test(_lru_init_test06)
};
return cmocka_run_group_tests(tests, NULL, NULL);
}