github/ocf
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Parallel eviction

Browse Source 
Eviction changes allowing to evict (remap) cachelines while
holding hash bucket write lock instead of global metadata
write lock.

As eviction (replacement) is now tightly coupled with request,
each request uses eviction size equal to number of its
unmapped cachelines.

Evicting without global metadata write lock is possible
thanks to the fact that remaping is always performed
while exclusively holding cacheline (read or write) lock.
So for a cacheline on LRU list we acquire cacheline lock,
safely resolve hash and consequently write-lock hash bucket.
Since cacheline lock is acquired under hash bucket (everywhere
except for new eviction implementation), we are certain that
noone acquires cacheline lock behind our back. Concurrent
eviction threads are eliminated by holding eviction list
lock for the duration of critial locking operations.

Signed-off-by: Adam Rutkowski <adam.j.rutkowski@intel.com>
This commit is contained in:
Adam Rutkowski
2021-03-05 11:20:47 +01:00
parent 1411314678
commit 81fc7ab5c5
20 changed files with 694 additions and 330 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
tests/unit/tests/engine/engine_common.c/prepare_clines_miss.c
View File

@@ -2,8 +2,8 @@
* <tested_file_path>src/engine/engine_common.c</tested_file_path>
* <tested_function>ocf_prepare_clines_miss</tested_function>
* <functions_to_leave>
* INSERT HERE LIST OF FUNCTIONS YOU WANT TO LEAVE
* ONE FUNCTION PER LINE
* ocf_prepare_clines_evict
* ocf_engine_evict
* </functions_to_leave>
*/
@@ -36,6 +36,11 @@
#include "engine/engine_common.c/prepare_clines_miss_generated_wraps.c"
struct ocf_cache_line_concurrency *__wrap_ocf_cache_line_concurrency(ocf_cache_t cache)
{
return NULL;
}
void __wrap_ocf_req_hash_lock_upgrade(struct ocf_request *req)
{
}
@@ -66,13 +71,6 @@ void __wrap_ocf_metadata_end_exclusive_access(
{
}
int __wrap_space_managment_evict_do(struct ocf_cache *cache,
struct ocf_request *req, uint32_t evict_cline_no)
{
function_called();
return mock();
}
bool __wrap_ocf_part_is_enabled(struct ocf_user_part *target_part)
{
return mock();
@@ -93,9 +91,21 @@ void __wrap_ocf_req_set_mapping_error(struct ocf_request *req)
function_called();
}
int __wrap_space_managment_evict_do(struct ocf_request *req)
{
function_called();
return mock();
}
uint32_t __wrap_ocf_engine_unmapped_count(struct ocf_request *req)
{
return 100;
}
static void ocf_prepare_clines_miss_test01(void **state)
{
struct ocf_request req = {};
struct ocf_cache cache;
struct ocf_request req = {.cache = &cache };
print_test_description("Target part is disabled and empty\n");
will_return(__wrap_ocf_part_is_enabled, false);
expect_function_call(__wrap_ocf_req_set_mapping_error);
@@ -104,7 +114,9 @@ static void ocf_prepare_clines_miss_test01(void **state)
static void ocf_prepare_clines_miss_test02(void **state)
{
struct ocf_request req = {};
struct ocf_cache cache;
struct ocf_request req = {.cache = &cache };
print_test_description("Target part is disabled but has cachelines assigned.\n");
print_test_description("\tMark mapping error\n");
@@ -116,20 +128,18 @@ static void ocf_prepare_clines_miss_test02(void **state)
static void ocf_prepare_clines_miss_test03(void **state)
{
struct ocf_request req = {};
struct ocf_cache cache;
struct ocf_request req = {.cache = &cache };
print_test_description("Target part is enabled but doesn't have enough space.\n");
print_test_description("\tEviction is ok and cachelines lock is acquired.\n");
will_return(__wrap_ocf_part_is_enabled, true);
will_return(__wrap_ocf_part_has_space, false);
will_return(__wrap_ocf_part_has_space, false);
will_return_always(__wrap_ocf_part_has_space, false);
expect_function_call(__wrap_space_managment_evict_do);
will_return(__wrap_space_managment_evict_do, LOOKUP_MAPPED);
will_return_always(__wrap_space_managment_evict_do, LOOKUP_INSERTED);
expect_function_call(__wrap_ocf_engine_map);
will_return(__wrap_ocf_req_test_mapping_error, false);
will_return_always(__wrap_ocf_req_test_mapping_error, false);
will_return(__wrap_lock_clines, 0);
expect_function_call(__wrap_lock_clines);
@@ -139,57 +149,38 @@ static void ocf_prepare_clines_miss_test03(void **state)
static void ocf_prepare_clines_miss_test04(void **state)
{
struct ocf_request req = {};
struct ocf_cache cache;
struct ocf_request req = {.cache = &cache };
print_test_description("Target part is enabled but doesn't have enough space.\n");
print_test_description("\tEviction failed\n");
will_return(__wrap_ocf_part_is_enabled, true);
will_return(__wrap_ocf_part_has_space, false);
will_return_always(__wrap_ocf_part_has_space, false);
will_return(__wrap_ocf_part_has_space, false);
expect_function_call(__wrap_space_managment_evict_do);
will_return(__wrap_space_managment_evict_do, LOOKUP_MISS);
expect_function_call(__wrap_ocf_req_set_mapping_error);
assert_int_equal(ocf_prepare_clines_miss(&req, NULL), -OCF_ERR_NO_LOCK);
}
static void ocf_prepare_clines_miss_test05(void **state)
{
struct ocf_request req = {};
print_test_description("Target part is enabled but doesn't have enough space.\n");
print_test_description("Eviction is ok, but mapping failed.\n");
will_return(__wrap_ocf_part_has_space, false);
will_return(__wrap_ocf_part_has_space, false);
expect_function_call(__wrap_space_managment_evict_do);
will_return(__wrap_space_managment_evict_do, LOOKUP_HIT);
will_return(__wrap_ocf_part_is_enabled, true);
expect_function_call(__wrap_ocf_engine_map);
will_return(__wrap_ocf_req_test_mapping_error, true);
will_return_always(__wrap_ocf_req_test_mapping_error, true);
assert_int_equal(ocf_prepare_clines_miss(&req, NULL), -OCF_ERR_NO_LOCK);
}
static void ocf_prepare_clines_miss_test06(void **state)
{
struct ocf_request req = {};
struct ocf_cache cache;
struct ocf_request req = {.cache = &cache };
print_test_description("Target part is enabled but doesn't have enough space.\n");
print_test_description("Eviction and mapping were ok, but failed to lock cachelines.\n");
will_return(__wrap_ocf_part_has_space, false);
will_return(__wrap_ocf_part_has_space, false);
will_return_always(__wrap_ocf_part_has_space, false);
expect_function_call(__wrap_space_managment_evict_do);
will_return(__wrap_space_managment_evict_do, LOOKUP_HIT);
will_return(__wrap_ocf_part_is_enabled, true);
expect_function_call(__wrap_ocf_engine_map);
will_return(__wrap_ocf_req_test_mapping_error, false);
will_return_always(__wrap_ocf_req_test_mapping_error, false);
expect_function_call(__wrap_lock_clines);
will_return(__wrap_lock_clines, -OCF_ERR_NO_LOCK);
@@ -201,20 +192,20 @@ static void ocf_prepare_clines_miss_test06(void **state)
static void ocf_prepare_clines_miss_test07(void **state)
{
struct ocf_request req = {};
struct ocf_cache cache;
struct ocf_request req = {.cache = &cache };
print_test_description("Target part is enabled but doesn't have enough space.\n");
print_test_description("Eviction and mapping were ok, lock not acquired.\n");
will_return(__wrap_ocf_part_has_space, false);
will_return(__wrap_ocf_part_has_space, false);
will_return_always(__wrap_ocf_part_has_space, false);
expect_function_call(__wrap_space_managment_evict_do);
will_return(__wrap_space_managment_evict_do, LOOKUP_HIT);
will_return(__wrap_ocf_part_is_enabled, true);
expect_function_call(__wrap_ocf_engine_map);
will_return(__wrap_ocf_req_test_mapping_error, false);
will_return_always(__wrap_ocf_req_test_mapping_error, false);
expect_function_call(__wrap_lock_clines);
will_return(__wrap_lock_clines, OCF_LOCK_NOT_ACQUIRED);
@@ -224,15 +215,17 @@ static void ocf_prepare_clines_miss_test07(void **state)
static void ocf_prepare_clines_miss_test08(void **state)
{
struct ocf_request req = {};
struct ocf_cache cache;
struct ocf_request req = {.cache = &cache };
print_test_description("Target part is enabled has enough space.\n");
print_test_description("\tMapping and cacheline lock are both ok\n");
will_return(__wrap_ocf_part_is_enabled, true);
will_return(__wrap_ocf_part_has_space, true);
will_return_always(__wrap_ocf_part_has_space, true);
expect_function_call(__wrap_ocf_engine_map);
will_return(__wrap_ocf_req_test_mapping_error, false);
will_return_always(__wrap_ocf_req_test_mapping_error, false);
expect_function_call(__wrap_lock_clines);
will_return(__wrap_lock_clines, OCF_LOCK_ACQUIRED);
@@ -247,7 +240,6 @@ int main(void)
cmocka_unit_test(ocf_prepare_clines_miss_test02),
cmocka_unit_test(ocf_prepare_clines_miss_test03),
cmocka_unit_test(ocf_prepare_clines_miss_test04),
cmocka_unit_test(ocf_prepare_clines_miss_test05),
cmocka_unit_test(ocf_prepare_clines_miss_test06),
cmocka_unit_test(ocf_prepare_clines_miss_test07),
cmocka_unit_test(ocf_prepare_clines_miss_test08)

42
tests/unit/tests/eviction/eviction.c/eviction.c
View File

@@ -27,9 +27,9 @@ struct test_cache
{
struct ocf_cache cache;
struct ocf_user_part_config part[OCF_IO_CLASS_MAX];
struct ocf_user_part upart[OCF_IO_CLASS_MAX];
uint32_t overflow[OCF_IO_CLASS_MAX];
uint32_t evictable[OCF_IO_CLASS_MAX];
uint32_t req_unmapped;
};
bool __wrap_ocf_eviction_can_evict(ocf_cache_t cache)
@@ -62,10 +62,12 @@ uint32_t __wrap_ocf_eviction_need_space(struct ocf_cache *cache,
tcache->overflow[part->id] -= overflown_consumed;
tcache->evictable[part->id] -= clines;
tcache->req_unmapped -= clines;
check_expected(part);
check_expected(clines);
function_called();
return mock();
}
@@ -157,7 +159,7 @@ static struct ocf_lst_entry *_list_getter(
{
struct test_cache* tcache = cache;
return &tcache->upart[idx].lst_valid;
return &tcache->cache.user_parts[idx].lst_valid;
}
static void init_part_list(struct test_cache *tcache)
@@ -165,23 +167,30 @@ static void init_part_list(struct test_cache *tcache)
unsigned i;
for (i = 0; i < OCF_IO_CLASS_MAX; i++) {
tcache->upart[i].id = i;
tcache->upart[i].config = &tcache->part[i];
tcache->upart[i].config->priority = i+1;
tcache->upart[i].config->flags.eviction = 1;
tcache->cache.user_parts[i].id = i;
tcache->cache.user_parts[i].config = &tcache->part[i];
tcache->cache.user_parts[i].config->priority = i+1;
tcache->cache.user_parts[i].config->flags.eviction = 1;
}
ocf_lst_init((ocf_cache_t)tcache, &tcache->cache.lst_part, OCF_IO_CLASS_MAX,
_list_getter, ocf_part_lst_cmp_valid);
for (i = 0; i < OCF_IO_CLASS_MAX; i++) {
ocf_lst_init_entry(&tcache->cache.lst_part, &tcache->upart[i].lst_valid);
ocf_lst_init_entry(&tcache->cache.lst_part, &tcache->cache.user_parts[i].lst_valid);
ocf_lst_add_tail(&tcache->cache.lst_part, i);
}
}
uint32_t __wrap_ocf_engine_unmapped_count(struct ocf_request *req)
{
struct test_cache* tcache = (struct test_cache*)req->cache;
return tcache->req_unmapped;
}
#define _expect_evict_call(tcache, part_id, req_count, ret_count) \
do { \
expect_value(__wrap_ocf_eviction_need_space, part, &tcache.upart[part_id]); \
expect_value(__wrap_ocf_eviction_need_space, part, &tcache.cache.user_parts[part_id]); \
expect_value(__wrap_ocf_eviction_need_space, clines, req_count); \
expect_function_call(__wrap_ocf_eviction_need_space); \
will_return(__wrap_ocf_eviction_need_space, ret_count); \
@@ -190,6 +199,7 @@ static void init_part_list(struct test_cache *tcache)
static void ocf_evict_do_test01(void **state)
{
struct test_cache tcache = {};
struct ocf_request req = {.cache = &tcache.cache, .part_id = 0 };
unsigned evicted;
print_test_description("one IO class, no overflow\n");
@@ -197,16 +207,17 @@ static void ocf_evict_do_test01(void **state)
init_part_list(&tcache);
tcache.evictable[10] = 100;
tcache.req_unmapped = 50;
_expect_evict_call(tcache, 10, 50, 50);
evicted = ocf_evict_do((ocf_cache_t *)&tcache, NULL, 50, &tcache.upart[0]);
evicted = ocf_evict_do(&req);
assert_int_equal(evicted, 50);
}
static void ocf_evict_do_test02(void **state)
{
struct test_cache tcache = {};
struct ocf_request req = {.cache = &tcache.cache, .part_id = 0 };
unsigned i;
unsigned evicted;
@@ -216,16 +227,18 @@ static void ocf_evict_do_test02(void **state)
tcache.evictable[10] = 100;
tcache.overflow[10] = 100;
tcache.req_unmapped = 50;
_expect_evict_call(tcache, 10, 50, 50);
evicted = ocf_evict_do((ocf_cache_t *)&tcache, NULL, 50, &tcache.upart[0]);
evicted = ocf_evict_do(&req);
assert_int_equal(evicted, 50);
}
static void ocf_evict_do_test03(void **state)
{
struct test_cache tcache = {};
struct ocf_request req = {.cache = &tcache.cache, .part_id = 0 };
unsigned i;
unsigned evicted;
@@ -237,19 +250,21 @@ static void ocf_evict_do_test03(void **state)
tcache.evictable[12] = 100;
tcache.evictable[16] = 100;
tcache.evictable[17] = 100;
tcache.req_unmapped = 350;
_expect_evict_call(tcache, 10, 100, 100);
_expect_evict_call(tcache, 12, 100, 100);
_expect_evict_call(tcache, 16, 100, 100);
_expect_evict_call(tcache, 17, 50, 50);
evicted = ocf_evict_do((ocf_cache_t *)&tcache, NULL, 350, &tcache.upart[0]);
evicted = ocf_evict_do(&req);
assert_int_equal(evicted, 350);
}
static void ocf_evict_do_test04(void **state)
{
struct test_cache tcache = {};
struct ocf_request req = {.cache = &tcache.cache, .part_id = 0 };
unsigned i;
unsigned evicted;
@@ -266,6 +281,7 @@ static void ocf_evict_do_test04(void **state)
tcache.evictable[17] = 100;
tcache.evictable[18] = 100;
tcache.overflow[18] = 100;
tcache.req_unmapped = 580;
_expect_evict_call(tcache, 12, 40, 40);
_expect_evict_call(tcache, 14, 100, 100);
@@ -275,7 +291,7 @@ static void ocf_evict_do_test04(void **state)
_expect_evict_call(tcache, 16, 100, 100);
_expect_evict_call(tcache, 17, 80, 80);
evicted = ocf_evict_do((ocf_cache_t *)&tcache, NULL, 580, &tcache.upart[0]);
evicted = ocf_evict_do(&req);
assert_int_equal(evicted, 580);
}
int main(void)

109
tests/unit/tests/eviction/lru.c/lru_iter.c
View File

@@ -10,6 +10,8 @@
* _lru_evp_set_empty
* _lru_evp_all_empty
* ocf_rotate_right
* lru_iter_eviction_next
* lru_iter_cleaning_next
* </functions_to_leave>
*/
@@ -157,7 +159,26 @@ void write_test_case_description(void)
test_case++;
}
/* transform cacheline numbers so that they remain unique but have
* assignment to list modulo OCF_NUM_EVICTION_LISTS */
for (test_case = 0; test_case < num_cases; test_case++) {
for (i = 0; i < OCF_NUM_EVICTION_LISTS; i++) {
j = 0;
while (test_cases[j][i][test_case] != -1) {
test_cases[j][i][test_case] = test_cases[j][i][test_case] *
OCF_NUM_EVICTION_LISTS + i;
j++;
}
}
}
#ifdef DEBUG
static bool desc_printed = false;
if (desc_printed)
return;
desc_printed = true;
for (test_case = 0; test_case < num_cases; test_case++) {
print_message("test case no %d\n", test_case);
for (i = 0; i < OCF_NUM_EVICTION_LISTS; i++) {
@@ -196,6 +217,11 @@ struct ocf_lru_list *__wrap_evp_lru_get_list(struct ocf_user_part *part,
list.num_nodes = i;
}
#ifdef DEBUG
print_message("list for case %u evp %u: head: %u tail %u elems %u\n",
current_case, evp, list.head, list.tail, list.num_nodes);
#endif
return &list;
}
@@ -245,6 +271,76 @@ union eviction_policy_meta *__wrap_ocf_metadata_get_eviction_policy(
}
void __wrap_add_lru_head(ocf_cache_t cache,
struct ocf_lru_list *list,
unsigned int collision_index)
{
unsigned list_head = list->head;
unsigned i, j = collision_index % OCF_NUM_EVICTION_LISTS;
i = 1;
while (test_cases[i][j][current_case] != -1)
i++;
test_cases[i+1][j][current_case] = -1;
while (i--)
test_cases[i + 1][j][current_case] = test_cases[i][j][current_case];
test_cases[0][j][current_case] = collision_index;
#ifdef DEBUG
print_message("case %u evp %u head set to %u\n", current_case, j, collision_index);
#endif
}
void __wrap_remove_lru_list(ocf_cache_t cache,
struct ocf_lru_list *list,
unsigned int collision_index)
{
bool found;
unsigned i, j;
found = false;
for (i = 0; i < OCF_NUM_EVICTION_LISTS; i++)
{
j = 0;
while (test_cases[j][i][current_case] != -1) {
if (!found && test_cases[j][i][current_case] == collision_index) {
assert_int_equal(test_cases[0][i][current_case], list->head);
found = true;
}
if (found)
test_cases[j][i][current_case] = test_cases[j+1][i][current_case];
j++;
}
if (found)
break;
}
assert(found);
#ifdef DEBUG
print_message("case %u removed %u from evp %u\n", current_case, collision_index, i);
#endif
}
bool __wrap__lru_lock(struct ocf_lru_iter *iter,
ocf_cache_line_t cache_line,
ocf_core_id_t *core_id, uint64_t *core_line)
{
return true;
}
bool __wrap__lru_trylock_cacheline(struct ocf_lru_iter *iter,
ocf_cache_line_t cline)
{
return true;
}
static void _lru_run_test(unsigned test_case)
{
unsigned start_pos;
@@ -258,6 +354,8 @@ static void _lru_run_test(unsigned test_case)
unsigned pos[OCF_NUM_EVICTION_LISTS];
unsigned i;
write_test_case_description();
for (i = 0; i < OCF_NUM_EVICTION_LISTS; i++)
{
pos[i] = -1;
@@ -265,12 +363,10 @@ static void _lru_run_test(unsigned test_case)
pos[i]++;
}
lru_iter_init(&iter, NULL, NULL, start_pos, false);
lru_iter_init(&iter, NULL, NULL, start_pos, false, false, false,
NULL, NULL);
do {
/* get cacheline from iterator */
cache_line = lru_iter_next(&iter);
/* check what is expected to be returned from iterator */
if (pos[curr_evp] == -1) {
i = 1;
@@ -294,6 +390,9 @@ static void _lru_run_test(unsigned test_case)
pos[curr_evp]--;
}
/* get cacheline from iterator */
cache_line = lru_iter_cleaning_next(&iter);
assert_int_equal(cache_line, expected_cache_line);
curr_evp = (curr_evp + 1) % OCF_NUM_EVICTION_LISTS;
@@ -475,7 +574,5 @@ int main(void)
print_message("Unit test for lru_iter_next\n");
write_test_case_description();
return cmocka_run_group_tests(tests, NULL, NULL);
}