Parallelize ALRU recovery

Signed-off-by: Robert Baldyga <robert.baldyga@intel.com>
This commit is contained in:
Robert Baldyga 2022-01-20 13:33:03 +01:00
parent 8cc71cc9cb
commit b70492ad3d
8 changed files with 287 additions and 217 deletions

View File

@ -323,6 +323,15 @@ int cleaning_policy_acp_initialize(struct ocf_cache *cache,
return 0;
}
void cleaning_policy_acp_recovery(ocf_cache_t cache,
ocf_cleaning_recovery_end_t cmpl, void *priv)
{
int result;
result = cleaning_policy_acp_initialize(cache, 1);
cmpl(priv, result);
}
int cleaning_policy_acp_set_cleaning_param(ocf_cache_t cache,
uint32_t param_id, uint32_t param_value)
{

View File

@ -12,6 +12,9 @@ void cleaning_policy_acp_setup(ocf_cache_t cache);
int cleaning_policy_acp_initialize(ocf_cache_t cache, int init_metadata);
void cleaning_policy_acp_recovery(ocf_cache_t cache,
ocf_cleaning_recovery_end_t cmpl, void *priv);
void cleaning_policy_acp_deinitialize(ocf_cache_t cache);
void cleaning_policy_acp_perform_cleaning(ocf_cache_t cache,

View File

@ -10,6 +10,7 @@
#include "../metadata/metadata.h"
#include "../utils/utils_cleaner.h"
#include "../utils/utils_user_part.h"
#include "../utils/utils_parallelize.h"
#include "../utils/utils_realloc.h"
#include "../concurrency/ocf_cache_line_concurrency.h"
#include "../ocf_def_priv.h"
@ -61,54 +62,63 @@ struct alru_context {
/* -- Start of ALRU functions -- */
/* Adds the given collision_index to the _head_ of the ALRU list */
static void add_alru_head(struct ocf_cache *cache, int partition_id,
unsigned int collision_index)
/* Appends given sublist to the _head_ of the ALRU list */
static void append_alru_head(ocf_cache_t cache, ocf_part_id_t part_id,
ocf_cache_line_t head, ocf_cache_line_t tail)
{
unsigned int curr_head_index;
unsigned int collision_table_entries = cache->device->collision_table_entries;
struct alru_cleaning_policy *part_alru = &cache->user_parts[partition_id]
.clean_pol->policy.alru;
struct alru_cleaning_policy_meta *alru;
ocf_cache_line_t terminator = cache->device->collision_table_entries;
struct alru_cleaning_policy *part_alru;
struct cleaning_policy_meta *meta;
struct alru_cleaning_policy_meta *old_head;
struct alru_cleaning_policy_meta *entry;
ENV_BUG_ON(!(collision_index < collision_table_entries));
part_alru = &cache->user_parts[part_id].clean_pol->policy.alru;
ENV_BUG_ON(env_atomic_read(&part_alru->size) < 0);
if (head == terminator && tail == terminator)
return;
ENV_WARN_ON(!metadata_test_dirty(cache, collision_index));
ENV_WARN_ON(!metadata_test_valid_any(cache, collision_index));
ENV_BUG_ON(head == terminator);
ENV_BUG_ON(tail == terminator);
/* First node to be added/ */
if (env_atomic_read(&part_alru->size) == 0) {
part_alru->lru_head = collision_index;
part_alru->lru_tail = collision_index;
alru = &ocf_metadata_get_cleaning_policy(cache,
collision_index)->meta.alru;
alru->lru_next = collision_table_entries;
alru->lru_prev = collision_table_entries;
alru->timestamp = env_ticks_to_secs(
env_get_tick_count());
if (part_alru->lru_head == terminator) {
part_alru->lru_head = head;
part_alru->lru_tail = tail;
} else {
/* Not the first node to be added. */
meta = ocf_metadata_get_cleaning_policy(cache, part_alru->lru_head);
old_head = &meta->meta.alru;
old_head->lru_prev = tail;
curr_head_index = part_alru->lru_head;
meta = ocf_metadata_get_cleaning_policy(cache, tail);
entry = &meta->meta.alru;
entry->lru_next = part_alru->lru_head;
ENV_BUG_ON(!(curr_head_index < collision_table_entries));
alru = &ocf_metadata_get_cleaning_policy(cache,
collision_index)->meta.alru;
alru->lru_next = curr_head_index;
alru->lru_prev = collision_table_entries;
alru->timestamp = env_ticks_to_secs(
env_get_tick_count());
alru = &ocf_metadata_get_cleaning_policy(cache,
curr_head_index)->meta.alru;
alru->lru_prev = collision_index;
part_alru->lru_head = collision_index;
part_alru->lru_head = head;
}
}
/* Adds the given collision_index to the _head_ of the ALRU list */
static void add_alru_head(ocf_cache_t cache, ocf_part_id_t part_id,
ocf_cache_line_t cline)
{
ocf_cache_line_t terminator = cache->device->collision_table_entries;
struct alru_cleaning_policy *part_alru;
struct cleaning_policy_meta *meta;
struct alru_cleaning_policy_meta *entry;
ENV_BUG_ON(!(cline < terminator));
ENV_WARN_ON(!metadata_test_dirty(cache, cline));
ENV_WARN_ON(!metadata_test_valid_any(cache, cline));
part_alru = &cache->user_parts[part_id].clean_pol->policy.alru;
meta = ocf_metadata_get_cleaning_policy(cache, cline);
entry = &meta->meta.alru;
entry->lru_next = terminator;
entry->lru_prev = terminator;
entry->timestamp = env_ticks_to_secs(env_get_tick_count());
append_alru_head(cache, part_id, cline, cline);
env_atomic_inc(&part_alru->size);
}
@ -375,24 +385,6 @@ static void _alru_rebuild(struct ocf_cache *cache)
}
}
static int cleaning_policy_alru_initialize_part(struct ocf_cache *cache,
struct ocf_user_part *user_part, int init_metadata)
{
struct alru_cleaning_policy *part_alru =
&user_part->clean_pol->policy.alru;
if (init_metadata) {
/* ALRU initialization */
env_atomic_set(&part_alru->size, 0);
part_alru->lru_head = cache->device->collision_table_entries;
part_alru->lru_tail = cache->device->collision_table_entries;
}
cache->device->runtime_meta->cleaning_thread_access = 0;
return 0;
}
void cleaning_policy_alru_setup(struct ocf_cache *cache)
{
struct alru_cleaning_policy_config *config;
@ -405,10 +397,8 @@ void cleaning_policy_alru_setup(struct ocf_cache *cache)
config->activity_threshold = OCF_ALRU_DEFAULT_ACTIVITY_THRESHOLD;
}
int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata)
int cleaning_policy_alru_init_common(ocf_cache_t cache)
{
struct ocf_user_part *user_part;
ocf_part_id_t part_id;
struct alru_context *ctx;
int error = 0;
unsigned i;
@ -432,13 +422,20 @@ int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata)
return error;
}
cache->device->runtime_meta->cleaning_thread_access = 0;
cache->cleaner.cleaning_policy_context = ctx;
for_each_user_part(cache, user_part, part_id) {
cleaning_policy_alru_initialize_part(cache,
user_part, init_metadata);
}
return 0;
}
int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata)
{
int result;
result = cleaning_policy_alru_init_common(cache);
if (result)
return result;
if (init_metadata)
_alru_rebuild(cache);
@ -448,6 +445,184 @@ int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata)
return 0;
}
#define OCF_ALRU_RECOVERY_SHARDS_CNT 32
struct ocf_alru_recovery_context {
ocf_cache_t cache;
struct {
struct {
ocf_cache_line_t head;
ocf_cache_line_t tail;
} part[OCF_USER_IO_CLASS_MAX];
} shard[OCF_ALRU_RECOVERY_SHARDS_CNT] __attribute__((aligned(64)));
ocf_cleaning_recovery_end_t cmpl;
void *priv;
};
static void add_alru_head_recovery(struct ocf_alru_recovery_context *context,
unsigned shard_id, ocf_core_id_t part_id,
ocf_cache_line_t cline)
{
ocf_cache_t cache = context->cache;
ocf_cache_line_t curr_head, terminator;
struct cleaning_policy_meta *meta;
struct alru_cleaning_policy_meta *entry;
struct alru_cleaning_policy_meta *next;
terminator = ocf_metadata_collision_table_entries(cache);
curr_head = context->shard[shard_id].part[part_id].head;
meta = ocf_metadata_get_cleaning_policy(cache, cline);
entry = &meta->meta.alru;
if (curr_head == terminator) {
/* First node to be added/ */
entry->lru_next = terminator;
entry->lru_prev = terminator;
entry->timestamp = env_ticks_to_secs(env_get_tick_count());
context->shard[shard_id].part[part_id].head = cline;
context->shard[shard_id].part[part_id].tail = cline;
} else {
/* Not the first node to be added. */
entry->lru_next = curr_head;
entry->lru_prev = terminator;
entry->timestamp = env_ticks_to_secs(env_get_tick_count());
meta = ocf_metadata_get_cleaning_policy(cache, curr_head);
next = &meta->meta.alru;
next->lru_prev = cline;
context->shard[shard_id].part[part_id].head = cline;
}
}
static int ocf_alru_recovery_handle(ocf_parallelize_t parallelize,
void *priv, unsigned shard_id, unsigned shards_cnt)
{
struct ocf_alru_recovery_context *context = priv;
ocf_cache_t cache = context->cache;
ocf_cache_line_t entries = cache->device->collision_table_entries;
ocf_cache_line_t terminator = entries;
unsigned part_size[OCF_USER_IO_CLASS_MAX] = {};
struct ocf_user_part *user_part;
struct alru_cleaning_policy *part_alru;
ocf_part_id_t part_id;
ocf_core_id_t core_id;
ocf_cache_line_t cline, portion;
uint64_t begin, end;
uint32_t step = 0;
int i;
portion = DIV_ROUND_UP((uint64_t)entries, shards_cnt);
begin = portion*shard_id;
end = OCF_MIN(portion*(shard_id + 1), entries);
for (i = 0; i < OCF_USER_IO_CLASS_MAX; i++) {
context->shard[shard_id].part[i].head = terminator;
context->shard[shard_id].part[i].tail = terminator;
}
for (cline = begin; cline < end; cline++) {
ocf_metadata_get_core_and_part_id(cache, cline,
&core_id, &part_id);
OCF_COND_RESCHED_DEFAULT(step);
if (core_id == OCF_CORE_MAX)
continue;
if (!metadata_test_dirty(cache, cline)) {
cleaning_policy_alru_init_cache_block(cache, cline);
} else {
add_alru_head_recovery(context, shard_id,
part_id, cline);
++part_size[part_id];
}
}
for_each_user_part(cache, user_part, part_id) {
part_alru = &user_part->clean_pol->policy.alru;
env_atomic_add(part_size[part_id], &part_alru->size);
}
return 0;
}
static void ocf_alru_recovery_finish(ocf_parallelize_t parallelize,
void *priv, int error)
{
struct ocf_alru_recovery_context *context = priv;
ocf_cache_t cache = context->cache;
ocf_part_id_t part_id;
ocf_cache_line_t head, tail;
unsigned shard;
if (error)
goto end;
for (part_id = 0; part_id < OCF_USER_IO_CLASS_MAX; part_id++) {
for (shard = 0; shard < OCF_ALRU_RECOVERY_SHARDS_CNT; shard++) {
head = context->shard[shard].part[part_id].head;
tail = context->shard[shard].part[part_id].tail;
append_alru_head(cache, part_id, head, tail);
}
}
ocf_kick_cleaner(cache);
end:
context->cmpl(context->priv, error);
ocf_parallelize_destroy(parallelize);
}
void cleaning_policy_alru_recovery(ocf_cache_t cache,
ocf_cleaning_recovery_end_t cmpl, void *priv)
{
struct ocf_alru_recovery_context *context;
ocf_parallelize_t parallelize;
struct alru_cleaning_policy *part_alru;
struct ocf_user_part *user_part;
ocf_part_id_t part_id;
int result;
result = ocf_parallelize_create(&parallelize, cache,
OCF_ALRU_RECOVERY_SHARDS_CNT, sizeof(*context),
ocf_alru_recovery_handle, ocf_alru_recovery_finish);
if (result) {
cmpl(priv, result);
return;
}
result = cleaning_policy_alru_init_common(cache);
if (result) {
ocf_parallelize_destroy(parallelize);
cmpl(priv, result);
return;
}
for_each_user_part(cache, user_part, part_id) {
/* ALRU initialization */
part_alru = &user_part->clean_pol->policy.alru;
env_atomic_set(&part_alru->size, 0);
part_alru->lru_head = cache->device->collision_table_entries;
part_alru->lru_tail = cache->device->collision_table_entries;
cache->device->runtime_meta->cleaning_thread_access = 0;
}
context = ocf_parallelize_get_priv(parallelize);
context->cache = cache;
context->cmpl = cmpl;
context->priv = priv;
ocf_parallelize_run(parallelize);
}
void cleaning_policy_alru_deinitialize(struct ocf_cache *cache)
{
struct alru_context *alru = cache->cleaner.cleaning_policy_context;

View File

@ -11,6 +11,8 @@
void cleaning_policy_alru_setup(ocf_cache_t cache);
int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata);
void cleaning_policy_alru_recovery(ocf_cache_t cache,
ocf_cleaning_recovery_end_t cmpl, void *priv);
void cleaning_policy_alru_deinitialize(ocf_cache_t cache);
void cleaning_policy_alru_init_cache_block(ocf_cache_t cache,
uint32_t cache_line);

View File

@ -54,4 +54,6 @@ void ocf_kick_cleaner(ocf_cache_t cache);
void ocf_stop_cleaner(ocf_cache_t cache);
typedef void (*ocf_cleaning_recovery_end_t)(void *priv, int error);
#endif

View File

@ -14,6 +14,8 @@
struct cleaning_policy_ops {
void (*setup)(ocf_cache_t cache);
int (*initialize)(ocf_cache_t cache, int init_metadata);
void (*recovery)(ocf_cache_t cache,
ocf_cleaning_recovery_end_t cmpl, void *priv);
void (*deinitialize)(ocf_cache_t cache);
int (*add_core)(ocf_cache_t cache, ocf_core_id_t core_id);
void (*remove_core)(ocf_cache_t cache, ocf_core_id_t core_id);
@ -43,6 +45,7 @@ static struct cleaning_policy_ops cleaning_policy_ops[ocf_cleaning_max] = {
.purge_range = cleaning_policy_alru_purge_range,
.set_hot_cache_line = cleaning_policy_alru_set_hot_cache_line,
.initialize = cleaning_policy_alru_initialize,
.recovery = cleaning_policy_alru_recovery,
.deinitialize = cleaning_policy_alru_deinitialize,
.set_cleaning_param = cleaning_policy_alru_set_cleaning_param,
.get_cleaning_param = cleaning_policy_alru_get_cleaning_param,
@ -56,6 +59,7 @@ static struct cleaning_policy_ops cleaning_policy_ops[ocf_cleaning_max] = {
.purge_range = cleaning_policy_acp_purge_range,
.set_hot_cache_line = cleaning_policy_acp_set_hot_cache_line,
.initialize = cleaning_policy_acp_initialize,
.recovery = cleaning_policy_acp_recovery,
.deinitialize = cleaning_policy_acp_deinitialize,
.set_cleaning_param = cleaning_policy_acp_set_cleaning_param,
.get_cleaning_param = cleaning_policy_acp_get_cleaning_param,
@ -87,6 +91,20 @@ static inline int ocf_cleaning_initialize(ocf_cache_t cache,
return cleaning_policy_ops[policy].initialize(cache, init_metadata);
}
static inline void ocf_cleaning_recovery(ocf_cache_t cache,
ocf_cleaning_t policy,
ocf_cleaning_recovery_end_t cmpl, void *priv)
{
ENV_BUG_ON(policy >= ocf_cleaning_max);
if (unlikely(!cleaning_policy_ops[policy].recovery)) {
cmpl(priv, 0);
return;
}
cleaning_policy_ops[policy].recovery(cache, cmpl, priv);
}
static inline void ocf_cleaning_deinitialize(ocf_cache_t cache)
{
ocf_cleaning_t policy;

View File

@ -642,23 +642,6 @@ static void ocf_mngt_rebuild_metadata(ocf_cache_t cache,
ocf_parallelize_run(parallelize);
}
static inline ocf_error_t _ocf_init_cleaning_policy(ocf_cache_t cache,
ocf_cleaning_t cleaning_policy,
enum ocf_metadata_shutdown_status shutdown_status)
{
ocf_error_t result;
if (shutdown_status == ocf_metadata_clean_shutdown)
result = ocf_cleaning_initialize(cache, cleaning_policy, 0);
else
result = ocf_cleaning_initialize(cache, cleaning_policy, 1);
if (result)
ocf_cache_log(cache, log_err, "Cannot initialize cleaning policy\n");
return result;
}
static void _ocf_mngt_load_rebuild_metadata_complete(void *priv, int error)
{
struct ocf_cache_attach_context *context = priv;
@ -682,6 +665,13 @@ static void _ocf_mngt_load_rebuild_metadata(ocf_pipeline_t pipeline,
ocf_pipeline_next(pipeline);
}
static void _ocf_mngt_cleaning_recovery_complete(void *priv, int error)
{
struct ocf_cache_attach_context *context = priv;
OCF_PL_NEXT_ON_SUCCESS_RET(context->pipeline, error);
}
static void _ocf_mngt_load_init_cleaning(ocf_pipeline_t pipeline,
void *priv, ocf_pipeline_arg_t arg)
{
@ -689,13 +679,14 @@ static void _ocf_mngt_load_init_cleaning(ocf_pipeline_t pipeline,
ocf_cache_t cache = context->cache;
ocf_error_t result;
result = _ocf_init_cleaning_policy(cache, cache->cleaner.policy,
context->metadata.shutdown_status);
if (context->metadata.shutdown_status == ocf_metadata_clean_shutdown) {
result = ocf_cleaning_initialize(cache,
cache->cleaner.policy, 0);
OCF_PL_NEXT_ON_SUCCESS_RET(pipeline, result);
}
if (result)
OCF_PL_FINISH_RET(pipeline, result);
ocf_pipeline_next(pipeline);
ocf_cleaning_recovery(cache, cache->cleaner.policy,
_ocf_mngt_cleaning_recovery_complete, context);
}
void _ocf_mngt_load_metadata_complete(void *priv, int error)

View File

@ -1,130 +0,0 @@
/*
* Copyright(c) 2012-2021 Intel Corporation
* SPDX-License-Identifier: BSD-3-Clause
*/
/*
<tested_file_path>src/cleaning/alru.c</tested_file_path>
<tested_function>cleaning_policy_alru_initialize_part</tested_function>
<functions_to_leave>
</functions_to_leave>
*/
#undef static
#undef inline
/*
* This headers must be in test source file. It's important that cmocka.h is
* last.
*/
#include <stdarg.h>
#include <stddef.h>
#include <setjmp.h>
#include <cmocka.h>
#include "print_desc.h"
/*
* Headers from tested target.
*/
#include "ocf/ocf.h"
#include "../ocf_cache_priv.h"
#include "cleaning.h"
#include "alru.h"
#include "../metadata/metadata.h"
#include "../utils/utils_cleaner.h"
#include "../utils/utils_user_part.h"
#include "../utils/utils_realloc.h"
#include "../concurrency/ocf_cache_line_concurrency.h"
#include "../ocf_def_priv.h"
#include "cleaning/alru.c/cleaning_policy_alru_initialize_part_test_generated_wraps.c"
static void cleaning_policy_alru_initialize_test01(void **state)
{
int result;
struct ocf_cache *cache;
ocf_part_id_t part_id = 0;
int collision_table_entries = 900729;
print_test_description("Check if all variables are set correctly");
cache = test_malloc(sizeof(*cache));
cache->user_parts[part_id].part.runtime = test_malloc(sizeof(struct ocf_part_runtime));
cache->user_parts[part_id].clean_pol = test_malloc(sizeof(*cache->user_parts[part_id].clean_pol));
cache->user_parts[part_id].part.id = part_id;
cache->device = test_malloc(sizeof(struct ocf_cache_device));
cache->device->runtime_meta = test_malloc(sizeof(struct ocf_superblock_runtime));
cache->device->collision_table_entries = collision_table_entries;
result = cleaning_policy_alru_initialize_part(cache, &cache->user_parts[part_id], 1, 1);
assert_int_equal(result, 0);
assert_int_equal(env_atomic_read(&cache->user_parts[part_id].clean_pol->policy.alru.size), 0);
assert_int_equal(cache->user_parts[part_id].clean_pol->policy.alru.lru_head, collision_table_entries);
assert_int_equal(cache->user_parts[part_id].clean_pol->policy.alru.lru_tail, collision_table_entries);
assert_int_equal(cache->device->runtime_meta->cleaning_thread_access, 0);
test_free(cache->device->runtime_meta);
test_free(cache->device);
test_free(cache->user_parts[part_id].clean_pol);
test_free(cache->user_parts[part_id].part.runtime);
test_free(cache);
}
static void cleaning_policy_alru_initialize_test02(void **state)
{
int result;
struct ocf_cache *cache;
ocf_part_id_t part_id = 0;
uint32_t collision_table_entries = 900729;
print_test_description("Check if only appropirate variables are changed");
cache = test_malloc(sizeof(*cache));
cache->user_parts[part_id].part.runtime = test_malloc(sizeof(struct ocf_part_runtime));
cache->user_parts[part_id].clean_pol = test_malloc(sizeof(*cache->user_parts[part_id].clean_pol));
cache->device = test_malloc(sizeof(struct ocf_cache_device));
cache->device->runtime_meta = test_malloc(sizeof(struct ocf_superblock_runtime));
env_atomic_set(&cache->user_parts[part_id].clean_pol->policy.alru.size, 1);
cache->user_parts[part_id].clean_pol->policy.alru.lru_head = -collision_table_entries;
cache->user_parts[part_id].clean_pol->policy.alru.lru_tail = -collision_table_entries;
result = cleaning_policy_alru_initialize_part(cache, &cache->user_parts[part_id], 0, 0);
assert_int_equal(result, 0);
assert_int_equal(env_atomic_read(&cache->user_parts[part_id].clean_pol->policy.alru.size), 1);
assert_int_equal(cache->user_parts[part_id].clean_pol->policy.alru.lru_head, -collision_table_entries);
assert_int_equal(cache->user_parts[part_id].clean_pol->policy.alru.lru_tail, -collision_table_entries);
assert_int_equal(cache->device->runtime_meta->cleaning_thread_access, 0);
test_free(cache->device->runtime_meta);
test_free(cache->device);
test_free(cache->user_parts[part_id].clean_pol);
test_free(cache->user_parts[part_id].part.runtime);
test_free(cache);
}
/*
* Main function. It runs tests.
*/
int main(void)
{
const struct CMUnitTest tests[] = {
cmocka_unit_test(cleaning_policy_alru_initialize_test01),
cmocka_unit_test(cleaning_policy_alru_initialize_test02)
};
print_message("Unit test of alru.c\n");
return cmocka_run_group_tests(tests, NULL, NULL);
}