From 036aca41b3aef8e9e9a97d33c7ec2a09edec0808 Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Wed, 12 Jan 2022 22:12:58 +0100 Subject: [PATCH] Parallelize ocf_lru_populate() Signed-off-by: Robert Baldyga --- src/mngt/ocf_mngt_cache.c | 131 +++++++++++++++++++++----------------- src/ocf_lru.c | 62 ++++++++++++++++-- src/ocf_lru.h | 6 +- 3 files changed, 132 insertions(+), 67 deletions(-) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index df7f50b..c67908c 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -215,14 +215,6 @@ static void __init_parts_attached(ocf_cache_t cache) ocf_lru_init(cache, &cache->free); } -static void __populate_free(ocf_cache_t cache) -{ - uint64_t free_clines = ocf_metadata_collision_table_entries(cache) - - ocf_get_cache_occupancy(cache); - - ocf_lru_populate(cache, free_clines); -} - static ocf_error_t __init_cleaning_policy(ocf_cache_t cache) { int i; @@ -300,29 +292,6 @@ static void __reset_stats(ocf_cache_t cache) } } -static ocf_error_t init_attached_data_structures(ocf_cache_t cache) -{ - ocf_error_t result; - - /* Lock to ensure consistency */ - - ocf_metadata_init_hash_table(cache); - ocf_metadata_init_collision(cache); - __init_parts_attached(cache); - __populate_free(cache); - - result = __init_cleaning_policy(cache); - if (result) { - ocf_cache_log(cache, log_err, - "Cannot initialize cleaning policy\n"); - return result; - } - - __setup_promotion_policy(cache); - - return 0; -} - static void init_attached_data_structures_recovery(ocf_cache_t cache, bool init_collision) { @@ -565,21 +534,29 @@ static inline ocf_error_t _ocf_init_cleaning_policy(ocf_cache_t cache, return result; } -static void _ocf_mngt_load_post_metadata_load(ocf_pipeline_t pipeline, +static void _ocf_mngt_load_rebuild_metadata(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; - ocf_error_t result; int ret; if (context->metadata.shutdown_status != ocf_metadata_clean_shutdown) { ret = _ocf_mngt_recovery_rebuild_metadata(cache); if (ret) OCF_PL_FINISH_RET(pipeline, ret); - __populate_free(cache); } + ocf_pipeline_next(pipeline); +} + +static void _ocf_mngt_load_init_cleaning(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) +{ + struct ocf_cache_attach_context *context = priv; + ocf_cache_t cache = context->cache; + ocf_error_t result; + result = _ocf_init_cleaning_policy(cache, cache->cleaner.policy, context->metadata.shutdown_status); @@ -1106,16 +1083,51 @@ static void _ocf_mngt_attach_prepare_metadata(ocf_pipeline_t pipeline, /** * @brief initializing cache anew (not loading or recovering) */ -static void _ocf_mngt_attach_init_instance(ocf_pipeline_t pipeline, +static void _ocf_mngt_attach_init_metadata(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) +{ + struct ocf_cache_attach_context *context = priv; + ocf_cache_t cache = context->cache; + + ocf_metadata_init_hash_table(cache); + ocf_metadata_init_collision(cache); + __init_parts_attached(cache); + + ocf_pipeline_next(pipeline); +} + +static void _ocf_mngt_attach_populate_free_complete(void *priv, int error) +{ + struct ocf_cache_attach_context *context = priv; + + OCF_PL_NEXT_ON_SUCCESS_RET(context->pipeline, error); +} + +static void _ocf_mngt_attach_populate_free(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) +{ + struct ocf_cache_attach_context *context = priv; + ocf_cache_t cache = context->cache; + + ocf_lru_populate(cache, _ocf_mngt_attach_populate_free_complete, + context); +} + +static void _ocf_mngt_attach_init_services(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; ocf_error_t result; - result = init_attached_data_structures(cache); - if (result) + result = __init_cleaning_policy(cache); + if (result) { + ocf_cache_log(cache, log_err, + "Cannot initialize cleaning policy\n"); OCF_PL_FINISH_RET(pipeline, result); + } + + __setup_promotion_policy(cache); /* In initial cache state there is no dirty data, so all dirty data is considered to be flushed @@ -1726,7 +1738,9 @@ struct ocf_pipeline_properties _ocf_mngt_cache_attach_pipeline_properties = { OCF_PL_STEP(_ocf_mngt_test_volume), OCF_PL_STEP(_ocf_mngt_init_cleaner), OCF_PL_STEP(_ocf_mngt_init_promotion), - OCF_PL_STEP(_ocf_mngt_attach_init_instance), + OCF_PL_STEP(_ocf_mngt_attach_init_metadata), + OCF_PL_STEP(_ocf_mngt_attach_populate_free), + OCF_PL_STEP(_ocf_mngt_attach_init_services), OCF_PL_STEP(_ocf_mngt_zero_superblock), OCF_PL_STEP(_ocf_mngt_attach_flush_metadata), OCF_PL_STEP(_ocf_mngt_attach_discard), @@ -1737,6 +1751,19 @@ struct ocf_pipeline_properties _ocf_mngt_cache_attach_pipeline_properties = { }, }; +static void _ocf_mngt_load_populate_free(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) +{ + struct ocf_cache_attach_context *context = priv; + + if (context->metadata.shutdown_status != ocf_metadata_clean_shutdown) { + _ocf_mngt_attach_populate_free(pipeline, priv, arg); + return; + } + + ocf_pipeline_next(pipeline); +} + struct ocf_pipeline_properties _ocf_mngt_cache_load_pipeline_properties = { .priv_size = sizeof(struct ocf_cache_attach_context), .finish = _ocf_mngt_cache_attach_finish, @@ -1754,7 +1781,9 @@ struct ocf_pipeline_properties _ocf_mngt_cache_load_pipeline_properties = { OCF_PL_STEP(_ocf_mngt_init_promotion), OCF_PL_STEP(_ocf_mngt_load_add_cores), OCF_PL_STEP(_ocf_mngt_load_metadata), - OCF_PL_STEP(_ocf_mngt_load_post_metadata_load), + OCF_PL_STEP(_ocf_mngt_load_rebuild_metadata), + OCF_PL_STEP(_ocf_mngt_load_populate_free), + OCF_PL_STEP(_ocf_mngt_load_init_cleaning), OCF_PL_STEP(_ocf_mngt_attach_shutdown_status), OCF_PL_STEP(_ocf_mngt_attach_flush_metadata), OCF_PL_STEP(_ocf_mngt_attach_shutdown_status), @@ -2096,7 +2125,6 @@ static void _ocf_mngt_standby_init_structures_attach(ocf_pipeline_t pipeline, ocf_cache_t cache = context->cache; init_attached_data_structures_recovery(cache, true); - __populate_free(cache); ocf_pipeline_next(pipeline); } @@ -2136,23 +2164,6 @@ static void _ocf_mngt_standby_recovery(ocf_pipeline_t pipeline, ret = _ocf_mngt_recovery_rebuild_metadata(cache); if (ret) OCF_PL_FINISH_RET(pipeline, ret); - __populate_free(cache); - - ocf_pipeline_next(pipeline); -} - -static void _ocf_mngt_standby_init_cleaning(ocf_pipeline_t pipeline, - void *priv, ocf_pipeline_arg_t arg) -{ - struct ocf_cache_attach_context *context = priv; - ocf_cache_t cache = context->cache; - ocf_error_t result; - - result = _ocf_init_cleaning_policy(cache, cache->cleaner.policy, - context->metadata.shutdown_status); - - if (result) - OCF_PL_FINISH_RET(pipeline, result); ocf_pipeline_next(pipeline); } @@ -2183,7 +2194,8 @@ struct ocf_pipeline_properties _ocf_mngt_cache_standby_attach_pipeline_propertie OCF_PL_STEP(_ocf_mngt_test_volume), OCF_PL_STEP(_ocf_mngt_init_cleaner), OCF_PL_STEP(_ocf_mngt_standby_init_structures_attach), - OCF_PL_STEP(_ocf_mngt_standby_init_cleaning), + OCF_PL_STEP(_ocf_mngt_attach_populate_free), + OCF_PL_STEP(_ocf_mngt_load_init_cleaning), OCF_PL_STEP(_ocf_mngt_standby_preapre_mempool), OCF_PL_STEP(_ocf_mngt_standby_init_pio_concurrency), OCF_PL_STEP(_ocf_mngt_zero_superblock), @@ -2211,10 +2223,11 @@ struct ocf_pipeline_properties _ocf_mngt_cache_standby_load_pipeline_properties OCF_PL_STEP(_ocf_mngt_load_metadata_recovery), OCF_PL_STEP(_ocf_mngt_init_cleaner), OCF_PL_STEP(_ocf_mngt_standby_init_structures_load), - OCF_PL_STEP(_ocf_mngt_standby_init_cleaning), + OCF_PL_STEP(_ocf_mngt_load_init_cleaning), OCF_PL_STEP(_ocf_mngt_standby_preapre_mempool), OCF_PL_STEP(_ocf_mngt_standby_init_pio_concurrency), OCF_PL_STEP(_ocf_mngt_standby_recovery), + OCF_PL_STEP(_ocf_mngt_attach_populate_free), OCF_PL_STEP(_ocf_mngt_standby_post_init), OCF_PL_STEP_TERMINATOR(), }, diff --git a/src/ocf_lru.c b/src/ocf_lru.c index 736cf8f..1b69cb7 100644 --- a/src/ocf_lru.c +++ b/src/ocf_lru.c @@ -7,6 +7,7 @@ #include "ocf_lru.h" #include "utils/utils_cleaner.h" #include "utils/utils_cache_line.h" +#include "utils/utils_parallelize.h" #include "concurrency/ocf_concurrency.h" #include "mngt/ocf_mngt_common.h" #include "engine/engine_zero.h" @@ -867,17 +868,27 @@ void ocf_lru_dirty_cline(ocf_cache_t cache, struct ocf_part *part, OCF_METADATA_LRU_WR_UNLOCK(cline); } -/* put invalid cachelines on freelist partition lru list */ -void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines) +struct ocf_lru_populate_context { + ocf_cache_t cache; + env_atomic curr_size; + + ocf_lru_populate_end_t cmpl; + void *priv; +}; + +static int ocf_lru_populate_handle(ocf_parallelize_t parallelize, + void *priv, unsigned shard_id, unsigned shards_cnt) { + struct ocf_lru_populate_context *context = priv; + ocf_cache_t cache = context->cache; ocf_cache_line_t cnt, cline; ocf_cache_line_t entries = ocf_metadata_collision_table_entries(cache); struct ocf_lru_list *list; - unsigned lru_list; + unsigned lru_list = shard_id; unsigned step = 0; cnt = 0; - for (cline = 0; cline < entries; cline++) { + for (cline = shard_id; cline < entries; cline += shards_cnt) { OCF_COND_RESCHED_DEFAULT(step); if (metadata_test_valid_any(cache, cline)) @@ -885,7 +896,6 @@ void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines) ocf_metadata_set_partition_id(cache, cline, PARTITION_FREELIST); - lru_list = (cline % OCF_NUM_LRU_LISTS); list = ocf_lru_get_list(&cache->free, lru_list, true); add_lru_head_nobalance(cache, list, cline); @@ -893,9 +903,47 @@ void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines) cnt++; } - ENV_BUG_ON(cnt != num_free_clines); + env_atomic_add(cnt, &context->curr_size); - env_atomic_set(&cache->free.runtime->curr_size, cnt); + return 0; +} + +static void ocf_lru_populate_finish(ocf_parallelize_t parallelize, + void *priv, int error) +{ + struct ocf_lru_populate_context *context = priv; + + env_atomic_set(&context->cache->free.runtime->curr_size, + env_atomic_read(&context->curr_size)); + + context->cmpl(context->priv, error); + + ocf_parallelize_destroy(parallelize); +} + +/* put invalid cachelines on freelist partition lru list */ +void ocf_lru_populate(ocf_cache_t cache, + ocf_lru_populate_end_t cmpl, void *priv) +{ + struct ocf_lru_populate_context *context; + ocf_parallelize_t parallelize; + int result; + + result = ocf_parallelize_create(¶llelize, cache, OCF_NUM_LRU_LISTS, + sizeof(*context), ocf_lru_populate_handle, + ocf_lru_populate_finish); + if (result) { + cmpl(priv, result); + return; + } + + context = ocf_parallelize_get_priv(parallelize); + context->cache = cache; + env_atomic_set(&context->curr_size, 0); + context->cmpl = cmpl; + context->priv = priv; + + ocf_parallelize_run(parallelize); } static bool _is_cache_line_acting(struct ocf_cache *cache, diff --git a/src/ocf_lru.h b/src/ocf_lru.h index e093ee8..79fc559 100644 --- a/src/ocf_lru.h +++ b/src/ocf_lru.h @@ -32,10 +32,14 @@ void ocf_lru_repart(ocf_cache_t cache, ocf_cache_line_t cline, struct ocf_part *src_upart, struct ocf_part *dst_upart); void ocf_lru_add_free(ocf_cache_t cache, ocf_cache_line_t cline); uint32_t ocf_lru_num_free(ocf_cache_t cache); -void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines); struct ocf_lru_list *ocf_lru_get_list(struct ocf_part *part, uint32_t lru_idx, bool clean); void ocf_lru_remove_locked(ocf_cache_t cache, struct ocf_lru_list *list, ocf_cache_line_t cline); +typedef void (*ocf_lru_populate_end_t)(void *priv, int error); + +void ocf_lru_populate(ocf_cache_t cache, + ocf_lru_populate_end_t cmpl, void *priv); + #endif