From 568c5654978b2d25ea6cc1a2428465bd55b8a005 Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Fri, 21 Jan 2022 16:34:35 +0100 Subject: [PATCH 01/15] Init properties before loading superblock Signed-off-by: Robert Baldyga --- src/mngt/ocf_mngt_cache.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 8b14bfe..4dce6ad 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -2428,6 +2428,7 @@ struct ocf_pipeline_properties _ocf_mngt_cache_activate_pipeline_properties = { .steps = { OCF_PL_STEP(_ocf_mngt_copy_uuid_data), OCF_PL_STEP(_ocf_mngt_activate_set_cache_device), + OCF_PL_STEP(_ocf_mngt_activate_init_properties), OCF_PL_STEP(_ocf_mngt_activate_compare_superblock), OCF_PL_STEP(_ocf_mngt_load_superblock), OCF_PL_STEP(_ocf_mngt_activate_check_superblock), From 25e2551964d578236481f3da25e82c71b4ef9cac Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Mon, 17 Jan 2022 20:48:17 +0100 Subject: [PATCH 02/15] Check core status during recovery based on core metadata Signed-off-by: Robert Baldyga --- src/mngt/ocf_mngt_cache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 4dce6ad..df7f50b 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -527,7 +527,7 @@ static int _ocf_mngt_rebuild_metadata(ocf_cache_t cache) continue; } - if (!cache->core[core_id].added) { + if (!cache->core[core_id].conf_meta->valid) { ocf_cache_log(cache, log_err, "Stale mapping in " "on-disk metadata - refusing to " "recover cache.\n"); From b82d30a0efff11fb79f2e4b24a2e3015da870516 Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Fri, 14 Jan 2022 12:19:07 +0100 Subject: [PATCH 03/15] Add missing hb lock functions implementation Signed-off-by: Robert Baldyga --- src/concurrency/ocf_metadata_concurrency.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/concurrency/ocf_metadata_concurrency.c b/src/concurrency/ocf_metadata_concurrency.c index 8c190c2..9eae499 100644 --- a/src/concurrency/ocf_metadata_concurrency.c +++ b/src/concurrency/ocf_metadata_concurrency.c @@ -299,6 +299,18 @@ void ocf_hb_cline_naked_unlock_wr(struct ocf_metadata_lock *metadata_lock, ocf_hb_id_naked_unlock(metadata_lock, hash, OCF_METADATA_WR); } +void ocf_hb_id_naked_lock_wr(struct ocf_metadata_lock *metadata_lock, + ocf_cache_line_t hash) +{ + ocf_hb_id_naked_lock(metadata_lock, hash, OCF_METADATA_WR); +} + +void ocf_hb_id_naked_unlock_wr(struct ocf_metadata_lock *metadata_lock, + ocf_cache_line_t hash) +{ + ocf_hb_id_naked_unlock(metadata_lock, hash, OCF_METADATA_WR); +} + /* common part of protected hash bucket lock routines */ static inline void ocf_hb_id_prot_lock_common( struct ocf_metadata_lock *metadata_lock, From a947127f550948272a09299fa666425ebcb7f2fe Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Fri, 14 Jan 2022 17:16:21 +0100 Subject: [PATCH 04/15] Introduce ocf_lru_add_free() function Signed-off-by: Robert Baldyga --- src/ocf_lru.c | 9 +++++++++ src/ocf_lru.h | 1 + 2 files changed, 10 insertions(+) diff --git a/src/ocf_lru.c b/src/ocf_lru.c index ad87f1a..fee3849 100644 --- a/src/ocf_lru.c +++ b/src/ocf_lru.c @@ -1027,3 +1027,12 @@ uint32_t ocf_lru_num_free(ocf_cache_t cache) { return env_atomic_read(&cache->free.runtime->curr_size); } + +void ocf_lru_add_free(ocf_cache_t cache, ocf_cache_line_t cline) +{ + uint32_t lru_list = (cline % OCF_NUM_LRU_LISTS); + struct ocf_lru_list *list; + + list = ocf_lru_get_list(&cache->free, lru_list, true); + add_lru_head_nobalance(cache, list, cline); +} diff --git a/src/ocf_lru.h b/src/ocf_lru.h index a8b0abd..e093ee8 100644 --- a/src/ocf_lru.h +++ b/src/ocf_lru.h @@ -30,6 +30,7 @@ void ocf_lru_clean(ocf_cache_t cache, struct ocf_user_part *user_part, ocf_queue_t io_queue, uint32_t count); void ocf_lru_repart(ocf_cache_t cache, ocf_cache_line_t cline, struct ocf_part *src_upart, struct ocf_part *dst_upart); +void ocf_lru_add_free(ocf_cache_t cache, ocf_cache_line_t cline); uint32_t ocf_lru_num_free(ocf_cache_t cache); void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines); struct ocf_lru_list *ocf_lru_get_list(struct ocf_part *part, From 57fd5c1f20e79bfd5bd204ff88924332b8b818b6 Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Wed, 12 Jan 2022 22:12:17 +0100 Subject: [PATCH 05/15] Introduce ocf_parallelize utility Introduce utility that allows to parallelize management operation across all available io queues. Signed-off-by: Robert Baldyga --- src/utils/utils_parallelize.c | 145 ++++++++++++++++++++++++++++++++++ src/utils/utils_parallelize.h | 32 ++++++++ 2 files changed, 177 insertions(+) create mode 100644 src/utils/utils_parallelize.c create mode 100644 src/utils/utils_parallelize.h diff --git a/src/utils/utils_parallelize.c b/src/utils/utils_parallelize.c new file mode 100644 index 0000000..7c5b129 --- /dev/null +++ b/src/utils/utils_parallelize.c @@ -0,0 +1,145 @@ +/* + * Copyright(c) 2012-2021 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "ocf/ocf.h" +#include "../engine/cache_engine.h" +#include "../engine/engine_common.h" +#include "../ocf_request.h" +#include "utils_parallelize.h" + +struct ocf_parallelize { + ocf_cache_t cache; + ocf_parallelize_handle_t handle; + ocf_parallelize_finish_t finish; + void *priv; + + unsigned shards_cnt; + env_atomic remaining; + env_atomic error; + + struct ocf_request *reqs[]; +}; + +static int _ocf_parallelize_hndl(struct ocf_request *req) +{ + ocf_parallelize_t parallelize = req->priv; + ocf_parallelize_finish_t finish; + void *priv; + int error; + + error = parallelize->handle(parallelize, parallelize->priv, + req->byte_position, parallelize->shards_cnt); + + env_atomic_cmpxchg(¶llelize->error, 0, error); + + if (env_atomic_dec_return(¶llelize->remaining)) + return 0; + + finish = parallelize->finish; + priv = parallelize->priv; + error = env_atomic_read(¶llelize->error); + + finish(parallelize, priv, error); + + return 0; +} + +static const struct ocf_io_if _io_if_parallelize = { + .read = _ocf_parallelize_hndl, + .write = _ocf_parallelize_hndl, +}; + +int ocf_parallelize_create(ocf_parallelize_t *parallelize, + ocf_cache_t cache, unsigned shards_cnt, uint32_t priv_size, + ocf_parallelize_handle_t handle, + ocf_parallelize_finish_t finish) +{ + ocf_parallelize_t tmp_parallelize; + ocf_queue_t queue; + size_t prl_size; + unsigned queue_count = 0; + int result, i; + + list_for_each_entry(queue, &cache->io_queues, list) + queue_count++; + + if (shards_cnt == 0) + shards_cnt = queue_count; + + prl_size = sizeof(*tmp_parallelize) + + shards_cnt * sizeof(*tmp_parallelize->reqs); + + tmp_parallelize = env_vzalloc(prl_size + priv_size); + if (!tmp_parallelize) + return -OCF_ERR_NO_MEM; + + if (priv_size > 0) + tmp_parallelize->priv = (void *)tmp_parallelize + prl_size; + + tmp_parallelize->cache = cache; + tmp_parallelize->handle = handle; + tmp_parallelize->finish = finish; + + tmp_parallelize->shards_cnt = shards_cnt; + env_atomic_set(&tmp_parallelize->remaining, shards_cnt); + env_atomic_set(&tmp_parallelize->error, 0); + + for (i = 0; i < shards_cnt;) { + list_for_each_entry(queue, &cache->io_queues, list) { + if (i == shards_cnt) + break; + tmp_parallelize->reqs[i] = ocf_req_new(queue, + NULL, 0, 0, 0); + if (!tmp_parallelize->reqs[i]) { + result = -OCF_ERR_NO_MEM; + goto err_reqs; + } + tmp_parallelize->reqs[i]->info.internal = true; + tmp_parallelize->reqs[i]->io_if = &_io_if_parallelize; + tmp_parallelize->reqs[i]->byte_position = i; + tmp_parallelize->reqs[i]->priv = tmp_parallelize; + i++; + } + } + + *parallelize = tmp_parallelize; + + return 0; + +err_reqs: + while (i--) + ocf_req_put(tmp_parallelize->reqs[i]); + env_vfree(tmp_parallelize); + + return result; +} + +void ocf_parallelize_destroy(ocf_parallelize_t parallelize) +{ + int i; + + for (i = 0; i < parallelize->shards_cnt; i++) + ocf_req_put(parallelize->reqs[i]); + + env_vfree(parallelize); +} + +void *ocf_parallelize_get_priv(ocf_parallelize_t parallelize) +{ + return parallelize->priv; +} + +void ocf_parallelize_set_priv(ocf_parallelize_t parallelize, void *priv) +{ + parallelize->priv = priv; +} + +void ocf_parallelize_run(ocf_parallelize_t parallelize) +{ + int i; + + for (i = 0; i < parallelize->shards_cnt; i++) + ocf_engine_push_req_front(parallelize->reqs[i], false); +} diff --git a/src/utils/utils_parallelize.h b/src/utils/utils_parallelize.h new file mode 100644 index 0000000..0066ef4 --- /dev/null +++ b/src/utils/utils_parallelize.h @@ -0,0 +1,32 @@ +/* + * Copyright(c) 2022 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef __UTILS_PARALLELIZE_H__ +#define __UTILS_PARALLELIZE_H__ + +#include "ocf/ocf.h" + +typedef struct ocf_parallelize *ocf_parallelize_t; + +typedef int (*ocf_parallelize_handle_t)(ocf_parallelize_t parallelize, + void *priv, unsigned shard_id, unsigned shards_cnt); + +typedef void (*ocf_parallelize_finish_t)(ocf_parallelize_t parallelize, + void *priv, int error); + +int ocf_parallelize_create(ocf_parallelize_t *parallelize, + ocf_cache_t cache, unsigned shards_cnt, uint32_t priv_size, + ocf_parallelize_handle_t handle, + ocf_parallelize_finish_t finish); + +void ocf_parallelize_destroy(ocf_parallelize_t parallelize); + +void ocf_parallelize_set_priv(ocf_parallelize_t parallelize, void *priv); + +void *ocf_parallelize_get_priv(ocf_parallelize_t parallelize); + +void ocf_parallelize_run(ocf_parallelize_t parallelize); + +#endif /* __UTILS_PARALLELIZE_H__ */ From 6611b25d1eba61e348d3a365c40107649294d57c Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Wed, 12 Jan 2022 09:06:42 +0100 Subject: [PATCH 06/15] Initialize LRU lists in domain of cache lines Signed-off-by: Robert Baldyga --- src/ocf_lru.c | 52 ++++++++++++--------------------------------------- 1 file changed, 12 insertions(+), 40 deletions(-) diff --git a/src/ocf_lru.c b/src/ocf_lru.c index fee3849..736cf8f 100644 --- a/src/ocf_lru.c +++ b/src/ocf_lru.c @@ -867,63 +867,35 @@ void ocf_lru_dirty_cline(ocf_cache_t cache, struct ocf_part *part, OCF_METADATA_LRU_WR_UNLOCK(cline); } -static ocf_cache_line_t next_phys_invalid(ocf_cache_t cache, - ocf_cache_line_t phys) -{ - ocf_cache_line_t lg; - ocf_cache_line_t collision_table_entries = - ocf_metadata_collision_table_entries(cache); - - if (phys == collision_table_entries) - return collision_table_entries; - - lg = ocf_metadata_map_phy2lg(cache, phys); - while (metadata_test_valid_any(cache, lg)) { - ++phys; - - if (phys == collision_table_entries) - break; - - lg = ocf_metadata_map_phy2lg(cache, phys); - } - - return phys; -} - /* put invalid cachelines on freelist partition lru list */ void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines) { - ocf_cache_line_t phys, cline; - ocf_cache_line_t collision_table_entries = - ocf_metadata_collision_table_entries(cache); + ocf_cache_line_t cnt, cline; + ocf_cache_line_t entries = ocf_metadata_collision_table_entries(cache); struct ocf_lru_list *list; unsigned lru_list; - unsigned i; unsigned step = 0; - phys = 0; - for (i = 0; i < num_free_clines; i++) { - /* find first invalid cacheline */ - phys = next_phys_invalid(cache, phys); - ENV_BUG_ON(phys == collision_table_entries); - cline = ocf_metadata_map_phy2lg(cache, phys); - ++phys; + cnt = 0; + for (cline = 0; cline < entries; cline++) { + OCF_COND_RESCHED_DEFAULT(step); + + if (metadata_test_valid_any(cache, cline)) + continue; ocf_metadata_set_partition_id(cache, cline, PARTITION_FREELIST); lru_list = (cline % OCF_NUM_LRU_LISTS); list = ocf_lru_get_list(&cache->free, lru_list, true); - add_lru_head(cache, list, cline); + add_lru_head_nobalance(cache, list, cline); - OCF_COND_RESCHED_DEFAULT(step); + cnt++; } - /* we should have reached the last invalid cache line */ - phys = next_phys_invalid(cache, phys); - ENV_BUG_ON(phys != collision_table_entries); + ENV_BUG_ON(cnt != num_free_clines); - env_atomic_set(&cache->free.runtime->curr_size, i); + env_atomic_set(&cache->free.runtime->curr_size, cnt); } static bool _is_cache_line_acting(struct ocf_cache *cache, From 036aca41b3aef8e9e9a97d33c7ec2a09edec0808 Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Wed, 12 Jan 2022 22:12:58 +0100 Subject: [PATCH 07/15] Parallelize ocf_lru_populate() Signed-off-by: Robert Baldyga --- src/mngt/ocf_mngt_cache.c | 131 +++++++++++++++++++++----------------- src/ocf_lru.c | 62 ++++++++++++++++-- src/ocf_lru.h | 6 +- 3 files changed, 132 insertions(+), 67 deletions(-) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index df7f50b..c67908c 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -215,14 +215,6 @@ static void __init_parts_attached(ocf_cache_t cache) ocf_lru_init(cache, &cache->free); } -static void __populate_free(ocf_cache_t cache) -{ - uint64_t free_clines = ocf_metadata_collision_table_entries(cache) - - ocf_get_cache_occupancy(cache); - - ocf_lru_populate(cache, free_clines); -} - static ocf_error_t __init_cleaning_policy(ocf_cache_t cache) { int i; @@ -300,29 +292,6 @@ static void __reset_stats(ocf_cache_t cache) } } -static ocf_error_t init_attached_data_structures(ocf_cache_t cache) -{ - ocf_error_t result; - - /* Lock to ensure consistency */ - - ocf_metadata_init_hash_table(cache); - ocf_metadata_init_collision(cache); - __init_parts_attached(cache); - __populate_free(cache); - - result = __init_cleaning_policy(cache); - if (result) { - ocf_cache_log(cache, log_err, - "Cannot initialize cleaning policy\n"); - return result; - } - - __setup_promotion_policy(cache); - - return 0; -} - static void init_attached_data_structures_recovery(ocf_cache_t cache, bool init_collision) { @@ -565,21 +534,29 @@ static inline ocf_error_t _ocf_init_cleaning_policy(ocf_cache_t cache, return result; } -static void _ocf_mngt_load_post_metadata_load(ocf_pipeline_t pipeline, +static void _ocf_mngt_load_rebuild_metadata(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; - ocf_error_t result; int ret; if (context->metadata.shutdown_status != ocf_metadata_clean_shutdown) { ret = _ocf_mngt_recovery_rebuild_metadata(cache); if (ret) OCF_PL_FINISH_RET(pipeline, ret); - __populate_free(cache); } + ocf_pipeline_next(pipeline); +} + +static void _ocf_mngt_load_init_cleaning(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) +{ + struct ocf_cache_attach_context *context = priv; + ocf_cache_t cache = context->cache; + ocf_error_t result; + result = _ocf_init_cleaning_policy(cache, cache->cleaner.policy, context->metadata.shutdown_status); @@ -1106,16 +1083,51 @@ static void _ocf_mngt_attach_prepare_metadata(ocf_pipeline_t pipeline, /** * @brief initializing cache anew (not loading or recovering) */ -static void _ocf_mngt_attach_init_instance(ocf_pipeline_t pipeline, +static void _ocf_mngt_attach_init_metadata(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) +{ + struct ocf_cache_attach_context *context = priv; + ocf_cache_t cache = context->cache; + + ocf_metadata_init_hash_table(cache); + ocf_metadata_init_collision(cache); + __init_parts_attached(cache); + + ocf_pipeline_next(pipeline); +} + +static void _ocf_mngt_attach_populate_free_complete(void *priv, int error) +{ + struct ocf_cache_attach_context *context = priv; + + OCF_PL_NEXT_ON_SUCCESS_RET(context->pipeline, error); +} + +static void _ocf_mngt_attach_populate_free(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) +{ + struct ocf_cache_attach_context *context = priv; + ocf_cache_t cache = context->cache; + + ocf_lru_populate(cache, _ocf_mngt_attach_populate_free_complete, + context); +} + +static void _ocf_mngt_attach_init_services(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; ocf_error_t result; - result = init_attached_data_structures(cache); - if (result) + result = __init_cleaning_policy(cache); + if (result) { + ocf_cache_log(cache, log_err, + "Cannot initialize cleaning policy\n"); OCF_PL_FINISH_RET(pipeline, result); + } + + __setup_promotion_policy(cache); /* In initial cache state there is no dirty data, so all dirty data is considered to be flushed @@ -1726,7 +1738,9 @@ struct ocf_pipeline_properties _ocf_mngt_cache_attach_pipeline_properties = { OCF_PL_STEP(_ocf_mngt_test_volume), OCF_PL_STEP(_ocf_mngt_init_cleaner), OCF_PL_STEP(_ocf_mngt_init_promotion), - OCF_PL_STEP(_ocf_mngt_attach_init_instance), + OCF_PL_STEP(_ocf_mngt_attach_init_metadata), + OCF_PL_STEP(_ocf_mngt_attach_populate_free), + OCF_PL_STEP(_ocf_mngt_attach_init_services), OCF_PL_STEP(_ocf_mngt_zero_superblock), OCF_PL_STEP(_ocf_mngt_attach_flush_metadata), OCF_PL_STEP(_ocf_mngt_attach_discard), @@ -1737,6 +1751,19 @@ struct ocf_pipeline_properties _ocf_mngt_cache_attach_pipeline_properties = { }, }; +static void _ocf_mngt_load_populate_free(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) +{ + struct ocf_cache_attach_context *context = priv; + + if (context->metadata.shutdown_status != ocf_metadata_clean_shutdown) { + _ocf_mngt_attach_populate_free(pipeline, priv, arg); + return; + } + + ocf_pipeline_next(pipeline); +} + struct ocf_pipeline_properties _ocf_mngt_cache_load_pipeline_properties = { .priv_size = sizeof(struct ocf_cache_attach_context), .finish = _ocf_mngt_cache_attach_finish, @@ -1754,7 +1781,9 @@ struct ocf_pipeline_properties _ocf_mngt_cache_load_pipeline_properties = { OCF_PL_STEP(_ocf_mngt_init_promotion), OCF_PL_STEP(_ocf_mngt_load_add_cores), OCF_PL_STEP(_ocf_mngt_load_metadata), - OCF_PL_STEP(_ocf_mngt_load_post_metadata_load), + OCF_PL_STEP(_ocf_mngt_load_rebuild_metadata), + OCF_PL_STEP(_ocf_mngt_load_populate_free), + OCF_PL_STEP(_ocf_mngt_load_init_cleaning), OCF_PL_STEP(_ocf_mngt_attach_shutdown_status), OCF_PL_STEP(_ocf_mngt_attach_flush_metadata), OCF_PL_STEP(_ocf_mngt_attach_shutdown_status), @@ -2096,7 +2125,6 @@ static void _ocf_mngt_standby_init_structures_attach(ocf_pipeline_t pipeline, ocf_cache_t cache = context->cache; init_attached_data_structures_recovery(cache, true); - __populate_free(cache); ocf_pipeline_next(pipeline); } @@ -2136,23 +2164,6 @@ static void _ocf_mngt_standby_recovery(ocf_pipeline_t pipeline, ret = _ocf_mngt_recovery_rebuild_metadata(cache); if (ret) OCF_PL_FINISH_RET(pipeline, ret); - __populate_free(cache); - - ocf_pipeline_next(pipeline); -} - -static void _ocf_mngt_standby_init_cleaning(ocf_pipeline_t pipeline, - void *priv, ocf_pipeline_arg_t arg) -{ - struct ocf_cache_attach_context *context = priv; - ocf_cache_t cache = context->cache; - ocf_error_t result; - - result = _ocf_init_cleaning_policy(cache, cache->cleaner.policy, - context->metadata.shutdown_status); - - if (result) - OCF_PL_FINISH_RET(pipeline, result); ocf_pipeline_next(pipeline); } @@ -2183,7 +2194,8 @@ struct ocf_pipeline_properties _ocf_mngt_cache_standby_attach_pipeline_propertie OCF_PL_STEP(_ocf_mngt_test_volume), OCF_PL_STEP(_ocf_mngt_init_cleaner), OCF_PL_STEP(_ocf_mngt_standby_init_structures_attach), - OCF_PL_STEP(_ocf_mngt_standby_init_cleaning), + OCF_PL_STEP(_ocf_mngt_attach_populate_free), + OCF_PL_STEP(_ocf_mngt_load_init_cleaning), OCF_PL_STEP(_ocf_mngt_standby_preapre_mempool), OCF_PL_STEP(_ocf_mngt_standby_init_pio_concurrency), OCF_PL_STEP(_ocf_mngt_zero_superblock), @@ -2211,10 +2223,11 @@ struct ocf_pipeline_properties _ocf_mngt_cache_standby_load_pipeline_properties OCF_PL_STEP(_ocf_mngt_load_metadata_recovery), OCF_PL_STEP(_ocf_mngt_init_cleaner), OCF_PL_STEP(_ocf_mngt_standby_init_structures_load), - OCF_PL_STEP(_ocf_mngt_standby_init_cleaning), + OCF_PL_STEP(_ocf_mngt_load_init_cleaning), OCF_PL_STEP(_ocf_mngt_standby_preapre_mempool), OCF_PL_STEP(_ocf_mngt_standby_init_pio_concurrency), OCF_PL_STEP(_ocf_mngt_standby_recovery), + OCF_PL_STEP(_ocf_mngt_attach_populate_free), OCF_PL_STEP(_ocf_mngt_standby_post_init), OCF_PL_STEP_TERMINATOR(), }, diff --git a/src/ocf_lru.c b/src/ocf_lru.c index 736cf8f..1b69cb7 100644 --- a/src/ocf_lru.c +++ b/src/ocf_lru.c @@ -7,6 +7,7 @@ #include "ocf_lru.h" #include "utils/utils_cleaner.h" #include "utils/utils_cache_line.h" +#include "utils/utils_parallelize.h" #include "concurrency/ocf_concurrency.h" #include "mngt/ocf_mngt_common.h" #include "engine/engine_zero.h" @@ -867,17 +868,27 @@ void ocf_lru_dirty_cline(ocf_cache_t cache, struct ocf_part *part, OCF_METADATA_LRU_WR_UNLOCK(cline); } -/* put invalid cachelines on freelist partition lru list */ -void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines) +struct ocf_lru_populate_context { + ocf_cache_t cache; + env_atomic curr_size; + + ocf_lru_populate_end_t cmpl; + void *priv; +}; + +static int ocf_lru_populate_handle(ocf_parallelize_t parallelize, + void *priv, unsigned shard_id, unsigned shards_cnt) { + struct ocf_lru_populate_context *context = priv; + ocf_cache_t cache = context->cache; ocf_cache_line_t cnt, cline; ocf_cache_line_t entries = ocf_metadata_collision_table_entries(cache); struct ocf_lru_list *list; - unsigned lru_list; + unsigned lru_list = shard_id; unsigned step = 0; cnt = 0; - for (cline = 0; cline < entries; cline++) { + for (cline = shard_id; cline < entries; cline += shards_cnt) { OCF_COND_RESCHED_DEFAULT(step); if (metadata_test_valid_any(cache, cline)) @@ -885,7 +896,6 @@ void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines) ocf_metadata_set_partition_id(cache, cline, PARTITION_FREELIST); - lru_list = (cline % OCF_NUM_LRU_LISTS); list = ocf_lru_get_list(&cache->free, lru_list, true); add_lru_head_nobalance(cache, list, cline); @@ -893,9 +903,47 @@ void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines) cnt++; } - ENV_BUG_ON(cnt != num_free_clines); + env_atomic_add(cnt, &context->curr_size); - env_atomic_set(&cache->free.runtime->curr_size, cnt); + return 0; +} + +static void ocf_lru_populate_finish(ocf_parallelize_t parallelize, + void *priv, int error) +{ + struct ocf_lru_populate_context *context = priv; + + env_atomic_set(&context->cache->free.runtime->curr_size, + env_atomic_read(&context->curr_size)); + + context->cmpl(context->priv, error); + + ocf_parallelize_destroy(parallelize); +} + +/* put invalid cachelines on freelist partition lru list */ +void ocf_lru_populate(ocf_cache_t cache, + ocf_lru_populate_end_t cmpl, void *priv) +{ + struct ocf_lru_populate_context *context; + ocf_parallelize_t parallelize; + int result; + + result = ocf_parallelize_create(¶llelize, cache, OCF_NUM_LRU_LISTS, + sizeof(*context), ocf_lru_populate_handle, + ocf_lru_populate_finish); + if (result) { + cmpl(priv, result); + return; + } + + context = ocf_parallelize_get_priv(parallelize); + context->cache = cache; + env_atomic_set(&context->curr_size, 0); + context->cmpl = cmpl; + context->priv = priv; + + ocf_parallelize_run(parallelize); } static bool _is_cache_line_acting(struct ocf_cache *cache, diff --git a/src/ocf_lru.h b/src/ocf_lru.h index e093ee8..79fc559 100644 --- a/src/ocf_lru.h +++ b/src/ocf_lru.h @@ -32,10 +32,14 @@ void ocf_lru_repart(ocf_cache_t cache, ocf_cache_line_t cline, struct ocf_part *src_upart, struct ocf_part *dst_upart); void ocf_lru_add_free(ocf_cache_t cache, ocf_cache_line_t cline); uint32_t ocf_lru_num_free(ocf_cache_t cache); -void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines); struct ocf_lru_list *ocf_lru_get_list(struct ocf_part *part, uint32_t lru_idx, bool clean); void ocf_lru_remove_locked(ocf_cache_t cache, struct ocf_lru_list *list, ocf_cache_line_t cline); +typedef void (*ocf_lru_populate_end_t)(void *priv, int error); + +void ocf_lru_populate(ocf_cache_t cache, + ocf_lru_populate_end_t cmpl, void *priv); + #endif From f3e4f8c2dbf7eca5471ca07fdfbe52cebe2a2664 Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Thu, 13 Jan 2022 15:06:59 +0100 Subject: [PATCH 08/15] Parallelize ocf_mngt_rebuild_metadata() Signed-off-by: Robert Baldyga --- src/mngt/ocf_mngt_cache.c | 166 +++++++++++++++++++++++++++++++------- 1 file changed, 137 insertions(+), 29 deletions(-) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index c67908c..7b1120c 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -17,10 +17,12 @@ #include "../utils/utils_cache_line.h" #include "../utils/utils_io.h" #include "../utils/utils_cache_line.h" +#include "../utils/utils_parallelize.h" #include "../utils/utils_pipeline.h" #include "../utils/utils_refcnt.h" #include "../utils/utils_async_lock.h" #include "../concurrency/ocf_concurrency.h" +#include "../concurrency/ocf_metadata_concurrency.h" #include "../ocf_lru.h" #include "../ocf_ctx_priv.h" #include "../cleaning/cleaning.h" @@ -464,18 +466,70 @@ static void _recovery_reset_cline_metadata(struct ocf_cache *cache, ocf_cleaning_init_cache_block(cache, cline); } -static int _ocf_mngt_rebuild_metadata(ocf_cache_t cache) +typedef void (*ocf_mngt_rebuild_metadata_end_t)(void *priv, int error); + +/* + * IMPORTANT: This value must match number of LRU lists so that adding + * cache lines to the list can be implemented without locking (each shard + * owns it's own LRU list). Don't change this value unless you are really + * sure you know what you're doing. + */ +#define OCF_MNGT_REBUILD_METADATA_SHARDS_CNT OCF_NUM_LRU_LISTS + +struct ocf_mngt_rebuild_metadata_context { + ocf_cache_t cache; + + struct { + env_atomic lines; + } core[OCF_CORE_MAX]; + + struct { + struct { + uint32_t lines; + } core[OCF_CORE_MAX]; + } shard[OCF_MNGT_REBUILD_METADATA_SHARDS_CNT]; + + ocf_mngt_rebuild_metadata_end_t cmpl; + void *priv; +}; + +static void ocf_mngt_cline_rebuild_metadata(ocf_cache_t cache, + ocf_core_id_t core_id, uint64_t core_line, + ocf_cache_line_t cline) { - ocf_cache_line_t cline; + ocf_part_id_t part_id = PARTITION_DEFAULT; + ocf_cache_line_t hash_index; + + ocf_metadata_set_partition_id(cache, cline, part_id); + + hash_index = ocf_metadata_hash_func(cache, core_line, core_id); + + ocf_hb_id_naked_lock_wr(&cache->metadata.lock, hash_index); + ocf_metadata_add_to_collision(cache, core_id, core_line, hash_index, + cline); + ocf_hb_id_naked_unlock_wr(&cache->metadata.lock, hash_index); + + ocf_lru_init_cline(cache, cline); + + ocf_lru_add(cache, cline); +} + +static int ocf_mngt_rebuild_metadata_handle(ocf_parallelize_t parallelize, + void *priv, unsigned shard_id, unsigned shards_cnt) +{ + struct ocf_mngt_rebuild_metadata_context *context = priv; + ocf_cache_t cache = context->cache; + ocf_cache_line_t begin, increment, cline; + ocf_core_t core; ocf_core_id_t core_id; uint64_t core_line; unsigned char step = 0; - const uint64_t collision_table_entries = - ocf_metadata_collision_table_entries(cache); + const uint64_t entries = ocf_metadata_collision_table_entries(cache); - ocf_metadata_start_exclusive_access(&cache->metadata.lock); + begin = shard_id; + increment = shards_cnt; - for (cline = 0; cline < collision_table_entries; cline++) { + for (cline = begin; cline < entries; cline += increment) { bool any_valid = true; OCF_COND_RESCHED(step, 128); @@ -504,17 +558,78 @@ static int _ocf_mngt_rebuild_metadata(ocf_cache_t cache) } /* Rebuild metadata for mapped cache line */ - ocf_cline_rebuild_metadata(cache, core_id, core_line, cline); + ocf_mngt_cline_rebuild_metadata(cache, core_id, + core_line, cline); + + context->shard[shard_id].core[core_id].lines++; } - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + for_each_core(cache, core, core_id) { + env_atomic_add(context->shard[shard_id].core[core_id].lines, + &context->core[core_id].lines); + } return 0; } -static int _ocf_mngt_recovery_rebuild_metadata(ocf_cache_t cache) +static void ocf_mngt_rebuild_metadata_finish(ocf_parallelize_t parallelize, + void *priv, int error) { - return _ocf_mngt_rebuild_metadata(cache); + struct ocf_mngt_rebuild_metadata_context *context = priv; + ocf_cache_t cache = context->cache; + ocf_part_id_t part_id = PARTITION_DEFAULT; + struct ocf_part_runtime *part; + ocf_core_t core; + ocf_core_id_t core_id; + uint32_t lines_total = 0; + + for_each_core(cache, core, core_id) { + uint32_t lines = env_atomic_read(&context->core[core_id].lines); + + env_atomic_set(&core->runtime_meta->cached_clines, lines); + env_atomic_set(&core->runtime_meta-> + part_counters[part_id].cached_clines, lines); + env_atomic_set(&core->runtime_meta->dirty_clines, lines); + env_atomic_set(&core->runtime_meta-> + part_counters[part_id].dirty_clines, lines); + if (lines) { + env_atomic64_set(&core->runtime_meta->dirty_since, + env_ticks_to_secs(env_get_tick_count())); + } + + lines_total += lines; + } + + part = cache->user_parts[part_id].part.runtime; + env_atomic_set(&part->curr_size, lines_total); + + context->cmpl(context->priv, error); + + ocf_parallelize_destroy(parallelize); +} + +static void ocf_mngt_rebuild_metadata(ocf_cache_t cache, + ocf_mngt_rebuild_metadata_end_t cmpl, void *priv) +{ + struct ocf_mngt_rebuild_metadata_context *context; + ocf_parallelize_t parallelize; + int result; + + result = ocf_parallelize_create(¶llelize, cache, + OCF_MNGT_REBUILD_METADATA_SHARDS_CNT, + sizeof(*context), ocf_mngt_rebuild_metadata_handle, + ocf_mngt_rebuild_metadata_finish); + if (result) { + cmpl(priv, result); + return; + } + + context = ocf_parallelize_get_priv(parallelize); + context->cache = cache; + context->cmpl = cmpl; + context->priv = priv; + + ocf_parallelize_run(parallelize); } static inline ocf_error_t _ocf_init_cleaning_policy(ocf_cache_t cache, @@ -534,17 +649,24 @@ static inline ocf_error_t _ocf_init_cleaning_policy(ocf_cache_t cache, return result; } +static void _ocf_mngt_load_rebuild_metadata_complete(void *priv, int error) +{ + struct ocf_cache_attach_context *context = priv; + + OCF_PL_NEXT_ON_SUCCESS_RET(context->pipeline, error); +} + static void _ocf_mngt_load_rebuild_metadata(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; - int ret; if (context->metadata.shutdown_status != ocf_metadata_clean_shutdown) { - ret = _ocf_mngt_recovery_rebuild_metadata(cache); - if (ret) - OCF_PL_FINISH_RET(pipeline, ret); + ocf_mngt_rebuild_metadata(cache, + _ocf_mngt_load_rebuild_metadata_complete, + context); + return; } ocf_pipeline_next(pipeline); @@ -2154,20 +2276,6 @@ static void _ocf_mngt_standby_init_pio_concurrency(ocf_pipeline_t pipeline, OCF_PL_NEXT_ON_SUCCESS_RET(context->pipeline, result); } -static void _ocf_mngt_standby_recovery(ocf_pipeline_t pipeline, - void *priv, ocf_pipeline_arg_t arg) -{ - struct ocf_cache_attach_context *context = priv; - ocf_cache_t cache = context->cache; - int ret; - - ret = _ocf_mngt_recovery_rebuild_metadata(cache); - if (ret) - OCF_PL_FINISH_RET(pipeline, ret); - - ocf_pipeline_next(pipeline); -} - static void _ocf_mngt_standby_post_init(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { @@ -2226,7 +2334,7 @@ struct ocf_pipeline_properties _ocf_mngt_cache_standby_load_pipeline_properties OCF_PL_STEP(_ocf_mngt_load_init_cleaning), OCF_PL_STEP(_ocf_mngt_standby_preapre_mempool), OCF_PL_STEP(_ocf_mngt_standby_init_pio_concurrency), - OCF_PL_STEP(_ocf_mngt_standby_recovery), + OCF_PL_STEP(_ocf_mngt_load_rebuild_metadata), OCF_PL_STEP(_ocf_mngt_attach_populate_free), OCF_PL_STEP(_ocf_mngt_standby_post_init), OCF_PL_STEP_TERMINATOR(), From 48bed40dd78e6c1e8d6e8ee290875fbb2e3dfc9f Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Fri, 14 Jan 2022 17:19:02 +0100 Subject: [PATCH 09/15] Reconstruct freelist during metadata rebuild Signed-off-by: Robert Baldyga --- src/mngt/ocf_mngt_cache.c | 51 ++++++++++++++++++--------------------- src/ocf_lru.c | 3 --- 2 files changed, 24 insertions(+), 30 deletions(-) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 7b1120c..4910964 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -456,16 +456,6 @@ err: OCF_PL_FINISH_RET(pipeline, -OCF_ERR_START_CACHE_FAIL); } -static void _recovery_reset_cline_metadata(struct ocf_cache *cache, - ocf_cache_line_t cline) -{ - ocf_metadata_set_core_info(cache, cline, OCF_CORE_MAX, ULLONG_MAX); - - metadata_init_status_bits(cache, cline); - - ocf_cleaning_init_cache_block(cache, cline); -} - typedef void (*ocf_mngt_rebuild_metadata_end_t)(void *priv, int error); /* @@ -489,10 +479,25 @@ struct ocf_mngt_rebuild_metadata_context { } core[OCF_CORE_MAX]; } shard[OCF_MNGT_REBUILD_METADATA_SHARDS_CNT]; + env_atomic free_lines; + ocf_mngt_rebuild_metadata_end_t cmpl; void *priv; }; +static void ocf_mngt_cline_reset_metadata(ocf_cache_t cache, + ocf_cache_line_t cline, uint32_t lru_list) +{ + ocf_metadata_set_core_info(cache, cline, OCF_CORE_MAX, ULLONG_MAX); + metadata_init_status_bits(cache, cline); + + ocf_metadata_set_partition_id(cache, cline, PARTITION_FREELIST); + + ocf_lru_add_free(cache, cline); + + ocf_cleaning_init_cache_block(cache, cline); +} + static void ocf_mngt_cline_rebuild_metadata(ocf_cache_t cache, ocf_core_id_t core_id, uint64_t core_line, ocf_cache_line_t cline) @@ -519,7 +524,7 @@ static int ocf_mngt_rebuild_metadata_handle(ocf_parallelize_t parallelize, { struct ocf_mngt_rebuild_metadata_context *context = priv; ocf_cache_t cache = context->cache; - ocf_cache_line_t begin, increment, cline; + ocf_cache_line_t begin, increment, cline, free_lines; ocf_core_t core; ocf_core_id_t core_id; uint64_t core_line; @@ -529,6 +534,7 @@ static int ocf_mngt_rebuild_metadata_handle(ocf_parallelize_t parallelize, begin = shard_id; increment = shards_cnt; + free_lines = 0; for (cline = begin; cline < entries; cline += increment) { bool any_valid = true; @@ -546,7 +552,8 @@ static int ocf_mngt_rebuild_metadata_handle(ocf_parallelize_t parallelize, any_valid = metadata_clear_valid_if_clean(cache, cline); if (!any_valid || core_id == OCF_CORE_MAX) { /* Reset metadata for not mapped or clean cache line */ - _recovery_reset_cline_metadata(cache, cline); + ocf_mngt_cline_reset_metadata(cache, cline, shard_id); + free_lines++; continue; } @@ -569,6 +576,8 @@ static int ocf_mngt_rebuild_metadata_handle(ocf_parallelize_t parallelize, &context->core[core_id].lines); } + env_atomic_add(free_lines, &context->free_lines); + return 0; } @@ -603,6 +612,9 @@ static void ocf_mngt_rebuild_metadata_finish(ocf_parallelize_t parallelize, part = cache->user_parts[part_id].part.runtime; env_atomic_set(&part->curr_size, lines_total); + env_atomic_set(&cache->free.runtime->curr_size, + env_atomic_read(&context->free_lines)); + context->cmpl(context->priv, error); ocf_parallelize_destroy(parallelize); @@ -1873,19 +1885,6 @@ struct ocf_pipeline_properties _ocf_mngt_cache_attach_pipeline_properties = { }, }; -static void _ocf_mngt_load_populate_free(ocf_pipeline_t pipeline, - void *priv, ocf_pipeline_arg_t arg) -{ - struct ocf_cache_attach_context *context = priv; - - if (context->metadata.shutdown_status != ocf_metadata_clean_shutdown) { - _ocf_mngt_attach_populate_free(pipeline, priv, arg); - return; - } - - ocf_pipeline_next(pipeline); -} - struct ocf_pipeline_properties _ocf_mngt_cache_load_pipeline_properties = { .priv_size = sizeof(struct ocf_cache_attach_context), .finish = _ocf_mngt_cache_attach_finish, @@ -1904,7 +1903,6 @@ struct ocf_pipeline_properties _ocf_mngt_cache_load_pipeline_properties = { OCF_PL_STEP(_ocf_mngt_load_add_cores), OCF_PL_STEP(_ocf_mngt_load_metadata), OCF_PL_STEP(_ocf_mngt_load_rebuild_metadata), - OCF_PL_STEP(_ocf_mngt_load_populate_free), OCF_PL_STEP(_ocf_mngt_load_init_cleaning), OCF_PL_STEP(_ocf_mngt_attach_shutdown_status), OCF_PL_STEP(_ocf_mngt_attach_flush_metadata), @@ -2335,7 +2333,6 @@ struct ocf_pipeline_properties _ocf_mngt_cache_standby_load_pipeline_properties OCF_PL_STEP(_ocf_mngt_standby_preapre_mempool), OCF_PL_STEP(_ocf_mngt_standby_init_pio_concurrency), OCF_PL_STEP(_ocf_mngt_load_rebuild_metadata), - OCF_PL_STEP(_ocf_mngt_attach_populate_free), OCF_PL_STEP(_ocf_mngt_standby_post_init), OCF_PL_STEP_TERMINATOR(), }, diff --git a/src/ocf_lru.c b/src/ocf_lru.c index 1b69cb7..7e45849 100644 --- a/src/ocf_lru.c +++ b/src/ocf_lru.c @@ -891,9 +891,6 @@ static int ocf_lru_populate_handle(ocf_parallelize_t parallelize, for (cline = shard_id; cline < entries; cline += shards_cnt) { OCF_COND_RESCHED_DEFAULT(step); - if (metadata_test_valid_any(cache, cline)) - continue; - ocf_metadata_set_partition_id(cache, cline, PARTITION_FREELIST); list = ocf_lru_get_list(&cache->free, lru_list, true); From 8cc71cc9cb22d2762ec0bef0d52c2e88fa629861 Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Fri, 21 Jan 2022 21:27:08 +0100 Subject: [PATCH 10/15] Remove ocf_cleaning_init_cache_block() from metadata rebuild Cleaning policy initializaton initializes metadata for all cache lines anyway, so this step is not needed. Signed-off-by: Robert Baldyga --- src/mngt/ocf_mngt_cache.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 4910964..71ec1c0 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -494,8 +494,6 @@ static void ocf_mngt_cline_reset_metadata(ocf_cache_t cache, ocf_metadata_set_partition_id(cache, cline, PARTITION_FREELIST); ocf_lru_add_free(cache, cline); - - ocf_cleaning_init_cache_block(cache, cline); } static void ocf_mngt_cline_rebuild_metadata(ocf_cache_t cache, From b70492ad3d6d7c18fe59e9b0ab6e13d7c7a6cbbc Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Thu, 20 Jan 2022 13:33:03 +0100 Subject: [PATCH 11/15] Parallelize ALRU recovery Signed-off-by: Robert Baldyga --- src/cleaning/acp.c | 9 + src/cleaning/acp.h | 3 + src/cleaning/alru.c | 303 ++++++++++++++---- src/cleaning/alru.h | 2 + src/cleaning/cleaning.h | 2 + src/cleaning/cleaning_ops.h | 18 ++ src/mngt/ocf_mngt_cache.c | 37 +-- ...leaning_policy_alru_initialize_part_test.c | 130 -------- 8 files changed, 287 insertions(+), 217 deletions(-) delete mode 100644 tests/unit/tests/cleaning/alru.c/cleaning_policy_alru_initialize_part_test.c diff --git a/src/cleaning/acp.c b/src/cleaning/acp.c index 0431367..bbddf29 100644 --- a/src/cleaning/acp.c +++ b/src/cleaning/acp.c @@ -323,6 +323,15 @@ int cleaning_policy_acp_initialize(struct ocf_cache *cache, return 0; } +void cleaning_policy_acp_recovery(ocf_cache_t cache, + ocf_cleaning_recovery_end_t cmpl, void *priv) +{ + int result; + + result = cleaning_policy_acp_initialize(cache, 1); + cmpl(priv, result); +} + int cleaning_policy_acp_set_cleaning_param(ocf_cache_t cache, uint32_t param_id, uint32_t param_value) { diff --git a/src/cleaning/acp.h b/src/cleaning/acp.h index 01142f5..52e9fa1 100644 --- a/src/cleaning/acp.h +++ b/src/cleaning/acp.h @@ -12,6 +12,9 @@ void cleaning_policy_acp_setup(ocf_cache_t cache); int cleaning_policy_acp_initialize(ocf_cache_t cache, int init_metadata); +void cleaning_policy_acp_recovery(ocf_cache_t cache, + ocf_cleaning_recovery_end_t cmpl, void *priv); + void cleaning_policy_acp_deinitialize(ocf_cache_t cache); void cleaning_policy_acp_perform_cleaning(ocf_cache_t cache, diff --git a/src/cleaning/alru.c b/src/cleaning/alru.c index 86ef0e4..4ca57b9 100644 --- a/src/cleaning/alru.c +++ b/src/cleaning/alru.c @@ -10,6 +10,7 @@ #include "../metadata/metadata.h" #include "../utils/utils_cleaner.h" #include "../utils/utils_user_part.h" +#include "../utils/utils_parallelize.h" #include "../utils/utils_realloc.h" #include "../concurrency/ocf_cache_line_concurrency.h" #include "../ocf_def_priv.h" @@ -61,54 +62,63 @@ struct alru_context { /* -- Start of ALRU functions -- */ -/* Adds the given collision_index to the _head_ of the ALRU list */ -static void add_alru_head(struct ocf_cache *cache, int partition_id, - unsigned int collision_index) +/* Appends given sublist to the _head_ of the ALRU list */ +static void append_alru_head(ocf_cache_t cache, ocf_part_id_t part_id, + ocf_cache_line_t head, ocf_cache_line_t tail) { - unsigned int curr_head_index; - unsigned int collision_table_entries = cache->device->collision_table_entries; - struct alru_cleaning_policy *part_alru = &cache->user_parts[partition_id] - .clean_pol->policy.alru; - struct alru_cleaning_policy_meta *alru; + ocf_cache_line_t terminator = cache->device->collision_table_entries; + struct alru_cleaning_policy *part_alru; + struct cleaning_policy_meta *meta; + struct alru_cleaning_policy_meta *old_head; + struct alru_cleaning_policy_meta *entry; - ENV_BUG_ON(!(collision_index < collision_table_entries)); + part_alru = &cache->user_parts[part_id].clean_pol->policy.alru; - ENV_BUG_ON(env_atomic_read(&part_alru->size) < 0); + if (head == terminator && tail == terminator) + return; - ENV_WARN_ON(!metadata_test_dirty(cache, collision_index)); - ENV_WARN_ON(!metadata_test_valid_any(cache, collision_index)); + ENV_BUG_ON(head == terminator); + ENV_BUG_ON(tail == terminator); - /* First node to be added/ */ - if (env_atomic_read(&part_alru->size) == 0) { - part_alru->lru_head = collision_index; - part_alru->lru_tail = collision_index; - - alru = &ocf_metadata_get_cleaning_policy(cache, - collision_index)->meta.alru; - alru->lru_next = collision_table_entries; - alru->lru_prev = collision_table_entries; - alru->timestamp = env_ticks_to_secs( - env_get_tick_count()); + if (part_alru->lru_head == terminator) { + part_alru->lru_head = head; + part_alru->lru_tail = tail; } else { - /* Not the first node to be added. */ + meta = ocf_metadata_get_cleaning_policy(cache, part_alru->lru_head); + old_head = &meta->meta.alru; + old_head->lru_prev = tail; - curr_head_index = part_alru->lru_head; + meta = ocf_metadata_get_cleaning_policy(cache, tail); + entry = &meta->meta.alru; + entry->lru_next = part_alru->lru_head; - ENV_BUG_ON(!(curr_head_index < collision_table_entries)); - - alru = &ocf_metadata_get_cleaning_policy(cache, - collision_index)->meta.alru; - alru->lru_next = curr_head_index; - alru->lru_prev = collision_table_entries; - alru->timestamp = env_ticks_to_secs( - env_get_tick_count()); - - alru = &ocf_metadata_get_cleaning_policy(cache, - curr_head_index)->meta.alru; - alru->lru_prev = collision_index; - - part_alru->lru_head = collision_index; + part_alru->lru_head = head; } +} + +/* Adds the given collision_index to the _head_ of the ALRU list */ +static void add_alru_head(ocf_cache_t cache, ocf_part_id_t part_id, + ocf_cache_line_t cline) +{ + ocf_cache_line_t terminator = cache->device->collision_table_entries; + struct alru_cleaning_policy *part_alru; + struct cleaning_policy_meta *meta; + struct alru_cleaning_policy_meta *entry; + + ENV_BUG_ON(!(cline < terminator)); + + ENV_WARN_ON(!metadata_test_dirty(cache, cline)); + ENV_WARN_ON(!metadata_test_valid_any(cache, cline)); + + part_alru = &cache->user_parts[part_id].clean_pol->policy.alru; + + meta = ocf_metadata_get_cleaning_policy(cache, cline); + entry = &meta->meta.alru; + entry->lru_next = terminator; + entry->lru_prev = terminator; + entry->timestamp = env_ticks_to_secs(env_get_tick_count()); + + append_alru_head(cache, part_id, cline, cline); env_atomic_inc(&part_alru->size); } @@ -375,24 +385,6 @@ static void _alru_rebuild(struct ocf_cache *cache) } } -static int cleaning_policy_alru_initialize_part(struct ocf_cache *cache, - struct ocf_user_part *user_part, int init_metadata) -{ - struct alru_cleaning_policy *part_alru = - &user_part->clean_pol->policy.alru; - - if (init_metadata) { - /* ALRU initialization */ - env_atomic_set(&part_alru->size, 0); - part_alru->lru_head = cache->device->collision_table_entries; - part_alru->lru_tail = cache->device->collision_table_entries; - } - - cache->device->runtime_meta->cleaning_thread_access = 0; - - return 0; -} - void cleaning_policy_alru_setup(struct ocf_cache *cache) { struct alru_cleaning_policy_config *config; @@ -405,10 +397,8 @@ void cleaning_policy_alru_setup(struct ocf_cache *cache) config->activity_threshold = OCF_ALRU_DEFAULT_ACTIVITY_THRESHOLD; } -int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata) +int cleaning_policy_alru_init_common(ocf_cache_t cache) { - struct ocf_user_part *user_part; - ocf_part_id_t part_id; struct alru_context *ctx; int error = 0; unsigned i; @@ -432,13 +422,20 @@ int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata) return error; } + cache->device->runtime_meta->cleaning_thread_access = 0; cache->cleaner.cleaning_policy_context = ctx; - for_each_user_part(cache, user_part, part_id) { - cleaning_policy_alru_initialize_part(cache, - user_part, init_metadata); - } + return 0; +} + +int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata) +{ + int result; + + result = cleaning_policy_alru_init_common(cache); + if (result) + return result; if (init_metadata) _alru_rebuild(cache); @@ -448,6 +445,184 @@ int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata) return 0; } +#define OCF_ALRU_RECOVERY_SHARDS_CNT 32 + +struct ocf_alru_recovery_context { + ocf_cache_t cache; + struct { + struct { + ocf_cache_line_t head; + ocf_cache_line_t tail; + } part[OCF_USER_IO_CLASS_MAX]; + } shard[OCF_ALRU_RECOVERY_SHARDS_CNT] __attribute__((aligned(64))); + + ocf_cleaning_recovery_end_t cmpl; + void *priv; +}; + +static void add_alru_head_recovery(struct ocf_alru_recovery_context *context, + unsigned shard_id, ocf_core_id_t part_id, + ocf_cache_line_t cline) +{ + ocf_cache_t cache = context->cache; + ocf_cache_line_t curr_head, terminator; + struct cleaning_policy_meta *meta; + struct alru_cleaning_policy_meta *entry; + struct alru_cleaning_policy_meta *next; + + terminator = ocf_metadata_collision_table_entries(cache); + curr_head = context->shard[shard_id].part[part_id].head; + + meta = ocf_metadata_get_cleaning_policy(cache, cline); + entry = &meta->meta.alru; + + if (curr_head == terminator) { + /* First node to be added/ */ + entry->lru_next = terminator; + entry->lru_prev = terminator; + entry->timestamp = env_ticks_to_secs(env_get_tick_count()); + + context->shard[shard_id].part[part_id].head = cline; + context->shard[shard_id].part[part_id].tail = cline; + } else { + /* Not the first node to be added. */ + entry->lru_next = curr_head; + entry->lru_prev = terminator; + entry->timestamp = env_ticks_to_secs(env_get_tick_count()); + + meta = ocf_metadata_get_cleaning_policy(cache, curr_head); + next = &meta->meta.alru; + + next->lru_prev = cline; + + context->shard[shard_id].part[part_id].head = cline; + } +} + +static int ocf_alru_recovery_handle(ocf_parallelize_t parallelize, + void *priv, unsigned shard_id, unsigned shards_cnt) +{ + struct ocf_alru_recovery_context *context = priv; + ocf_cache_t cache = context->cache; + ocf_cache_line_t entries = cache->device->collision_table_entries; + ocf_cache_line_t terminator = entries; + unsigned part_size[OCF_USER_IO_CLASS_MAX] = {}; + struct ocf_user_part *user_part; + struct alru_cleaning_policy *part_alru; + ocf_part_id_t part_id; + ocf_core_id_t core_id; + ocf_cache_line_t cline, portion; + uint64_t begin, end; + uint32_t step = 0; + int i; + + portion = DIV_ROUND_UP((uint64_t)entries, shards_cnt); + begin = portion*shard_id; + end = OCF_MIN(portion*(shard_id + 1), entries); + + for (i = 0; i < OCF_USER_IO_CLASS_MAX; i++) { + context->shard[shard_id].part[i].head = terminator; + context->shard[shard_id].part[i].tail = terminator; + } + + for (cline = begin; cline < end; cline++) { + ocf_metadata_get_core_and_part_id(cache, cline, + &core_id, &part_id); + + OCF_COND_RESCHED_DEFAULT(step); + + if (core_id == OCF_CORE_MAX) + continue; + + if (!metadata_test_dirty(cache, cline)) { + cleaning_policy_alru_init_cache_block(cache, cline); + } else { + add_alru_head_recovery(context, shard_id, + part_id, cline); + ++part_size[part_id]; + } + } + + for_each_user_part(cache, user_part, part_id) { + part_alru = &user_part->clean_pol->policy.alru; + env_atomic_add(part_size[part_id], &part_alru->size); + } + + return 0; +} + +static void ocf_alru_recovery_finish(ocf_parallelize_t parallelize, + void *priv, int error) +{ + struct ocf_alru_recovery_context *context = priv; + ocf_cache_t cache = context->cache; + ocf_part_id_t part_id; + ocf_cache_line_t head, tail; + unsigned shard; + + if (error) + goto end; + + for (part_id = 0; part_id < OCF_USER_IO_CLASS_MAX; part_id++) { + for (shard = 0; shard < OCF_ALRU_RECOVERY_SHARDS_CNT; shard++) { + head = context->shard[shard].part[part_id].head; + tail = context->shard[shard].part[part_id].tail; + + append_alru_head(cache, part_id, head, tail); + } + } + + ocf_kick_cleaner(cache); + +end: + context->cmpl(context->priv, error); + + ocf_parallelize_destroy(parallelize); +} + +void cleaning_policy_alru_recovery(ocf_cache_t cache, + ocf_cleaning_recovery_end_t cmpl, void *priv) +{ + struct ocf_alru_recovery_context *context; + ocf_parallelize_t parallelize; + struct alru_cleaning_policy *part_alru; + struct ocf_user_part *user_part; + ocf_part_id_t part_id; + int result; + + result = ocf_parallelize_create(¶llelize, cache, + OCF_ALRU_RECOVERY_SHARDS_CNT, sizeof(*context), + ocf_alru_recovery_handle, ocf_alru_recovery_finish); + if (result) { + cmpl(priv, result); + return; + } + + + result = cleaning_policy_alru_init_common(cache); + if (result) { + ocf_parallelize_destroy(parallelize); + cmpl(priv, result); + return; + } + + for_each_user_part(cache, user_part, part_id) { + /* ALRU initialization */ + part_alru = &user_part->clean_pol->policy.alru; + env_atomic_set(&part_alru->size, 0); + part_alru->lru_head = cache->device->collision_table_entries; + part_alru->lru_tail = cache->device->collision_table_entries; + cache->device->runtime_meta->cleaning_thread_access = 0; + } + + context = ocf_parallelize_get_priv(parallelize); + context->cache = cache; + context->cmpl = cmpl; + context->priv = priv; + + ocf_parallelize_run(parallelize); +} + void cleaning_policy_alru_deinitialize(struct ocf_cache *cache) { struct alru_context *alru = cache->cleaner.cleaning_policy_context; diff --git a/src/cleaning/alru.h b/src/cleaning/alru.h index c87dbe9..b72fa3a 100644 --- a/src/cleaning/alru.h +++ b/src/cleaning/alru.h @@ -11,6 +11,8 @@ void cleaning_policy_alru_setup(ocf_cache_t cache); int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata); +void cleaning_policy_alru_recovery(ocf_cache_t cache, + ocf_cleaning_recovery_end_t cmpl, void *priv); void cleaning_policy_alru_deinitialize(ocf_cache_t cache); void cleaning_policy_alru_init_cache_block(ocf_cache_t cache, uint32_t cache_line); diff --git a/src/cleaning/cleaning.h b/src/cleaning/cleaning.h index c48d50e..f85c002 100644 --- a/src/cleaning/cleaning.h +++ b/src/cleaning/cleaning.h @@ -54,4 +54,6 @@ void ocf_kick_cleaner(ocf_cache_t cache); void ocf_stop_cleaner(ocf_cache_t cache); +typedef void (*ocf_cleaning_recovery_end_t)(void *priv, int error); + #endif diff --git a/src/cleaning/cleaning_ops.h b/src/cleaning/cleaning_ops.h index 72bfae8..134d324 100644 --- a/src/cleaning/cleaning_ops.h +++ b/src/cleaning/cleaning_ops.h @@ -14,6 +14,8 @@ struct cleaning_policy_ops { void (*setup)(ocf_cache_t cache); int (*initialize)(ocf_cache_t cache, int init_metadata); + void (*recovery)(ocf_cache_t cache, + ocf_cleaning_recovery_end_t cmpl, void *priv); void (*deinitialize)(ocf_cache_t cache); int (*add_core)(ocf_cache_t cache, ocf_core_id_t core_id); void (*remove_core)(ocf_cache_t cache, ocf_core_id_t core_id); @@ -43,6 +45,7 @@ static struct cleaning_policy_ops cleaning_policy_ops[ocf_cleaning_max] = { .purge_range = cleaning_policy_alru_purge_range, .set_hot_cache_line = cleaning_policy_alru_set_hot_cache_line, .initialize = cleaning_policy_alru_initialize, + .recovery = cleaning_policy_alru_recovery, .deinitialize = cleaning_policy_alru_deinitialize, .set_cleaning_param = cleaning_policy_alru_set_cleaning_param, .get_cleaning_param = cleaning_policy_alru_get_cleaning_param, @@ -56,6 +59,7 @@ static struct cleaning_policy_ops cleaning_policy_ops[ocf_cleaning_max] = { .purge_range = cleaning_policy_acp_purge_range, .set_hot_cache_line = cleaning_policy_acp_set_hot_cache_line, .initialize = cleaning_policy_acp_initialize, + .recovery = cleaning_policy_acp_recovery, .deinitialize = cleaning_policy_acp_deinitialize, .set_cleaning_param = cleaning_policy_acp_set_cleaning_param, .get_cleaning_param = cleaning_policy_acp_get_cleaning_param, @@ -87,6 +91,20 @@ static inline int ocf_cleaning_initialize(ocf_cache_t cache, return cleaning_policy_ops[policy].initialize(cache, init_metadata); } +static inline void ocf_cleaning_recovery(ocf_cache_t cache, + ocf_cleaning_t policy, + ocf_cleaning_recovery_end_t cmpl, void *priv) +{ + ENV_BUG_ON(policy >= ocf_cleaning_max); + + if (unlikely(!cleaning_policy_ops[policy].recovery)) { + cmpl(priv, 0); + return; + } + + cleaning_policy_ops[policy].recovery(cache, cmpl, priv); +} + static inline void ocf_cleaning_deinitialize(ocf_cache_t cache) { ocf_cleaning_t policy; diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 71ec1c0..133c96d 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -642,23 +642,6 @@ static void ocf_mngt_rebuild_metadata(ocf_cache_t cache, ocf_parallelize_run(parallelize); } -static inline ocf_error_t _ocf_init_cleaning_policy(ocf_cache_t cache, - ocf_cleaning_t cleaning_policy, - enum ocf_metadata_shutdown_status shutdown_status) -{ - ocf_error_t result; - - if (shutdown_status == ocf_metadata_clean_shutdown) - result = ocf_cleaning_initialize(cache, cleaning_policy, 0); - else - result = ocf_cleaning_initialize(cache, cleaning_policy, 1); - - if (result) - ocf_cache_log(cache, log_err, "Cannot initialize cleaning policy\n"); - - return result; -} - static void _ocf_mngt_load_rebuild_metadata_complete(void *priv, int error) { struct ocf_cache_attach_context *context = priv; @@ -682,6 +665,13 @@ static void _ocf_mngt_load_rebuild_metadata(ocf_pipeline_t pipeline, ocf_pipeline_next(pipeline); } +static void _ocf_mngt_cleaning_recovery_complete(void *priv, int error) +{ + struct ocf_cache_attach_context *context = priv; + + OCF_PL_NEXT_ON_SUCCESS_RET(context->pipeline, error); +} + static void _ocf_mngt_load_init_cleaning(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { @@ -689,13 +679,14 @@ static void _ocf_mngt_load_init_cleaning(ocf_pipeline_t pipeline, ocf_cache_t cache = context->cache; ocf_error_t result; - result = _ocf_init_cleaning_policy(cache, cache->cleaner.policy, - context->metadata.shutdown_status); + if (context->metadata.shutdown_status == ocf_metadata_clean_shutdown) { + result = ocf_cleaning_initialize(cache, + cache->cleaner.policy, 0); + OCF_PL_NEXT_ON_SUCCESS_RET(pipeline, result); + } - if (result) - OCF_PL_FINISH_RET(pipeline, result); - - ocf_pipeline_next(pipeline); + ocf_cleaning_recovery(cache, cache->cleaner.policy, + _ocf_mngt_cleaning_recovery_complete, context); } void _ocf_mngt_load_metadata_complete(void *priv, int error) diff --git a/tests/unit/tests/cleaning/alru.c/cleaning_policy_alru_initialize_part_test.c b/tests/unit/tests/cleaning/alru.c/cleaning_policy_alru_initialize_part_test.c deleted file mode 100644 index a2c3823..0000000 --- a/tests/unit/tests/cleaning/alru.c/cleaning_policy_alru_initialize_part_test.c +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright(c) 2012-2021 Intel Corporation - * SPDX-License-Identifier: BSD-3-Clause - */ -/* -src/cleaning/alru.c -cleaning_policy_alru_initialize_part - - -*/ - -#undef static -#undef inline -/* - * This headers must be in test source file. It's important that cmocka.h is - * last. - */ -#include -#include -#include -#include -#include "print_desc.h" - -/* - * Headers from tested target. - */ -#include "ocf/ocf.h" -#include "../ocf_cache_priv.h" -#include "cleaning.h" -#include "alru.h" -#include "../metadata/metadata.h" -#include "../utils/utils_cleaner.h" -#include "../utils/utils_user_part.h" -#include "../utils/utils_realloc.h" -#include "../concurrency/ocf_cache_line_concurrency.h" -#include "../ocf_def_priv.h" - -#include "cleaning/alru.c/cleaning_policy_alru_initialize_part_test_generated_wraps.c" - - -static void cleaning_policy_alru_initialize_test01(void **state) -{ - int result; - struct ocf_cache *cache; - ocf_part_id_t part_id = 0; - - int collision_table_entries = 900729; - - print_test_description("Check if all variables are set correctly"); - - cache = test_malloc(sizeof(*cache)); - cache->user_parts[part_id].part.runtime = test_malloc(sizeof(struct ocf_part_runtime)); - cache->user_parts[part_id].clean_pol = test_malloc(sizeof(*cache->user_parts[part_id].clean_pol)); - cache->user_parts[part_id].part.id = part_id; - cache->device = test_malloc(sizeof(struct ocf_cache_device)); - cache->device->runtime_meta = test_malloc(sizeof(struct ocf_superblock_runtime)); - - cache->device->collision_table_entries = collision_table_entries; - - result = cleaning_policy_alru_initialize_part(cache, &cache->user_parts[part_id], 1, 1); - - assert_int_equal(result, 0); - - assert_int_equal(env_atomic_read(&cache->user_parts[part_id].clean_pol->policy.alru.size), 0); - assert_int_equal(cache->user_parts[part_id].clean_pol->policy.alru.lru_head, collision_table_entries); - assert_int_equal(cache->user_parts[part_id].clean_pol->policy.alru.lru_tail, collision_table_entries); - - assert_int_equal(cache->device->runtime_meta->cleaning_thread_access, 0); - - test_free(cache->device->runtime_meta); - test_free(cache->device); - test_free(cache->user_parts[part_id].clean_pol); - test_free(cache->user_parts[part_id].part.runtime); - test_free(cache); -} - -static void cleaning_policy_alru_initialize_test02(void **state) -{ - int result; - struct ocf_cache *cache; - ocf_part_id_t part_id = 0; - - uint32_t collision_table_entries = 900729; - - print_test_description("Check if only appropirate variables are changed"); - - cache = test_malloc(sizeof(*cache)); - cache->user_parts[part_id].part.runtime = test_malloc(sizeof(struct ocf_part_runtime)); - cache->user_parts[part_id].clean_pol = test_malloc(sizeof(*cache->user_parts[part_id].clean_pol)); - cache->device = test_malloc(sizeof(struct ocf_cache_device)); - cache->device->runtime_meta = test_malloc(sizeof(struct ocf_superblock_runtime)); - - env_atomic_set(&cache->user_parts[part_id].clean_pol->policy.alru.size, 1); - cache->user_parts[part_id].clean_pol->policy.alru.lru_head = -collision_table_entries; - cache->user_parts[part_id].clean_pol->policy.alru.lru_tail = -collision_table_entries; - - result = cleaning_policy_alru_initialize_part(cache, &cache->user_parts[part_id], 0, 0); - - assert_int_equal(result, 0); - - assert_int_equal(env_atomic_read(&cache->user_parts[part_id].clean_pol->policy.alru.size), 1); - assert_int_equal(cache->user_parts[part_id].clean_pol->policy.alru.lru_head, -collision_table_entries); - assert_int_equal(cache->user_parts[part_id].clean_pol->policy.alru.lru_tail, -collision_table_entries); - - assert_int_equal(cache->device->runtime_meta->cleaning_thread_access, 0); - - test_free(cache->device->runtime_meta); - test_free(cache->device); - test_free(cache->user_parts[part_id].clean_pol); - test_free(cache->user_parts[part_id].part.runtime); - test_free(cache); -} - -/* - * Main function. It runs tests. - */ -int main(void) -{ - const struct CMUnitTest tests[] = { - cmocka_unit_test(cleaning_policy_alru_initialize_test01), - cmocka_unit_test(cleaning_policy_alru_initialize_test02) - }; - - print_message("Unit test of alru.c\n"); - - return cmocka_run_group_tests(tests, NULL, NULL); -} - - - From 93391c78d8a837c3dfef6af74d33146a45f1a52e Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Mon, 24 Jan 2022 13:55:16 +0100 Subject: [PATCH 12/15] Parallelize ACP recovery Signed-off-by: Robert Baldyga --- src/cleaning/acp.c | 191 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 188 insertions(+), 3 deletions(-) diff --git a/src/cleaning/acp.c b/src/cleaning/acp.c index bbddf29..1a3a56f 100644 --- a/src/cleaning/acp.c +++ b/src/cleaning/acp.c @@ -9,6 +9,7 @@ #include "../metadata/metadata.h" #include "../utils/utils_cleaner.h" #include "../utils/utils_cache_line.h" +#include "../utils/utils_parallelize.h" #include "../ocf_request.h" #include "../cleaning/acp.h" #include "../engine/engine_common.h" @@ -270,8 +271,7 @@ void cleaning_policy_acp_setup(struct ocf_cache *cache) config->flush_max_buffers = OCF_ACP_DEFAULT_FLUSH_MAX_BUFFERS; } -int cleaning_policy_acp_initialize(struct ocf_cache *cache, - int init_metadata) +int cleaning_policy_acp_init_common(ocf_cache_t cache) { struct acp_context *acp; int err, i; @@ -317,18 +317,203 @@ int cleaning_policy_acp_initialize(struct ocf_cache *cache, } } + return 0; +} + +int cleaning_policy_acp_initialize(ocf_cache_t cache, int init_metadata) +{ + int result; + + result = cleaning_policy_acp_init_common(cache); + if (result) + return result; + _acp_rebuild(cache); ocf_kick_cleaner(cache); return 0; } +#define OCF_ACP_RECOVERY_SHARDS_CNT 32 + +struct ocf_acp_recovery_context { + ocf_cache_t cache; + + struct { + uint16_t *chunk[OCF_CORE_MAX]; + struct { + struct list_head chunk_list; + } bucket[ACP_MAX_BUCKETS]; + } shard[OCF_ACP_RECOVERY_SHARDS_CNT]; + + ocf_cleaning_recovery_end_t cmpl; + void *priv; +}; + +static int ocf_acp_recovery_handle(ocf_parallelize_t parallelize, + void *priv, unsigned shard_id, unsigned shards_cnt) +{ + struct ocf_acp_recovery_context *context = priv; + ocf_cache_t cache = context->cache; + ocf_cache_line_t entries = cache->device->collision_table_entries; + ocf_cache_line_t cline, portion; + uint64_t begin, end; + struct acp_cleaning_policy_meta *acp_meta; + struct acp_chunk_info *chunk; + ocf_core_id_t core_id; + uint32_t step = 0; + + portion = DIV_ROUND_UP((uint64_t)entries, shards_cnt); + begin = portion*shard_id; + end = OCF_MIN(portion*(shard_id + 1), entries); + + for (cline = begin; cline < end; cline++) { + ocf_metadata_get_core_and_part_id(cache, cline, &core_id, NULL); + + OCF_COND_RESCHED_DEFAULT(step); + + if (core_id == OCF_CORE_MAX) + continue; + + if (!metadata_test_dirty(cache, cline)) { + cleaning_policy_acp_init_cache_block(cache, cline); + continue; + } + + acp_meta = _acp_meta_get(cache, cline); + acp_meta->dirty = 1; + + chunk = _acp_get_chunk(cache, cline); + context->shard[shard_id].chunk[core_id][chunk->chunk_id]++; + } + + return 0; +} + +static void ocf_acp_recovery_chunk(struct ocf_acp_recovery_context *context, + struct acp_chunk_info *chunk) +{ + ocf_cache_t cache = context->cache; + struct acp_context *acp = _acp_get_ctx_from_cache(cache); + struct acp_bucket *bucket; + unsigned shard_id; + uint8_t bucket_id; + + chunk->num_dirty = 0; + for (shard_id = 0; shard_id < OCF_ACP_RECOVERY_SHARDS_CNT; shard_id++) { + chunk->num_dirty += context->shard[shard_id] + .chunk[chunk->core_id][chunk->chunk_id]; + } + + for (bucket_id = 0; bucket_id < ACP_MAX_BUCKETS; bucket_id++) { + bucket = &acp->bucket_info[bucket_id]; + if (chunk->num_dirty < bucket->threshold) + break; + } + + bucket = &acp->bucket_info[--bucket_id]; + chunk->bucket_id = bucket_id; + + list_move_tail(&chunk->list, &bucket->chunk_list); +} + +static void ocf_acp_recovery_finish(ocf_parallelize_t parallelize, + void *priv, int error) +{ + struct ocf_acp_recovery_context *context = priv; + ocf_cache_t cache = context->cache; + struct acp_context *acp = _acp_get_ctx_from_cache(cache); + ocf_core_id_t core_id; + ocf_core_t core; + uint64_t core_size; + uint64_t num_chunks; + uint64_t chunk_id; + uint32_t step = 0; + + for_each_core(cache, core, core_id) { + core_size = core->conf_meta->length; + num_chunks = OCF_DIV_ROUND_UP(core_size, ACP_CHUNK_SIZE); + + for (chunk_id = 0; chunk_id < num_chunks; chunk_id++) { + ocf_acp_recovery_chunk(context, + &acp->chunk_info[core_id][chunk_id]); + OCF_COND_RESCHED_DEFAULT(step); + } + } + + ocf_cache_log(cache, log_info, "Finished rebuilding ACP metadata\n"); + + ocf_kick_cleaner(cache); + + context->cmpl(context->priv, error); + + for_each_core(cache, core, core_id) { + if (context->shard[0].chunk[core_id]) + env_vfree(context->shard[0].chunk[core_id]); + } + + ocf_parallelize_destroy(parallelize); +} + void cleaning_policy_acp_recovery(ocf_cache_t cache, ocf_cleaning_recovery_end_t cmpl, void *priv) { + struct ocf_acp_recovery_context *context; + ocf_parallelize_t parallelize; + ocf_core_id_t core_id; + ocf_core_t core; + unsigned shards_cnt = OCF_ACP_RECOVERY_SHARDS_CNT; + unsigned shard_id; + uint64_t core_size; + uint64_t num_chunks; + uint16_t *chunks; int result; - result = cleaning_policy_acp_initialize(cache, 1); + result = ocf_parallelize_create(¶llelize, cache, + OCF_ACP_RECOVERY_SHARDS_CNT, sizeof(*context), + ocf_acp_recovery_handle, ocf_acp_recovery_finish); + if (result) { + cmpl(priv, result); + return; + } + + context = ocf_parallelize_get_priv(parallelize); + context->cache = cache; + context->cmpl = cmpl; + context->priv = priv; + + for_each_core(cache, core, core_id) { + core_size = core->conf_meta->length; + num_chunks = OCF_DIV_ROUND_UP(core_size, ACP_CHUNK_SIZE); + + chunks = env_vzalloc(sizeof(*chunks) * num_chunks * shards_cnt); + if (!chunks) { + result = -OCF_ERR_NO_MEM; + goto err; + } + + for (shard_id = 0; shard_id < shards_cnt; shard_id++) { + context->shard[shard_id].chunk[core_id] = + &chunks[num_chunks * shard_id]; + } + } + + result = cleaning_policy_acp_init_common(cache); + if (result) + goto err; + + ocf_parallelize_run(parallelize); + + return; + +err: + for_each_core(cache, core, core_id) { + if (context->shard[0].chunk[core_id]) + env_vfree(context->shard[0].chunk[core_id]); + } + + ocf_parallelize_destroy(parallelize); + cmpl(priv, result); } From 481e5b7b9bc6fb8370f1f8a927e4862840bac843 Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Sat, 29 Jan 2022 23:21:32 +0100 Subject: [PATCH 13/15] Introduce bisect generator utility Signed-off-by: Robert Baldyga --- src/utils/utils_generator.c | 187 ++++++++++++++++++++++++++++++++++++ src/utils/utils_generator.h | 23 +++++ 2 files changed, 210 insertions(+) create mode 100644 src/utils/utils_generator.c create mode 100644 src/utils/utils_generator.h diff --git a/src/utils/utils_generator.c b/src/utils/utils_generator.c new file mode 100644 index 0000000..32b42a2 --- /dev/null +++ b/src/utils/utils_generator.c @@ -0,0 +1,187 @@ +#include "utils_generator.h" + +/** + * @brief Reverse bits of 32-bit value + * + * @param[in] x Value to be reversed + * + * @return Reversed value + */ +static inline uint32_t bitreverse32(register uint32_t x) +{ + x = (((x & 0xaaaaaaaa) >> 1) | ((x & 0x55555555) << 1)); + x = (((x & 0xcccccccc) >> 2) | ((x & 0x33333333) << 2)); + x = (((x & 0xf0f0f0f0) >> 4) | ((x & 0x0f0f0f0f) << 4)); + x = (((x & 0xff00ff00) >> 8) | ((x & 0x00ff00ff) << 8)); + return((x >> 16) | (x << 16)); +} + +/** + * @brief Initialize bisect generator state + * + * @param[in] generator Pointer to generator structure + * @param[in] limit Limit of the value returned by generator (maximum value + * returned by the generator is limit - 1) + * @param[in] offset Offset at which generator should start + * + * @return Reversed value + */ +void ocf_generator_bisect_init( + struct ocf_generator_bisect_state *generator, + uint32_t limit, uint32_t offset) +{ + unsigned clz; + uint32_t maplen; + + clz = __builtin_clz(limit - 1); + maplen = 1 << (32 - clz); + + generator->curr = (uint64_t)offset * maplen / limit; + generator->limit = limit; +} + +/** + * @brief Generate next value of bisect generator + * + * This function calculates next value of the generator. The generator + * pattern is based on order of indexes in array visited with bisection + * algorithm, where always the left child is visited first at every depth. + * This can be imagined as a special implementation of BFS done on a full + * binary tree, where visiting nodes on each depth level is done the same + * order as original array (so the algorithm is recursive at each level). + * + * Example: + * + * 1. We generate array of all values for number of bits needed to express + * limit value - 1. + * + * For limit==14 (4 bits) it would be: + * [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + * + * 2. We take first element of the array, and then build full binary tree + * from other elements (it should always be possible to build a full + * binary tree, as number of remaining elements is always 2^n-1). + * + * The full binary tree for example array looks like this: + * + * Depth 0 -> 8 + * / \ + * / \ + * / \ + * / \ + * / \ + * / \ + * / \ + * / \ + * Depth 1 -> 4 12 + * / \ / \ + * / \ / \ + * / \ / \ + * Depth 2 -> 2 6 10 14 + * / \ / \ / \ / \ + * Depth 3 -> 1 3 5 7 9 11 13 15 + * + * 3. We traverse the tree: + * a) If depth level has one element, we take it. + * b) If depth level has two elements, we take left and then right. + * c) If depth level has more than two elements, we repeat steps + * from 2 on array built from elements on that level left to right. + * + * At level 0 we take 8, at level 1 we take 4 and 12, and at level 3 we + * take 2 and build tree like this one: + * + * 10 + * / \ + * 6 14 + * + * Then at level 0 of that tree we take 10, at level 1 we take 6 and 14, + * and then we go back the original tree. + * + * At level 3 we take 1 and we build another tree from remaining elements + * of that level: + * + * 9 + * / \ + * / \ + * / \ + * 5 13 + * / \ / \ + * 3 7 11 15 + * + * Repeating step 3 on that tree we get elements 9, then 5 and 13, and then + * by running steps from 2 on the lowest level of that tree we get elements + * in following order: 3, 11, 7 and 15. + * + * So the entire sequence would be as follows: + * [0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15] + * + * This algorithm is however quite complex, and it can be simplified + * significantly thanks to properties of the result sequence. Note that + * when this sequence is written in binary it looks like this: + * + * 0 0000 + * 8 1000 + * 4 0100 + * 12 1100 + * 2 0010 + * 10 1010 + * 6 0110 + * 14 1110 + * 1 0001 + * 9 1001 + * 5 0101 + * 13 1101 + * 3 0011 + * 11 1011 + * 7 0111 + * 15 1111 + * + * So in its binary representation it looks like a mirror image of binary + * representation of the original sequence: + * + * 0 0000 0000 0 + * 8 1000 0001 1 + * 4 0100 0010 2 + * 12 1100 0011 3 + * 2 0010 0100 4 + * 10 1010 0101 5 + * 6 0110 0110 6 + * 14 1110 0111 7 + * 1 0001 1000 8 + * 9 1001 1001 9 + * 5 0101 1010 10 + * 13 1101 1011 11 + * 3 0011 1100 12 + * 11 1011 1101 13 + * 7 0111 1110 14 + * 15 1111 1111 15 + * + * With that knowledge we can easily calculate the next result value by just + * reversing order of bits for each value in original sequence. + * + * As a result we are left with sequence that contains all the numbers that + * can be expressed with number of bits reqiured to express limit - 1. The only + * thing we need to do at that point is to just filter out values that do not + * fit within the limit. + * + * @param[in] generator Pointer to generator structure + * + * @return Generated value + */ +uint32_t ocf_generator_bisect_next( + struct ocf_generator_bisect_state *generator) +{ + unsigned clz; + uint32_t maplen; + uint32_t value; + + clz = __builtin_clz(generator->limit - 1); + maplen = 1 << (32 - clz); + + do { + value = bitreverse32(generator->curr) >> clz; + generator->curr = (generator->curr + 1) % maplen; + } while (value >= generator->limit); + + return value; +} diff --git a/src/utils/utils_generator.h b/src/utils/utils_generator.h new file mode 100644 index 0000000..068602d --- /dev/null +++ b/src/utils/utils_generator.h @@ -0,0 +1,23 @@ +/* + * Copyright(c) 2022 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef __UTILS_GENERATOR_H__ +#define __UTILS_GENERATOR_H__ + +#include "ocf/ocf.h" + +struct ocf_generator_bisect_state { + uint32_t curr; + uint32_t limit; +}; + +void ocf_generator_bisect_init( + struct ocf_generator_bisect_state *generator, + uint32_t limit, uint32_t offset); + +uint32_t ocf_generator_bisect_next( + struct ocf_generator_bisect_state *generator); + +#endif /* __UTILS_GENERATOR_H__ */ From 53f6cffe7f2ec41df081288a0d47ad25163e74c2 Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Sun, 30 Jan 2022 19:20:00 +0100 Subject: [PATCH 14/15] tests: Add unit tests for bisect generator Signed-off-by: Robert Baldyga --- .../utils_generator_bisect.c | 214 ++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 tests/unit/tests/utils/utils_generator.c/utils_generator_bisect.c diff --git a/tests/unit/tests/utils/utils_generator.c/utils_generator_bisect.c b/tests/unit/tests/utils/utils_generator.c/utils_generator_bisect.c new file mode 100644 index 0000000..0f9e853 --- /dev/null +++ b/tests/unit/tests/utils/utils_generator.c/utils_generator_bisect.c @@ -0,0 +1,214 @@ +/* + * Copyright(c) 2022 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* + * src/utils/utils_generator.c + * ocf_generator_bisect_next + * + * bitreverse32 + * ocf_generator_bisect_init + * + */ + +#undef static + +#undef inline + + +#include +#include +#include +#include +#include "print_desc.h" + +#include "utils_generator.h" + +#include "utils/utils_generator.c/utils_generator_bisect_generated_wraps.c" + +static void ocf_generator_bisect_test01(void **state) +{ + struct { + uint32_t values[16]; + uint32_t limit; + } expected_output[] = { + { + .values = { 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15 }, + .limit = 16, + }, + { + .values = { 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7 }, + .limit = 15, + }, + { + .values = { 0, 8, 4, 12, 2, 10, 6, 1, 9, 5, 13, 3, 11, 7 }, + .limit = 14, + }, + { + .values = { 0, 8, 4, 12, 2, 10, 6, 1, 9, 5, 3, 11, 7 }, + .limit = 13, + }, + { + .values = { 0, 8, 4, 2, 10, 6, 1, 9, 5, 3, 11, 7 }, + .limit = 12, + }, + { + .values = { 0, 8, 4, 2, 10, 6, 1, 9, 5, 3, 7 }, + .limit = 11, + }, + { + .values = { 0, 8, 4, 2, 6, 1, 9, 5, 3, 7 }, + .limit = 10, + }, + { + .values = { 0, 8, 4, 2, 6, 1, 5, 3, 7 }, + .limit = 9, + }, + { + .values = { 0, 4, 2, 6, 1, 5, 3, 7 }, + .limit = 8, + }, + { + .values = { 0, 4, 2, 6, 1, 5, 3 }, + .limit = 7, + }, + { + .values = { 0, 4, 2, 1, 5, 3 }, + .limit = 6, + }, + { + .values = { 0, 4, 2, 1, 3 }, + .limit = 5, + }, + { + .values = { 0, 2, 1, 3 }, + .limit = 4, + }, + { + .values = { 0, 2, 1 }, + .limit = 3, + }, + { + .values = { 0, 1 }, + .limit = 2, + }, + { + .values = { 0 }, + .limit = 1, + }, + }; + struct ocf_generator_bisect_state generator; + uint32_t value; + int i, j; + + print_test_description("Check if sequence order is correct"); + + for (i = 0; i < sizeof(expected_output)/sizeof(*expected_output); i++) { + ocf_generator_bisect_init(&generator, + expected_output[i].limit, 0); + + for (j = 0; j < expected_output[i].limit; j++) { + value = ocf_generator_bisect_next(&generator); + assert_int_equal(value, expected_output[i].values[j]); + } + } +} + +static void ocf_generator_bisect_test02(void **state) +{ + struct { + uint32_t values[16]; + uint32_t limit; + uint32_t offset; + } expected_output[] = { + { + .values = { 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15, 0 }, + .limit = 16, + .offset = 1, + }, + { + .values = { 15, 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7 }, + .limit = 16, + .offset = 15, + }, + { + .values = { 1, 9, 5, 13, 3, 11, 7, 15, 0, 8, 4, 12, 2, 10, 6, 14 }, + .limit = 16, + .offset = 8, + }, + { + .values = { 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 0 }, + .limit = 15, + .offset = 1, + }, + { + .values = { 7, 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11 }, + .limit = 15, + .offset = 14, + }, + { + .values = { 1, 9, 5, 13, 3, 11, 7, 0, 8, 4, 12, 2, 10, 6, 14 }, + .limit = 15, + .offset = 8, + }, + { + .values = { 8, 4, 2, 10, 6, 1, 9, 5, 3, 11, 7, 0 }, + .limit = 12, + .offset = 1, + }, + { + .values = { 7, 0, 8, 4, 2, 10, 6, 1, 9, 5, 3, 11 }, + .limit = 12, + .offset = 11, + }, + { + .values = { 1, 9, 5, 3, 11, 7, 0, 8, 4, 2, 10, 6 }, + .limit = 12, + .offset = 6, + }, + { + .values = { 8, 4, 2, 6, 1, 5, 3, 7, 0 }, + .limit = 9, + .offset = 1, + }, + { + .values = { 7, 0, 8, 4, 2, 6, 1, 5, 3 }, + .limit = 9, + .offset = 8, + }, + { + .values = { 1, 5, 3, 7, 0, 8, 4, 2, 6 }, + .limit = 9, + .offset = 5, + }, + }; + struct ocf_generator_bisect_state generator; + uint32_t value; + int i, j; + + print_test_description("Check if offset works correctly"); + + for (i = 0; i < sizeof(expected_output)/sizeof(*expected_output); i++) { + ocf_generator_bisect_init(&generator, + expected_output[i].limit, + expected_output[i].offset); + + for (j = 0; j < expected_output[i].limit; j++) { + value = ocf_generator_bisect_next(&generator); + assert_int_equal(value, expected_output[i].values[j]); + } + } +} + +int main(void) +{ + const struct CMUnitTest tests[] = { + cmocka_unit_test(ocf_generator_bisect_test01), + cmocka_unit_test(ocf_generator_bisect_test02), + }; + + print_message("Unit tests for generator bisect\n"); + + return cmocka_run_group_tests(tests, NULL, NULL); +} From 6a665ea6b17da34ff11a150ef8bb8a4788058459 Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Wed, 26 Jan 2022 19:05:28 +0100 Subject: [PATCH 15/15] Shuffle entries within freelists Signed-off-by: Robert Baldyga --- src/ocf_lru.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/ocf_lru.c b/src/ocf_lru.c index 7e45849..0062023 100644 --- a/src/ocf_lru.c +++ b/src/ocf_lru.c @@ -7,6 +7,7 @@ #include "ocf_lru.h" #include "utils/utils_cleaner.h" #include "utils/utils_cache_line.h" +#include "utils/utils_generator.h" #include "utils/utils_parallelize.h" #include "concurrency/ocf_concurrency.h" #include "mngt/ocf_mngt_common.h" @@ -883,17 +884,29 @@ static int ocf_lru_populate_handle(ocf_parallelize_t parallelize, ocf_cache_t cache = context->cache; ocf_cache_line_t cnt, cline; ocf_cache_line_t entries = ocf_metadata_collision_table_entries(cache); + struct ocf_generator_bisect_state generator; struct ocf_lru_list *list; unsigned lru_list = shard_id; unsigned step = 0; + uint32_t portion, offset; + uint32_t i, idx; + + portion = DIV_ROUND_UP((uint64_t)entries, shards_cnt); + offset = shard_id * portion / shards_cnt; + ocf_generator_bisect_init(&generator, portion, offset); + + list = ocf_lru_get_list(&cache->free, lru_list, true); cnt = 0; - for (cline = shard_id; cline < entries; cline += shards_cnt) { + for (i = 0; i < portion; i++) { OCF_COND_RESCHED_DEFAULT(step); - ocf_metadata_set_partition_id(cache, cline, PARTITION_FREELIST); + idx = ocf_generator_bisect_next(&generator); + cline = idx * shards_cnt + shard_id; + if (cline >= entries) + continue; - list = ocf_lru_get_list(&cache->free, lru_list, true); + ocf_metadata_set_partition_id(cache, cline, PARTITION_FREELIST); add_lru_head_nobalance(cache, list, cline);