From ca9d8072dfeeff971555b2dca3f6ec490442d0a9 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Fri, 23 Sep 2022 07:09:07 +0200 Subject: [PATCH 01/11] posix env: assert in rwsem unlock Instead of ignoring `pthread_rwlock_unlock()` return value assert that is must succeed. The function returns an error eg. when there is an attempt to unlock the resource from a different thread than it was originally locked which is illegal in userspace. Signed-off-by: Michal Mielewczyk --- env/posix/ocf_env.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env/posix/ocf_env.h b/env/posix/ocf_env.h index 4fa163a..e1ab63e 100644 --- a/env/posix/ocf_env.h +++ b/env/posix/ocf_env.h @@ -306,7 +306,7 @@ static inline int env_rwsem_init(env_rwsem *s) static inline void env_rwsem_up_read(env_rwsem *s) { - pthread_rwlock_unlock(&s->lock); + ENV_BUG_ON(pthread_rwlock_unlock(&s->lock)); } static inline void env_rwsem_down_read(env_rwsem *s) From ef997b47fa703629fd85c927244f7d3ae6172a4c Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Fri, 16 Sep 2022 13:00:58 +0200 Subject: [PATCH 02/11] Fix whitespaces Signed-off-by: Michal Mielewczyk --- src/metadata/metadata.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/metadata/metadata.c b/src/metadata/metadata.c index d5bff02..5d1f6eb 100644 --- a/src/metadata/metadata.c +++ b/src/metadata/metadata.c @@ -994,7 +994,7 @@ static bool ocf_check_if_cleaner_enabled(ocf_pipeline_t pipeline, void* priv, ocf_pipeline_arg_t arg) { struct ocf_metadata_context *context = priv; - + return !context->cache->conf_meta->cleaner_disabled; } @@ -1002,7 +1002,7 @@ struct ocf_pipeline_properties ocf_metadata_flush_all_pipeline_props = { .priv_size = sizeof(struct ocf_metadata_context), .finish = ocf_metadata_flush_all_finish, .steps = { - + OCF_PL_STEP_COND_ARG_INT(ocf_check_if_cleaner_enabled, ocf_metadata_flush_segment, metadata_segment_cleaning), @@ -1156,7 +1156,7 @@ struct ocf_pipeline_properties ocf_metadata_load_all_pipeline_props = { metadata_segment_cleaning), OCF_PL_STEP_FOREACH(ocf_metadata_load_segment, ocf_metadata_load_all_args), - + OCF_PL_STEP_COND_ARG_INT(ocf_check_if_cleaner_enabled, ocf_metadata_check_crc, metadata_segment_cleaning), From 21d5da83d975a1bdbf6e4f7312dfe0b5d52767d7 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Mon, 26 Sep 2022 14:03:08 +0200 Subject: [PATCH 03/11] A utility for counting queues Signed-off-by: Michal Mielewczyk --- src/ocf_cache.c | 13 ++++++++++++- src/ocf_cache_priv.h | 5 ++++- src/utils/utils_parallelize.c | 3 +-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/ocf_cache.c b/src/ocf_cache.c index be11244..f0665ac 100644 --- a/src/ocf_cache.c +++ b/src/ocf_cache.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ @@ -25,6 +25,17 @@ ocf_volume_t ocf_cache_get_front_volume(ocf_cache_t cache) return cache->device ? &cache->device->front_volume : NULL; } +uint32_t ocf_cache_get_queue_count(ocf_cache_t cache) +{ + ocf_queue_t queue = NULL; + uint32_t cnt = 0; + + list_for_each_entry(queue, &cache->io_queues, list) + cnt++; + + return cnt; +} + int ocf_cache_set_name(ocf_cache_t cache, const char *src, size_t src_size) { int result; diff --git a/src/ocf_cache_priv.h b/src/ocf_cache_priv.h index 041bc6f..807483a 100644 --- a/src/ocf_cache_priv.h +++ b/src/ocf_cache_priv.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ @@ -19,6 +19,7 @@ #include "ocf_stats_priv.h" #include "cleaning/cleaning.h" #include "ocf_logger_priv.h" +#include "ocf_queue_priv.h" #include "promotion/promotion.h" #define DIRTY_FLUSHED 1 @@ -175,6 +176,8 @@ static inline uint64_t ocf_get_cache_occupancy(ocf_cache_t cache) return result; } +uint32_t ocf_cache_get_queue_count(ocf_cache_t cache); + int ocf_cache_set_name(ocf_cache_t cache, const char *src, size_t src_size); int ocf_cache_volume_type_init(ocf_ctx_t ctx); diff --git a/src/utils/utils_parallelize.c b/src/utils/utils_parallelize.c index 8265aa5..fc0ce6e 100644 --- a/src/utils/utils_parallelize.c +++ b/src/utils/utils_parallelize.c @@ -57,8 +57,7 @@ int ocf_parallelize_create(ocf_parallelize_t *parallelize, unsigned queue_count = 0; int result, i; - list_for_each_entry(queue, &cache->io_queues, list) - queue_count++; + queue_count = ocf_cache_get_queue_count(cache); if (shards_cnt == 0) shards_cnt = queue_count; From ca70ea3fff5779857ff35edb93a14f75cce2f68e Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Thu, 15 Sep 2022 12:04:55 +0200 Subject: [PATCH 04/11] Deinit cleaning policy if attaching cache failed Normally cleaning policy would be deinitialized during stopping cache which is one of steps of error handling e.g in case of failed cache activation. But since `cache_stop()` may be called only for an attached cache instance, cleaning policy needs to deinitialized explicitly. Signed-off-by: Michal Mielewczyk --- src/mngt/ocf_mngt_cache.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 7cc1aa8..68f93cf 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -130,6 +130,9 @@ struct ocf_cache_attach_context { bool promotion_initialized : 1; /*!< Promotion policy has been started */ + bool cleaning_initialized : 1; + /*!< Cleaning policy has been initialized */ + bool cores_opened : 1; /*!< underlying cores are opened (happens only during * load or recovery @@ -1233,6 +1236,7 @@ static void _ocf_mngt_attach_init_services(ocf_pipeline_t pipeline, considered to be flushed */ cache->conf_meta->dirty_flushed = true; + context->flags.cleaning_initialized = true; ocf_pipeline_next(pipeline); } @@ -1728,6 +1732,9 @@ static void _ocf_mngt_attach_handle_error( if (context->flags.promotion_initialized) __deinit_promotion_policy(cache); + if (context->flags.cleaning_initialized) + __deinit_cleaning_policy(cache); + if (context->flags.cores_opened) _ocf_mngt_deinit_added_cores(context); From f8e8d74539b0a54a4f4ab9b334208f479ba8643c Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Thu, 15 Sep 2022 13:16:08 +0200 Subject: [PATCH 05/11] attach: setup promotion policy before cleaning Signed-off-by: Michal Mielewczyk --- src/mngt/ocf_mngt_cache.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 68f93cf..cee7c1b 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -1220,6 +1220,8 @@ static void _ocf_mngt_attach_init_services(ocf_pipeline_t pipeline, ocf_cache_t cache = context->cache; ocf_error_t result; + __setup_promotion_policy(cache); + if (context->metadata.cleaner_disabled) __set_cleaning_policy(cache, ocf_cleaning_nop); @@ -1230,8 +1232,6 @@ static void _ocf_mngt_attach_init_services(ocf_pipeline_t pipeline, OCF_PL_FINISH_RET(pipeline, result); } - __setup_promotion_policy(cache); - /* In initial cache state there is no dirty data, so all dirty data is considered to be flushed */ From da67112b17d143c614a6b715f9a9e0e2701a1d83 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Fri, 16 Sep 2022 12:10:57 +0200 Subject: [PATCH 06/11] load: `init_structures` as a separate step Signed-off-by: Michal Mielewczyk --- src/mngt/ocf_mngt_cache.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index cee7c1b..cc025a9 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -677,6 +677,20 @@ static void _ocf_mngt_load_init_cleaning(ocf_pipeline_t pipeline, _ocf_mngt_cleaning_recovery_complete, context); } +static void _ocf_mngt_load_init_structures(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) +{ + struct ocf_cache_attach_context *context = priv; + ocf_cache_t cache = context->cache; + + if (context->metadata.shutdown_status == ocf_metadata_clean_shutdown) + OCF_PL_NEXT_RET(pipeline); + + init_attached_data_structures_recovery(cache, true); + + ocf_pipeline_next(context->pipeline); +} + void _ocf_mngt_load_metadata_complete(void *priv, int error) { struct ocf_cache_attach_context *context = priv; @@ -728,8 +742,6 @@ static void _ocf_mngt_load_metadata(ocf_pipeline_t pipeline, } else { ocf_cache_log(cache, log_warn, "ERROR: Cache device did not shut down properly!\n"); - init_attached_data_structures_recovery(cache, true); - ocf_cache_log(cache, log_info, "Initiating recovery sequence...\n"); _ocf_mngt_load_metadata_recovery(pipeline, priv, arg); } @@ -1826,6 +1838,7 @@ struct ocf_pipeline_properties _ocf_mngt_cache_load_pipeline_properties = { OCF_PL_STEP(_ocf_mngt_init_cleaner), OCF_PL_STEP(_ocf_mngt_init_promotion), OCF_PL_STEP(_ocf_mngt_load_add_cores), + OCF_PL_STEP(_ocf_mngt_load_init_structures), OCF_PL_STEP(_ocf_mngt_load_metadata), OCF_PL_STEP(_ocf_mngt_load_rebuild_metadata), OCF_PL_STEP(_ocf_mngt_load_init_cleaning), From b50bd1b50621f84f688ea7deaeecec6ca5d68fab Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Fri, 16 Sep 2022 12:56:25 +0200 Subject: [PATCH 07/11] Initialize metadata structures in pipelines Initializing metadata in an asynchronous manner will allow to use parallelization utilities in the future commits Signed-off-by: Michal Mielewczyk --- src/metadata/metadata.c | 16 ++++- src/metadata/metadata.h | 16 ++++- src/mngt/ocf_mngt_cache.c | 143 ++++++++++++++++++++++++++++++++------ 3 files changed, 148 insertions(+), 27 deletions(-) diff --git a/src/metadata/metadata.c b/src/metadata/metadata.c index 5d1f6eb..4f79546 100644 --- a/src/metadata/metadata.c +++ b/src/metadata/metadata.c @@ -831,22 +831,33 @@ static inline void _ocf_init_collision_entry(struct ocf_cache *cache, /* * Initialize collision table */ -void ocf_metadata_init_collision(struct ocf_cache *cache) +void ocf_metadata_init_collision(ocf_pipeline_t pipeline, void *priv, + ocf_pipeline_arg_t arg) { + struct ocf_init_metadata_context *context = priv; + ocf_cache_t cache = context->cache; unsigned int i; unsigned int step = 0; + if (context->skip_collision) + OCF_PL_NEXT_RET(pipeline); + for (i = 0; i < cache->device->collision_table_entries; i++) { _ocf_init_collision_entry(cache, i); OCF_COND_RESCHED_DEFAULT(step); } + + ocf_pipeline_next(context->pipeline); } /* * Initialize hash table */ -void ocf_metadata_init_hash_table(struct ocf_cache *cache) +void ocf_metadata_init_hash_table(ocf_pipeline_t pipeline, void *priv, + ocf_pipeline_arg_t arg) { + struct ocf_init_metadata_context *context = priv; + ocf_cache_t cache = context->cache; unsigned int i; unsigned int hash_table_entries = cache->device->hash_table_entries; ocf_cache_line_t invalid_idx = cache->device->collision_table_entries; @@ -860,6 +871,7 @@ void ocf_metadata_init_hash_table(struct ocf_cache *cache) ocf_metadata_set_hash(cache, i, invalid_idx); } + ocf_pipeline_next(context->pipeline); } /* diff --git a/src/metadata/metadata.h b/src/metadata/metadata.h index f26115d..6546702 100644 --- a/src/metadata/metadata.h +++ b/src/metadata/metadata.h @@ -25,6 +25,16 @@ #define CLEAN 2 #define DIRTY 3 +typedef void (*ocf_mngt_init_metadata_end_t)(void *priv, int error); + +struct ocf_init_metadata_context { + ocf_mngt_init_metadata_end_t cmpl; + ocf_pipeline_t pipeline; + ocf_cache_t cache; + uint8_t skip_collision; + void *priv; +}; + /** * @brief Initialize metadata * @@ -60,14 +70,16 @@ void ocf_metadata_init_freelist_partition(struct ocf_cache *cache); * * @param cache - Cache instance */ -void ocf_metadata_init_hash_table(struct ocf_cache *cache); +void ocf_metadata_init_hash_table(ocf_pipeline_t pipeline, void *priv, + ocf_pipeline_arg_t arg); /** * @brief Initialize collision table * * @param cache - Cache instance */ -void ocf_metadata_init_collision(struct ocf_cache *cache); +void ocf_metadata_init_collision(ocf_pipeline_t pipeline, void *priv, + ocf_pipeline_arg_t arg); /** * @brief De-Initialize metadata diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index cc025a9..5cb5bd6 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -203,14 +203,19 @@ static void __init_partitions(ocf_cache_t cache) } } -static void __init_parts_attached(ocf_cache_t cache) +static void _init_parts_attached(ocf_pipeline_t pipeline, void *priv, + ocf_pipeline_arg_t arg) { + struct ocf_init_metadata_context *context = priv; + ocf_cache_t cache = context->cache; ocf_part_id_t part_id; for (part_id = 0; part_id < OCF_USER_IO_CLASS_MAX; part_id++) ocf_lru_init(cache, &cache->user_parts[part_id].part); ocf_lru_init(cache, &cache->free); + + ocf_pipeline_next(pipeline); } static ocf_error_t __init_cleaning_policy(ocf_cache_t cache) @@ -270,8 +275,11 @@ static void __init_metadata_version(ocf_cache_t cache) cache->conf_meta->metadata_version = METADATA_VERSION(); } -static void __reset_stats(ocf_cache_t cache) +static void _reset_stats(ocf_pipeline_t pipeline, void *priv, + ocf_pipeline_arg_t arg) { + struct ocf_init_metadata_context *context = priv; + ocf_cache_t cache = context->cache; ocf_core_t core; ocf_core_id_t core_id; ocf_part_id_t i; @@ -288,17 +296,65 @@ static void __reset_stats(ocf_cache_t cache) part_counters[i].dirty_clines, 0); } } + + ocf_pipeline_next(pipeline); } -static void init_attached_data_structures_recovery(ocf_cache_t cache, - bool init_collision) +static void _init_metadata_version(ocf_pipeline_t pipeline, void *priv, + ocf_pipeline_arg_t arg) { - ocf_metadata_init_hash_table(cache); - if (init_collision) - ocf_metadata_init_collision(cache); - __init_parts_attached(cache); - __reset_stats(cache); + struct ocf_init_metadata_context *context = priv; + ocf_cache_t cache = context->cache; + __init_metadata_version(cache); + + ocf_pipeline_next(pipeline); +} + +static void _ocf_mngt_init_metadata_finish(ocf_pipeline_t pipeline, + void *priv, int error) +{ + struct ocf_init_metadata_context *context = priv; + + context->cmpl(context->priv, error); + + ocf_pipeline_destroy(pipeline); +} + +struct ocf_pipeline_properties ocf_init_attached_recovery_props = { + .priv_size = sizeof(struct ocf_init_metadata_context), + .finish = _ocf_mngt_init_metadata_finish, + .steps = { + OCF_PL_STEP(ocf_metadata_init_hash_table), + OCF_PL_STEP(ocf_metadata_init_collision), + OCF_PL_STEP(_init_parts_attached), + OCF_PL_STEP(_reset_stats), + OCF_PL_STEP(_init_metadata_version), + OCF_PL_STEP_TERMINATOR(), + }, +}; + +static void init_attached_data_structures_recovery(ocf_cache_t cache, + ocf_mngt_init_metadata_end_t cmpl, void *priv, bool skip_collision) +{ + struct ocf_init_metadata_context *context; + ocf_pipeline_t pipeline; + int result; + + result = ocf_pipeline_create(&pipeline, cache, + &ocf_init_attached_recovery_props); + if (result) + OCF_CMPL_RET(priv, result); + + context = ocf_pipeline_get_priv(pipeline); + + context->cmpl = cmpl; + context->priv = priv; + context->pipeline = pipeline; + context->cache = cache; + context->skip_collision = skip_collision; + + OCF_PL_NEXT_RET(pipeline); } /**************************************************************** @@ -677,6 +733,20 @@ static void _ocf_mngt_load_init_cleaning(ocf_pipeline_t pipeline, _ocf_mngt_cleaning_recovery_complete, context); } +static void _ocf_mngt_init_metadata_complete(void *priv, int error) +{ + struct ocf_cache_attach_context *context = priv; + ocf_cache_t cache = context->cache; + + if (error) { + ocf_cache_log(cache, log_err, + "ERROR: Cannot initialize cache metadata\n"); + OCF_PL_FINISH_RET(context->pipeline, -OCF_ERR_NO_MEM); + } + + ocf_pipeline_next(context->pipeline); +} + static void _ocf_mngt_load_init_structures(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { @@ -686,9 +756,8 @@ static void _ocf_mngt_load_init_structures(ocf_pipeline_t pipeline, if (context->metadata.shutdown_status == ocf_metadata_clean_shutdown) OCF_PL_NEXT_RET(pipeline); - init_attached_data_structures_recovery(cache, true); - - ocf_pipeline_next(context->pipeline); + init_attached_data_structures_recovery(cache, + _ocf_mngt_init_metadata_complete, context, false); } void _ocf_mngt_load_metadata_complete(void *priv, int error) @@ -1192,6 +1261,39 @@ static void _ocf_mngt_attach_prepare_metadata(ocf_pipeline_t pipeline, ocf_pipeline_next(pipeline); } +struct ocf_pipeline_properties ocf_init_metadata_pipeline_props = { + .priv_size = sizeof(struct ocf_init_metadata_context), + .finish = _ocf_mngt_init_metadata_finish, + .steps = { + OCF_PL_STEP(ocf_metadata_init_hash_table), + OCF_PL_STEP(ocf_metadata_init_collision), + OCF_PL_STEP(_init_parts_attached), + OCF_PL_STEP_TERMINATOR(), + }, +}; + +static void _ocf_mngt_init_metadata(ocf_cache_t cache, + ocf_mngt_init_metadata_end_t cmpl, void *priv) +{ + struct ocf_init_metadata_context *context; + ocf_pipeline_t pipeline; + int result; + + result = ocf_pipeline_create(&pipeline, cache, + &ocf_init_metadata_pipeline_props); + if (result) + OCF_CMPL_RET(priv, result); + + context = ocf_pipeline_get_priv(pipeline); + + context->cmpl = cmpl; + context->priv = priv; + context->pipeline = pipeline; + context->cache = cache; + + OCF_PL_NEXT_RET(pipeline); +} + /** * @brief initializing cache anew (not loading or recovering) */ @@ -1201,11 +1303,8 @@ static void _ocf_mngt_attach_init_metadata(ocf_pipeline_t pipeline, struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; - ocf_metadata_init_hash_table(cache); - ocf_metadata_init_collision(cache); - __init_parts_attached(cache); - - ocf_pipeline_next(pipeline); + _ocf_mngt_init_metadata(cache, _ocf_mngt_init_metadata_complete, + context); } static void _ocf_mngt_attach_populate_free_complete(void *priv, int error) @@ -2179,9 +2278,8 @@ static void _ocf_mngt_standby_init_structures_attach(ocf_pipeline_t pipeline, struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; - init_attached_data_structures_recovery(cache, true); - - ocf_pipeline_next(pipeline); + init_attached_data_structures_recovery(cache, + _ocf_mngt_init_metadata_complete, context, false); } static void _ocf_mngt_standby_init_structures_load(ocf_pipeline_t pipeline, @@ -2190,9 +2288,8 @@ static void _ocf_mngt_standby_init_structures_load(ocf_pipeline_t pipeline, struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; - init_attached_data_structures_recovery(cache, false); - - ocf_pipeline_next(pipeline); + init_attached_data_structures_recovery(cache, + _ocf_mngt_init_metadata_complete, context, true); } static void _ocf_mngt_standby_init_pio_concurrency(ocf_pipeline_t pipeline, From 4dbf740f5b32b0c394ab7e6a2f9a0d3b43271cb0 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Fri, 16 Sep 2022 13:36:49 +0200 Subject: [PATCH 08/11] Parallelize initializing collision section Signed-off-by: Michal Mielewczyk --- src/metadata/metadata.c | 58 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/src/metadata/metadata.c b/src/metadata/metadata.c index 4f79546..eba81b0 100644 --- a/src/metadata/metadata.c +++ b/src/metadata/metadata.c @@ -18,6 +18,7 @@ #include "../utils/utils_cache_line.h" #include "../utils/utils_io.h" #include "../utils/utils_pipeline.h" +#include "../utils/utils_parallelize.h" #define OCF_METADATA_DEBUG 0 @@ -831,23 +832,66 @@ static inline void _ocf_init_collision_entry(struct ocf_cache *cache, /* * Initialize collision table */ + +static int ocf_metadata_init_collision_handle(ocf_parallelize_t parallelize, + void *priv, unsigned shard_id, unsigned shards_cnt) +{ + struct ocf_init_metadata_context *context = priv; + ocf_cache_t cache = context->cache; + ocf_cache_line_t collision_table_entries = cache->device->collision_table_entries; + uint32_t entry, portion, begin, end, step=0; + + portion = OCF_DIV_ROUND_UP((uint64_t)collision_table_entries, shards_cnt); + begin = portion*shard_id; + end = OCF_MIN(portion*(shard_id + 1), collision_table_entries); + + for (entry = begin; entry < end; entry++) { + OCF_COND_RESCHED_DEFAULT(step); + + if (entry >= collision_table_entries) + break; + + _ocf_init_collision_entry(cache, entry); + } + + return 0; +} + +static void ocf_metadata_init_finish(ocf_parallelize_t parallelize, + void *priv, int error) +{ + struct ocf_init_metadata_context *context = priv; + + ocf_pipeline_next(context->pipeline); + + ocf_parallelize_destroy(parallelize); +} + void ocf_metadata_init_collision(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { struct ocf_init_metadata_context *context = priv; + struct ocf_init_metadata_context *parallel_context; ocf_cache_t cache = context->cache; - unsigned int i; - unsigned int step = 0; + ocf_parallelize_t parallelize; + int result; if (context->skip_collision) OCF_PL_NEXT_RET(pipeline); - for (i = 0; i < cache->device->collision_table_entries; i++) { - _ocf_init_collision_entry(cache, i); - OCF_COND_RESCHED_DEFAULT(step); - } + result = ocf_parallelize_create(¶llelize, cache, + ocf_cache_get_queue_count(cache), sizeof(*context), + ocf_metadata_init_collision_handle, + ocf_metadata_init_finish); + if (result) + OCF_PL_FINISH_RET(pipeline, result); - ocf_pipeline_next(context->pipeline); + parallel_context = ocf_parallelize_get_priv(parallelize); + + parallel_context->pipeline = pipeline; + parallel_context->cache = cache; + + ocf_parallelize_run(parallelize); } /* From 8faf74169a2670cfcb3e2c25fa17bca500329e02 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Fri, 16 Sep 2022 13:41:29 +0200 Subject: [PATCH 09/11] Parallelize initializing hash table Signed-off-by: Michal Mielewczyk --- src/metadata/metadata.c | 52 +++++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/src/metadata/metadata.c b/src/metadata/metadata.c index eba81b0..96a9aeb 100644 --- a/src/metadata/metadata.c +++ b/src/metadata/metadata.c @@ -897,25 +897,53 @@ void ocf_metadata_init_collision(ocf_pipeline_t pipeline, void *priv, /* * Initialize hash table */ +static int ocf_metadata_init_hash_table_handle(ocf_parallelize_t parallelize, + void *priv, unsigned shard_id, unsigned shards_cnt) +{ + struct ocf_init_metadata_context *context = priv; + ocf_cache_t cache = context->cache; + uint32_t hash_table_entries = cache->device->hash_table_entries; + ocf_cache_line_t invalid_idx = cache->device->collision_table_entries; + uint32_t entry, portion, begin, end, step=0; + + portion = OCF_DIV_ROUND_UP((uint64_t)hash_table_entries, shards_cnt); + begin = portion*shard_id; + end = OCF_MIN(portion*(shard_id + 1), hash_table_entries); + + for (entry = begin; entry < end; entry++) { + OCF_COND_RESCHED_DEFAULT(step); + + if (entry >= hash_table_entries) + break; + + ocf_metadata_set_hash(cache, entry, invalid_idx); + } + + return 0; +} + void ocf_metadata_init_hash_table(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { struct ocf_init_metadata_context *context = priv; + struct ocf_init_metadata_context *parallel_context; ocf_cache_t cache = context->cache; - unsigned int i; - unsigned int hash_table_entries = cache->device->hash_table_entries; - ocf_cache_line_t invalid_idx = cache->device->collision_table_entries; + ocf_parallelize_t parallelize; + int result; - /* Init hash table */ - for (i = 0; i < hash_table_entries; i++) { - /* hash_table contains indexes from collision_table - * thus it shall be initialized in improper values - * from collision_table - **/ - ocf_metadata_set_hash(cache, i, invalid_idx); - } + result = ocf_parallelize_create(¶llelize, cache, + ocf_cache_get_queue_count(cache), sizeof(*context), + ocf_metadata_init_hash_table_handle, + ocf_metadata_init_finish); + if (result) + OCF_PL_FINISH_RET(pipeline, result); - ocf_pipeline_next(context->pipeline); + parallel_context = ocf_parallelize_get_priv(parallelize); + + parallel_context->pipeline = pipeline; + parallel_context->cache = cache; + + ocf_parallelize_run(parallelize); } /* From c0e99e1f7930093b14c9a7a3c969f94df66c520c Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Thu, 15 Sep 2022 09:48:02 +0200 Subject: [PATCH 10/11] cleaning: rename `recovery` to `populate` The function not only recovers cleaning policy metadata but is also utilized to initialize data structures so more generic name is actually more accurate Signed-off-by: Michal Mielewczyk --- src/cleaning/acp.c | 34 +++++++++++++++++----------------- src/cleaning/acp.h | 6 +++--- src/cleaning/alru.c | 32 ++++++++++++++++---------------- src/cleaning/alru.h | 6 +++--- src/cleaning/cleaning.h | 4 ++-- src/cleaning/cleaning_ops.h | 18 +++++++++--------- src/mngt/ocf_mngt_cache.c | 6 +++--- 7 files changed, 53 insertions(+), 53 deletions(-) diff --git a/src/cleaning/acp.c b/src/cleaning/acp.c index 07f6a76..efde479 100644 --- a/src/cleaning/acp.c +++ b/src/cleaning/acp.c @@ -334,9 +334,9 @@ int cleaning_policy_acp_initialize(ocf_cache_t cache, int init_metadata) return 0; } -#define OCF_ACP_RECOVERY_SHARDS_CNT 32 +#define OCF_ACP_POPULATE_SHARDS_CNT 32 -struct ocf_acp_recovery_context { +struct ocf_acp_populate_context { ocf_cache_t cache; struct { @@ -344,16 +344,16 @@ struct ocf_acp_recovery_context { struct { struct list_head chunk_list; } bucket[ACP_MAX_BUCKETS]; - } shard[OCF_ACP_RECOVERY_SHARDS_CNT]; + } shard[OCF_ACP_POPULATE_SHARDS_CNT]; - ocf_cleaning_recovery_end_t cmpl; + ocf_cleaning_populate_end_t cmpl; void *priv; }; -static int ocf_acp_recovery_handle(ocf_parallelize_t parallelize, +static int ocf_acp_populate_handle(ocf_parallelize_t parallelize, void *priv, unsigned shard_id, unsigned shards_cnt) { - struct ocf_acp_recovery_context *context = priv; + struct ocf_acp_populate_context *context = priv; ocf_cache_t cache = context->cache; ocf_cache_line_t entries = cache->device->collision_table_entries; ocf_cache_line_t cline, portion; @@ -390,7 +390,7 @@ static int ocf_acp_recovery_handle(ocf_parallelize_t parallelize, return 0; } -static void ocf_acp_recovery_chunk(struct ocf_acp_recovery_context *context, +static void ocf_acp_populate_chunk(struct ocf_acp_populate_context *context, struct acp_chunk_info *chunk) { ocf_cache_t cache = context->cache; @@ -400,7 +400,7 @@ static void ocf_acp_recovery_chunk(struct ocf_acp_recovery_context *context, uint8_t bucket_id; chunk->num_dirty = 0; - for (shard_id = 0; shard_id < OCF_ACP_RECOVERY_SHARDS_CNT; shard_id++) { + for (shard_id = 0; shard_id < OCF_ACP_POPULATE_SHARDS_CNT; shard_id++) { chunk->num_dirty += context->shard[shard_id] .chunk[chunk->core_id][chunk->chunk_id]; } @@ -417,10 +417,10 @@ static void ocf_acp_recovery_chunk(struct ocf_acp_recovery_context *context, list_move_tail(&chunk->list, &bucket->chunk_list); } -static void ocf_acp_recovery_finish(ocf_parallelize_t parallelize, +static void ocf_acp_populate_finish(ocf_parallelize_t parallelize, void *priv, int error) { - struct ocf_acp_recovery_context *context = priv; + struct ocf_acp_populate_context *context = priv; ocf_cache_t cache = context->cache; struct acp_context *acp = _acp_get_ctx_from_cache(cache); ocf_core_id_t core_id; @@ -435,7 +435,7 @@ static void ocf_acp_recovery_finish(ocf_parallelize_t parallelize, num_chunks = OCF_DIV_ROUND_UP(core_size, ACP_CHUNK_SIZE); for (chunk_id = 0; chunk_id < num_chunks; chunk_id++) { - ocf_acp_recovery_chunk(context, + ocf_acp_populate_chunk(context, &acp->chunk_info[core_id][chunk_id]); OCF_COND_RESCHED_DEFAULT(step); } @@ -455,14 +455,14 @@ static void ocf_acp_recovery_finish(ocf_parallelize_t parallelize, ocf_parallelize_destroy(parallelize); } -void cleaning_policy_acp_recovery(ocf_cache_t cache, - ocf_cleaning_recovery_end_t cmpl, void *priv) +void cleaning_policy_acp_populate(ocf_cache_t cache, + ocf_cleaning_populate_end_t cmpl, void *priv) { - struct ocf_acp_recovery_context *context; + struct ocf_acp_populate_context *context; ocf_parallelize_t parallelize; ocf_core_id_t core_id; ocf_core_t core; - unsigned shards_cnt = OCF_ACP_RECOVERY_SHARDS_CNT; + unsigned shards_cnt = OCF_ACP_POPULATE_SHARDS_CNT; unsigned shard_id; uint64_t core_size; uint64_t num_chunks; @@ -470,8 +470,8 @@ void cleaning_policy_acp_recovery(ocf_cache_t cache, int result; result = ocf_parallelize_create(¶llelize, cache, - OCF_ACP_RECOVERY_SHARDS_CNT, sizeof(*context), - ocf_acp_recovery_handle, ocf_acp_recovery_finish); + OCF_ACP_POPULATE_SHARDS_CNT, sizeof(*context), + ocf_acp_populate_handle, ocf_acp_populate_finish); if (result) { cmpl(priv, result); return; diff --git a/src/cleaning/acp.h b/src/cleaning/acp.h index 52e9fa1..62fb11e 100644 --- a/src/cleaning/acp.h +++ b/src/cleaning/acp.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ #ifndef __LAYER_CLEANING_POLICY_AGGRESSIVE_H__ @@ -12,8 +12,8 @@ void cleaning_policy_acp_setup(ocf_cache_t cache); int cleaning_policy_acp_initialize(ocf_cache_t cache, int init_metadata); -void cleaning_policy_acp_recovery(ocf_cache_t cache, - ocf_cleaning_recovery_end_t cmpl, void *priv); +void cleaning_policy_acp_populate(ocf_cache_t cache, + ocf_cleaning_populate_end_t cmpl, void *priv); void cleaning_policy_acp_deinitialize(ocf_cache_t cache); diff --git a/src/cleaning/alru.c b/src/cleaning/alru.c index a2ec098..9536b7e 100644 --- a/src/cleaning/alru.c +++ b/src/cleaning/alru.c @@ -445,22 +445,22 @@ int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata) return 0; } -#define OCF_ALRU_RECOVERY_SHARDS_CNT 32 +#define OCF_ALRU_POPULATE_SHARDS_CNT 32 -struct ocf_alru_recovery_context { +struct ocf_alru_populate_context { ocf_cache_t cache; struct { struct { ocf_cache_line_t head; ocf_cache_line_t tail; } part[OCF_USER_IO_CLASS_MAX]; - } shard[OCF_ALRU_RECOVERY_SHARDS_CNT] __attribute__((aligned(64))); + } shard[OCF_ALRU_POPULATE_SHARDS_CNT] __attribute__((aligned(64))); - ocf_cleaning_recovery_end_t cmpl; + ocf_cleaning_populate_end_t cmpl; void *priv; }; -static void add_alru_head_recovery(struct ocf_alru_recovery_context *context, +static void add_alru_head_populate(struct ocf_alru_populate_context *context, unsigned shard_id, ocf_core_id_t part_id, ocf_cache_line_t cline) { @@ -499,10 +499,10 @@ static void add_alru_head_recovery(struct ocf_alru_recovery_context *context, } } -static int ocf_alru_recovery_handle(ocf_parallelize_t parallelize, +static int ocf_alru_populate_handle(ocf_parallelize_t parallelize, void *priv, unsigned shard_id, unsigned shards_cnt) { - struct ocf_alru_recovery_context *context = priv; + struct ocf_alru_populate_context *context = priv; ocf_cache_t cache = context->cache; ocf_cache_line_t entries = cache->device->collision_table_entries; ocf_cache_line_t terminator = entries; @@ -537,7 +537,7 @@ static int ocf_alru_recovery_handle(ocf_parallelize_t parallelize, if (!metadata_test_dirty(cache, cline)) { cleaning_policy_alru_init_cache_block(cache, cline); } else { - add_alru_head_recovery(context, shard_id, + add_alru_head_populate(context, shard_id, part_id, cline); ++part_size[part_id]; } @@ -551,10 +551,10 @@ static int ocf_alru_recovery_handle(ocf_parallelize_t parallelize, return 0; } -static void ocf_alru_recovery_finish(ocf_parallelize_t parallelize, +static void ocf_alru_populate_finish(ocf_parallelize_t parallelize, void *priv, int error) { - struct ocf_alru_recovery_context *context = priv; + struct ocf_alru_populate_context *context = priv; ocf_cache_t cache = context->cache; ocf_part_id_t part_id; ocf_cache_line_t head, tail; @@ -564,7 +564,7 @@ static void ocf_alru_recovery_finish(ocf_parallelize_t parallelize, goto end; for (part_id = 0; part_id < OCF_USER_IO_CLASS_MAX; part_id++) { - for (shard = 0; shard < OCF_ALRU_RECOVERY_SHARDS_CNT; shard++) { + for (shard = 0; shard < OCF_ALRU_POPULATE_SHARDS_CNT; shard++) { head = context->shard[shard].part[part_id].head; tail = context->shard[shard].part[part_id].tail; @@ -580,10 +580,10 @@ end: ocf_parallelize_destroy(parallelize); } -void cleaning_policy_alru_recovery(ocf_cache_t cache, - ocf_cleaning_recovery_end_t cmpl, void *priv) +void cleaning_policy_alru_populate(ocf_cache_t cache, + ocf_cleaning_populate_end_t cmpl, void *priv) { - struct ocf_alru_recovery_context *context; + struct ocf_alru_populate_context *context; ocf_parallelize_t parallelize; struct alru_cleaning_policy *part_alru; struct ocf_user_part *user_part; @@ -591,8 +591,8 @@ void cleaning_policy_alru_recovery(ocf_cache_t cache, int result; result = ocf_parallelize_create(¶llelize, cache, - OCF_ALRU_RECOVERY_SHARDS_CNT, sizeof(*context), - ocf_alru_recovery_handle, ocf_alru_recovery_finish); + OCF_ALRU_POPULATE_SHARDS_CNT, sizeof(*context), + ocf_alru_populate_handle, ocf_alru_populate_finish); if (result) { cmpl(priv, result); return; diff --git a/src/cleaning/alru.h b/src/cleaning/alru.h index b72fa3a..49adf9e 100644 --- a/src/cleaning/alru.h +++ b/src/cleaning/alru.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ #ifndef __LAYER_CLEANING_POLICY_ALRU_H__ @@ -11,8 +11,8 @@ void cleaning_policy_alru_setup(ocf_cache_t cache); int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata); -void cleaning_policy_alru_recovery(ocf_cache_t cache, - ocf_cleaning_recovery_end_t cmpl, void *priv); +void cleaning_policy_alru_populate(ocf_cache_t cache, + ocf_cleaning_populate_end_t cmpl, void *priv); void cleaning_policy_alru_deinitialize(ocf_cache_t cache); void cleaning_policy_alru_init_cache_block(ocf_cache_t cache, uint32_t cache_line); diff --git a/src/cleaning/cleaning.h b/src/cleaning/cleaning.h index f85c002..99aa1f1 100644 --- a/src/cleaning/cleaning.h +++ b/src/cleaning/cleaning.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ @@ -54,6 +54,6 @@ void ocf_kick_cleaner(ocf_cache_t cache); void ocf_stop_cleaner(ocf_cache_t cache); -typedef void (*ocf_cleaning_recovery_end_t)(void *priv, int error); +typedef void (*ocf_cleaning_populate_end_t)(void *priv, int error); #endif diff --git a/src/cleaning/cleaning_ops.h b/src/cleaning/cleaning_ops.h index 134d324..6ba7bcf 100644 --- a/src/cleaning/cleaning_ops.h +++ b/src/cleaning/cleaning_ops.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ @@ -14,8 +14,8 @@ struct cleaning_policy_ops { void (*setup)(ocf_cache_t cache); int (*initialize)(ocf_cache_t cache, int init_metadata); - void (*recovery)(ocf_cache_t cache, - ocf_cleaning_recovery_end_t cmpl, void *priv); + void (*populate)(ocf_cache_t cache, + ocf_cleaning_populate_end_t cmpl, void *priv); void (*deinitialize)(ocf_cache_t cache); int (*add_core)(ocf_cache_t cache, ocf_core_id_t core_id); void (*remove_core)(ocf_cache_t cache, ocf_core_id_t core_id); @@ -45,7 +45,7 @@ static struct cleaning_policy_ops cleaning_policy_ops[ocf_cleaning_max] = { .purge_range = cleaning_policy_alru_purge_range, .set_hot_cache_line = cleaning_policy_alru_set_hot_cache_line, .initialize = cleaning_policy_alru_initialize, - .recovery = cleaning_policy_alru_recovery, + .populate = cleaning_policy_alru_populate, .deinitialize = cleaning_policy_alru_deinitialize, .set_cleaning_param = cleaning_policy_alru_set_cleaning_param, .get_cleaning_param = cleaning_policy_alru_get_cleaning_param, @@ -59,7 +59,7 @@ static struct cleaning_policy_ops cleaning_policy_ops[ocf_cleaning_max] = { .purge_range = cleaning_policy_acp_purge_range, .set_hot_cache_line = cleaning_policy_acp_set_hot_cache_line, .initialize = cleaning_policy_acp_initialize, - .recovery = cleaning_policy_acp_recovery, + .populate = cleaning_policy_acp_populate, .deinitialize = cleaning_policy_acp_deinitialize, .set_cleaning_param = cleaning_policy_acp_set_cleaning_param, .get_cleaning_param = cleaning_policy_acp_get_cleaning_param, @@ -91,18 +91,18 @@ static inline int ocf_cleaning_initialize(ocf_cache_t cache, return cleaning_policy_ops[policy].initialize(cache, init_metadata); } -static inline void ocf_cleaning_recovery(ocf_cache_t cache, +static inline void ocf_cleaning_populate(ocf_cache_t cache, ocf_cleaning_t policy, - ocf_cleaning_recovery_end_t cmpl, void *priv) + ocf_cleaning_populate_end_t cmpl, void *priv) { ENV_BUG_ON(policy >= ocf_cleaning_max); - if (unlikely(!cleaning_policy_ops[policy].recovery)) { + if (unlikely(!cleaning_policy_ops[policy].populate)) { cmpl(priv, 0); return; } - cleaning_policy_ops[policy].recovery(cache, cmpl, priv); + cleaning_policy_ops[policy].populate(cache, cmpl, priv); } static inline void ocf_cleaning_deinitialize(ocf_cache_t cache) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 5cb5bd6..2e8afd1 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -709,7 +709,7 @@ static void _ocf_mngt_load_rebuild_metadata(ocf_pipeline_t pipeline, ocf_pipeline_next(pipeline); } -static void _ocf_mngt_cleaning_recovery_complete(void *priv, int error) +static void _ocf_mngt_cleaning_populate_complete(void *priv, int error) { struct ocf_cache_attach_context *context = priv; @@ -729,8 +729,8 @@ static void _ocf_mngt_load_init_cleaning(ocf_pipeline_t pipeline, OCF_PL_NEXT_ON_SUCCESS_RET(pipeline, result); } - ocf_cleaning_recovery(cache, cache->cleaner.policy, - _ocf_mngt_cleaning_recovery_complete, context); + ocf_cleaning_populate(cache, cache->cleaner.policy, + _ocf_mngt_cleaning_populate_complete, context); } static void _ocf_mngt_init_metadata_complete(void *priv, int error) From 7b8093aa34439808b56e00ab636f9a09186b46ed Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Thu, 15 Sep 2022 13:04:25 +0200 Subject: [PATCH 11/11] Refactor cleaning policies initialization Don't populate cleaning policies during initialization procedure so the user has to call the latter explicitly. Until now cleaning policies could be populated in two ways: - implicitly during cleaning policy initialization, - explicitly be calling populate. The difference was that the former was single threaded. This patch removes the functionally redundant and less efficient code. Signed-off-by: Michal Mielewczyk --- src/cleaning/acp.c | 47 ++------------------------- src/cleaning/acp.h | 2 +- src/cleaning/alru.c | 63 ++----------------------------------- src/cleaning/alru.h | 2 +- src/cleaning/cleaning_ops.h | 4 +-- src/mngt/ocf_mngt_cache.c | 45 ++++++++++++++++++-------- src/mngt/ocf_mngt_flush.c | 33 ++++++++++++++----- 7 files changed, 67 insertions(+), 129 deletions(-) diff --git a/src/cleaning/acp.c b/src/cleaning/acp.c index efde479..841bf47 100644 --- a/src/cleaning/acp.c +++ b/src/cleaning/acp.c @@ -235,32 +235,6 @@ void cleaning_policy_acp_deinitialize(struct ocf_cache *cache) cache->cleaner.cleaning_policy_context = NULL; } -static void _acp_rebuild(struct ocf_cache *cache) -{ - ocf_cache_line_t cline; - ocf_core_id_t cline_core_id; - uint32_t step = 0; - - for (cline = 0; cline < cache->device->collision_table_entries; cline++) { - ocf_metadata_get_core_and_part_id(cache, cline, &cline_core_id, - NULL); - - OCF_COND_RESCHED_DEFAULT(step); - - if (cline_core_id == OCF_CORE_MAX) - continue; - - cleaning_policy_acp_init_cache_block(cache, cline); - - if (!metadata_test_dirty(cache, cline)) - continue; - - cleaning_policy_acp_set_hot_cache_line(cache, cline); - } - - ocf_cache_log(cache, log_info, "Finished rebuilding ACP metadata\n"); -} - void cleaning_policy_acp_setup(struct ocf_cache *cache) { struct acp_cleaning_policy_config *config; @@ -271,7 +245,7 @@ void cleaning_policy_acp_setup(struct ocf_cache *cache) config->flush_max_buffers = OCF_ACP_DEFAULT_FLUSH_MAX_BUFFERS; } -int cleaning_policy_acp_init_common(ocf_cache_t cache) +int cleaning_policy_acp_initialize(ocf_cache_t cache, int kick_cleaner) { struct acp_context *acp; int err, i; @@ -317,19 +291,8 @@ int cleaning_policy_acp_init_common(ocf_cache_t cache) } } - return 0; -} - -int cleaning_policy_acp_initialize(ocf_cache_t cache, int init_metadata) -{ - int result; - - result = cleaning_policy_acp_init_common(cache); - if (result) - return result; - - _acp_rebuild(cache); - ocf_kick_cleaner(cache); + if (kick_cleaner) + ocf_kick_cleaner(cache); return 0; } @@ -498,10 +461,6 @@ void cleaning_policy_acp_populate(ocf_cache_t cache, } } - result = cleaning_policy_acp_init_common(cache); - if (result) - goto err; - ocf_parallelize_run(parallelize); return; diff --git a/src/cleaning/acp.h b/src/cleaning/acp.h index 62fb11e..d260501 100644 --- a/src/cleaning/acp.h +++ b/src/cleaning/acp.h @@ -10,7 +10,7 @@ void cleaning_policy_acp_setup(ocf_cache_t cache); -int cleaning_policy_acp_initialize(ocf_cache_t cache, int init_metadata); +int cleaning_policy_acp_initialize(ocf_cache_t cache, int kick_cleaner); void cleaning_policy_acp_populate(ocf_cache_t cache, ocf_cleaning_populate_end_t cmpl, void *priv); diff --git a/src/cleaning/alru.c b/src/cleaning/alru.c index 9536b7e..d3065c5 100644 --- a/src/cleaning/alru.c +++ b/src/cleaning/alru.c @@ -349,42 +349,6 @@ void cleaning_policy_alru_set_hot_cache_line(struct ocf_cache *cache, env_spinlock_unlock(&ctx->list_lock[part_id]); } -static void _alru_rebuild(struct ocf_cache *cache) -{ - struct ocf_user_part *user_part; - struct alru_cleaning_policy *part_alru; - ocf_part_id_t part_id; - ocf_core_id_t core_id; - ocf_cache_line_t cline; - uint32_t step = 0; - - for_each_user_part(cache, user_part, part_id) { - /* ALRU initialization */ - part_alru = &user_part->clean_pol->policy.alru; - env_atomic_set(&part_alru->size, 0); - part_alru->lru_head = cache->device->collision_table_entries; - part_alru->lru_tail = cache->device->collision_table_entries; - cache->device->runtime_meta->cleaning_thread_access = 0; - } - - for (cline = 0; cline < cache->device->collision_table_entries; cline++) { - ocf_metadata_get_core_and_part_id(cache, cline, &core_id, - NULL); - - OCF_COND_RESCHED_DEFAULT(step); - - if (core_id == OCF_CORE_MAX) - continue; - - cleaning_policy_alru_init_cache_block(cache, cline); - - if (!metadata_test_dirty(cache, cline)) - continue; - - cleaning_policy_alru_set_hot_cache_line(cache, cline); - } -} - void cleaning_policy_alru_setup(struct ocf_cache *cache) { struct alru_cleaning_policy_config *config; @@ -397,7 +361,7 @@ void cleaning_policy_alru_setup(struct ocf_cache *cache) config->activity_threshold = OCF_ALRU_DEFAULT_ACTIVITY_THRESHOLD; } -int cleaning_policy_alru_init_common(ocf_cache_t cache) +int cleaning_policy_alru_initialize(ocf_cache_t cache, int kick_cleaner) { struct alru_context *ctx; int error = 0; @@ -426,21 +390,8 @@ int cleaning_policy_alru_init_common(ocf_cache_t cache) cache->cleaner.cleaning_policy_context = ctx; - return 0; -} - -int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata) -{ - int result; - - result = cleaning_policy_alru_init_common(cache); - if (result) - return result; - - if (init_metadata) - _alru_rebuild(cache); - - ocf_kick_cleaner(cache); + if (kick_cleaner) + ocf_kick_cleaner(cache); return 0; } @@ -598,14 +549,6 @@ void cleaning_policy_alru_populate(ocf_cache_t cache, return; } - - result = cleaning_policy_alru_init_common(cache); - if (result) { - ocf_parallelize_destroy(parallelize); - cmpl(priv, result); - return; - } - for_each_user_part(cache, user_part, part_id) { /* ALRU initialization */ part_alru = &user_part->clean_pol->policy.alru; diff --git a/src/cleaning/alru.h b/src/cleaning/alru.h index 49adf9e..8d8c6b5 100644 --- a/src/cleaning/alru.h +++ b/src/cleaning/alru.h @@ -10,7 +10,7 @@ #include "alru_structs.h" void cleaning_policy_alru_setup(ocf_cache_t cache); -int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata); +int cleaning_policy_alru_initialize(ocf_cache_t cache, int kick_cleaner); void cleaning_policy_alru_populate(ocf_cache_t cache, ocf_cleaning_populate_end_t cmpl, void *priv); void cleaning_policy_alru_deinitialize(ocf_cache_t cache); diff --git a/src/cleaning/cleaning_ops.h b/src/cleaning/cleaning_ops.h index 6ba7bcf..60493a8 100644 --- a/src/cleaning/cleaning_ops.h +++ b/src/cleaning/cleaning_ops.h @@ -81,14 +81,14 @@ static inline void ocf_cleaning_setup(ocf_cache_t cache, ocf_cleaning_t policy) } static inline int ocf_cleaning_initialize(ocf_cache_t cache, - ocf_cleaning_t policy, int init_metadata) + ocf_cleaning_t policy, int kick_cleaner) { ENV_BUG_ON(policy >= ocf_cleaning_max); if (unlikely(!cleaning_policy_ops[policy].initialize)) return 0; - return cleaning_policy_ops[policy].initialize(cache, init_metadata); + return cleaning_policy_ops[policy].initialize(cache, kick_cleaner); } static inline void ocf_cleaning_populate(ocf_cache_t cache, diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 2e8afd1..b249fdb 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -229,7 +229,7 @@ static ocf_error_t __init_cleaning_policy(ocf_cache_t cache) for (i = 0; i < ocf_cleaning_max; i++) ocf_cleaning_setup(cache, i); - return ocf_cleaning_initialize(cache, cache->cleaner.policy, 1); + return ocf_cleaning_initialize(cache, cache->cleaner.policy, false); } static void __deinit_cleaning_policy(ocf_cache_t cache) @@ -724,13 +724,19 @@ static void _ocf_mngt_load_init_cleaning(ocf_pipeline_t pipeline, ocf_error_t result; if (context->metadata.shutdown_status == ocf_metadata_clean_shutdown) { - result = ocf_cleaning_initialize(cache, - cache->cleaner.policy, 0); + /* Cleaning policy structures have been loaded so no need to populate + them for the second time */ + result = ocf_cleaning_initialize(cache, cache->cleaner.policy, true); OCF_PL_NEXT_ON_SUCCESS_RET(pipeline, result); - } - ocf_cleaning_populate(cache, cache->cleaner.policy, - _ocf_mngt_cleaning_populate_complete, context); + } else { + result = ocf_cleaning_initialize(cache, cache->cleaner.policy, false); + if (result) + OCF_PL_FINISH_RET(pipeline, result); + + ocf_cleaning_populate(cache, cache->cleaner.policy, + _ocf_mngt_cleaning_populate_complete, context); + } } static void _ocf_mngt_init_metadata_complete(void *priv, int error) @@ -1324,6 +1330,24 @@ static void _ocf_mngt_attach_populate_free(ocf_pipeline_t pipeline, context); } +static void _ocf_mngt_cleaning_populate_init_complete(void *priv, int error) +{ + struct ocf_cache_attach_context *context = priv; + ocf_cache_t cache = context->cache; + + if (error) + OCF_PL_FINISH_RET(context->pipeline, error); + + /* In initial cache state there is no dirty data, so all dirty data is + considered to be flushed + */ + cache->conf_meta->dirty_flushed = true; + + context->flags.cleaning_initialized = true; + + OCF_PL_NEXT_RET(context->pipeline); +} + static void _ocf_mngt_attach_init_services(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { @@ -1343,13 +1367,8 @@ static void _ocf_mngt_attach_init_services(ocf_pipeline_t pipeline, OCF_PL_FINISH_RET(pipeline, result); } - /* In initial cache state there is no dirty data, so all dirty data is - considered to be flushed - */ - cache->conf_meta->dirty_flushed = true; - context->flags.cleaning_initialized = true; - - ocf_pipeline_next(pipeline); + ocf_cleaning_populate(cache, cache->cleaner.policy, + _ocf_mngt_cleaning_populate_init_complete, context); } uint64_t _ocf_mngt_calculate_ram_needed(ocf_cache_line_size_t line_size, diff --git a/src/mngt/ocf_mngt_flush.c b/src/mngt/ocf_mngt_flush.c index 5702ffc..aa894bb 100644 --- a/src/mngt/ocf_mngt_flush.c +++ b/src/mngt/ocf_mngt_flush.c @@ -961,18 +961,16 @@ static void _ocf_mngt_deinit_clean_policy(ocf_pipeline_t pipeline, void *priv, _ocf_mngt_cleaning_deinit_complete, context); } -static void _ocf_mngt_init_clean_policy(ocf_pipeline_t pipeline, void *priv, - ocf_pipeline_arg_t arg) +static void _ocf_mngt_cleaning_init_complete(void *priv, int error) { - int result; struct ocf_mngt_cache_set_cleaning_context *context = priv; + ocf_pipeline_t pipeline = context->pipeline; ocf_cache_t cache = context->cache; ocf_cleaning_t old_policy = context->old_policy; ocf_cleaning_t new_policy = context->new_policy; ocf_cleaning_t emergency_policy = ocf_cleaning_nop; - result = ocf_cleaning_initialize(cache, new_policy, 1); - if (result) { + if (error) { ocf_cache_log(cache, log_info, "Failed to initialize %s cleaning " "policy. Setting %s instead\n", ocf_cleaning_get_name(new_policy), @@ -986,16 +984,35 @@ static void _ocf_mngt_init_clean_policy(ocf_pipeline_t pipeline, void *priv, __set_cleaning_policy(cache, new_policy); - ocf_refcnt_unfreeze(&cache->cleaner.refcnt); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + OCF_PL_NEXT_ON_SUCCESS_RET(pipeline, error); - OCF_PL_NEXT_ON_SUCCESS_RET(pipeline, result); +} + +static void _ocf_mngt_init_clean_policy(ocf_pipeline_t pipeline, void *priv, + ocf_pipeline_arg_t arg) +{ + int result; + struct ocf_mngt_cache_set_cleaning_context *context = priv; + ocf_cache_t cache = context->cache; + ocf_cleaning_t new_policy = context->new_policy; + + result = ocf_cleaning_initialize(cache, new_policy, false); + if (result) { + _ocf_mngt_cleaning_init_complete(context, result); + } else { + ocf_cleaning_populate(cache, new_policy, + _ocf_mngt_cleaning_init_complete, context); + } } static void _ocf_mngt_set_cleaning_finish(ocf_pipeline_t pipeline, void *priv, int error) { struct ocf_mngt_cache_set_cleaning_context *context = priv; + ocf_cache_t cache = context->cache; + + ocf_refcnt_unfreeze(&cache->cleaner.refcnt); + ocf_metadata_end_exclusive_access(&cache->metadata.lock); context->cmpl(context->priv, error);