From 348b0f9ab889b41f57d0ba4e5cac219198fd387a Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Thu, 11 Apr 2019 13:16:16 -0400 Subject: [PATCH] Async wait for cleaner completion Signed-off-by: Adam Rutkowski --- inc/ocf_cache.h | 7 -- src/eviction/lru.c | 52 ++++---- src/mngt/ocf_mngt_core.c | 250 +++++++++++++++++++++++++------------- src/ocf_cache.c | 28 ----- src/ocf_cache_priv.h | 3 +- src/utils/utils_cleaner.c | 47 +++++++ src/utils/utils_cleaner.h | 38 ++++++ 7 files changed, 281 insertions(+), 144 deletions(-) diff --git a/inc/ocf_cache.h b/inc/ocf_cache.h index efd6ea1..d2964f8 100644 --- a/inc/ocf_cache.h +++ b/inc/ocf_cache.h @@ -160,13 +160,6 @@ bool ocf_cache_is_device_attached(ocf_cache_t cache); */ bool ocf_cache_is_running(ocf_cache_t cache); -/** - * @brief Check if cleaning triggered by eviction runs on the cache - * - * @param[in] cache Cache object - */ -bool ocf_cache_has_pending_cleaning(ocf_cache_t cache); - /** * @brief Get cache mode of given cache object * diff --git a/src/eviction/lru.c b/src/eviction/lru.c index 604e1fc..3b1c21e 100644 --- a/src/eviction/lru.c +++ b/src/eviction/lru.c @@ -273,9 +273,9 @@ void evp_lru_rm_cline(ocf_cache_t cache, ocf_cache_line_t cline) static void evp_lru_clean_end(void *private_data, int error) { - env_atomic *cleaning_in_progress = private_data; + struct ocf_refcnt *counter = private_data; - env_atomic_set(cleaning_in_progress, 0); + ocf_refcnt_dec(counter); } static int evp_lru_clean_getter(ocf_cache_t cache, @@ -309,32 +309,40 @@ static int evp_lru_clean_getter(ocf_cache_t cache, static void evp_lru_clean(ocf_cache_t cache, ocf_queue_t io_queue, ocf_part_id_t part_id, uint32_t count) { - env_atomic *progress = &cache->cleaning[part_id]; + struct ocf_refcnt *counter = &cache->refcnt.cleaning[part_id]; struct ocf_user_part *part = &cache->user_parts[part_id]; + struct ocf_cleaner_attribs attribs = { + .cache_line_lock = true, + .do_sort = true, + + .cmpl_context = counter, + .cmpl_fn = evp_lru_clean_end, + + .getter = evp_lru_clean_getter, + .getter_context = &attribs, + .getter_item = part->runtime->eviction.policy.lru.dirty_tail, + + .count = count > 32 ? 32 : count, + + .io_queue = io_queue + }; + int cnt; if (ocf_mngt_is_cache_locked(cache)) return; - if (env_atomic_cmpxchg(progress, 0, 1) == 0) { - /* Initialize attributes for cleaner */ - struct ocf_cleaner_attribs attribs = { - .cache_line_lock = true, - .do_sort = true, - - .cmpl_context = progress, - .cmpl_fn = evp_lru_clean_end, - - .getter = evp_lru_clean_getter, - .getter_context = &attribs, - .getter_item = part->runtime->eviction.policy.lru.dirty_tail, - - .count = count > 32 ? 32 : count, - - .io_queue = io_queue - }; - - ocf_cleaner_fire(cache, &attribs); + cnt = ocf_refcnt_inc(counter); + if (!cnt) { + /* cleaner disabled by mngmt operation */ + return; } + if (cnt > 1) { + /* cleaning already running for this partition */ + ocf_refcnt_dec(counter); + return; + } + + ocf_cleaner_fire(cache, &attribs); } static void evp_lru_zero_line_complete(struct ocf_request *ocf_req, int error) diff --git a/src/mngt/ocf_mngt_core.c b/src/mngt/ocf_mngt_core.c index aef375a..d3b9e8b 100644 --- a/src/mngt/ocf_mngt_core.c +++ b/src/mngt/ocf_mngt_core.c @@ -560,32 +560,14 @@ err_pipeline: OCF_CMPL_RET(cache, NULL, priv, result); } -/* - * Synchronously wait until cleaning triggered by eviction finishes. - * TODO: Replace it with asynchronous mechanism. - */ -static int _ocf_cleaning_wait_for_finish(ocf_cache_t cache, int32_t timeout_ms) -{ - if (!ocf_cache_is_device_attached(cache)) - return 0; - - while (ocf_cache_has_pending_cleaning(cache)) { - env_msleep(20); - - timeout_ms -= 20; - if (timeout_ms <= 0) - return -EBUSY; - } - - return 0; -} - struct ocf_mngt_cache_remove_core_context { ocf_mngt_cache_remove_core_end_t cmpl; void *priv; ocf_pipeline_t pipeline; ocf_cache_t cache; + ocf_core_t core; const char *core_name; + struct ocf_cleaner_wait_context cleaner_wait; }; static void ocf_mngt_cache_remove_core_finish(ocf_pipeline_t pipeline, @@ -602,19 +584,13 @@ static void ocf_mngt_cache_remove_core_finish(ocf_pipeline_t pipeline, context->core_name); } + ocf_cleaner_refcnt_unfreeze(cache); + context->cmpl(context->priv, error); ocf_pipeline_destroy(context->pipeline); } -struct ocf_pipeline_properties ocf_mngt_cache_remove_core_pipeline_props = { - .priv_size = sizeof(struct ocf_mngt_cache_remove_core_context), - .finish = ocf_mngt_cache_remove_core_finish, - .steps = { - OCF_PL_STEP_TERMINATOR(), - }, -}; - static void ocf_mngt_cache_remove_core_flush_sb_complete(void *priv, int error) { struct ocf_mngt_cache_remove_core_context *context = priv; @@ -623,39 +599,13 @@ static void ocf_mngt_cache_remove_core_flush_sb_complete(void *priv, int error) error ? -OCF_ERR_WRITE_CACHE : 0); } -void ocf_mngt_cache_remove_core(ocf_core_t core, - ocf_mngt_cache_remove_core_end_t cmpl, void *priv) +static void _ocf_mngt_cache_remove_core(ocf_pipeline_t pipeline, void *priv, + ocf_pipeline_arg_t arg) { - struct ocf_mngt_cache_remove_core_context *context; - ocf_pipeline_t pipeline; - ocf_cache_t cache; - ocf_core_id_t core_id; - int result; - - OCF_CHECK_NULL(core); - - cache = ocf_core_get_cache(core); - core_id = ocf_core_get_id(core); - - if (!cache->mngt_queue) - OCF_CMPL_RET(cache, -OCF_ERR_INVAL); - - /* TODO: Make this asynchronous */ - if (_ocf_cleaning_wait_for_finish(cache, 60 * 1000)) - OCF_CMPL_RET(priv, -OCF_ERR_CACHE_IN_USE); - - result = ocf_pipeline_create(&pipeline, cache, - &ocf_mngt_cache_remove_core_pipeline_props); - if (result) - OCF_CMPL_RET(priv, result); - - context = ocf_pipeline_get_priv(pipeline); - - context->cmpl = cmpl; - context->priv = priv; - context->pipeline = pipeline; - context->cache = cache; - context->core_name = ocf_core_get_name(core); + struct ocf_mngt_cache_remove_core_context *context = priv; + ocf_cache_t cache = context->cache; + ocf_core_t core = context->core; + ocf_core_id_t core_id = ocf_core_get_id(core); ocf_core_log(core, log_debug, "Removing core\n"); @@ -675,53 +625,183 @@ void ocf_mngt_cache_remove_core(ocf_core_t core, ocf_mngt_cache_remove_core_flush_sb_complete, context); } -static int _ocf_mngt_cache_detach_core(ocf_core_t core) +static void ocf_mngt_cache_remove_core_wait_cleaning_complete(void *priv) { - struct ocf_cache *cache = core->volume.cache; - ocf_core_id_t core_id = ocf_core_get_id(core); - int status; - - status = cache_mng_core_close(cache, core_id); - if (!status) { - cache->ocf_core_inactive_count++; - env_bit_set(ocf_cache_state_incomplete, - &cache->cache_state); - } - - return status; + ocf_pipeline_t pipeline = priv; + ocf_pipeline_next(pipeline); } -void ocf_mngt_cache_detach_core(ocf_core_t core, - ocf_mngt_cache_detach_core_end_t cmpl, void *priv) +static void ocf_mngt_cache_remove_core_wait_cleaning(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) { + struct ocf_mngt_cache_remove_core_context *context = priv; + ocf_cache_t cache = context->cache; + + if (!ocf_cache_is_device_attached(cache)) + OCF_PL_NEXT_RET(pipeline); + + ocf_cleaner_refcnt_freeze(cache); + ocf_cleaner_refcnt_register_zero_cb(cache, &context->cleaner_wait, + ocf_mngt_cache_remove_core_wait_cleaning_complete, + pipeline); +} + +struct ocf_pipeline_properties ocf_mngt_cache_remove_core_pipeline_props = { + .priv_size = sizeof(struct ocf_mngt_cache_remove_core_context), + .finish = ocf_mngt_cache_remove_core_finish, + .steps = { + OCF_PL_STEP(ocf_mngt_cache_remove_core_wait_cleaning), + OCF_PL_STEP(_ocf_mngt_cache_remove_core), + OCF_PL_STEP_TERMINATOR(), + }, +}; + +void ocf_mngt_cache_remove_core(ocf_core_t core, + ocf_mngt_cache_remove_core_end_t cmpl, void *priv) +{ + struct ocf_mngt_cache_remove_core_context *context; + ocf_pipeline_t pipeline; ocf_cache_t cache; - const char *core_name; int result; OCF_CHECK_NULL(core); cache = ocf_core_get_cache(core); - core_name = ocf_core_get_name(core); if (!cache->mngt_queue) OCF_CMPL_RET(cache, -OCF_ERR_INVAL); - /* TODO: Make this asynchronous */ - if (_ocf_cleaning_wait_for_finish(cache, 60 * 1000)) - OCF_CMPL_RET(priv, -OCF_ERR_CACHE_IN_USE); + result = ocf_pipeline_create(&pipeline, cache, + &ocf_mngt_cache_remove_core_pipeline_props); + if (result) + OCF_CMPL_RET(priv, result); + + context = ocf_pipeline_get_priv(pipeline); + + context->cmpl = cmpl; + context->priv = priv; + context->pipeline = pipeline; + context->cache = cache; + context->core = core; + context->core_name = ocf_core_get_name(core); + + ocf_pipeline_next(pipeline); +} + +struct ocf_mngt_cache_detach_core_context { + ocf_mngt_cache_detach_core_end_t cmpl; + void *priv; + ocf_pipeline_t pipeline; + ocf_cache_t cache; + ocf_core_t core; + const char *core_name; + struct ocf_cleaner_wait_context cleaner_wait; +}; + +static void _ocf_mngt_cache_detach_core(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) +{ + struct ocf_mngt_cache_remove_core_context *context = priv; + ocf_cache_t cache = context->cache; + ocf_core_t core = context->core; + ocf_core_id_t core_id = ocf_core_get_id(core); + int status; ocf_core_log(core, log_debug, "Detaching core\n"); - result = _ocf_mngt_cache_detach_core(core); - if (!result) { + status = cache_mng_core_close(cache, core_id); + + if (status) + OCF_PL_FINISH_RET(pipeline, status); + + cache->ocf_core_inactive_count++; + env_bit_set(ocf_cache_state_incomplete, + &cache->cache_state); + ocf_pipeline_next(pipeline); +} + +static void ocf_mngt_cache_detach_core_finish(ocf_pipeline_t pipeline, + void *priv, int error) +{ + struct ocf_mngt_cache_remove_core_context *context = priv; + ocf_cache_t cache = context->cache; + + if (!error) { ocf_cache_log(cache, log_info, "Core %s successfully detached\n", - core_name); + context->core_name); } else { ocf_cache_log(cache, log_err, "Detaching core %s failed\n", - core_name); + context->core_name); } - OCF_CMPL_RET(priv, result); + ocf_cleaner_refcnt_unfreeze(context->cache); + + context->cmpl(context->priv, error); + + ocf_pipeline_destroy(context->pipeline); +} + +static void ocf_mngt_cache_detach_core_wait_cleaning_complete(void *priv) +{ + ocf_pipeline_t pipeline = priv; + ocf_pipeline_next(pipeline); +} + +static void ocf_mngt_cache_detach_core_wait_cleaning(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) +{ + struct ocf_mngt_cache_remove_core_context *context = priv; + ocf_cache_t cache = context->cache; + + if (!ocf_cache_is_device_attached(cache)) + OCF_PL_NEXT_RET(pipeline); + + ocf_cleaner_refcnt_freeze(cache); + ocf_cleaner_refcnt_register_zero_cb(cache, &context->cleaner_wait, + ocf_mngt_cache_detach_core_wait_cleaning_complete, + pipeline); +} + +struct ocf_pipeline_properties ocf_mngt_cache_detach_core_pipeline_props = { + .priv_size = sizeof(struct ocf_mngt_cache_detach_core_context), + .finish = ocf_mngt_cache_detach_core_finish, + .steps = { + OCF_PL_STEP(ocf_mngt_cache_detach_core_wait_cleaning), + OCF_PL_STEP(_ocf_mngt_cache_detach_core), + OCF_PL_STEP_TERMINATOR(), + }, +}; + +void ocf_mngt_cache_detach_core(ocf_core_t core, + ocf_mngt_cache_detach_core_end_t cmpl, void *priv) +{ + struct ocf_mngt_cache_detach_core_context *context; + ocf_pipeline_t pipeline; + ocf_cache_t cache; + int result; + + OCF_CHECK_NULL(core); + + cache = ocf_core_get_cache(core); + + if (!cache->mngt_queue) + OCF_CMPL_RET(cache, -OCF_ERR_INVAL); + + result = ocf_pipeline_create(&pipeline, cache, + &ocf_mngt_cache_detach_core_pipeline_props); + if (result) + OCF_CMPL_RET(priv, result); + + context = ocf_pipeline_get_priv(pipeline); + + context->cmpl = cmpl; + context->priv = priv; + context->pipeline = pipeline; + context->cache = ocf_core_get_cache(core); + context->core = core; + context->core_name = ocf_core_get_name(core); + + ocf_pipeline_next(pipeline); } int ocf_mngt_core_set_uuid(ocf_core_t core, const struct ocf_volume_uuid *uuid) diff --git a/src/ocf_cache.c b/src/ocf_cache.c index eadf234..0970f75 100644 --- a/src/ocf_cache.c +++ b/src/ocf_cache.c @@ -53,34 +53,6 @@ bool ocf_cache_is_device_attached(ocf_cache_t cache) return !ocf_refcnt_frozen(&cache->refcnt.metadata); } -/* - * This is temporary workaround allowing to check if cleaning triggered - * by eviction policy is running on the cache. This information is needed - * to remove core from cache properly. - * - * TODO: Replace this with asynchronous notification to which remove/detach - * core pipelines can subscribe. - */ -bool ocf_cache_has_pending_cleaning(ocf_cache_t cache) -{ - struct ocf_user_part *curr_part; - ocf_part_id_t part_id; - bool cleaning_active = false; - - OCF_CHECK_NULL(cache); - - OCF_METADATA_LOCK_RD(); - for_each_part(cache, curr_part, part_id) { - if (env_atomic_read(&cache->cleaning[part_id])) { - cleaning_active = true; - break; - } - } - OCF_METADATA_UNLOCK_RD(); - - return cleaning_active; -} - ocf_cache_mode_t ocf_cache_get_mode(ocf_cache_t cache) { OCF_CHECK_NULL(cache); diff --git a/src/ocf_cache_priv.h b/src/ocf_cache_priv.h index 609c73b..aa0b824 100644 --- a/src/ocf_cache_priv.h +++ b/src/ocf_cache_priv.h @@ -171,6 +171,7 @@ struct ocf_cache { struct { struct ocf_refcnt dirty; struct ocf_refcnt metadata; + struct ocf_refcnt cleaning[OCF_IO_CLASS_MAX]; } refcnt; uint32_t fallback_pt_error_threshold; @@ -193,8 +194,6 @@ struct ocf_cache { env_atomic flush_in_progress; - env_atomic cleaning[OCF_IO_CLASS_MAX]; - struct ocf_cleaner cleaner; struct ocf_metadata_updater metadata_updater; diff --git a/src/utils/utils_cleaner.c b/src/utils/utils_cleaner.c index f221a4c..14384da 100644 --- a/src/utils/utils_cleaner.c +++ b/src/utils/utils_cleaner.c @@ -8,6 +8,7 @@ #include "../engine/engine_common.h" #include "../concurrency/ocf_concurrency.h" #include "utils_cleaner.h" +#include "utils_part.h" #include "utils_req.h" #include "utils_io.h" #include "utils_cache_line.h" @@ -1014,3 +1015,49 @@ void ocf_cleaner_sort_flush_containers(struct flush_container *fctbl, _ocf_cleaner_swap); } } + +void ocf_cleaner_refcnt_freeze(ocf_cache_t cache) +{ + struct ocf_user_part *curr_part; + ocf_part_id_t part_id; + + for_each_part(cache, curr_part, part_id) + ocf_refcnt_freeze(&cache->refcnt.cleaning[part_id]); +} + +void ocf_cleaner_refcnt_unfreeze(ocf_cache_t cache) +{ + struct ocf_user_part *curr_part; + ocf_part_id_t part_id; + + for_each_part(cache, curr_part, part_id) + ocf_refcnt_unfreeze(&cache->refcnt.cleaning[part_id]); +} + +static void ocf_cleaner_refcnt_register_zero_cb_finish(void *priv) +{ + struct ocf_cleaner_wait_context *ctx = priv; + + if (!env_atomic_dec_return(&ctx->waiting)) + ctx->cb(ctx->priv); +} + +void ocf_cleaner_refcnt_register_zero_cb(ocf_cache_t cache, + struct ocf_cleaner_wait_context *ctx, + ocf_cleaner_refcnt_zero_cb_t cb, void *priv) +{ + struct ocf_user_part *curr_part; + ocf_part_id_t part_id; + + env_atomic_set(&ctx->waiting, 1); + ctx->cb = cb; + ctx->priv = priv; + + for_each_part(cache, curr_part, part_id) { + env_atomic_inc(&ctx->waiting); + ocf_refcnt_register_zero_cb(&cache->refcnt.cleaning[part_id], + ocf_cleaner_refcnt_register_zero_cb_finish, ctx); + } + + ocf_cleaner_refcnt_register_zero_cb_finish(ctx); +} diff --git a/src/utils/utils_cleaner.h b/src/utils/utils_cleaner.h index 5773779..80d19a9 100644 --- a/src/utils/utils_cleaner.h +++ b/src/utils/utils_cleaner.h @@ -79,6 +79,18 @@ struct flush_container { struct ocf_mngt_cache_flush_context *context; }; +typedef void (*ocf_cleaner_refcnt_zero_cb_t)(void *priv); + +/** + * @brief Context for ocf_cleaner_refcnt_register_zero_cb + */ +struct ocf_cleaner_wait_context +{ + env_atomic waiting; + ocf_cleaner_refcnt_zero_cb_t cb; + void *priv; +}; + /** * @brief Run cleaning procedure * @@ -119,4 +131,30 @@ void ocf_cleaner_sort_sectors(struct flush_data *tbl, uint32_t num); void ocf_cleaner_sort_flush_containers(struct flush_container *fctbl, uint32_t num); +/** + * @brief Disable incrementing of cleaner reference counters + * + * @param cache - Cache instance + */ +void ocf_cleaner_refcnt_freeze(ocf_cache_t cache); + +/** + * @brief Enable incrementing of cleaner reference counters + * + * @param cache - Cache instance + */ +void ocf_cleaner_refcnt_unfreeze(ocf_cache_t cache); + +/** + * @brief Register callback for cleaner reference counters dropping to 0 + * + * @param cache - Cache instance + * @param ctx - Routine private context, allocated by caller to avoid ENOMEM + * @param cb - Caller callback + * @param priv - Caller callback private data + */ +void ocf_cleaner_refcnt_register_zero_cb(ocf_cache_t cache, + struct ocf_cleaner_wait_context *ctx, + ocf_cleaner_refcnt_zero_cb_t cb, void *priv); + #endif /* UTILS_CLEANER_H_ */