diff --git a/src/utils/utils_refcnt.c b/env/posix/ocf_env_refcnt.c similarity index 63% rename from src/utils/utils_refcnt.c rename to env/posix/ocf_env_refcnt.c index 54d0379..69045e6 100644 --- a/src/utils/utils_refcnt.c +++ b/env/posix/ocf_env_refcnt.c @@ -4,49 +4,54 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#include "../utils/utils_refcnt.h" +#include "ocf_env_refcnt.h" -void ocf_refcnt_init(struct ocf_refcnt *rc) +int env_refcnt_init(struct env_refcnt *rc, const char *name, size_t name_len) { env_atomic_set(&rc->counter, 0); env_atomic_set(&rc->freeze, 0); env_atomic_set(&rc->callback, 0); rc->cb = NULL; + + return 0; } -int ocf_refcnt_dec(struct ocf_refcnt *rc) +void env_refcnt_deinit(struct env_refcnt *rc) +{ + +} + +void env_refcnt_dec(struct env_refcnt *rc) { int val = env_atomic_dec_return(&rc->counter); ENV_BUG_ON(val < 0); if (!val && env_atomic_cmpxchg(&rc->callback, 1, 0)) rc->cb(rc->priv); - - return val; } -int ocf_refcnt_inc(struct ocf_refcnt *rc) +bool env_refcnt_inc(struct env_refcnt *rc) { int val; if (!env_atomic_read(&rc->freeze)) { val = env_atomic_inc_return(&rc->counter); if (!env_atomic_read(&rc->freeze)) - return val; + return !!val; else - ocf_refcnt_dec(rc); + env_refcnt_dec(rc); } return 0; } -void ocf_refcnt_freeze(struct ocf_refcnt *rc) +void env_refcnt_freeze(struct env_refcnt *rc) { env_atomic_inc(&rc->freeze); } -void ocf_refcnt_register_zero_cb(struct ocf_refcnt *rc, ocf_refcnt_cb_t cb, +void env_refcnt_register_zero_cb(struct env_refcnt *rc, env_refcnt_cb_t cb, void *priv) { ENV_BUG_ON(!env_atomic_read(&rc->freeze)); @@ -56,21 +61,21 @@ void ocf_refcnt_register_zero_cb(struct ocf_refcnt *rc, ocf_refcnt_cb_t cb, rc->cb = cb; rc->priv = priv; env_atomic_set(&rc->callback, 1); - ocf_refcnt_dec(rc); + env_refcnt_dec(rc); } -void ocf_refcnt_unfreeze(struct ocf_refcnt *rc) +void env_refcnt_unfreeze(struct env_refcnt *rc) { int val = env_atomic_dec_return(&rc->freeze); ENV_BUG_ON(val < 0); } -bool ocf_refcnt_frozen(struct ocf_refcnt *rc) +bool env_refcnt_frozen(struct env_refcnt *rc) { return !!env_atomic_read(&rc->freeze); } -bool ocf_refcnt_zeroed(struct ocf_refcnt *rc) +bool env_refcnt_zeroed(struct env_refcnt *rc) { return (env_atomic_read(&rc->counter) == 0); } diff --git a/env/posix/ocf_env_refcnt.h b/env/posix/ocf_env_refcnt.h new file mode 100644 index 0000000..0395ed6 --- /dev/null +++ b/env/posix/ocf_env_refcnt.h @@ -0,0 +1,54 @@ +/* + * Copyright(c) 2019-2021 Intel Corporation + * Copyright(c) 2024-2025 Huawei Technologies + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef __OCF_ENV_REFCNT_H__ +#define __OCF_ENV_REFCNT_H__ + +#include "ocf_env.h" + +typedef void (*env_refcnt_cb_t)(void *priv); + +struct env_refcnt +{ + env_atomic counter __attribute__((aligned(64))); + env_atomic freeze; + env_atomic callback; + env_refcnt_cb_t cb; + void *priv; +}; + +/* Initialize reference counter */ +int env_refcnt_init(struct env_refcnt *rc, const char *name, size_t name_len); + +void env_refcnt_deinit(struct env_refcnt *rc); + +/* Try to increment counter. Returns true if successfull, false + * if counter is frozen */ +bool env_refcnt_inc(struct env_refcnt *rc); + +/* Decrement reference counter */ +void env_refcnt_dec(struct env_refcnt *rc); + +/* Disallow incrementing of underlying counter - attempts to increment counter + * will be failing until env_refcnt_unfreeze is calleed. + * It's ok to call freeze multiple times, in which case counter is frozen + * until all freeze calls are offset by a corresponding unfreeze.*/ +void env_refcnt_freeze(struct env_refcnt *rc); + +/* Cancel the effect of single env_refcnt_freeze call */ +void env_refcnt_unfreeze(struct env_refcnt *rc); + +bool env_refcnt_frozen(struct env_refcnt *rc); + +bool env_refcnt_zeroed(struct env_refcnt *rc); + +/* Register callback to be called when reference counter drops to 0. + * Must be called after counter is frozen. + * Cannot be called until previously regsitered callback had fired. */ +void env_refcnt_register_zero_cb(struct env_refcnt *rc, env_refcnt_cb_t cb, + void *priv); + +#endif // __OCF_ENV_REFCNT_H__ diff --git a/src/cleaning/acp_structs.h b/src/cleaning/acp_structs.h index b4d2154..3467481 100644 --- a/src/cleaning/acp_structs.h +++ b/src/cleaning/acp_structs.h @@ -1,11 +1,12 @@ /* * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2023-2025 Huawei Technologies Co., Ltd. * SPDX-License-Identifier: BSD-3-Clause */ #ifndef __CLEANING_AGGRESSIVE_STRUCTS_H__ #define __CLEANING_AGGRESSIVE_STRUCTS_H__ -#include "../utils/utils_cleaner.h" +#include "ocf_env_headers.h" /* TODO: remove acp metadata */ struct acp_cleaning_policy_meta { diff --git a/src/cleaning/cleaning.h b/src/cleaning/cleaning.h index 99aa1f1..2764639 100644 --- a/src/cleaning/cleaning.h +++ b/src/cleaning/cleaning.h @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2023-2025 Huawei Technologies Co., Ltd. * SPDX-License-Identifier: BSD-3-Clause */ @@ -9,8 +10,8 @@ #include "alru_structs.h" #include "nop_structs.h" #include "acp_structs.h" +#include "ocf_env_refcnt.h" #include "ocf/ocf_cleaner.h" -#include "../utils/utils_refcnt.h" #define CLEANING_POLICY_CONFIG_BYTES 256 #define CLEANING_POLICY_TYPE_MAX 4 @@ -40,7 +41,7 @@ struct cleaning_policy_meta { }; struct ocf_cleaner { - struct ocf_refcnt refcnt __attribute__((aligned(64))); + struct env_refcnt refcnt; ocf_cleaning_t policy; void *cleaning_policy_context; ocf_queue_t io_queue; diff --git a/src/cleaning/cleaning_ops.h b/src/cleaning/cleaning_ops.h index 60493a8..9153da9 100644 --- a/src/cleaning/cleaning_ops.h +++ b/src/cleaning/cleaning_ops.h @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2023-2025 Huawei Technologies Co., Ltd. * SPDX-License-Identifier: BSD-3-Clause */ @@ -9,7 +10,7 @@ #include "../metadata/metadata_superblock.h" #include "../metadata/metadata_structs.h" #include "../ocf_cache_priv.h" -#include "../utils/utils_refcnt.h" +#include "ocf_env_refcnt.h" struct cleaning_policy_ops { void (*setup)(ocf_cache_t cache); @@ -125,7 +126,7 @@ static inline int ocf_cleaning_add_core(ocf_cache_t cache, ocf_cleaning_t policy; int result = 0; - if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt))) + if (unlikely(!env_refcnt_inc(&cache->cleaner.refcnt))) return -OCF_ERR_NO_LOCK; policy = cache->cleaner.policy; @@ -138,7 +139,7 @@ static inline int ocf_cleaning_add_core(ocf_cache_t cache, result = cleaning_policy_ops[policy].add_core(cache, core_id); unlock: - ocf_refcnt_dec(&cache->cleaner.refcnt); + env_refcnt_dec(&cache->cleaner.refcnt); return result; } @@ -148,7 +149,7 @@ static inline void ocf_cleaning_remove_core(ocf_cache_t cache, { ocf_cleaning_t policy; - if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt))) + if (unlikely(!env_refcnt_inc(&cache->cleaner.refcnt))) return; policy = cache->cleaner.policy; @@ -161,7 +162,7 @@ static inline void ocf_cleaning_remove_core(ocf_cache_t cache, cleaning_policy_ops[policy].remove_core(cache, core_id); unlock: - ocf_refcnt_dec(&cache->cleaner.refcnt); + env_refcnt_dec(&cache->cleaner.refcnt); } static inline void ocf_cleaning_init_cache_block(ocf_cache_t cache, @@ -169,7 +170,7 @@ static inline void ocf_cleaning_init_cache_block(ocf_cache_t cache, { ocf_cleaning_t policy; - if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt))) + if (unlikely(!env_refcnt_inc(&cache->cleaner.refcnt))) return; policy = cache->cleaner.policy; @@ -181,7 +182,7 @@ static inline void ocf_cleaning_init_cache_block(ocf_cache_t cache, cleaning_policy_ops[policy].init_cache_block(cache, cache_line); unlock: - ocf_refcnt_dec(&cache->cleaner.refcnt); + env_refcnt_dec(&cache->cleaner.refcnt); } static inline void ocf_cleaning_purge_cache_block(ocf_cache_t cache, @@ -189,7 +190,7 @@ static inline void ocf_cleaning_purge_cache_block(ocf_cache_t cache, { ocf_cleaning_t policy; - if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt))) + if (unlikely(!env_refcnt_inc(&cache->cleaner.refcnt))) return; policy = cache->cleaner.policy; @@ -201,7 +202,7 @@ static inline void ocf_cleaning_purge_cache_block(ocf_cache_t cache, cleaning_policy_ops[policy].purge_cache_block(cache, cache_line); unlock: - ocf_refcnt_dec(&cache->cleaner.refcnt); + env_refcnt_dec(&cache->cleaner.refcnt); } static inline void ocf_cleaning_purge_range(ocf_cache_t cache, @@ -209,7 +210,7 @@ static inline void ocf_cleaning_purge_range(ocf_cache_t cache, { ocf_cleaning_t policy; - if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt))) + if (unlikely(!env_refcnt_inc(&cache->cleaner.refcnt))) return; policy = cache->cleaner.policy; @@ -222,7 +223,7 @@ static inline void ocf_cleaning_purge_range(ocf_cache_t cache, end_byte); unlock: - ocf_refcnt_dec(&cache->cleaner.refcnt); + env_refcnt_dec(&cache->cleaner.refcnt); } static inline void ocf_cleaning_set_hot_cache_line(ocf_cache_t cache, @@ -230,7 +231,7 @@ static inline void ocf_cleaning_set_hot_cache_line(ocf_cache_t cache, { ocf_cleaning_t policy; - if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt))) + if (unlikely(!env_refcnt_inc(&cache->cleaner.refcnt))) return; policy = cache->cleaner.policy; @@ -242,7 +243,7 @@ static inline void ocf_cleaning_set_hot_cache_line(ocf_cache_t cache, cleaning_policy_ops[policy].set_hot_cache_line(cache, cache_line); unlock: - ocf_refcnt_dec(&cache->cleaner.refcnt); + env_refcnt_dec(&cache->cleaner.refcnt); } static inline int ocf_cleaning_set_param(ocf_cache_t cache, @@ -274,7 +275,7 @@ static inline void ocf_cleaning_perform_cleaning(ocf_cache_t cache, { ocf_cleaning_t policy; - if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt))) { + if (unlikely(!env_refcnt_inc(&cache->cleaner.refcnt))) { cmpl(&cache->cleaner, 1000); return; } @@ -283,14 +284,14 @@ static inline void ocf_cleaning_perform_cleaning(ocf_cache_t cache, ENV_BUG_ON(policy >= ocf_cleaning_max); if (unlikely(!cleaning_policy_ops[policy].perform_cleaning)) { - ocf_refcnt_dec(&cache->cleaner.refcnt); + env_refcnt_dec(&cache->cleaner.refcnt); cmpl(&cache->cleaner, 1000); return; } cleaning_policy_ops[policy].perform_cleaning(cache, cmpl); - ocf_refcnt_dec(&cache->cleaner.refcnt); + env_refcnt_dec(&cache->cleaner.refcnt); } static inline const char *ocf_cleaning_get_name(ocf_cleaning_t policy) diff --git a/src/engine/cache_engine.c b/src/engine/cache_engine.c index 2ab7972..5acb58e 100644 --- a/src/engine/cache_engine.c +++ b/src/engine/cache_engine.c @@ -1,6 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation - * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2024-2025 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -22,7 +22,6 @@ #include "engine_flush.h" #include "engine_discard.h" #include "../utils/utils_user_part.h" -#include "../utils/utils_refcnt.h" #include "../ocf_request.h" #include "../metadata/metadata.h" #include "../ocf_space.h" diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 710e1b1..5566228 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -5,6 +5,8 @@ */ #include "ocf/ocf.h" +#include "ocf_env.h" +#include "ocf_env_refcnt.h" #include "ocf_mngt_common.h" #include "ocf_mngt_core_priv.h" #include "../ocf_priv.h" @@ -20,8 +22,8 @@ #include "../utils/utils_cache_line.h" #include "../utils/utils_parallelize.h" #include "../utils/utils_pipeline.h" -#include "../utils/utils_refcnt.h" #include "../utils/utils_async_lock.h" +#include "../utils/utils_cleaner.h" #include "../concurrency/ocf_concurrency.h" #include "../concurrency/ocf_metadata_concurrency.h" #include "../ocf_lru.h" @@ -188,7 +190,7 @@ static void __init_partitions(ocf_cache_t cache) /* Add other partition to the cache and make it as dummy */ for (i_part = 0; i_part < OCF_USER_IO_CLASS_MAX; i_part++) { - ocf_refcnt_freeze(&cache->user_parts[i_part].cleaning.counter); + env_refcnt_freeze(&cache->user_parts[i_part].cleaning.counter); if (i_part == PARTITION_DEFAULT) continue; @@ -217,21 +219,30 @@ static void _init_parts_attached(ocf_pipeline_t pipeline, void *priv, static ocf_error_t __init_cleaning_policy(ocf_cache_t cache) { + int result; int i; OCF_ASSERT_PLUGGED(cache); - ocf_refcnt_init(&cache->cleaner.refcnt); + result = env_refcnt_init(&cache->cleaner.refcnt, "cleaner", + sizeof("cleaner")); + if (result) + return result; for (i = 0; i < ocf_cleaning_max; i++) ocf_cleaning_setup(cache, i); - return ocf_cleaning_initialize(cache, cache->cleaner.policy, false); + result = ocf_cleaning_initialize(cache, cache->cleaner.policy, false); + if (result) + env_refcnt_deinit(&cache->cleaner.refcnt); + + return result; } static void __deinit_cleaning_policy(ocf_cache_t cache) { ocf_cleaning_deinitialize(cache); + env_refcnt_deinit(&cache->cleaner.refcnt); } static void __setup_promotion_policy(ocf_cache_t cache) @@ -425,7 +436,9 @@ static void _ocf_mngt_load_add_cores(ocf_pipeline_t pipeline, * Attach bottom device to core structure * in cache */ + env_refcnt_freeze(&tvolume->refcnt); ocf_volume_move(&core->volume, tvolume); + env_refcnt_unfreeze(&tvolume->refcnt); ocf_mngt_core_pool_remove(cache->owner, tvolume); core->opened = true; @@ -720,6 +733,12 @@ static void _ocf_mngt_load_init_cleaning(ocf_pipeline_t pipeline, ocf_cache_t cache = context->cache; ocf_error_t result; + result = env_refcnt_init(&cache->cleaner.refcnt, "cleaner", sizeof("cleaner")); + if (result) { + ocf_cache_log(cache, log_err, "Cannot initialize cleaner refcount\n"); + OCF_PL_FINISH_RET(pipeline, result); + } + if (context->metadata.shutdown_status == ocf_metadata_clean_shutdown) { /* Cleaning policy structures have been loaded so no need to populate them for the second time */ @@ -827,7 +846,8 @@ static int _ocf_mngt_init_new_cache(struct ocf_cache_mngt_init_params *params, char *new_cache_name) { ocf_cache_t cache = env_vzalloc(sizeof(*cache)); - int result; + int result = 0; + int i = 0; if (!cache) { ocf_log(params->ctx, log_err, "Failed to allocate cache %s\n", @@ -843,13 +863,37 @@ static int _ocf_mngt_init_new_cache(struct ocf_cache_mngt_init_params *params, goto alloc_err; } + result = env_refcnt_init(&cache->refcnt.cache, "cache", sizeof("cache")); + if (result) + goto lock_init_err; + + result = env_refcnt_init(&cache->refcnt.dirty, "dirty", sizeof("dirty")); + if (result) + goto dirty_refcnt_err; + + result = env_refcnt_init(&cache->refcnt.metadata, "metadata", sizeof("metadata")); + if (result) + goto metadata_refcnt_err; + + result = env_refcnt_init(&cache->refcnt.d2c, "d2c", sizeof("d2c")); + if (result) + goto d2c_refcnt_err; + + for (i = 0; i < OCF_USER_IO_CLASS_MAX; i++) { + result = env_refcnt_init(&cache->user_parts[i].cleaning.counter, + "cleaning", sizeof("cleaning")); + if (result) + goto cleaning_refcnt_err; + env_atomic_set(&cache->user_parts[i].cleaning.cleaner_running, 0); + } + /* Lock cache during setup - this trylock should always succeed */ result = ocf_mngt_cache_trylock(cache); if (result) { ocf_log(params->ctx, log_crit, "Failed to lock the newly created cache %s\n", new_cache_name); - goto lock_init_err; + goto cleaning_refcnt_err; } if (env_mutex_init(&cache->flush_mutex)) { @@ -869,7 +913,7 @@ static int _ocf_mngt_init_new_cache(struct ocf_cache_mngt_init_params *params, goto mutex_err; } - result = !ocf_refcnt_inc(&cache->refcnt.cache); + result = !env_refcnt_inc(&cache->refcnt.cache); if (result) { ocf_log(params->ctx, log_crit, "Failed to increment %s refcnt\n", @@ -877,10 +921,8 @@ static int _ocf_mngt_init_new_cache(struct ocf_cache_mngt_init_params *params, goto cache_refcnt_inc_err; } - /* start with freezed metadata ref counter to indicate detached device*/ - ocf_refcnt_freeze(&cache->refcnt.metadata); - - ocf_refcnt_init(&cache->refcnt.d2c); + /* start with frozen metadata ref counter to indicate detached device*/ + env_refcnt_freeze(&cache->refcnt.metadata); env_atomic_set(&(cache->last_access_ms), env_ticks_to_msecs(env_get_tick_count())); @@ -898,6 +940,17 @@ mutex_err: env_mutex_destroy(&cache->flush_mutex); lock_err: ocf_mngt_cache_unlock(cache); +cleaning_refcnt_err: + for (; i >= 0; i--) { + env_refcnt_deinit(&cache->user_parts[i].cleaning.counter); + } + env_refcnt_deinit(&cache->refcnt.d2c); +d2c_refcnt_err: + env_refcnt_deinit(&cache->refcnt.metadata); +metadata_refcnt_err: + env_refcnt_deinit(&cache->refcnt.dirty); +dirty_refcnt_err: + env_refcnt_deinit(&cache->refcnt.cache); lock_init_err: ocf_mngt_cache_lock_deinit(cache); alloc_err: @@ -1377,8 +1430,10 @@ static void _ocf_mngt_cleaning_populate_init_complete(void *priv, int error) struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; - if (error) + if (error) { + env_refcnt_deinit(&cache->cleaner.refcnt); OCF_PL_FINISH_RET(context->pipeline, error); + } /* In initial cache state there is no dirty data, so all dirty data is considered to be flushed @@ -1459,6 +1514,7 @@ uint64_t ocf_mngt_get_ram_needed(ocf_cache_t cache, static void _ocf_mngt_init_handle_error(ocf_ctx_t ctx, struct ocf_cache_mngt_init_params *params) { + int i; ocf_cache_t cache = params->cache; if (!params->flags.cache_alloc) @@ -1471,6 +1527,14 @@ static void _ocf_mngt_init_handle_error(ocf_ctx_t ctx, if (params->flags.cache_locked) ocf_mngt_cache_unlock(cache); + for (i = 0; i < OCF_USER_IO_CLASS_MAX; i++) + env_refcnt_deinit(&cache->user_parts[i].cleaning.counter); + + env_refcnt_deinit(&cache->refcnt.d2c); + env_refcnt_deinit(&cache->refcnt.metadata); + env_refcnt_deinit(&cache->refcnt.dirty); + env_refcnt_deinit(&cache->refcnt.cache); + ocf_mngt_cache_lock_deinit(cache); if (params->flags.metadata_inited) @@ -1907,7 +1971,7 @@ static void _ocf_mngt_attach_post_init_finish(void *priv) struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; - ocf_refcnt_unfreeze(&cache->refcnt.d2c); + env_refcnt_unfreeze(&cache->refcnt.d2c); env_atomic_set(&cache->attach_pt, 0); @@ -1925,10 +1989,10 @@ static void _ocf_mngt_attach_post_init(ocf_pipeline_t pipeline, env_atomic_set(&cache->attach_pt, 1); ocf_cleaner_refcnt_unfreeze(cache); - ocf_refcnt_unfreeze(&cache->refcnt.metadata); + env_refcnt_unfreeze(&cache->refcnt.metadata); - ocf_refcnt_freeze(&cache->refcnt.d2c); - ocf_refcnt_register_zero_cb(&cache->refcnt.d2c, + env_refcnt_freeze(&cache->refcnt.d2c); + env_refcnt_register_zero_cb(&cache->refcnt.d2c, _ocf_mngt_attach_post_init_finish, context); } @@ -2081,22 +2145,14 @@ struct ocf_mngt_cache_unplug_context { int cache_write_error; }; -static void ocf_mngt_cache_stop_wait_metadata_io_finish(void *priv) -{ - struct ocf_mngt_cache_unplug_context *context = priv; - - ocf_pipeline_next(context->pipeline); -} - static void ocf_mngt_cache_stop_wait_metadata_io(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { struct ocf_mngt_cache_unplug_context *context = priv; - ocf_cache_t cache = context->cache; + struct env_refcnt *refcnt = &context->cache->refcnt.metadata; - ocf_refcnt_freeze(&cache->refcnt.metadata); - ocf_refcnt_register_zero_cb(&cache->refcnt.metadata, - ocf_mngt_cache_stop_wait_metadata_io_finish, context); + env_refcnt_freeze(refcnt); + ocf_mngt_continue_pipeline_on_zero_refcnt(refcnt, context->pipeline); } static void ocf_mngt_cache_stop_check_dirty(ocf_pipeline_t pipeline, @@ -2257,14 +2313,35 @@ static void ocf_mngt_cache_stop_put_io_queues(ocf_pipeline_t pipeline, ocf_pipeline_next(pipeline); } +static void _ocf_mngt_cache_dealloc(void *priv) +{ + ocf_cache_t cache = priv; + ocf_ctx_t ctx; + unsigned i; + + ctx = cache->owner; + ocf_metadata_deinit(cache); + + env_refcnt_deinit(&cache->refcnt.cache); + env_refcnt_deinit(&cache->refcnt.dirty); + env_refcnt_deinit(&cache->refcnt.metadata); + for (i = 0; i < OCF_USER_IO_CLASS_MAX; i++) + env_refcnt_deinit(&cache->user_parts[i].cleaning.counter); + + env_vfree(cache); + ocf_ctx_put(ctx); +} + static void ocf_mngt_cache_remove(ocf_ctx_t ctx, ocf_cache_t cache) { - /* Mark device uninitialized */ - ocf_refcnt_freeze(&cache->refcnt.cache); - - /* Deinitialize locks */ + /* Deinitialize cache lock */ ocf_mngt_cache_lock_deinit(cache); + /* Mark device uninitialized */ + env_refcnt_freeze(&cache->refcnt.cache); + env_refcnt_register_zero_cb(&cache->refcnt.cache, + _ocf_mngt_cache_dealloc, cache); + env_spinlock_destroy(&cache->io_queues_lock); env_mutex_destroy(&cache->flush_mutex); @@ -2289,7 +2366,7 @@ static void ocf_mngt_cache_stop_finish(ocf_pipeline_t pipeline, ocf_mngt_cache_remove(context->ctx, cache); } else { /* undo metadata counter freeze */ - ocf_refcnt_unfreeze(&cache->refcnt.metadata); + env_refcnt_unfreeze(&cache->refcnt.metadata); env_bit_clear(ocf_cache_state_stopping, &cache->cache_state); env_bit_set(ocf_cache_state_running, &cache->cache_state); @@ -2446,7 +2523,7 @@ static void _ocf_mngt_standby_post_init(ocf_pipeline_t pipeline, struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; - ocf_refcnt_unfreeze(&cache->refcnt.metadata); + env_refcnt_unfreeze(&cache->refcnt.metadata); ocf_pipeline_next(pipeline); } @@ -2506,22 +2583,14 @@ struct ocf_cache_standby_detach_context { void *priv; }; -static void _ocf_mngt_standby_detach_wait_metadata_io_finish(void *priv) -{ - struct ocf_cache_standby_detach_context *context = priv; - - ocf_pipeline_next(context->pipeline); -} - static void _ocf_mngt_standby_detach_wait_metadata_io(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { struct ocf_cache_standby_detach_context *context = priv; - ocf_cache_t cache = context->cache; + struct env_refcnt *refcnt = &context->cache->refcnt.metadata; - ocf_refcnt_freeze(&cache->refcnt.metadata); - ocf_refcnt_register_zero_cb(&cache->refcnt.metadata, - _ocf_mngt_standby_detach_wait_metadata_io_finish, context); + env_refcnt_freeze(refcnt); + ocf_mngt_continue_pipeline_on_zero_refcnt(refcnt, context->pipeline); } static void _ocf_mngt_activate_set_cache_device(ocf_pipeline_t pipeline, @@ -2661,7 +2730,7 @@ static void _ocf_mngt_activate_handle_error( ocf_volume_deinit(&cache->device->volume); if (context->flags.metadata_frozen) - ocf_refcnt_unfreeze(&cache->refcnt.metadata); + env_refcnt_unfreeze(&cache->refcnt.metadata); } static void _ocf_mngt_cache_activate_finish(ocf_pipeline_t pipeline, @@ -2786,7 +2855,7 @@ static void ocf_mngt_stop_standby_stop_prepare(ocf_pipeline_t pipeline, struct ocf_mngt_cache_unplug_context *context = priv; ocf_cache_t cache = context->cache; - context->close_volume = !ocf_refcnt_frozen(&cache->refcnt.metadata); + context->close_volume = !env_refcnt_frozen(&cache->refcnt.metadata); ocf_pipeline_next(pipeline); } @@ -2832,7 +2901,7 @@ static void ocf_mngt_cache_standby_deinit_cache_volume(ocf_pipeline_t pipeline, struct ocf_mngt_cache_unplug_context *context = priv; ocf_cache_t cache = context->cache; - if (!ocf_refcnt_frozen(&cache->refcnt.metadata)) { + if (!env_refcnt_frozen(&cache->refcnt.metadata)) { ocf_volume_deinit(&cache->device->volume); env_vfree(cache->device); @@ -2992,7 +3061,7 @@ static void _ocf_mngt_cache_standby_activate(ocf_cache_t cache, if (!ocf_cache_is_standby(cache)) OCF_CMPL_RET(cache, priv1, priv2, -OCF_ERR_CACHE_EXIST); - if (!ocf_refcnt_frozen(&cache->refcnt.metadata)) + if (!env_refcnt_frozen(&cache->refcnt.metadata)) OCF_CMPL_RET(cache, priv1, priv2, OCF_ERR_STANDBY_ATTACHED); result = ocf_pipeline_create(&pipeline, cache, @@ -3361,7 +3430,7 @@ void ocf_mngt_cache_standby_detach(ocf_cache_t cache, if (!ocf_cache_is_standby(cache)) OCF_CMPL_RET(priv, -OCF_ERR_CACHE_EXIST); - if (ocf_refcnt_frozen(&cache->refcnt.metadata)) + if (env_refcnt_frozen(&cache->refcnt.metadata)) OCF_CMPL_RET(priv, -OCF_ERR_INVAL); _ocf_mngt_cache_standby_detach(cache, cmpl, priv); @@ -3753,21 +3822,14 @@ static void ocf_mngt_cache_detach_flush(ocf_pipeline_t pipeline, ocf_mngt_cache_flush(cache, ocf_mngt_cache_detach_flush_cmpl, context); } -static void ocf_mngt_cache_detach_stop_cache_io_finish(void *priv) -{ - struct ocf_mngt_cache_unplug_context *context = priv; - ocf_pipeline_next(context->pipeline); -} - static void ocf_mngt_cache_detach_stop_cache_io(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { struct ocf_mngt_cache_unplug_context *context = priv; - ocf_cache_t cache = context->cache; + struct env_refcnt *refcnt = &context->cache->refcnt.metadata; - ocf_refcnt_freeze(&cache->refcnt.metadata); - ocf_refcnt_register_zero_cb(&cache->refcnt.metadata, - ocf_mngt_cache_detach_stop_cache_io_finish, context); + env_refcnt_freeze(refcnt); + ocf_mngt_continue_pipeline_on_zero_refcnt(refcnt, context->pipeline); } static void ocf_mngt_cache_detach_stop_cleaner_io_finish(void *priv) @@ -3840,7 +3902,7 @@ static void ocf_mngt_cache_detach_finish(ocf_pipeline_t pipeline, struct ocf_mngt_cache_unplug_context *context = priv; ocf_cache_t cache = context->cache; - ocf_refcnt_unfreeze(&cache->refcnt.dirty); + env_refcnt_unfreeze(&cache->refcnt.dirty); if (!error) { if (!context->cache_write_error) { @@ -3915,7 +3977,7 @@ void ocf_mngt_cache_detach(ocf_cache_t cache, context->cache = cache; /* prevent dirty io */ - ocf_refcnt_freeze(&cache->refcnt.dirty); + env_refcnt_freeze(&cache->refcnt.dirty); ocf_pipeline_next(pipeline); } diff --git a/src/mngt/ocf_mngt_common.c b/src/mngt/ocf_mngt_common.c index 611aee2..1733e20 100644 --- a/src/mngt/ocf_mngt_common.c +++ b/src/mngt/ocf_mngt_common.c @@ -1,10 +1,11 @@ /* * Copyright(c) 2012-2021 Intel Corporation - * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2024-2025 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ #include "ocf/ocf.h" +#include "ocf_env_refcnt.h" #include "ocf_mngt_common.h" #include "ocf_mngt_core_priv.h" #include "../ocf_priv.h" @@ -137,19 +138,11 @@ void cache_mngt_core_remove_from_cache(ocf_core_t core) if (!core->opened && --cache->ocf_core_inactive_count == 0) env_bit_clear(ocf_cache_state_incomplete, &cache->cache_state); } - void ocf_mngt_cache_put(ocf_cache_t cache) { - ocf_ctx_t ctx; - OCF_CHECK_NULL(cache); - if (ocf_refcnt_dec(&cache->refcnt.cache) == 0) { - ctx = cache->owner; - ocf_metadata_deinit(cache); - env_vfree(cache); - ocf_ctx_put(ctx); - } + env_refcnt_dec(&cache->refcnt.cache); } void __set_cleaning_policy(ocf_cache_t cache, @@ -181,7 +174,7 @@ int ocf_mngt_cache_get_by_name(ocf_ctx_t ctx, const char *name, size_t name_len, if (instance) { /* if cache is either fully initialized or during recovery */ - if (!ocf_refcnt_inc(&instance->refcnt.cache)) { + if (!env_refcnt_inc(&instance->refcnt.cache)) { /* Cache not initialized yet */ instance = NULL; } @@ -245,9 +238,15 @@ static void _ocf_mngt_cache_lock(ocf_cache_t cache, if (ocf_mngt_cache_get(cache)) OCF_CMPL_RET(cache, priv, -OCF_ERR_CACHE_NOT_EXIST); + if (!env_refcnt_inc(&cache->refcnt.lock)) { + ocf_mngt_cache_put(cache); + OCF_CMPL_RET(cache, priv, -OCF_ERR_CACHE_NOT_EXIST); + } + waiter = ocf_async_lock_new_waiter(&cache->lock, _ocf_mngt_cache_lock_complete); if (!waiter) { + env_refcnt_dec(&cache->refcnt.lock); ocf_mngt_cache_put(cache); OCF_CMPL_RET(cache, priv, -OCF_ERR_NO_MEM); } @@ -259,20 +258,26 @@ static void _ocf_mngt_cache_lock(ocf_cache_t cache, context->priv = priv; lock_fn(waiter); + env_refcnt_dec(&cache->refcnt.lock); } static int _ocf_mngt_cache_trylock(ocf_cache_t cache, ocf_trylock_fn_t trylock_fn, ocf_unlock_fn_t unlock_fn) { - int result; + int result = 0; if (ocf_mngt_cache_get(cache)) return -OCF_ERR_CACHE_NOT_EXIST; + if (!env_refcnt_inc(&cache->refcnt.lock)) { + ocf_mngt_cache_put(cache); + return -OCF_ERR_CACHE_NOT_EXIST; + } + result = trylock_fn(&cache->lock); if (result) { ocf_mngt_cache_put(cache); - return result; + goto out; } if (env_bit_test(ocf_cache_state_stopping, &cache->cache_state)) { @@ -281,6 +286,8 @@ static int _ocf_mngt_cache_trylock(ocf_cache_t cache, result = -OCF_ERR_CACHE_NOT_EXIST; } +out: + env_refcnt_dec(&cache->refcnt.lock); return result; } @@ -293,13 +300,38 @@ static void _ocf_mngt_cache_unlock(ocf_cache_t cache, int ocf_mngt_cache_lock_init(ocf_cache_t cache) { - return ocf_async_lock_init(&cache->lock, + int result; + + result = env_refcnt_init(&cache->refcnt.lock, "lock", sizeof("lock")); + if (result) + return result; + + result = ocf_async_lock_init(&cache->lock, sizeof(struct ocf_mngt_cache_lock_context)); + if (result) + env_refcnt_deinit(&cache->refcnt.lock); + + return result; +} + +static void _ocf_mngt_cache_lock_deinit(void *priv) +{ + ocf_cache_t cache = priv; + + ocf_async_lock_deinit(&cache->lock); + env_refcnt_dec(&cache->refcnt.cache); + env_refcnt_deinit(&cache->refcnt.lock); } void ocf_mngt_cache_lock_deinit(ocf_cache_t cache) { - ocf_async_lock_deinit(&cache->lock); + bool cache_get; + + cache_get = env_refcnt_inc(&cache->refcnt.cache); + ENV_BUG_ON(!cache_get); + env_refcnt_freeze(&cache->refcnt.lock); + env_refcnt_register_zero_cb(&cache->refcnt.lock, + _ocf_mngt_cache_lock_deinit, cache); } void ocf_mngt_cache_lock(ocf_cache_t cache, @@ -358,7 +390,7 @@ bool ocf_mngt_cache_is_locked(ocf_cache_t cache) /* if cache is either fully initialized or during recovery */ static bool _ocf_mngt_cache_try_get(ocf_cache_t cache) { - return !!ocf_refcnt_inc(&cache->refcnt.cache); + return env_refcnt_inc(&cache->refcnt.cache); } int ocf_mngt_cache_get(ocf_cache_t cache) @@ -483,3 +515,16 @@ int ocf_mngt_cache_visit_reverse(ocf_ctx_t ocf_ctx, return result; } + +static void _ocf_mngt_continue_pipeline_on_zero_refcnt_cb(void *priv) +{ + ocf_pipeline_next((ocf_pipeline_t)priv); +} + +void ocf_mngt_continue_pipeline_on_zero_refcnt(struct env_refcnt *refcnt, + ocf_pipeline_t pipeline) +{ + env_refcnt_register_zero_cb(refcnt, + _ocf_mngt_continue_pipeline_on_zero_refcnt_cb, + pipeline); +} diff --git a/src/mngt/ocf_mngt_common.h b/src/mngt/ocf_mngt_common.h index 751a2e0..4e0adc4 100644 --- a/src/mngt/ocf_mngt_common.h +++ b/src/mngt/ocf_mngt_common.h @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2025 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -7,6 +8,9 @@ #ifndef __OCF_MNGT_COMMON_H__ #define __OCF_MNGT_COMMON_H__ +#include "ocf_env_refcnt.h" +#include "../utils/utils_pipeline.h" + void cache_mngt_core_deinit(ocf_core_t core); void cache_mngt_core_remove_from_meta(ocf_core_t core); @@ -33,4 +37,7 @@ bool ocf_mngt_cache_is_locked(ocf_cache_t cache); void __set_cleaning_policy(ocf_cache_t cache, ocf_cleaning_t new_cleaning_policy); +void ocf_mngt_continue_pipeline_on_zero_refcnt(struct env_refcnt *refcnt, + ocf_pipeline_t pipeline); + #endif /* __OCF_MNGT_COMMON_H__ */ diff --git a/src/mngt/ocf_mngt_core.c b/src/mngt/ocf_mngt_core.c index 96ee7f4..c32e033 100644 --- a/src/mngt/ocf_mngt_core.c +++ b/src/mngt/ocf_mngt_core.c @@ -11,6 +11,7 @@ #include "../metadata/metadata.h" #include "../engine/cache_engine.h" #include "../utils/utils_pipeline.h" +#include "../utils/utils_cleaner.h" #include "../ocf_stats_priv.h" #include "../ocf_def_priv.h" #include "../cleaning/cleaning_ops.h" diff --git a/src/mngt/ocf_mngt_flush.c b/src/mngt/ocf_mngt_flush.c index a0ca152..6ecca4b 100644 --- a/src/mngt/ocf_mngt_flush.c +++ b/src/mngt/ocf_mngt_flush.c @@ -1,10 +1,11 @@ /* * Copyright(c) 2012-2022 Intel Corporation - * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2023-2025 Huawei Technologies Co., Ltd. * SPDX-License-Identifier: BSD-3-Clause */ #include "ocf/ocf.h" +#include "ocf_env_refcnt.h" #include "ocf_mngt_common.h" #include "../ocf_priv.h" #include "../metadata/metadata.h" @@ -15,7 +16,6 @@ #include "../utils/utils_cache_line.h" #include "../utils/utils_user_part.h" #include "../utils/utils_pipeline.h" -#include "../utils/utils_refcnt.h" #include "../ocf_request.h" #include "../ocf_def_priv.h" @@ -82,12 +82,6 @@ struct ocf_mngt_cache_flush_context struct flush_containers_context fcs; }; -static void _ocf_mngt_begin_flush_complete(void *priv) -{ - struct ocf_mngt_cache_flush_context *context = priv; - ocf_pipeline_next(context->pipeline); -} - static void _ocf_mngt_begin_flush(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { @@ -102,11 +96,11 @@ static void _ocf_mngt_begin_flush(ocf_pipeline_t pipeline, void *priv, OCF_PL_FINISH_RET(pipeline, -OCF_ERR_FLUSH_IN_PROGRESS); context->flags.lock = true; - ocf_refcnt_freeze(&cache->refcnt.dirty); + env_refcnt_freeze(&cache->refcnt.dirty); context->flags.freeze = true; - ocf_refcnt_register_zero_cb(&cache->refcnt.dirty, - _ocf_mngt_begin_flush_complete, context); + ocf_mngt_continue_pipeline_on_zero_refcnt(&cache->refcnt.dirty, + context->pipeline); } bool ocf_mngt_core_is_dirty(ocf_core_t core) @@ -615,7 +609,7 @@ static void _ocf_mngt_flush_finish(ocf_pipeline_t pipeline, void *priv, ocf_core_t core = context->core; if (context->flags.freeze) - ocf_refcnt_unfreeze(&cache->refcnt.dirty); + env_refcnt_unfreeze(&cache->refcnt.dirty); if (context->flags.lock) env_mutex_unlock(&cache->flush_mutex); @@ -956,8 +950,8 @@ static void _ocf_mngt_deinit_clean_policy(ocf_pipeline_t pipeline, void *priv, ocf_metadata_start_exclusive_access(&cache->metadata.lock); - ocf_refcnt_freeze(&cache->cleaner.refcnt); - ocf_refcnt_register_zero_cb(&cache->cleaner.refcnt, + env_refcnt_freeze(&cache->cleaner.refcnt); + env_refcnt_register_zero_cb(&cache->cleaner.refcnt, _ocf_mngt_cleaning_deinit_complete, context); } @@ -1011,7 +1005,7 @@ static void _ocf_mngt_set_cleaning_finish(ocf_pipeline_t pipeline, void *priv, struct ocf_mngt_cache_set_cleaning_context *context = priv; ocf_cache_t cache = context->cache; - ocf_refcnt_unfreeze(&cache->cleaner.refcnt); + env_refcnt_unfreeze(&cache->cleaner.refcnt); ocf_metadata_end_exclusive_access(&cache->metadata.lock); context->cmpl(context->priv, error); diff --git a/src/ocf_cache.c b/src/ocf_cache.c index 2610940..718d5c8 100644 --- a/src/ocf_cache.c +++ b/src/ocf_cache.c @@ -1,10 +1,11 @@ /* * Copyright(c) 2012-2022 Intel Corporation - * Copyright(c) 2023-2024 Huawei Technologies + * Copyright(c) 2023-2025 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ #include "ocf/ocf.h" +#include "ocf_env_refcnt.h" #include "metadata/metadata.h" #include "metadata/metadata.h" #include "engine/cache_engine.h" @@ -133,7 +134,7 @@ int ocf_cache_get_info(ocf_cache_t cache, struct ocf_cache_info *info) info->attached = ocf_cache_is_device_attached(cache); info->standby_detached = ocf_cache_is_standby(cache) && - ocf_refcnt_frozen(&cache->refcnt.metadata); + env_refcnt_frozen(&cache->refcnt.metadata); if (info->attached && !info->standby_detached) { info->volume_type = ocf_ctx_get_volume_type_id(cache->owner, cache->device->volume.type); @@ -297,8 +298,8 @@ static void ocf_cache_volume_io_complete_generic(struct ocf_request *req, { ocf_cache_t cache = req->cache; - ocf_refcnt_dec(&cache->refcnt.metadata); ocf_io_end_func(req, error); + env_refcnt_dec(&cache->refcnt.metadata); } static void ocf_cache_io_complete(struct ocf_request *req, int error) @@ -311,8 +312,8 @@ static void ocf_cache_io_complete(struct ocf_request *req, int error) if (env_atomic_dec_return(&req->req_remaining)) return; - ocf_refcnt_dec(&cache->refcnt.metadata); ocf_io_end_func(req, req->error); + env_refcnt_dec(&cache->refcnt.metadata); } static void ocf_cache_volume_submit_io(ocf_io_t io) @@ -321,7 +322,7 @@ static void ocf_cache_volume_submit_io(ocf_io_t io) ocf_cache_t cache = req->cache; int result; - if (!ocf_refcnt_inc(&cache->refcnt.metadata)) { + if (!env_refcnt_inc(&cache->refcnt.metadata)) { ocf_io_end_func(io, -OCF_ERR_IO); return; } @@ -343,6 +344,7 @@ static void ocf_cache_volume_submit_io(ocf_io_t io) "Metadata update error (error=%d)!\n", result); } + // TODO pass it actually // TODO why the result is not passed to io_cmpl??? ocf_cache_io_complete(req, 0); } @@ -352,7 +354,7 @@ static void ocf_cache_volume_submit_flush(ocf_io_t io) struct ocf_request *req = ocf_io_to_req(io); ocf_cache_t cache = req->cache; - if (!ocf_refcnt_inc(&cache->refcnt.metadata)) { + if (!env_refcnt_inc(&cache->refcnt.metadata)) { ocf_io_end_func(io, -OCF_ERR_IO); return; } @@ -371,7 +373,7 @@ static void ocf_cache_volume_submit_discard(ocf_io_t io) struct ocf_request *req = ocf_io_to_req(io); ocf_cache_t cache = req->cache; - if (!ocf_refcnt_inc(&cache->refcnt.metadata)) { + if (!env_refcnt_inc(&cache->refcnt.metadata)) { ocf_io_end_func(io, -OCF_ERR_IO); return; } @@ -487,6 +489,6 @@ int ocf_cache_volume_type_init(ocf_ctx_t ctx) bool ocf_dbg_cache_is_settled(ocf_cache_t cache) { - return ocf_refcnt_zeroed(&cache->refcnt.metadata) && - ocf_refcnt_zeroed(&cache->refcnt.d2c); + return env_refcnt_zeroed(&cache->refcnt.metadata) && + env_refcnt_zeroed(&cache->refcnt.d2c); } diff --git a/src/ocf_cache_priv.h b/src/ocf_cache_priv.h index b4ab8b6..49b856c 100644 --- a/src/ocf_cache_priv.h +++ b/src/ocf_cache_priv.h @@ -1,6 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation - * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2024-2025 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -15,7 +15,6 @@ #include "metadata/metadata_structs.h" #include "utils/utils_list.h" #include "utils/utils_pipeline.h" -#include "utils/utils_refcnt.h" #include "utils/utils_async_lock.h" #include "ocf_stats_priv.h" #include "cleaning/cleaning.h" @@ -79,14 +78,17 @@ struct ocf_cache { struct { /* cache get/put counter */ - struct ocf_refcnt cache __attribute__((aligned(64))); + struct env_refcnt cache; /* # of requests potentially dirtying cachelines */ - struct ocf_refcnt dirty __attribute__((aligned(64))); + struct env_refcnt dirty; /* # of requests accessing attached metadata, excluding * management reqs */ - struct ocf_refcnt metadata __attribute__((aligned(64))); + struct env_refcnt metadata; /* # of requests in d2c mode */ - struct ocf_refcnt d2c; + struct env_refcnt d2c; + /* # of unsettled cache lock operations (lock not acquired, + * waiter not added yet) */ + struct env_refcnt lock; } refcnt; struct { diff --git a/src/ocf_core.c b/src/ocf_core.c index bdf7378..0748642 100644 --- a/src/ocf_core.c +++ b/src/ocf_core.c @@ -1,10 +1,11 @@ /* * Copyright(c) 2012-2021 Intel Corporation - * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2023-2025 Huawei Technologies Co., Ltd. * SPDX-License-Identifier: BSD-3-Clause */ #include "ocf/ocf.h" +#include "ocf_env_refcnt.h" #include "ocf_priv.h" #include "ocf_core_priv.h" #include "ocf_io_priv.h" @@ -173,7 +174,7 @@ static inline void dec_counter_if_req_was_dirty(struct ocf_request *req) return; req->dirty = 0; - ocf_refcnt_dec(&req->cache->refcnt.dirty); + env_refcnt_dec(&req->cache->refcnt.dirty); } static inline int ocf_core_validate_io(ocf_io_t io) diff --git a/src/ocf_io.c b/src/ocf_io.c index 6ca76f9..0d5c65b 100644 --- a/src/ocf_io.c +++ b/src/ocf_io.c @@ -1,6 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation - * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2024-2025 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -10,6 +10,10 @@ #include "ocf_volume_priv.h" #include "ocf_core_priv.h" #include "utils/utils_io_allocator.h" +#include "ocf_env_refcnt.h" +#ifdef OCF_DEBUG_STATS +#include "ocf_stats_priv.h" +#endif int ocf_io_allocator_default_init(ocf_io_allocator_t allocator, const char *name) @@ -79,13 +83,13 @@ ocf_io_t ocf_io_new(ocf_volume_t volume, ocf_queue_t queue, if ((addr % sector_size) || (bytes % sector_size)) return NULL; - if (!ocf_refcnt_inc(&volume->refcnt)) + if (!env_refcnt_inc(&volume->refcnt)) return NULL; req = ocf_io_allocator_new(&volume->type->allocator, volume, queue, addr, bytes, dir); if (!req) { - ocf_refcnt_dec(&volume->refcnt); + env_refcnt_dec(&volume->refcnt); return NULL; } @@ -131,7 +135,7 @@ void ocf_io_put(ocf_io_t io) ocf_io_allocator_del(&volume->type->allocator, (void *)req); - ocf_refcnt_dec(&volume->refcnt); + env_refcnt_dec(&volume->refcnt); } ocf_volume_t ocf_io_get_volume(ocf_io_t io) diff --git a/src/ocf_lru.c b/src/ocf_lru.c index 4bad30d..a0ba95c 100644 --- a/src/ocf_lru.c +++ b/src/ocf_lru.c @@ -1,10 +1,12 @@ /* * Copyright(c) 2012-2022 Intel Corporation - * Copyright(c) 2023-2024 Huawei Technologies Co., Ltd. + * Copyright(c) 2023-2025 Huawei Technologies Co., Ltd. * SPDX-License-Identifier: BSD-3-Clause */ +#include "ocf_env.h" #include "ocf_space.h" +#include "ocf_env_refcnt.h" #include "ocf_lru.h" #include "utils/utils_cleaner.h" #include "utils/utils_cache_line.h" @@ -582,7 +584,8 @@ static void ocf_lru_clean_end(void *private_data, int error) entries[i].cache_line); } - ocf_refcnt_dec(&ctx->counter); + env_atomic_set(&ctx->cleaner_running, 0); + env_refcnt_dec(&ctx->counter); } void ocf_lru_clean(ocf_cache_t cache, struct ocf_user_part *user_part, @@ -600,21 +603,20 @@ void ocf_lru_clean(ocf_cache_t cache, struct ocf_user_part *user_part, struct flush_data *entries = ctx->entries; struct ocf_lru_iter iter; unsigned lru_idx; - int cnt; unsigned i; unsigned lock_idx; if (ocf_mngt_cache_is_locked(cache)) return; - cnt = ocf_refcnt_inc(&ctx->counter); - if (!cnt) { + + if (unlikely(!env_refcnt_inc(&ctx->counter))) { /* cleaner disabled by management operation */ return; } - if (cnt > 1) { + if (env_atomic_cmpxchg(&ctx->cleaner_running, 0, 1) != 0) { /* cleaning already running for this partition */ - ocf_refcnt_dec(&ctx->counter); + env_refcnt_dec(&ctx->counter); return; } @@ -641,7 +643,7 @@ void ocf_lru_clean(ocf_cache_t cache, struct ocf_user_part *user_part, ocf_metadata_end_shared_access(&cache->metadata.lock, lock_idx); if (i == 0) { - ocf_refcnt_dec(&ctx->counter); + env_refcnt_dec(&ctx->counter); return; } diff --git a/src/ocf_part.h b/src/ocf_part.h index 8c32fd5..c2d4b30 100644 --- a/src/ocf_part.h +++ b/src/ocf_part.h @@ -1,6 +1,6 @@ /* * Copyright(c) 2012-2021 Intel Corporation - * Copyright(c) 2023-2024 Huawei Technologies Co., Ltd. + * Copyright(c) 2023-2025 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -11,6 +11,7 @@ #include "utils/utils_cleaner.h" #include "cleaning/cleaning.h" #include "ocf_space.h" +#include "ocf_env_refcnt.h" #define OCF_NUM_PARTITIONS OCF_USER_IO_CLASS_MAX + 2 @@ -70,7 +71,8 @@ struct ocf_lru_iter struct ocf_part_cleaning_ctx { ocf_cache_t cache; - struct ocf_refcnt counter; + struct env_refcnt counter; + env_atomic cleaner_running; struct flush_data entries[OCF_EVICTION_CLEAN_SIZE]; }; diff --git a/src/ocf_request.c b/src/ocf_request.c index 6e38327..9b7ce4d 100644 --- a/src/ocf_request.c +++ b/src/ocf_request.c @@ -1,6 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation - * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2024-2025 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -10,6 +10,7 @@ #include "concurrency/ocf_metadata_concurrency.h" #include "engine/engine_common.h" #include "utils/utils_cache_line.h" +#include "ocf_env_refcnt.h" #define OCF_UTILS_RQ_DEBUG 0 @@ -120,7 +121,7 @@ struct ocf_request *ocf_req_new_cleaner(ocf_cache_t cache, ocf_queue_t queue, struct ocf_request *req; bool map_allocated = true, is_mngt = false; - if (!ocf_refcnt_inc(&cache->refcnt.metadata)) + if (!env_refcnt_inc(&cache->refcnt.metadata)) return NULL; if (unlikely(ocf_queue_is_mngt(queue))) { @@ -137,7 +138,7 @@ struct ocf_request *ocf_req_new_cleaner(ocf_cache_t cache, ocf_queue_t queue, } if (!req) { - ocf_refcnt_dec(&cache->refcnt.metadata); + env_refcnt_dec(&cache->refcnt.metadata); return NULL; } req->is_mngt = is_mngt; @@ -192,8 +193,8 @@ struct ocf_request *ocf_req_new(ocf_queue_t queue, ocf_core_t core, ocf_queue_get(queue); - if (!ocf_refcnt_inc(&cache->refcnt.metadata)) { - if (!ocf_refcnt_inc(&cache->refcnt.d2c)) + if (!env_refcnt_inc(&cache->refcnt.metadata)) { + if (!env_refcnt_inc(&cache->refcnt.d2c)) ENV_BUG(); req = ocf_req_new_d2c(queue, core, addr, bytes, rw); if (unlikely(!req)) { @@ -220,7 +221,7 @@ struct ocf_request *ocf_req_new(ocf_queue_t queue, ocf_core_t core, } if (unlikely(!req)) { - ocf_refcnt_dec(&cache->refcnt.metadata); + env_refcnt_dec(&cache->refcnt.metadata); ocf_queue_put(queue); return NULL; } @@ -261,7 +262,7 @@ struct ocf_request *ocf_req_new_cache(ocf_cache_t cache, ocf_queue_t queue, ENV_BUG_ON(ocf_queue_is_mngt(queue)); - if (!ocf_refcnt_inc(&cache->refcnt.metadata)) + if (!env_refcnt_inc(&cache->refcnt.metadata)) return NULL; ocf_queue_get(queue); @@ -281,7 +282,7 @@ struct ocf_request *ocf_req_new_cache(ocf_cache_t cache, ocf_queue_t queue, } if (unlikely(!req)) { - ocf_refcnt_dec(&cache->refcnt.metadata); + env_refcnt_dec(&cache->refcnt.metadata); ocf_queue_put(queue); return NULL; } @@ -387,9 +388,9 @@ void ocf_req_put(struct ocf_request *req) OCF_DEBUG_TRACE(req->cache); if (req->d2c) - ocf_refcnt_dec(&req->cache->refcnt.d2c); + env_refcnt_dec(&req->cache->refcnt.d2c); else if (!req->is_mngt || req->cleaner) - ocf_refcnt_dec(&req->cache->refcnt.metadata); + env_refcnt_dec(&req->cache->refcnt.metadata); if (unlikely(req->is_mngt)) { env_free(req); @@ -405,7 +406,7 @@ void ocf_req_put(struct ocf_request *req) int ocf_req_set_dirty(struct ocf_request *req) { - req->dirty = !!ocf_refcnt_inc(&req->cache->refcnt.dirty); + req->dirty = !!env_refcnt_inc(&req->cache->refcnt.dirty); return req->dirty ? 0 : -OCF_ERR_AGAIN; } diff --git a/src/ocf_volume.c b/src/ocf_volume.c index 3d8f6da..fe5180c 100644 --- a/src/ocf_volume.c +++ b/src/ocf_volume.c @@ -1,6 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation - * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2024-2025 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -9,6 +9,7 @@ #include "ocf_volume_priv.h" #include "ocf_core_priv.h" #include "ocf_request.h" +#include "ocf_env_refcnt.h" #include "ocf_io_priv.h" #include "ocf_env.h" @@ -111,8 +112,11 @@ int ocf_volume_init(ocf_volume_t volume, ocf_volume_type_t type, volume->uuid.data = NULL; volume->uuid_copy = false; - ocf_refcnt_init(&volume->refcnt); - ocf_refcnt_freeze(&volume->refcnt); + ret = env_refcnt_init(&volume->refcnt, "volume", sizeof("volume")); + if (ret) + goto err1; + + env_refcnt_freeze(&volume->refcnt); if (!uuid) return 0; @@ -123,7 +127,7 @@ int ocf_volume_init(ocf_volume_t volume, ocf_volume_type_t type, data = env_vmalloc(uuid->size); if (!data) { ret = -OCF_ERR_NO_MEM; - goto err; + goto err2; } volume->uuid.data = data; @@ -131,7 +135,7 @@ int ocf_volume_init(ocf_volume_t volume, ocf_volume_type_t type, ret = env_memcpy(data, uuid->size, uuid->data, uuid->size); if (ret) { ret = -OCF_ERR_INVAL; - goto err; + goto err3; } } else { volume->uuid.data = uuid->data; @@ -142,19 +146,22 @@ int ocf_volume_init(ocf_volume_t volume, ocf_volume_type_t type, if (volume->type->properties->ops.on_init) { ret = volume->type->properties->ops.on_init(volume); if (ret) - goto err; + goto err3; } return 0; -err: - ocf_refcnt_unfreeze(&volume->refcnt); - env_free(volume->priv); - volume->priv = NULL; +err3: if (volume->uuid_copy && volume->uuid.data) env_vfree(volume->uuid.data); volume->uuid.data = NULL; volume->uuid.size = 0; +err2: + env_refcnt_unfreeze(&volume->refcnt); + env_refcnt_deinit(&volume->refcnt); +err1: + env_free(volume->priv); + volume->priv = NULL; return ret; } @@ -168,6 +175,7 @@ void ocf_volume_deinit(ocf_volume_t volume) env_free(volume->priv); volume->priv = NULL; volume->type = NULL; + env_refcnt_deinit(&volume->refcnt); if (volume->uuid_copy && volume->uuid.data) { env_vfree(volume->uuid.data); @@ -181,7 +189,14 @@ void ocf_volume_move(ocf_volume_t volume, ocf_volume_t from) OCF_CHECK_NULL(volume); OCF_CHECK_NULL(from); - ocf_volume_deinit(volume); + ENV_BUG_ON(!env_refcnt_zeroed(&volume->refcnt)); + ENV_BUG_ON(!env_refcnt_zeroed(&from->refcnt)); + + env_free(volume->priv); + if (volume->uuid_copy && volume->uuid.data) + env_vfree(volume->uuid.data); + + /* volume->refcnt is not reinitialized */ volume->opened = from->opened; volume->type = from->type; @@ -190,7 +205,8 @@ void ocf_volume_move(ocf_volume_t volume, ocf_volume_t from) volume->priv = from->priv; volume->cache = from->cache; volume->features = from->features; - volume->refcnt = from->refcnt; + env_refcnt_init(&volume->refcnt, "volume", sizeof("volume")); + env_refcnt_freeze(&volume->refcnt); /* * Deinitialize original volume without freeing resources. @@ -442,7 +458,7 @@ int ocf_volume_open(ocf_volume_t volume, void *volume_params) if (ret) return ret; - ocf_refcnt_unfreeze(&volume->refcnt); + env_refcnt_unfreeze(&volume->refcnt); volume->opened = true; return 0; @@ -465,8 +481,8 @@ void ocf_volume_close(ocf_volume_t volume) return; env_completion_init(&cmpl); - ocf_refcnt_freeze(&volume->refcnt); - ocf_refcnt_register_zero_cb(&volume->refcnt, ocf_volume_close_end, + env_refcnt_freeze(&volume->refcnt); + env_refcnt_register_zero_cb(&volume->refcnt, ocf_volume_close_end, &cmpl); env_completion_wait(&cmpl); env_completion_destroy(&cmpl); diff --git a/src/ocf_volume_priv.h b/src/ocf_volume_priv.h index 4440fa3..dea2afe 100644 --- a/src/ocf_volume_priv.h +++ b/src/ocf_volume_priv.h @@ -1,6 +1,6 @@ /* * Copyright(c) 2012-2021 Intel Corporation - * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2024-2025 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -8,8 +8,8 @@ #define __OCF_VOLUME_PRIV_H__ #include "ocf_env.h" +#include "ocf_env_refcnt.h" #include "ocf_io_priv.h" -#include "utils/utils_refcnt.h" #include "utils/utils_io_allocator.h" struct ocf_volume_extended { @@ -36,7 +36,7 @@ struct ocf_volume { void *priv; ocf_cache_t cache; struct list_head core_pool_item; - struct ocf_refcnt refcnt __attribute__((aligned(64))); + struct env_refcnt refcnt; } __attribute__((aligned(64))); int ocf_volume_type_init(struct ocf_volume_type **type, diff --git a/src/utils/utils_cleaner.c b/src/utils/utils_cleaner.c index 3178efd..79ce59b 100644 --- a/src/utils/utils_cleaner.c +++ b/src/utils/utils_cleaner.c @@ -1,6 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation - * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2024-2025 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -14,6 +14,7 @@ #include "utils_io.h" #include "utils_cache_line.h" #include "../ocf_queue_priv.h" +#include "ocf_env_refcnt.h" #define OCF_UTILS_CLEANER_DEBUG 0 @@ -840,7 +841,7 @@ void ocf_cleaner_refcnt_freeze(ocf_cache_t cache) ocf_part_id_t part_id; for_each_user_part(cache, curr_part, part_id) - ocf_refcnt_freeze(&curr_part->cleaning.counter); + env_refcnt_freeze(&curr_part->cleaning.counter); } void ocf_cleaner_refcnt_unfreeze(ocf_cache_t cache) @@ -849,7 +850,7 @@ void ocf_cleaner_refcnt_unfreeze(ocf_cache_t cache) ocf_part_id_t part_id; for_each_user_part(cache, curr_part, part_id) - ocf_refcnt_unfreeze(&curr_part->cleaning.counter); + env_refcnt_unfreeze(&curr_part->cleaning.counter); } static void ocf_cleaner_refcnt_register_zero_cb_finish(void *priv) @@ -873,7 +874,7 @@ void ocf_cleaner_refcnt_register_zero_cb(ocf_cache_t cache, for_each_user_part(cache, curr_part, part_id) { env_atomic_inc(&ctx->waiting); - ocf_refcnt_register_zero_cb(&curr_part->cleaning.counter, + env_refcnt_register_zero_cb(&curr_part->cleaning.counter, ocf_cleaner_refcnt_register_zero_cb_finish, ctx); } diff --git a/src/utils/utils_refcnt.h b/src/utils/utils_refcnt.h deleted file mode 100644 index 0871854..0000000 --- a/src/utils/utils_refcnt.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright(c) 2019-2021 Intel Corporation - * Copyright(c) 2024 Huawei Technologies - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef __OCF_REFCNT_H__ -#define __OCF_REFCNT_H__ - -#include "ocf_env.h" - -typedef void (*ocf_refcnt_cb_t)(void *priv); - -struct ocf_refcnt -{ - env_atomic counter; - env_atomic freeze; - env_atomic callback; - ocf_refcnt_cb_t cb; - void *priv; -}; - -/* Initialize reference counter */ -void ocf_refcnt_init(struct ocf_refcnt *rc); - -/* Try to increment counter. Returns counter value (> 0) if successfull, 0 - * if counter is frozen */ -int ocf_refcnt_inc(struct ocf_refcnt *rc); - -/* Decrement reference counter and return post-decrement value */ -int ocf_refcnt_dec(struct ocf_refcnt *rc); - -/* Disallow incrementing of underlying counter - attempts to increment counter - * will be failing until ocf_refcnt_unfreeze is calleed. - * It's ok to call freeze multiple times, in which case counter is frozen - * until all freeze calls are offset by a corresponding unfreeze.*/ -void ocf_refcnt_freeze(struct ocf_refcnt *rc); - -/* Cancel the effect of single ocf_refcnt_freeze call */ -void ocf_refcnt_unfreeze(struct ocf_refcnt *rc); - -bool ocf_refcnt_frozen(struct ocf_refcnt *rc); - -bool ocf_refcnt_zeroed(struct ocf_refcnt *rc); - -/* Register callback to be called when reference counter drops to 0. - * Must be called after counter is frozen. - * Cannot be called until previously regsitered callback had fired. */ -void ocf_refcnt_register_zero_cb(struct ocf_refcnt *rc, ocf_refcnt_cb_t cb, - void *priv); - -#endif // __OCF_REFCNT_H__