Async wait for cleaner completion

Signed-off-by: Adam Rutkowski <adam.j.rutkowski@intel.com>

inc/ocf_cache.h

@@ -160,13 +160,6 @@ bool ocf_cache_is_device_attached(ocf_cache_t cache);
  */
 bool ocf_cache_is_running(ocf_cache_t cache);
 
-/**
- * @brief Check if cleaning triggered by eviction runs on the cache
- *
- * @param[in] cache Cache object
- */
-bool ocf_cache_has_pending_cleaning(ocf_cache_t cache);
-
 /**
  * @brief Get cache mode of given cache object
  *

src/eviction/lru.c

@@ -273,9 +273,9 @@ void evp_lru_rm_cline(ocf_cache_t cache, ocf_cache_line_t cline)
 
 static void evp_lru_clean_end(void *private_data, int error)
 {
-	env_atomic *cleaning_in_progress = private_data;
+	struct ocf_refcnt *counter = private_data;
 
-	env_atomic_set(cleaning_in_progress, 0);
+	ocf_refcnt_dec(counter);
 }
 
 static int evp_lru_clean_getter(ocf_cache_t cache,
@@ -309,19 +309,13 @@ static int evp_lru_clean_getter(ocf_cache_t cache,
 static void evp_lru_clean(ocf_cache_t cache, ocf_queue_t io_queue,
 		ocf_part_id_t part_id, uint32_t count)
 {
-	env_atomic *progress = &cache->cleaning[part_id];
+	struct ocf_refcnt *counter = &cache->refcnt.cleaning[part_id];
 	struct ocf_user_part *part = &cache->user_parts[part_id];
-
-	if (ocf_mngt_is_cache_locked(cache))
-		return;
-
-	if (env_atomic_cmpxchg(progress, 0, 1) == 0) {
-		/* Initialize attributes for cleaner */
 	struct ocf_cleaner_attribs attribs = {
 		.cache_line_lock = true,
 		.do_sort = true,
 
-			.cmpl_context = progress,
+		.cmpl_context = counter,
 		.cmpl_fn = evp_lru_clean_end,
 
 		.getter = evp_lru_clean_getter,
@@ -332,9 +326,23 @@ static void evp_lru_clean(ocf_cache_t cache, ocf_queue_t io_queue,
 
 		.io_queue = io_queue
 	};
+	int cnt;
+
+	if (ocf_mngt_is_cache_locked(cache))
+		return;
+
+	cnt = ocf_refcnt_inc(counter);
+	if (!cnt) {
+		/* cleaner disabled by mngmt operation */
+		return;
+	}
+	if (cnt > 1) {
+		/* cleaning already running for this partition */
+		ocf_refcnt_dec(counter);
+		return;
+	}
 
 	ocf_cleaner_fire(cache, &attribs);
-	}
 }
 
 static void evp_lru_zero_line_complete(struct ocf_request *ocf_req, int error)

src/mngt/ocf_mngt_core.c

@@ -560,32 +560,14 @@ err_pipeline:
 	OCF_CMPL_RET(cache, NULL, priv, result);
 }
 
-/*
- * Synchronously wait until cleaning triggered by eviction finishes.
- * TODO: Replace it with asynchronous mechanism.
- */
-static int _ocf_cleaning_wait_for_finish(ocf_cache_t cache, int32_t timeout_ms)
-{
-	if (!ocf_cache_is_device_attached(cache))
-		return 0;
-
-	while (ocf_cache_has_pending_cleaning(cache)) {
-		env_msleep(20);
-
-		timeout_ms -= 20;
-		if (timeout_ms <= 0)
-			return -EBUSY;
-	}
-
-	return 0;
-}
-
 struct ocf_mngt_cache_remove_core_context {
 	ocf_mngt_cache_remove_core_end_t cmpl;
 	void *priv;
 	ocf_pipeline_t pipeline;
 	ocf_cache_t cache;
+	ocf_core_t core;
 	const char *core_name;
+	struct ocf_cleaner_wait_context cleaner_wait;
 };
 
 static void ocf_mngt_cache_remove_core_finish(ocf_pipeline_t pipeline,
@@ -602,19 +584,13 @@ static void ocf_mngt_cache_remove_core_finish(ocf_pipeline_t pipeline,
 				context->core_name);
 	}
 
+	ocf_cleaner_refcnt_unfreeze(cache);
+
 	context->cmpl(context->priv, error);
 
 	ocf_pipeline_destroy(context->pipeline);
 }
 
-struct ocf_pipeline_properties ocf_mngt_cache_remove_core_pipeline_props = {
-	.priv_size = sizeof(struct ocf_mngt_cache_remove_core_context),
-	.finish = ocf_mngt_cache_remove_core_finish,
-	.steps = {
-		OCF_PL_STEP_TERMINATOR(),
-	},
-};
-
 static void ocf_mngt_cache_remove_core_flush_sb_complete(void *priv, int error)
 {
 	struct ocf_mngt_cache_remove_core_context *context = priv;
@@ -623,39 +599,13 @@ static void ocf_mngt_cache_remove_core_flush_sb_complete(void *priv, int error)
 			error ? -OCF_ERR_WRITE_CACHE : 0);
 }
 
-void ocf_mngt_cache_remove_core(ocf_core_t core,
+static void _ocf_mngt_cache_remove_core(ocf_pipeline_t pipeline, void *priv,
-		ocf_mngt_cache_remove_core_end_t cmpl, void *priv)
+		ocf_pipeline_arg_t arg)
 {
-	struct ocf_mngt_cache_remove_core_context *context;
+	struct ocf_mngt_cache_remove_core_context *context = priv;
-	ocf_pipeline_t pipeline;
+	ocf_cache_t cache = context->cache;
-	ocf_cache_t cache;
+	ocf_core_t core = context->core;
-	ocf_core_id_t core_id;
+	ocf_core_id_t core_id = ocf_core_get_id(core);
-	int result;
-
-	OCF_CHECK_NULL(core);
-
-	cache = ocf_core_get_cache(core);
-	core_id = ocf_core_get_id(core);
-
-	if (!cache->mngt_queue)
-		OCF_CMPL_RET(cache, -OCF_ERR_INVAL);
-
-	/* TODO: Make this asynchronous */
-	if (_ocf_cleaning_wait_for_finish(cache, 60 * 1000))
-		OCF_CMPL_RET(priv, -OCF_ERR_CACHE_IN_USE);
-
-	result = ocf_pipeline_create(&pipeline, cache,
-			&ocf_mngt_cache_remove_core_pipeline_props);
-	if (result)
-		OCF_CMPL_RET(priv, result);
-
-	context = ocf_pipeline_get_priv(pipeline);
-
-	context->cmpl = cmpl;
-	context->priv = priv;
-	context->pipeline = pipeline;
-	context->cache = cache;
-	context->core_name = ocf_core_get_name(core);
 
 	ocf_core_log(core, log_debug, "Removing core\n");
 
@@ -675,53 +625,183 @@ void ocf_mngt_cache_remove_core(ocf_core_t core,
 			ocf_mngt_cache_remove_core_flush_sb_complete, context);
 }
 
-static int _ocf_mngt_cache_detach_core(ocf_core_t core)
+static void ocf_mngt_cache_remove_core_wait_cleaning_complete(void *priv)
 {
-	struct ocf_cache *cache = core->volume.cache;
+	ocf_pipeline_t pipeline = priv;
-	ocf_core_id_t core_id = ocf_core_get_id(core);
+	ocf_pipeline_next(pipeline);
-	int status;
-
-	status = cache_mng_core_close(cache, core_id);
-	if (!status) {
-		cache->ocf_core_inactive_count++;
-		env_bit_set(ocf_cache_state_incomplete,
-				&cache->cache_state);
-	}
-
-	return status;
 }
 
-void ocf_mngt_cache_detach_core(ocf_core_t core,
+static void ocf_mngt_cache_remove_core_wait_cleaning(ocf_pipeline_t pipeline,
-		ocf_mngt_cache_detach_core_end_t cmpl, void *priv)
+		void *priv, ocf_pipeline_arg_t arg)
 {
+	struct ocf_mngt_cache_remove_core_context *context = priv;
+	ocf_cache_t cache = context->cache;
+
+	if (!ocf_cache_is_device_attached(cache))
+		OCF_PL_NEXT_RET(pipeline);
+
+	ocf_cleaner_refcnt_freeze(cache);
+	ocf_cleaner_refcnt_register_zero_cb(cache, &context->cleaner_wait,
+			ocf_mngt_cache_remove_core_wait_cleaning_complete,
+			pipeline);
+}
+
+struct ocf_pipeline_properties ocf_mngt_cache_remove_core_pipeline_props = {
+	.priv_size = sizeof(struct ocf_mngt_cache_remove_core_context),
+	.finish = ocf_mngt_cache_remove_core_finish,
+	.steps = {
+		OCF_PL_STEP(ocf_mngt_cache_remove_core_wait_cleaning),
+		OCF_PL_STEP(_ocf_mngt_cache_remove_core),
+		OCF_PL_STEP_TERMINATOR(),
+	},
+};
+
+void ocf_mngt_cache_remove_core(ocf_core_t core,
+		ocf_mngt_cache_remove_core_end_t cmpl, void *priv)
+{
+	struct ocf_mngt_cache_remove_core_context *context;
+	ocf_pipeline_t pipeline;
 	ocf_cache_t cache;
-	const char *core_name;
 	int result;
 
 	OCF_CHECK_NULL(core);
 
 	cache = ocf_core_get_cache(core);
-	core_name = ocf_core_get_name(core);
 
 	if (!cache->mngt_queue)
 		OCF_CMPL_RET(cache, -OCF_ERR_INVAL);
 
-	/* TODO: Make this asynchronous */
+	result = ocf_pipeline_create(&pipeline, cache,
-	if (_ocf_cleaning_wait_for_finish(cache, 60 * 1000))
+			&ocf_mngt_cache_remove_core_pipeline_props);
-		OCF_CMPL_RET(priv, -OCF_ERR_CACHE_IN_USE);
+	if (result)
+		OCF_CMPL_RET(priv, result);
+
+	context = ocf_pipeline_get_priv(pipeline);
+
+	context->cmpl = cmpl;
+	context->priv = priv;
+	context->pipeline = pipeline;
+	context->cache = cache;
+	context->core = core;
+	context->core_name = ocf_core_get_name(core);
+
+	ocf_pipeline_next(pipeline);
+}
+
+struct ocf_mngt_cache_detach_core_context {
+	ocf_mngt_cache_detach_core_end_t cmpl;
+	void *priv;
+	ocf_pipeline_t pipeline;
+	ocf_cache_t cache;
+	ocf_core_t core;
+	const char *core_name;
+	struct ocf_cleaner_wait_context cleaner_wait;
+};
+
+static void _ocf_mngt_cache_detach_core(ocf_pipeline_t pipeline,
+		void *priv, ocf_pipeline_arg_t arg)
+{
+	struct ocf_mngt_cache_remove_core_context *context = priv;
+	ocf_cache_t cache = context->cache;
+	ocf_core_t core = context->core;
+	ocf_core_id_t core_id = ocf_core_get_id(core);
+	int status;
 
 	ocf_core_log(core, log_debug, "Detaching core\n");
 
-	result = _ocf_mngt_cache_detach_core(core);
+	status = cache_mng_core_close(cache, core_id);
-	if (!result) {
+
+	if (status)
+		OCF_PL_FINISH_RET(pipeline, status);
+
+	cache->ocf_core_inactive_count++;
+	env_bit_set(ocf_cache_state_incomplete,
+			&cache->cache_state);
+	ocf_pipeline_next(pipeline);
+}
+
+static void ocf_mngt_cache_detach_core_finish(ocf_pipeline_t pipeline,
+		void *priv, int error)
+{
+	struct ocf_mngt_cache_remove_core_context *context = priv;
+	ocf_cache_t cache = context->cache;
+
+	if (!error) {
 		ocf_cache_log(cache, log_info, "Core %s successfully detached\n",
-				core_name);
+				context->core_name);
 	} else {
 		ocf_cache_log(cache, log_err, "Detaching core %s failed\n",
-				core_name);
+				context->core_name);
 	}
 
+	ocf_cleaner_refcnt_unfreeze(context->cache);
+
+	context->cmpl(context->priv, error);
+
+	ocf_pipeline_destroy(context->pipeline);
+}
+
+static void ocf_mngt_cache_detach_core_wait_cleaning_complete(void *priv)
+{
+	ocf_pipeline_t pipeline = priv;
+	ocf_pipeline_next(pipeline);
+}
+
+static void ocf_mngt_cache_detach_core_wait_cleaning(ocf_pipeline_t pipeline,
+		void *priv, ocf_pipeline_arg_t arg)
+{
+	struct ocf_mngt_cache_remove_core_context *context = priv;
+	ocf_cache_t cache = context->cache;
+
+	if (!ocf_cache_is_device_attached(cache))
+		OCF_PL_NEXT_RET(pipeline);
+
+	ocf_cleaner_refcnt_freeze(cache);
+	ocf_cleaner_refcnt_register_zero_cb(cache, &context->cleaner_wait,
+			ocf_mngt_cache_detach_core_wait_cleaning_complete,
+			pipeline);
+}
+
+struct ocf_pipeline_properties ocf_mngt_cache_detach_core_pipeline_props = {
+	.priv_size = sizeof(struct ocf_mngt_cache_detach_core_context),
+	.finish = ocf_mngt_cache_detach_core_finish,
+	.steps = {
+		OCF_PL_STEP(ocf_mngt_cache_detach_core_wait_cleaning),
+		OCF_PL_STEP(_ocf_mngt_cache_detach_core),
+		OCF_PL_STEP_TERMINATOR(),
+	},
+};
+
+void ocf_mngt_cache_detach_core(ocf_core_t core,
+		ocf_mngt_cache_detach_core_end_t cmpl, void *priv)
+{
+	struct ocf_mngt_cache_detach_core_context *context;
+	ocf_pipeline_t pipeline;
+	ocf_cache_t cache;
+	int result;
+
+	OCF_CHECK_NULL(core);
+
+	cache = ocf_core_get_cache(core);
+
+	if (!cache->mngt_queue)
+		OCF_CMPL_RET(cache, -OCF_ERR_INVAL);
+
+	result = ocf_pipeline_create(&pipeline, cache,
+			&ocf_mngt_cache_detach_core_pipeline_props);
+	if (result)
 		OCF_CMPL_RET(priv, result);
+
+	context = ocf_pipeline_get_priv(pipeline);
+
+	context->cmpl = cmpl;
+	context->priv = priv;
+	context->pipeline = pipeline;
+	context->cache = ocf_core_get_cache(core);
+	context->core = core;
+	context->core_name = ocf_core_get_name(core);
+
+	ocf_pipeline_next(pipeline);
 }
 
 int ocf_mngt_core_set_uuid(ocf_core_t core, const struct ocf_volume_uuid *uuid)

src/ocf_cache.c

@@ -53,34 +53,6 @@ bool ocf_cache_is_device_attached(ocf_cache_t cache)
 	return !ocf_refcnt_frozen(&cache->refcnt.metadata);
 }
 
-/*
- * This is temporary workaround allowing to check if cleaning triggered
- * by eviction policy is running on the cache. This information is needed
- * to remove core from cache properly.
- *
- * TODO: Replace this with asynchronous notification to which remove/detach
- * core pipelines can subscribe.
- */
-bool ocf_cache_has_pending_cleaning(ocf_cache_t cache)
-{
-	struct ocf_user_part *curr_part;
-	ocf_part_id_t part_id;
-	bool cleaning_active = false;
-
-	OCF_CHECK_NULL(cache);
-
-	OCF_METADATA_LOCK_RD();
-	for_each_part(cache, curr_part, part_id) {
-		if (env_atomic_read(&cache->cleaning[part_id])) {
-			cleaning_active = true;
-			break;
-		}
-	}
-	OCF_METADATA_UNLOCK_RD();
-
-	return cleaning_active;
-}
-
 ocf_cache_mode_t ocf_cache_get_mode(ocf_cache_t cache)
 {
 	OCF_CHECK_NULL(cache);

src/ocf_cache_priv.h

@@ -171,6 +171,7 @@ struct ocf_cache {
 	struct {
 		struct ocf_refcnt dirty;
 		struct ocf_refcnt metadata;
+		struct ocf_refcnt cleaning[OCF_IO_CLASS_MAX];
 	} refcnt;
 
 	uint32_t fallback_pt_error_threshold;
@@ -193,8 +194,6 @@ struct ocf_cache {
 
 	env_atomic flush_in_progress;
 
-	env_atomic cleaning[OCF_IO_CLASS_MAX];
-
 	struct ocf_cleaner cleaner;
 	struct ocf_metadata_updater metadata_updater;
 

src/utils/utils_cleaner.c

@@ -8,6 +8,7 @@
 #include "../engine/engine_common.h"
 #include "../concurrency/ocf_concurrency.h"
 #include "utils_cleaner.h"
+#include "utils_part.h"
 #include "utils_req.h"
 #include "utils_io.h"
 #include "utils_cache_line.h"
@@ -1014,3 +1015,49 @@ void ocf_cleaner_sort_flush_containers(struct flush_container *fctbl,
 				_ocf_cleaner_swap);
 	}
 }
+
+void ocf_cleaner_refcnt_freeze(ocf_cache_t cache)
+{
+	struct ocf_user_part *curr_part;
+	ocf_part_id_t part_id;
+
+	for_each_part(cache, curr_part, part_id)
+		ocf_refcnt_freeze(&cache->refcnt.cleaning[part_id]);
+}
+
+void ocf_cleaner_refcnt_unfreeze(ocf_cache_t cache)
+{
+	struct ocf_user_part *curr_part;
+	ocf_part_id_t part_id;
+
+	for_each_part(cache, curr_part, part_id)
+		ocf_refcnt_unfreeze(&cache->refcnt.cleaning[part_id]);
+}
+
+static void ocf_cleaner_refcnt_register_zero_cb_finish(void *priv)
+{
+	struct ocf_cleaner_wait_context *ctx = priv;
+
+	if (!env_atomic_dec_return(&ctx->waiting))
+		ctx->cb(ctx->priv);
+}
+
+void ocf_cleaner_refcnt_register_zero_cb(ocf_cache_t cache,
+		struct ocf_cleaner_wait_context *ctx,
+		ocf_cleaner_refcnt_zero_cb_t cb, void *priv)
+{
+	struct ocf_user_part *curr_part;
+	ocf_part_id_t part_id;
+
+	env_atomic_set(&ctx->waiting, 1);
+	ctx->cb = cb;
+	ctx->priv = priv;
+
+	for_each_part(cache, curr_part, part_id) {
+		env_atomic_inc(&ctx->waiting);
+		ocf_refcnt_register_zero_cb(&cache->refcnt.cleaning[part_id],
+				ocf_cleaner_refcnt_register_zero_cb_finish, ctx);
+	}
+
+	ocf_cleaner_refcnt_register_zero_cb_finish(ctx);
+}

src/utils/utils_cleaner.h

@@ -79,6 +79,18 @@ struct flush_container {
 	struct ocf_mngt_cache_flush_context *context;
 };
 
+typedef void (*ocf_cleaner_refcnt_zero_cb_t)(void *priv);
+
+/**
+ * @brief Context for ocf_cleaner_refcnt_register_zero_cb
+ */
+struct ocf_cleaner_wait_context
+{
+	env_atomic waiting;
+	ocf_cleaner_refcnt_zero_cb_t cb;
+	void *priv;
+};
+
 /**
  * @brief Run cleaning procedure
  *
@@ -119,4 +131,30 @@ void ocf_cleaner_sort_sectors(struct flush_data *tbl, uint32_t num);
 void ocf_cleaner_sort_flush_containers(struct flush_container *fctbl,
 		uint32_t num);
 
+/**
+ * @brief Disable incrementing of cleaner reference counters
+ *
+ * @param cache - Cache instance
+ */
+void ocf_cleaner_refcnt_freeze(ocf_cache_t cache);
+
+/**
+ * @brief Enable incrementing of cleaner reference counters
+ *
+ * @param cache - Cache instance
+ */
+void ocf_cleaner_refcnt_unfreeze(ocf_cache_t cache);
+
+/**
+ * @brief Register callback for cleaner reference counters dropping to 0
+ *
+ * @param cache - Cache instance
+ * @param ctx - Routine private context, allocated by caller to avoid ENOMEM
+ * @param cb - Caller callback
+ * @param priv - Caller callback private data
+ */
+void ocf_cleaner_refcnt_register_zero_cb(ocf_cache_t cache,
+		struct ocf_cleaner_wait_context *ctx,
+		ocf_cleaner_refcnt_zero_cb_t cb, void *priv);
+
 #endif /* UTILS_CLEANER_H_ */