Use ref counter to track attach state

Signed-off-by: Adam Rutkowski <adam.j.rutkowski@intel.com>

src/engine/cache_engine.c

@@ -158,7 +158,7 @@ bool ocf_fallback_pt_is_on(ocf_cache_t cache)
 
 static inline bool ocf_seq_cutoff_is_on(ocf_cache_t cache)
 {
-	if (!env_atomic_read(&cache->attached))
+	if (!ocf_cache_is_device_attached(cache))
 		return false;
 
 	return (cache->device->freelist_part->curr_size <= SEQ_CUTOFF_FULL_MARGIN);

src/mngt/ocf_mngt_cache.c

@@ -562,6 +562,9 @@ static int _ocf_mngt_init_new_cache(struct ocf_cachemng_init_params *params)
 	env_atomic_set(&cache->ref_count, 1);
 	cache->owner = params->ctx;
 
+	/* start with freezed metadata ref counter to indicate detached device*/
+	ocf_refcnt_freeze(&cache->refcnt.metadata);
+
 	/* Copy all required initialization parameters */
 	cache->cache_id = params->id;
 
@@ -1511,9 +1514,7 @@ static void _ocf_mngt_attach_post_init(ocf_pipeline_t pipeline,
 		context->flags.cleaner_started = true;
 	}
 
-	env_waitqueue_init(&cache->pending_cache_wq);
-
-	env_atomic_set(&cache->attached, 1);
+	ocf_refcnt_unfreeze(&cache->refcnt.metadata);
 
 	ocf_pipeline_next(context->pipeline);
 }
@@ -1769,7 +1770,6 @@ static void _ocf_mngt_cache_unplug_complete(void *priv, int error)
 
 	env_vfree(cache->device);
 	cache->device = NULL;
-	env_atomic_set(&cache->attached, 0);
 
 	/* TODO: this should be removed from detach after 'attached' stats
 		are better separated in statistics */
@@ -1960,7 +1960,7 @@ static void ocf_mngt_cache_stop_unplug(ocf_pipeline_t pipeline,
 	struct ocf_mngt_cache_stop_context *context = priv;
 	ocf_cache_t cache = context->cache;
 
-	if (!env_atomic_read(&cache->attached))
+	if (!ocf_cache_is_device_attached(cache))
 		OCF_PL_NEXT_RET(pipeline);
 
 	_ocf_mngt_cache_unplug(cache, true, &context->unplug_context,
@@ -2295,19 +2295,21 @@ static void ocf_mngt_cache_detach_flush(ocf_pipeline_t pipeline,
 			context);
 }
 
+static void ocf_mngt_cache_detach_wait_pending_finish(void *priv)
+{
+	struct ocf_mngt_cache_detach_context *context = priv;
+	ocf_pipeline_next(context->pipeline);
+}
+
 static void ocf_mngt_cache_detach_wait_pending(ocf_pipeline_t pipeline,
 		void *priv, ocf_pipeline_arg_t arg)
 {
 	struct ocf_mngt_cache_detach_context *context = priv;
 	ocf_cache_t cache = context->cache;
 
-	env_atomic_set(&cache->attached, 0);
-
-	/* FIXME: This should be asynchronous! */
-	env_waitqueue_wait(cache->pending_cache_wq,
-			!env_atomic_read(&cache->pending_cache_requests));
-
-	ocf_pipeline_next(context->pipeline);
+	ocf_refcnt_freeze(&cache->refcnt.metadata);
+	ocf_refcnt_register_zero_cb(&cache->refcnt.metadata,
+			ocf_mngt_cache_detach_wait_pending_finish, context);
 }
 
 static void ocf_mngt_cache_detach_update_metadata(ocf_pipeline_t pipeline,
@@ -2406,7 +2408,7 @@ void ocf_mngt_cache_detach(ocf_cache_t cache,
 	if (!cache->mngt_queue)
 		OCF_CMPL_RET(cache, priv, -OCF_ERR_INVAL);
 
-	if (!env_atomic_read(&cache->attached))
+	if (!ocf_cache_is_device_attached(cache))
 		OCF_CMPL_RET(cache, priv, -OCF_ERR_INVAL);
 
 	result = ocf_pipeline_create(&pipeline, cache,

src/ocf_cache.c

@@ -50,7 +50,7 @@ bool ocf_cache_is_running(ocf_cache_t cache)
 bool ocf_cache_is_device_attached(ocf_cache_t cache)
 {
 	OCF_CHECK_NULL(cache);
-	return env_atomic_read(&(cache)->attached);
+	return !ocf_refcnt_frozen(&cache->refcnt.metadata);
 }
 
 void ocf_cache_wait_for_io_finish(ocf_cache_t cache)

src/ocf_cache_priv.h

@@ -172,11 +172,9 @@ struct ocf_cache {
 
 	struct {
 		struct ocf_refcnt dirty;
+		struct ocf_refcnt metadata;
 	} refcnt;
 
-	env_atomic pending_cache_requests;
-	env_waitqueue pending_cache_wq;
-
 	uint32_t fallback_pt_error_threshold;
 	env_atomic fallback_pt_error_counter;
 
@@ -197,9 +195,6 @@ struct ocf_cache {
 
 	env_atomic flush_in_progress;
 
-	/* 1 if cache device attached, 0 otherwise */
-	env_atomic attached;
-
 	env_atomic cleaning[OCF_IO_CLASS_MAX];
 
 	struct ocf_cleaner cleaner;

src/utils/utils_refcnt.c

@@ -59,3 +59,8 @@ void ocf_refcnt_unfreeze(struct ocf_refcnt *rc)
 	int val = env_atomic_dec_return(&rc->freeze);
 	ENV_BUG_ON(val < 0);
 }
+
+bool ocf_refcnt_frozen(struct ocf_refcnt *rc)
+{
+	return !!env_atomic_read(&rc->freeze);
+}

src/utils/utils_refcnt.h

@@ -37,6 +37,8 @@ void ocf_refcnt_freeze(struct ocf_refcnt *rc);
 /* Cancel the effect of single ocf_refcnt_freeze call */
 void ocf_refcnt_unfreeze(struct ocf_refcnt *rc);
 
+bool ocf_refcnt_frozen(struct ocf_refcnt *rc);
+
 /* Register callback to be called when reference counter drops to 0.
  * Must be called after counter is freezed.
  * Cannot be called until previously regsitered callback had fired. */

src/utils/utils_req.c

@@ -151,24 +151,6 @@ static env_allocator *_ocf_req_get_allocator(
 	return ocf_ctx->resources.req->allocator[idx];
 }
 
-static void start_cache_req(struct ocf_request *req)
-{
-	ocf_cache_t cache = req->cache;
-
-	if (req->queue == req->cache->mngt_queue)
-		return;
-
-	req->d2c = 1;
-	if (env_atomic_read(&cache->attached)) {
-		req->d2c = 0;
-		env_atomic_inc(&cache->pending_cache_requests);
-		if (!env_atomic_read(&cache->attached)) {
-			req->d2c = 1;
-			env_atomic_dec(&cache->pending_cache_requests);
-		}
-	}
-}
-
 struct ocf_request *ocf_req_new(ocf_queue_t queue, ocf_core_t core,
 		uint64_t addr, uint32_t bytes, int rw)
 {
@@ -212,7 +194,8 @@ struct ocf_request *ocf_req_new(ocf_queue_t queue, ocf_core_t core,
 	if (queue != cache->mngt_queue)
 		env_atomic_inc(&cache->pending_requests);
 
-	start_cache_req(req);
+	req->d2c = (queue != cache->mngt_queue) && !ocf_refcnt_inc(
+			&cache->refcnt.metadata);
 
 	env_atomic_set(&req->ref_count, 1);
 
@@ -292,10 +275,8 @@ void ocf_req_put(struct ocf_request *req)
 
 	OCF_DEBUG_TRACE(req->cache);
 
-	if (!req->d2c && !env_atomic_dec_return(
-			&req->cache->pending_cache_requests)) {
-		env_waitqueue_wake_up(&req->cache->pending_cache_wq);
-	}
+	if (!req->d2c && req->io_queue != req->cache->mngt_queue)
+		ocf_refcnt_dec(&req->cache->refcnt.metadata);
 
 	if (req->io_queue != req->cache->mngt_queue)
 		env_atomic_dec(&req->cache->pending_requests);