Introduce asynchronous cache lock

Signed-off-by: Robert Baldyga <robert.baldyga@intel.com>

src/cleaning/cleaning.c

@@ -111,7 +111,7 @@ static void ocf_cleaner_run_complete(ocf_cleaner_t cleaner, uint32_t interval)
 {
 	ocf_cache_t cache = ocf_cleaner_get_cache(cleaner);
 
-	env_rwsem_up_write(&cache->lock);
+	ocf_async_unlock(&cache->lock);
 	cleaner->end(cleaner, interval);
 
 	ocf_queue_put(cleaner->io_queue);
@@ -131,19 +131,19 @@ void ocf_cleaner_run(ocf_cleaner_t cleaner, ocf_queue_t queue)
 	 * (error, etc.).
 	 */
 	if (!env_bit_test(ocf_cache_state_running, &cache->cache_state) ||
-			ocf_mngt_is_cache_locked(cache)) {
+			ocf_mngt_cache_is_locked(cache)) {
 		cleaner->end(cleaner, SLEEP_TIME_MS);
 		return;
 	}
 
 	/* Sleep in case there is management operation in progress. */
-	if (env_rwsem_down_write_trylock(&cache->lock)) {
+	if (ocf_mngt_cache_trylock(cache)) {
 		cleaner->end(cleaner, SLEEP_TIME_MS);
 		return;
 	}
 
 	if (_ocf_cleaner_run_check_dirty_inactive(cache)) {
-		env_rwsem_up_write(&cache->lock);
+		ocf_mngt_cache_unlock(cache);
 		cleaner->end(cleaner, SLEEP_TIME_MS);
 		return;
 	}

src/eviction/lru.c

@@ -328,7 +328,7 @@ static void evp_lru_clean(ocf_cache_t cache, ocf_queue_t io_queue,
 	};
 	int cnt;
 
-	if (ocf_mngt_is_cache_locked(cache))
+	if (ocf_mngt_cache_is_locked(cache))
 		return;
 
 	cnt = ocf_refcnt_inc(counter);

src/mngt/ocf_mngt_cache.c

@@ -17,6 +17,7 @@
 #include "../utils/utils_cache_line.h"
 #include "../utils/utils_pipeline.h"
 #include "../utils/utils_refcnt.h"
+#include "../utils/utils_async_lock.h"
 #include "../concurrency/ocf_concurrency.h"
 #include "../eviction/ops.h"
 #include "../ocf_ctx_priv.h"
@@ -551,8 +552,15 @@ static int _ocf_mngt_init_new_cache(struct ocf_cachemng_init_params *params)
 	if (!cache)
 		return -OCF_ERR_NO_MEM;
 
-	if (env_rwsem_init(&cache->lock) ||
-			env_mutex_init(&cache->flush_mutex)) {
+	if (ocf_mngt_cache_lock_init(cache)) {
+		env_vfree(cache);
+		return -OCF_ERR_NO_MEM;
+	}
+
+	/* Lock cache during setup - this trylock should always succeed */
+	ENV_BUG_ON(ocf_mngt_cache_trylock(cache));
+
+	if (env_mutex_init(&cache->flush_mutex)) {
 		env_vfree(cache);
 		return -OCF_ERR_NO_MEM;
 	}
@@ -694,7 +702,6 @@ static int _ocf_mngt_init_prepare_cache(struct ocf_cachemng_init_params *param,
 	cache->backfill.max_queue_size = cfg->backfill.max_queue_size;
 	cache->backfill.queue_unblock_size = cfg->backfill.queue_unblock_size;
 
-	env_rwsem_down_write(&cache->lock); /* Lock cache during setup */
 	param->flags.cache_locked = true;
 
 	cache->pt_unaligned_io = cfg->pt_unaligned_io;
@@ -1216,17 +1223,11 @@ static int _ocf_mngt_cache_start(ocf_ctx_t ctx, ocf_cache_t *cache,
 
 	ocf_ctx_get(ctx);
 
-	if (params.locked) {
-		/* Increment reference counter to match cache_lock /
-		   cache_unlock convention. User is expected to call
-		   ocf_mngt_cache_unlock in future which would up the
-		   semaphore as well as decrement ref_count. */
-		ocf_refcnt_inc(&(*cache)->refcnt.cache);
-	} else {
+	if (!params.locked) {
 		/* User did not request to lock cache instance after creation -
-		   up the semaphore here since we have acquired the lock to
+		   unlock it here since we have acquired the lock to
 		   perform management operations. */
-		env_rwsem_up_write(&(*cache)->lock);
+		ocf_mngt_cache_unlock(*cache);
 		params.flags.cache_locked = false;
 	}
 

src/mngt/ocf_mngt_common.c

@@ -126,6 +126,7 @@ void ocf_mngt_cache_put(ocf_cache_t cache)
 	if (ocf_refcnt_dec(&cache->refcnt.cache) == 0) {
 		ctx = cache->owner;
 		ocf_metadata_deinit(cache);
+		ocf_mngt_cache_lock_deinit(cache);
 		env_vfree(cache);
 		ocf_ctx_put(ctx);
 	}
@@ -175,94 +176,166 @@ int ocf_mngt_cache_get_by_id(ocf_ctx_t ocf_ctx, ocf_cache_id_t id, ocf_cache_t *
 	return error;
 }
 
-bool ocf_mngt_is_cache_locked(ocf_cache_t cache)
+typedef void (*ocf_lock_fn_t)(ocf_async_lock_waiter_t waiter);
+
+typedef int (*ocf_trylock_fn_t)(ocf_async_lock_t lock);
+
+typedef void (*ocf_unlock_fn_t)(ocf_async_lock_t lock);
+
+struct ocf_mngt_cache_lock_context {
+	ocf_cache_t cache;
+	ocf_unlock_fn_t unlock_fn;
+	ocf_mngt_cache_lock_end_t cmpl;
+	void *priv;
+};
+
+static void _ocf_mngt_cache_lock_complete(
+		ocf_async_lock_waiter_t waiter, int error)
 {
-	if (env_rwsem_is_locked(&cache->lock))
-		return true;
+	struct ocf_mngt_cache_lock_context *context;
+	ocf_cache_t cache;
 
-	if (env_atomic_read(&cache->lock_waiter))
-		return true;
+	context = ocf_async_lock_waiter_get_priv(waiter);
+	cache = context->cache;
 
-	return false;
+	if (error) {
+		ocf_mngt_cache_put(cache);
+		goto out;
+	}
+
+	if (env_bit_test(ocf_cache_state_stopping, &cache->cache_state)) {
+		/* Cache already stopping, do not allow any operation */
+		context->unlock_fn(ocf_async_lock_waiter_get_lock(waiter));
+		ocf_mngt_cache_put(cache);
+		error = -OCF_ERR_CACHE_NOT_EXIST;
+	}
+
+out:
+	context->cmpl(context->cache, context->priv, error);
+}
+
+static void _ocf_mngt_cache_lock(ocf_cache_t cache,
+		ocf_mngt_cache_lock_end_t cmpl, void *priv,
+		ocf_lock_fn_t lock_fn, ocf_unlock_fn_t unlock_fn)
+{
+	ocf_async_lock_waiter_t waiter;
+	struct ocf_mngt_cache_lock_context *context;
+
+	if (ocf_mngt_cache_get(cache))
+		OCF_CMPL_RET(cache, priv, -OCF_ERR_CACHE_NOT_EXIST);
+
+	waiter = ocf_async_lock_new_waiter(&cache->lock,
+			_ocf_mngt_cache_lock_complete);
+	if (!waiter) {
+		ocf_mngt_cache_put(cache);
+		OCF_CMPL_RET(cache, priv, -OCF_ERR_NO_MEM);
+	}
+
+	context = ocf_async_lock_waiter_get_priv(waiter);
+	context->cache = cache;
+	context->unlock_fn = unlock_fn;
+	context->cmpl = cmpl;
+	context->priv = priv;
+
+	lock_fn(waiter);
+}
+
+static int _ocf_mngt_cache_trylock(ocf_cache_t cache,
+		ocf_trylock_fn_t trylock_fn, ocf_unlock_fn_t unlock_fn)
+{
+	int result;
+
+	if (ocf_mngt_cache_get(cache))
+		return -OCF_ERR_CACHE_NOT_EXIST;
+
+	result = trylock_fn(&cache->lock);
+	if (result)
+		return result;
+
+	if (env_bit_test(ocf_cache_state_stopping, &cache->cache_state)) {
+		/* Cache already stopping, do not allow any operation */
+		unlock_fn(&cache->lock);
+		return -OCF_ERR_CACHE_NOT_EXIST;
+	}
+
+	return 0;
 }
 
 static void _ocf_mngt_cache_unlock(ocf_cache_t cache,
-		void (*unlock_fn)(env_rwsem *s))
+		ocf_unlock_fn_t unlock_fn)
 {
 	unlock_fn(&cache->lock);
 	ocf_mngt_cache_put(cache);
 }
 
-void ocf_mngt_cache_unlock(ocf_cache_t cache)
+int ocf_mngt_cache_lock_init(ocf_cache_t cache)
 {
-	OCF_CHECK_NULL(cache);
-	_ocf_mngt_cache_unlock(cache, env_rwsem_up_write);
-}
+	int result;
 
-void ocf_mngt_cache_read_unlock(ocf_cache_t cache)
-{
-	OCF_CHECK_NULL(cache);
-	_ocf_mngt_cache_unlock(cache, env_rwsem_up_read);
-}
-
-static int _ocf_mngt_cache_lock(ocf_cache_t cache, int (*lock_fn)(env_rwsem *s),
-		void (*unlock_fn)(env_rwsem *s))
-{
-	int ret;
-
-	/* Increment reference counter */
-	if (!ocf_refcnt_inc(&cache->refcnt.cache))
-		return -OCF_ERR_CACHE_NOT_EXIST;
-
-	env_atomic_inc(&cache->lock_waiter);
-	ret = lock_fn(&cache->lock);
-	env_atomic_dec(&cache->lock_waiter);
-
-	if (ret) {
-		ocf_mngt_cache_put(cache);
-		return ret;
-	}
-
-	if (env_bit_test(ocf_cache_state_stopping, &cache->cache_state)) {
-		/* Cache already stooping, do not allow any operation */
-		ret = -OCF_ERR_CACHE_NOT_EXIST;
-		goto unlock;
-	}
+	result = ocf_async_lock_init(&cache->lock,
+			sizeof(struct ocf_mngt_cache_lock_context));
+	if (result)
+		return result;
 
 	return 0;
-
-unlock:
-	_ocf_mngt_cache_unlock(cache, unlock_fn);
-
-	return ret;
 }
 
-int ocf_mngt_cache_lock(ocf_cache_t cache)
+void ocf_mngt_cache_lock_deinit(ocf_cache_t cache)
 {
-	OCF_CHECK_NULL(cache);
-	return _ocf_mngt_cache_lock(cache, env_rwsem_down_write_interruptible,
-			env_rwsem_up_write);
+	ocf_async_lock_deinit(&cache->lock);
 }
 
-int ocf_mngt_cache_read_lock(ocf_cache_t cache)
+void ocf_mngt_cache_lock(ocf_cache_t cache,
+		ocf_mngt_cache_lock_end_t cmpl, void *priv)
 {
 	OCF_CHECK_NULL(cache);
-	return _ocf_mngt_cache_lock(cache, env_rwsem_down_read_interruptible,
-			env_rwsem_up_read);
+
+	_ocf_mngt_cache_lock(cache, cmpl, priv,
+			ocf_async_lock, ocf_async_unlock);
 }
 
 int ocf_mngt_cache_trylock(ocf_cache_t cache)
 {
 	OCF_CHECK_NULL(cache);
-	return _ocf_mngt_cache_lock(cache, env_rwsem_down_write_trylock,
-			env_rwsem_up_write);
+
+	return _ocf_mngt_cache_trylock(cache,
+			ocf_async_trylock, ocf_async_unlock);
+}
+
+void ocf_mngt_cache_unlock(ocf_cache_t cache)
+{
+	OCF_CHECK_NULL(cache);
+
+	_ocf_mngt_cache_unlock(cache, ocf_async_unlock);
+}
+
+void ocf_mngt_cache_read_lock(ocf_cache_t cache,
+		ocf_mngt_cache_lock_end_t cmpl, void *priv)
+{
+	OCF_CHECK_NULL(cache);
+
+	_ocf_mngt_cache_lock(cache, cmpl, priv,
+			ocf_async_read_lock, ocf_async_read_unlock);
 }
 
 int ocf_mngt_cache_read_trylock(ocf_cache_t cache)
 {
 	OCF_CHECK_NULL(cache);
-	return _ocf_mngt_cache_lock(cache, env_rwsem_down_read_trylock,
-			env_rwsem_up_read);
+
+	return _ocf_mngt_cache_trylock(cache,
+			ocf_async_read_trylock, ocf_async_read_unlock);
+}
+
+void ocf_mngt_cache_read_unlock(ocf_cache_t cache)
+{
+	OCF_CHECK_NULL(cache);
+
+	_ocf_mngt_cache_unlock(cache, ocf_async_read_unlock);
+}
+
+bool ocf_mngt_cache_is_locked(ocf_cache_t cache)
+{
+	return ocf_async_is_locked(&cache->lock);
 }
 
 /* if cache is either fully initialized or during recovery */
@@ -305,7 +378,6 @@ static int _ocf_mngt_cache_get_list_cpy(ocf_ctx_t ocf_ctx, ocf_cache_t **list,
 	}
 
 	list_for_each_entry(iter, &ocf_ctx->caches, list) {
-
 		if (_ocf_mngt_cache_try_get(iter))
 			(*list)[i++] = iter;
 	}

src/mngt/ocf_mngt_common.h

@@ -28,6 +28,9 @@ int ocf_mngt_add_partition_to_cache(struct ocf_cache *cache,
 		ocf_part_id_t part_id, const char *name, uint32_t min_size,
 		uint32_t max_size, uint8_t priority, bool valid);
 
-bool ocf_mngt_is_cache_locked(ocf_cache_t cache);
+int ocf_mngt_cache_lock_init(ocf_cache_t cache);
+void ocf_mngt_cache_lock_deinit(ocf_cache_t cache);
+
+bool ocf_mngt_cache_is_locked(ocf_cache_t cache);
 
 #endif /* __OCF_MNGT_COMMON_H__ */

src/ocf_cache_priv.h

@@ -15,6 +15,7 @@
 #include "metadata/metadata_updater_priv.h"
 #include "utils/utils_list.h"
 #include "utils/utils_refcnt.h"
+#include "utils/utils_async_lock.h"
 #include "ocf_stats_priv.h"
 #include "cleaning/cleaning.h"
 #include "ocf_logger_priv.h"
@@ -200,10 +201,11 @@ struct ocf_cache {
 	struct ocf_cleaner cleaner;
 	struct ocf_metadata_updater metadata_updater;
 
-	env_rwsem lock;
-	env_atomic lock_waiter;
-	/*!< most of the time this variable is set to 0, unless user requested
-	 *!< interruption of flushing process via ioctl/
+	struct ocf_async_lock lock;
+
+	/*
+	 * Most of the time this variable is set to 0, unless user requested
+	 * interruption of flushing process.
 	 */
 	int flushing_interrupted;
 	env_mutex flush_mutex;

src/utils/utils_async_lock.c

@@ -0,0 +1,238 @@
+/*
+ * Copyright(c) 2019 Intel Corporation
+ * SPDX-License-Identifier: BSD-3-Clause-Clear
+ */
+
+#include "utils_async_lock.h"
+
+struct ocf_async_lock_waiter {
+	struct list_head list;
+	ocf_async_lock_t lock;
+	bool write_lock;
+	ocf_async_lock_end_t cmpl;
+};
+
+void _ocf_async_lock_collect_waiters(ocf_async_lock_t lock,
+		struct list_head *waiters)
+{
+	ocf_async_lock_waiter_t iter, temp;
+
+	list_for_each_entry_safe(iter, temp, &lock->waiters, list) {
+		if (!iter->write_lock) {
+			list_move_tail(&iter->list, waiters);
+			lock->rd++;
+		} else {
+			if (!lock->rd) {
+				list_move_tail(&iter->list, waiters);
+				lock->wr = 1;
+			}
+			break;
+		}
+	}
+}
+
+void _ocf_async_lock_run_waiters(struct ocf_async_lock *lock,
+		struct list_head *waiters, int status)
+{
+	ocf_async_lock_waiter_t iter, temp;
+
+	/* TODO: Should we run waiters asynchronously? */
+
+	list_for_each_entry_safe(iter, temp, waiters, list) {
+		list_del(&iter->list);
+		iter->cmpl(iter, status);
+		env_vfree(iter);
+	}
+}
+
+int ocf_async_lock_init(struct ocf_async_lock *lock, uint32_t waiter_priv_size)
+{
+	int result;
+
+	result = env_mutex_init(&lock->mutex);
+	if (result)
+		return result;
+
+	INIT_LIST_HEAD(&lock->waiters);
+	lock->rd = 0;
+	lock->wr = 0;
+	lock->waiter_priv_size = waiter_priv_size;
+
+	return 0;
+}
+
+void ocf_async_lock_deinit(struct ocf_async_lock *lock)
+{
+	struct list_head waiters;
+	ocf_async_lock_waiter_t iter, temp;
+
+	INIT_LIST_HEAD(&waiters);
+
+	env_mutex_lock(&lock->mutex);
+	list_for_each_entry_safe(iter, temp, &lock->waiters, list)
+		list_move_tail(&iter->list, &waiters);
+	env_mutex_unlock(&lock->mutex);
+
+	_ocf_async_lock_run_waiters(lock, &waiters, -OCF_ERR_NO_LOCK);
+}
+
+ocf_async_lock_waiter_t ocf_async_lock_new_waiter(ocf_async_lock_t lock,
+		ocf_async_lock_end_t cmpl)
+{
+	ocf_async_lock_waiter_t waiter;
+
+	waiter = env_vmalloc(sizeof(*waiter) + lock->waiter_priv_size);
+	if (!waiter)
+		return NULL;
+
+	waiter->lock = lock;
+	waiter->cmpl = cmpl;
+
+	return waiter;
+}
+
+ocf_async_lock_t ocf_async_lock_waiter_get_lock(ocf_async_lock_waiter_t waiter)
+{
+	return waiter->lock;
+}
+
+void *ocf_async_lock_waiter_get_priv(ocf_async_lock_waiter_t waiter)
+{
+	return (void *)waiter + sizeof(*waiter);
+}
+
+static int _ocf_async_trylock(struct ocf_async_lock *lock)
+{
+	if (lock->wr || lock->rd)
+		return -OCF_ERR_NO_LOCK;
+
+	lock->wr = 1;
+	return 0;
+}
+
+void ocf_async_lock(ocf_async_lock_waiter_t waiter)
+{
+	ocf_async_lock_t lock = waiter->lock;
+	int result;
+
+	env_mutex_lock(&lock->mutex);
+
+	result = _ocf_async_trylock(lock);
+	if (!result) {
+		env_mutex_unlock(&lock->mutex);
+		waiter->cmpl(waiter, 0);
+		env_vfree(waiter);
+		return;
+	}
+
+	waiter->write_lock = true;
+	list_add_tail(&waiter->list, &lock->waiters);
+
+	env_mutex_unlock(&lock->mutex);
+}
+
+int ocf_async_trylock(struct ocf_async_lock *lock)
+{
+	int result;
+
+	env_mutex_lock(&lock->mutex);
+	result = _ocf_async_trylock(lock);
+	env_mutex_unlock(&lock->mutex);
+
+	return result;
+}
+
+void ocf_async_unlock(struct ocf_async_lock *lock)
+{
+	struct list_head waiters;
+
+	INIT_LIST_HEAD(&waiters);
+
+	env_mutex_lock(&lock->mutex);
+
+	ENV_BUG_ON(lock->rd);
+	ENV_BUG_ON(!lock->wr);
+
+	lock->wr = 0;
+
+	_ocf_async_lock_collect_waiters(lock, &waiters);
+
+	env_mutex_unlock(&lock->mutex);
+
+	_ocf_async_lock_run_waiters(lock, &waiters, 0);
+}
+
+static int _ocf_async_read_trylock(struct ocf_async_lock *lock)
+{
+	if (lock->wr || !list_empty(&lock->waiters))
+		return -OCF_ERR_NO_LOCK;
+
+	lock->rd++;
+	return 0;
+}
+
+void ocf_async_read_lock(ocf_async_lock_waiter_t waiter)
+{
+	ocf_async_lock_t lock = waiter->lock;
+	int result;
+
+	env_mutex_lock(&lock->mutex);
+
+	result = _ocf_async_read_trylock(lock);
+	if (!result) {
+		env_mutex_unlock(&lock->mutex);
+		waiter->cmpl(waiter, 0);
+		env_vfree(waiter);
+		return;
+	}
+
+	waiter->write_lock = false;
+	list_add_tail(&waiter->list, &lock->waiters);
+
+	env_mutex_unlock(&lock->mutex);
+}
+
+int ocf_async_read_trylock(struct ocf_async_lock *lock)
+{
+	int result;
+
+	env_mutex_lock(&lock->mutex);
+	result = _ocf_async_read_trylock(lock);
+	env_mutex_unlock(&lock->mutex);
+
+	return result;
+}
+
+void ocf_async_read_unlock(struct ocf_async_lock *lock)
+{
+	struct list_head waiters;
+
+	INIT_LIST_HEAD(&waiters);
+
+	env_mutex_lock(&lock->mutex);
+
+	ENV_BUG_ON(!lock->rd);
+	ENV_BUG_ON(lock->wr);
+
+	if (--lock->rd) {
+		env_mutex_unlock(&lock->mutex);
+		return;
+	}
+
+	_ocf_async_lock_collect_waiters(lock, &waiters);
+
+	env_mutex_unlock(&lock->mutex);
+
+	_ocf_async_lock_run_waiters(lock, &waiters, 0);
+}
+
+bool ocf_async_is_locked(struct ocf_async_lock *lock)
+{
+	bool locked;
+
+	env_mutex_lock(&lock->mutex);
+	locked = lock->rd || lock->wr;
+	env_mutex_unlock(&lock->mutex);
+
+	return locked;
+}

src/utils/utils_async_lock.h

@@ -0,0 +1,50 @@
+/*
+ * Copyright(c) 2019 Intel Corporation
+ * SPDX-License-Identifier: BSD-3-Clause-Clear
+ */
+
+#ifndef __UTILS_ASYNC_LOCK_H__
+#define __UTILS_ASYNC_LOCK_H__
+
+#include "ocf_env.h"
+
+struct ocf_async_lock {
+	struct list_head waiters;
+	env_mutex mutex;
+	uint32_t rd;
+	uint32_t wr;
+	uint32_t waiter_priv_size;
+};
+
+typedef struct ocf_async_lock *ocf_async_lock_t;
+
+typedef struct ocf_async_lock_waiter *ocf_async_lock_waiter_t;
+
+typedef void (*ocf_async_lock_end_t)(ocf_async_lock_waiter_t waiter, int error);
+
+int ocf_async_lock_init(ocf_async_lock_t lock, uint32_t waiter_priv_size);
+
+void ocf_async_lock_deinit(ocf_async_lock_t lock);
+
+ocf_async_lock_waiter_t ocf_async_lock_new_waiter(ocf_async_lock_t lock,
+		ocf_async_lock_end_t cmpl);
+
+ocf_async_lock_t ocf_async_lock_waiter_get_lock(ocf_async_lock_waiter_t waiter);
+
+void *ocf_async_lock_waiter_get_priv(ocf_async_lock_waiter_t waiter);
+
+void ocf_async_lock(ocf_async_lock_waiter_t waiter);
+
+int ocf_async_trylock(struct ocf_async_lock *lock);
+
+void ocf_async_unlock(struct ocf_async_lock *lock);
+
+void ocf_async_read_lock(ocf_async_lock_waiter_t waiter);
+
+int ocf_async_read_trylock(struct ocf_async_lock *lock);
+
+void ocf_async_read_unlock(struct ocf_async_lock *lock);
+
+bool ocf_async_is_locked(struct ocf_async_lock *lock);
+
+#endif /* __UTILS_ASYNC_LOCK_H__ */

src/utils/utils_pipeline.c

@@ -1,3 +1,8 @@
+/*
+ * Copyright(c) 2019 Intel Corporation
+ * SPDX-License-Identifier: BSD-3-Clause-Clear
+ */
+
 #include "ocf/ocf.h"
 #include "../engine/cache_engine.h"
 #include "../engine/engine_common.h"