Change alock API to include slow/fast lock callbacks

Signed-off-by: Kozlowski Mateusz <mateusz.kozlowski@intel.com>

src/concurrency/ocf_cache_line_concurrency.c

@@ -25,24 +25,6 @@ static bool ocf_cl_lock_line_is_acting(struct ocf_alock *alock,
 	return req->map[index].status != LOOKUP_MISS;
 }
 
-static bool ocf_cl_lock_line_is_locked(struct ocf_alock *alock,
-		struct ocf_request *req, unsigned index, int rw)
-{
-	if (rw == OCF_WRITE)
-		return req->map[index].wr_locked;
-	else
-		return req->map[index].rd_locked;
-}
-
-static void ocf_cl_lock_line_mark_locked(struct ocf_alock *alock,
-		struct ocf_request *req, unsigned index, int rw, bool locked)
-{
-	if (rw == OCF_WRITE)
-		req->map[index].wr_locked = locked;
-	else
-		req->map[index].rd_locked = locked;
-}
-
 static ocf_cache_line_t ocf_cl_lock_line_get_entry(
 		struct ocf_alock *alock, struct ocf_request *req,
 		unsigned index)
@@ -50,12 +32,118 @@ static ocf_cache_line_t ocf_cl_lock_line_get_entry(
 	return req->map[index].coll_idx;
 }
 
+static int ocf_cl_lock_line_fast(struct ocf_alock *alock,
+		struct ocf_request *req, int rw)
+{
+	int32_t i;
+	ocf_cache_line_t entry;
+	int ret = OCF_LOCK_ACQUIRED;
+
+	for (i = 0; i < req->core_line_count; i++) {
+		if (!ocf_cl_lock_line_needs_lock(alock, req, i)) {
+			/* nothing to lock */
+			continue;
+		}
+
+		entry = ocf_cl_lock_line_get_entry(alock, req, i);
+		ENV_BUG_ON(ocf_alock_is_index_locked(alock, req, i));
+
+		if (rw == OCF_WRITE) {
+			if (ocf_alock_trylock_entry_wr(alock, entry)) {
+				/* cache entry locked */
+				ocf_alock_mark_index_locked(alock, req, i, true);
+			} else {
+				/* Not possible to lock all cachelines */
+				ret = OCF_LOCK_NOT_ACQUIRED;
+				break;
+			}
+		} else {
+			if (ocf_alock_trylock_entry_rd_idle(alock, entry)) {
+				/* cache entry locked */
+				ocf_alock_mark_index_locked(alock, req, i, true);
+			} else {
+				/* Not possible to lock all cachelines */
+				ret = OCF_LOCK_NOT_ACQUIRED;
+				break;
+			}
+		}
+	}
+
+	/* Check if request is locked */
+	if (ret == OCF_LOCK_NOT_ACQUIRED) {
+		/* Request is not locked, discard acquired locks */
+		for (; i >= 0; i--) {
+			if (!ocf_cl_lock_line_needs_lock(alock, req, i))
+				continue;
+
+			entry = ocf_cl_lock_line_get_entry(alock, req, i);
+
+			if (ocf_alock_is_index_locked(alock, req, i)) {
+
+				if (rw == OCF_WRITE) {
+					ocf_alock_unlock_one_wr(alock, entry);
+				} else {
+					ocf_alock_unlock_one_rd(alock, entry);
+				}
+				ocf_alock_mark_index_locked(alock, req, i, false);
+			}
+		}
+	}
+
+	return ret;
+}
+
+static int ocf_cl_lock_line_slow(struct ocf_alock *alock,
+		struct ocf_request *req, int rw, ocf_req_async_lock_cb cmpl)
+{
+	int32_t i;
+	ocf_cache_line_t entry;
+	int ret = 0;
+
+	for (i = 0; i < req->core_line_count; i++) {
+
+		if (!ocf_cl_lock_line_needs_lock(alock, req, i)) {
+			/* nothing to lock */
+			env_atomic_dec(&req->lock_remaining);
+			continue;
+		}
+
+		entry = ocf_cl_lock_line_get_entry(alock, req, i);
+		ENV_BUG_ON(ocf_alock_is_index_locked(alock, req, i));
+
+
+		if (rw == OCF_WRITE) {
+			if (!ocf_alock_lock_one_wr(alock, entry, cmpl, req, i)) {
+				/* lock not acquired and not added to wait list */
+				ret = -OCF_ERR_NO_MEM;
+				goto err;
+			}
+		} else {
+			if (!ocf_alock_lock_one_rd(alock, entry, cmpl, req, i)) {
+				/* lock not acquired and not added to wait list */
+				ret = -OCF_ERR_NO_MEM;
+				goto err;
+			}
+		}
+	}
+
+	return ret;
+
+err:
+	for (; i >= 0; i--) {
+		if (!ocf_cl_lock_line_needs_lock(alock, req, i))
+			continue;
+
+		entry = ocf_cl_lock_line_get_entry(alock, req, i);
+		ocf_alock_waitlist_remove_entry(alock, req, i, entry, rw);
+	}
+
+	return ret;
+}
+
 static struct ocf_alock_lock_cbs ocf_cline_conc_cbs = {
-		.line_needs_lock = ocf_cl_lock_line_needs_lock,
-		.line_is_acting = ocf_cl_lock_line_is_acting,
-		.line_is_locked = ocf_cl_lock_line_is_locked,
-		.line_mark_locked = ocf_cl_lock_line_mark_locked,
-		.line_get_entry = ocf_cl_lock_line_get_entry
+		.lock_entries_fast = ocf_cl_lock_line_fast,
+		.lock_entries_slow = ocf_cl_lock_line_slow
 };
 
 bool ocf_cache_line_try_lock_rd(struct ocf_alock *alock,
@@ -95,17 +183,48 @@ int ocf_req_async_lock_wr(struct ocf_alock *alock,
 
 void ocf_req_unlock_rd(struct ocf_alock *alock, struct ocf_request *req)
 {
-	ocf_alock_unlock_rd(alock, req);
+	int32_t i;
+	ocf_cache_line_t entry;
+
+	for (i = 0; i < req->core_line_count; i++) {
+		if (!ocf_cl_lock_line_is_acting(alock, req, i))
+			continue;
+
+		if (!ocf_alock_is_index_locked(alock, req, i))
+			continue;
+
+		entry = ocf_cl_lock_line_get_entry(alock, req, i);
+
+		ocf_alock_unlock_one_rd(alock, entry);
+		ocf_alock_mark_index_locked(alock, req, i, false);
+	}
 }
 
 void ocf_req_unlock_wr(struct ocf_alock *alock, struct ocf_request *req)
 {
-	ocf_alock_unlock_wr(alock, req);
+	int32_t i;
+	ocf_cache_line_t entry;
+
+	for (i = 0; i < req->core_line_count; i++) {
+		if (!ocf_cl_lock_line_is_acting(alock, req, i))
+			continue;
+
+		if (!ocf_alock_is_index_locked(alock, req, i))
+			continue;
+
+		entry = ocf_cl_lock_line_get_entry(alock, req, i);
+
+		ocf_alock_unlock_one_wr(alock, entry);
+		ocf_alock_mark_index_locked(alock, req, i, false);
+	}
 }
 
 void ocf_req_unlock(struct ocf_alock *alock, struct ocf_request *req)
 {
-	ocf_alock_unlock(alock, req);
+	if (req->alock_rw == OCF_WRITE)
+		ocf_req_unlock_wr(alock, req);
+	else
+		ocf_req_unlock_rd(alock, req);
 }
 
 bool ocf_cache_line_are_waiters(struct ocf_alock *alock,

src/concurrency/ocf_mio_concurrency.c

@@ -16,43 +16,7 @@ struct ocf_mio_alock
 	unsigned num_pages;
 };
 
-static bool ocf_mio_lock_line_needs_lock(struct ocf_alock *alock,
-		struct ocf_request *req, unsigned index)
-{
-	return true;
-}
-
-static bool ocf_mio_lock_line_is_acting(struct ocf_alock *alock,
-		struct ocf_request *req, unsigned index)
-{
-	return true;
-}
-
-static bool ocf_mio_lock_line_is_locked(struct ocf_alock *alock,
-		struct ocf_request *req, unsigned index, int rw)
-{
-	struct metadata_io_request *m_req = (struct metadata_io_request *)req;
-
-	if (rw == OCF_WRITE)
-		return env_bit_test(index, &m_req->map);
-	else
-		return false;
-}
-
-static void ocf_mio_lock_line_mark_locked(struct ocf_alock *alock,
-		struct ocf_request *req, unsigned index, int rw, bool locked)
-{
-	struct metadata_io_request *m_req = (struct metadata_io_request *)req;
-
-	if (rw == OCF_READ)
-		return;
-	if (locked)
-		env_bit_set(index, &m_req->map);
-	else
-		env_bit_clear(index, &m_req->map);
-}
-
-static ocf_cache_line_t ocf_mio_lock_line_get_entry(
+static ocf_cache_line_t ocf_mio_lock_get_entry(
 		struct ocf_alock *alock, struct ocf_request *req,
 		unsigned index)
 {
@@ -66,12 +30,77 @@ static ocf_cache_line_t ocf_mio_lock_line_get_entry(
 	return page - mio_alock->first_page;
 }
 
+static int ocf_mio_lock_fast(struct ocf_alock *alock,
+		struct ocf_request *req, int rw)
+{
+	ocf_cache_line_t entry;
+	int ret = OCF_LOCK_ACQUIRED;
+	int32_t i;
+	ENV_BUG_ON(rw != OCF_WRITE);
+
+	for (i = 0; i < req->core_line_count; i++) {
+		entry = ocf_mio_lock_get_entry(alock, req, i);
+		ENV_BUG_ON(ocf_alock_is_index_locked(alock, req, i));
+
+		if (ocf_alock_trylock_entry_wr(alock, entry)) {
+			/* cache entry locked */
+			ocf_alock_mark_index_locked(alock, req, i, true);
+		} else {
+			/* Not possible to lock all cachelines */
+			ret = OCF_LOCK_NOT_ACQUIRED;
+			break;
+		}
+	}
+
+	/* Check if request is locked */
+	if (ret == OCF_LOCK_NOT_ACQUIRED) {
+		/* Request is not locked, discard acquired locks */
+		for (; i >= 0; i--) {
+			entry = ocf_mio_lock_get_entry(alock, req, i);
+
+			if (ocf_alock_is_index_locked(alock, req, i)) {
+				ocf_alock_unlock_one_wr(alock, entry);
+				ocf_alock_mark_index_locked(alock, req, i, false);
+			}
+		}
+	}
+
+	return ret;
+}
+
+static int ocf_mio_lock_slow(struct ocf_alock *alock,
+		struct ocf_request *req, int rw, ocf_req_async_lock_cb cmpl)
+{
+	int32_t i;
+	ocf_cache_line_t entry;
+	int ret = 0;
+	ENV_BUG_ON(rw != OCF_WRITE);
+
+	for (i = 0; i < req->core_line_count; i++) {
+		entry = ocf_mio_lock_get_entry(alock, req, i);
+		ENV_BUG_ON(ocf_alock_is_index_locked(alock, req, i));
+
+		if (!ocf_alock_lock_one_wr(alock, entry, cmpl, req, i)) {
+			/* lock not acquired and not added to wait list */
+			ret = -OCF_ERR_NO_MEM;
+			goto err;
+		}
+	}
+
+	return ret;
+
+err:
+	for (; i >= 0; i--) {
+		entry = ocf_mio_lock_get_entry(alock, req, i);
+		ocf_alock_waitlist_remove_entry(alock, req, i, entry, OCF_WRITE);
+	}
+
+	return ret;
+}
+
 static struct ocf_alock_lock_cbs ocf_mio_conc_cbs = {
-		.line_needs_lock = ocf_mio_lock_line_needs_lock,
-		.line_is_acting = ocf_mio_lock_line_is_acting,
-		.line_is_locked = ocf_mio_lock_line_is_locked,
-		.line_mark_locked = ocf_mio_lock_line_mark_locked,
-		.line_get_entry = ocf_mio_lock_line_get_entry
+		.lock_entries_fast = ocf_mio_lock_fast,
+		.lock_entries_slow = ocf_mio_lock_slow
 };
 
 int ocf_mio_async_lock(struct ocf_alock *alock,
@@ -84,8 +113,21 @@ int ocf_mio_async_lock(struct ocf_alock *alock,
 void ocf_mio_async_unlock(struct ocf_alock *alock,
 		struct metadata_io_request *m_req)
 {
-	ocf_alock_unlock_wr(alock, &m_req->req);
-	m_req->map = 0;
+	ocf_cache_line_t entry;
+	struct ocf_request *req = &m_req->req;
+	int i;
+
+	for (i = 0; i < req->core_line_count; i++) {
+		if (!ocf_alock_is_index_locked(alock, req, i))
+			continue;
+
+		entry = ocf_mio_lock_get_entry(alock, req, i);
+
+		ocf_alock_unlock_one_wr(alock, entry);
+		ocf_alock_mark_index_locked(alock, req, i, false);
+	}
+
+	m_req->alock_status = 0;
 }
 
 
@@ -107,15 +149,17 @@ int ocf_mio_concurrency_init(struct ocf_alock **self,
 	if (ret < 0)
 		return ret;
 	if (ret >= ALLOCATOR_NAME_MAX)
-		return -ENOSPC;
+		return -OCF_ERR_NO_MEM;
 
 	alock = env_vzalloc(base_size + sizeof(struct ocf_mio_alock));
 	if (!alock)
 		return -OCF_ERR_NO_MEM;
 
 	ret = ocf_alock_init_inplace(alock, num_pages, name, &ocf_mio_conc_cbs, cache);
-	if (ret)
+	if (ret) {
+		env_free(alock);
 		return ret;
+	}
 
 	mio_alock = (void*)alock + base_size;
 	mio_alock->first_page = first_page;

src/concurrency/ocf_mio_concurrency.h

@@ -3,8 +3,8 @@
  * SPDX-License-Identifier: BSD-3-Clause-Clear
  */
 
-#ifndef OCF_COLLISION_UPDATE_CONCURRENCY_H_
-#define OCF_COLLISION_UPDATE_CONCURRENCY_H_
+#ifndef OCF_MIO_CONCURRENCY_H_
+#define OCF_MIO_CONCURRENCY_H_
 
 #include "../utils/utils_alock.h"