Split global metadata lock

Divide single global lock instance into 4 to reduce contention
in multiple read-locks scenario.

Signed-off-by: Adam Rutkowski <adam.j.rutkowski@intel.com>

src/cleaning/acp.c

@@ -13,6 +13,7 @@
 #include "../cleaning/acp.h"
 #include "../engine/engine_common.h"
 #include "../concurrency/ocf_cache_line_concurrency.h"
+#include "../concurrency/ocf_metadata_concurrency.h"
 #include "cleaning_priv.h"
 
 #define OCF_ACP_DEBUG 0
@@ -385,8 +386,11 @@ static ocf_cache_line_t _acp_trylock_dirty(struct ocf_cache *cache,
 {
 	struct ocf_map_info info;
 	bool locked = false;
+	unsigned lock_idx = ocf_metadata_concurrency_next_idx(
+			cache->cleaner.io_queue);
 
-	ocf_hb_cline_prot_lock_rd(&cache->metadata.lock, core_id, core_line);
+	ocf_hb_cline_prot_lock_rd(&cache->metadata.lock, lock_idx, core_id,
+			core_line);
 
 	ocf_engine_lookup_map_entry(cache, &info, core_id,
 			core_line);
@@ -397,7 +401,8 @@ static ocf_cache_line_t _acp_trylock_dirty(struct ocf_cache *cache,
 		locked = true;
 	}
 
-	ocf_hb_cline_prot_unlock_rd(&cache->metadata.lock, core_id, core_line);
+	ocf_hb_cline_prot_unlock_rd(&cache->metadata.lock, lock_idx, core_id,
+			core_line);
 
 	return locked ? info.coll_idx : cache->device->collision_table_entries;
 }

src/concurrency/ocf_metadata_concurrency.c

@@ -5,12 +5,14 @@
 
 #include "ocf_metadata_concurrency.h"
 #include "../metadata/metadata_misc.h"
+#include "../ocf_queue_priv.h"
 
 int ocf_metadata_concurrency_init(struct ocf_metadata_lock *metadata_lock)
 {
 	int err = 0;
 	unsigned evp_iter;
 	unsigned part_iter;
+	unsigned global_iter;
 
 	for (evp_iter = 0; evp_iter < OCF_NUM_EVICTION_LISTS; evp_iter++) {
 		err = env_spinlock_init(&metadata_lock->eviction[evp_iter]);
@@ -20,22 +22,29 @@ int ocf_metadata_concurrency_init(struct ocf_metadata_lock *metadata_lock)
 
 	env_rwlock_init(&metadata_lock->status);
 
-	err = env_rwsem_init(&metadata_lock->global);
+	for (global_iter = 0; global_iter < OCF_NUM_GLOBAL_META_LOCKS;
-	if (err)
+			global_iter++) {
-		goto rwsem_err;
+		err = env_rwsem_init(&metadata_lock->global[global_iter]);
+		if (err)
+			goto global_err;
+	}
 
 	for (part_iter = 0; part_iter < OCF_IO_CLASS_MAX; part_iter++) {
 		err = env_spinlock_init(&metadata_lock->partition[part_iter]);
 		if (err)
-			goto spinlocks_err;
+			goto partition_err;
 	}
 
 	return err;
 
-spinlocks_err:
+partition_err:
 	while (part_iter--)
 		env_spinlock_destroy(&metadata_lock->partition[part_iter]);
-rwsem_err:
+
+global_err:
+	while (global_iter--)
+		env_rwsem_destroy(&metadata_lock->global[global_iter]);
+
 	env_rwlock_destroy(&metadata_lock->status);
 
 eviction_err:
@@ -55,8 +64,10 @@ void ocf_metadata_concurrency_deinit(struct ocf_metadata_lock *metadata_lock)
 	for (i = 0; i < OCF_NUM_EVICTION_LISTS; i++)
 		env_spinlock_destroy(&metadata_lock->eviction[i]);
 
+	for (i = 0; i < OCF_NUM_GLOBAL_META_LOCKS; i++)
+		env_rwsem_destroy(&metadata_lock->global[i]);
+
 	env_rwlock_destroy(&metadata_lock->status);
-	env_rwsem_destroy(&metadata_lock->global);
 }
 
 int ocf_metadata_concurrency_attached_init(
@@ -140,34 +151,68 @@ void ocf_metadata_concurrency_attached_deinit(
 void ocf_metadata_start_exclusive_access(
 		struct ocf_metadata_lock *metadata_lock)
 {
-        env_rwsem_down_write(&metadata_lock->global);
+	unsigned i;
+
+	for (i = 0; i < OCF_NUM_GLOBAL_META_LOCKS; i++) {
+		env_rwsem_down_write(&metadata_lock->global[i]);
+	}
 }
 
 int ocf_metadata_try_start_exclusive_access(
 		struct ocf_metadata_lock *metadata_lock)
 {
-	return env_rwsem_down_write_trylock(&metadata_lock->global);
+	unsigned i;
+	int error;
+
+	for (i = 0; i < OCF_NUM_GLOBAL_META_LOCKS; i++) {
+		error =  env_rwsem_down_write_trylock(&metadata_lock->global[i]);
+		if (error)
+			break;
+	}
+
+	if (error) {
+		while (i--) {
+			env_rwsem_up_write(&metadata_lock->global[i]);
+		}
+	}
+
+	return error;
 }
 
 void ocf_metadata_end_exclusive_access(
 		struct ocf_metadata_lock *metadata_lock)
 {
-        env_rwsem_up_write(&metadata_lock->global);
+	unsigned i;
+
+	for (i = OCF_NUM_GLOBAL_META_LOCKS; i > 0; i--)
+	        env_rwsem_up_write(&metadata_lock->global[i - 1]);
 }
 
-void ocf_metadata_start_shared_access(struct ocf_metadata_lock *metadata_lock)
+/* lock_idx determines which of underlying R/W locks is acquired for read. The goal
+   is to spread calls across all available underlying locks to reduce contention
+   on one single RW semaphor primitive. Technically any value is correct, but
+   picking wisely would allow for higher read througput:
+   * free running per-cpu counter sounds good,
+   * for rarely excercised code paths (e.g. management) any value would do.
+*/
+void ocf_metadata_start_shared_access(
+		struct ocf_metadata_lock *metadata_lock,
+		unsigned lock_idx)
 {
-        env_rwsem_down_read(&metadata_lock->global);
+        env_rwsem_down_read(&metadata_lock->global[lock_idx]);
 }
 
-int ocf_metadata_try_start_shared_access(struct ocf_metadata_lock *metadata_lock)
+int ocf_metadata_try_start_shared_access(
+		struct ocf_metadata_lock *metadata_lock,
+		unsigned lock_idx)
 {
-	return env_rwsem_down_read_trylock(&metadata_lock->global);
+	return env_rwsem_down_read_trylock(&metadata_lock->global[lock_idx]);
 }
 
-void ocf_metadata_end_shared_access(struct ocf_metadata_lock *metadata_lock)
+void ocf_metadata_end_shared_access(struct ocf_metadata_lock *metadata_lock,
+		unsigned lock_idx)
 {
-        env_rwsem_up_read(&metadata_lock->global);
+        env_rwsem_up_read(&metadata_lock->global[lock_idx]);
 }
 
 /* NOTE: Calling 'naked' lock/unlock requires caller to hold global metadata
@@ -267,74 +312,74 @@ void ocf_hb_cline_naked_unlock_wr(struct ocf_metadata_lock *metadata_lock,
 /* common part of protected hash bucket lock routines */
 static inline void ocf_hb_id_prot_lock_common(
 		struct ocf_metadata_lock *metadata_lock,
-		ocf_cache_line_t hash, int rw)
+		uint32_t lock_idx, ocf_cache_line_t hash, int rw)
 {
-	ocf_metadata_start_shared_access(metadata_lock);
+	ocf_metadata_start_shared_access(metadata_lock, lock_idx);
 	ocf_hb_id_naked_lock(metadata_lock, hash, rw);
 }
 
 /* common part of protected hash bucket unlock routines */
 static inline void ocf_hb_id_prot_unlock_common(
 		struct ocf_metadata_lock *metadata_lock,
-		ocf_cache_line_t hash, int rw)
+		uint32_t lock_idx, ocf_cache_line_t hash, int rw)
 {
 	ocf_hb_id_naked_unlock(metadata_lock, hash, rw);
-	ocf_metadata_end_shared_access(metadata_lock);
+	ocf_metadata_end_shared_access(metadata_lock, lock_idx);
 }
 
 /* NOTE: caller can lock at most one hash bucket at a time using protected
 	variants of lock routines. */
 void ocf_hb_cline_prot_lock_wr(struct ocf_metadata_lock *metadata_lock,
-		uint32_t core_id, uint64_t core_line)
+		uint32_t lock_idx, uint32_t core_id, uint64_t core_line)
 {
 	ocf_cache_line_t hash = ocf_metadata_hash_func(metadata_lock->cache,
 			core_line, core_id);
 
-	ocf_hb_id_prot_lock_common(metadata_lock, hash,
+	ocf_hb_id_prot_lock_common(metadata_lock, lock_idx,
-			OCF_METADATA_WR);
+			hash, OCF_METADATA_WR);
 }
 
 void ocf_hb_cline_prot_unlock_wr(struct ocf_metadata_lock *metadata_lock,
-		uint32_t core_id, uint64_t core_line)
+		uint32_t lock_idx, uint32_t core_id, uint64_t core_line)
 {
 	ocf_cache_line_t hash = ocf_metadata_hash_func(metadata_lock->cache,
 			core_line, core_id);
 
-	ocf_hb_id_prot_unlock_common(metadata_lock, hash,
+	ocf_hb_id_prot_unlock_common(metadata_lock, lock_idx,
-			OCF_METADATA_WR);
+			hash, OCF_METADATA_WR);
 }
 
 void ocf_hb_cline_prot_lock_rd(struct ocf_metadata_lock *metadata_lock,
-		uint32_t core_id, uint64_t core_line)
+		uint32_t lock_idx, uint32_t core_id, uint64_t core_line)
 {
 	ocf_cache_line_t hash = ocf_metadata_hash_func(metadata_lock->cache,
 			core_line, core_id);
 
-	ocf_hb_id_prot_lock_common(metadata_lock, hash,
+	ocf_hb_id_prot_lock_common(metadata_lock, lock_idx,
-			OCF_METADATA_RD);
+			hash, OCF_METADATA_RD);
 }
 
 void ocf_hb_cline_prot_unlock_rd(struct ocf_metadata_lock *metadata_lock,
-		uint32_t core_id, uint64_t core_line)
+		uint32_t lock_idx, uint32_t core_id, uint64_t core_line)
 {
 	ocf_cache_line_t hash = ocf_metadata_hash_func(metadata_lock->cache,
 			core_line, core_id);
 
-	ocf_hb_id_prot_unlock_common(metadata_lock, hash,
+	ocf_hb_id_prot_unlock_common(metadata_lock, lock_idx,
-			OCF_METADATA_RD);
+			hash, OCF_METADATA_RD);
 }
 
-void ocf_hb_hash_prot_lock_wr(struct ocf_metadata_lock *metadata_lock,
+void ocf_hb_id_prot_lock_wr(struct ocf_metadata_lock *metadata_lock,
-		ocf_cache_line_t hash)
+		unsigned lock_idx, ocf_cache_line_t hash)
 {
-	ocf_hb_id_prot_lock_common(metadata_lock, hash,
+	ocf_hb_id_prot_lock_common(metadata_lock, lock_idx, hash,
 			OCF_METADATA_WR);
 }
 
-void ocf_hb_hash_prot_unlock_wr(struct ocf_metadata_lock *metadata_lock,
+void ocf_hb_id_prot_unlock_wr(struct ocf_metadata_lock *metadata_lock,
-		ocf_cache_line_t hash)
+		unsigned lock_idx, ocf_cache_line_t hash)
 {
-	ocf_hb_id_prot_unlock_common(metadata_lock, hash,
+	ocf_hb_id_prot_unlock_common(metadata_lock, lock_idx, hash,
 			OCF_METADATA_WR);
 }
 
@@ -392,7 +437,8 @@ void ocf_hb_req_prot_lock_rd(struct ocf_request *req)
 {
 	ocf_cache_line_t hash;
 
-	ocf_metadata_start_shared_access(&req->cache->metadata.lock);
+	ocf_metadata_start_shared_access(&req->cache->metadata.lock,
+			req->lock_idx);
 	for_each_req_hash_asc(req, hash) {
 		ocf_hb_id_naked_lock(&req->cache->metadata.lock, hash,
 				OCF_METADATA_RD);
@@ -407,14 +453,16 @@ void ocf_hb_req_prot_unlock_rd(struct ocf_request *req)
 		ocf_hb_id_naked_unlock(&req->cache->metadata.lock, hash,
 				OCF_METADATA_RD);
 	}
-	ocf_metadata_end_shared_access(&req->cache->metadata.lock);
+	ocf_metadata_end_shared_access(&req->cache->metadata.lock,
+			req->lock_idx);
 }
 
 void ocf_hb_req_prot_lock_wr(struct ocf_request *req)
 {
 	ocf_cache_line_t hash;
 
-	ocf_metadata_start_shared_access(&req->cache->metadata.lock);
+	ocf_metadata_start_shared_access(&req->cache->metadata.lock,
+			req->lock_idx);
 	for_each_req_hash_asc(req, hash) {
 		ocf_hb_id_naked_lock(&req->cache->metadata.lock, hash,
 				OCF_METADATA_WR);
@@ -443,7 +491,8 @@ void ocf_hb_req_prot_unlock_wr(struct ocf_request *req)
 		ocf_hb_id_naked_unlock(&req->cache->metadata.lock, hash,
 				OCF_METADATA_WR);
 	}
-	ocf_metadata_end_shared_access(&req->cache->metadata.lock);
+	ocf_metadata_end_shared_access(&req->cache->metadata.lock,
+			req->lock_idx);
 }
 
 void ocf_collision_start_shared_access(struct ocf_metadata_lock *metadata_lock,

src/concurrency/ocf_metadata_concurrency.h

@@ -4,6 +4,7 @@
  */
 #include "../ocf_cache_priv.h"
 #include "../eviction/eviction.h"
+#include "../ocf_queue_priv.h"
 
 #ifndef __OCF_METADATA_CONCURRENCY_H__
 #define __OCF_METADATA_CONCURRENCY_H__
@@ -11,6 +12,11 @@
 #define OCF_METADATA_RD 0
 #define OCF_METADATA_WR 1
 
+static inline unsigned ocf_metadata_concurrency_next_idx(ocf_queue_t q)
+{
+	return q->lock_idx++ % OCF_NUM_GLOBAL_META_LOCKS;
+}
+
 int ocf_metadata_concurrency_init(struct ocf_metadata_lock *metadata_lock);
 
 void ocf_metadata_concurrency_deinit(struct ocf_metadata_lock *metadata_lock);
@@ -90,13 +96,16 @@ void ocf_metadata_end_exclusive_access(
 		struct ocf_metadata_lock *metadata_lock);
 
 int ocf_metadata_try_start_shared_access(
-		struct ocf_metadata_lock *metadata_lock);
+		struct ocf_metadata_lock *metadata_lock,
+		unsigned lock_idx);
 
 void ocf_metadata_start_shared_access(
-		struct ocf_metadata_lock *metadata_lock);
+		struct ocf_metadata_lock *metadata_lock,
+		unsigned lock_idx);
 
 void ocf_metadata_end_shared_access(
-		struct ocf_metadata_lock *metadata_lock);
+		struct ocf_metadata_lock *metadata_lock,
+		unsigned lock_idx);
 
 static inline void ocf_metadata_status_bits_lock(
 		struct ocf_metadata_lock *metadata_lock, int rw)
@@ -137,14 +146,19 @@ static inline void ocf_metadata_status_bits_unlock(
 				OCF_METADATA_WR)
 
 void ocf_hb_cline_prot_lock_rd(struct ocf_metadata_lock *metadata_lock,
-		uint32_t core_id, uint64_t core_line);
+		uint32_t lock_idx, uint32_t core_id, uint64_t core_line);
 void ocf_hb_cline_prot_unlock_rd(struct ocf_metadata_lock *metadata_lock,
-		uint32_t core_id, uint64_t core_line);
+		uint32_t lock_idx, uint32_t core_id, uint64_t core_line);
+
+void ocf_hb_cline_prot_lock_wr(struct ocf_metadata_lock *metadata_lock,
+		uint32_t lock_idx, uint32_t core_id, uint64_t core_line);
+void ocf_hb_cline_prot_unlock_wr(struct ocf_metadata_lock *metadata_lock,
+		uint32_t lock_idx, uint32_t core_id, uint64_t core_line);
 
 void ocf_hb_id_prot_lock_wr(struct ocf_metadata_lock *metadata_lock,
-		ocf_cache_line_t hash);
+		unsigned lock_idx, ocf_cache_line_t hash);
 void ocf_hb_id_prot_unlock_wr(struct ocf_metadata_lock *metadata_lock,
-		ocf_cache_line_t hash);
+		unsigned lock_idx, ocf_cache_line_t hash);
 
 /* caller must hold global metadata read lock */
 bool ocf_hb_cline_naked_trylock_rd(struct ocf_metadata_lock *metadata_lock,
@@ -152,12 +166,6 @@ bool ocf_hb_cline_naked_trylock_rd(struct ocf_metadata_lock *metadata_lock,
 void ocf_hb_cline_naked_unlock_rd(struct ocf_metadata_lock *metadata_lock,
 		uint32_t core_id, uint64_t core_line);
 
-void ocf_hb_cline_prot_lock_wr(struct ocf_metadata_lock *metadata_lock,
-		uint32_t core_id, uint64_t core_line);
-void ocf_hb_cline_prot_unlock_wr(struct ocf_metadata_lock *metadata_lock,
-		uint32_t core_id, uint64_t core_line);
-
-/* caller must hold global metadata read lock */
 bool ocf_hb_cline_naked_trylock_wr(struct ocf_metadata_lock *metadata_lock,
 		uint32_t core_id, uint64_t core_line);
 void ocf_hb_cline_naked_unlock_wr(struct ocf_metadata_lock *metadata_lock,

src/eviction/eviction.h

@@ -9,7 +9,6 @@
 #include "ocf/ocf.h"
 #include "lru.h"
 #include "lru_structs.h"
-#include "../ocf_request.h"
 
 #define OCF_TO_EVICTION_MIN 128UL
 #define OCF_PENDING_EVICTION_LIMIT 512UL
@@ -17,6 +16,7 @@
 #define OCF_NUM_EVICTION_LISTS 32
 
 struct ocf_user_part;
+struct ocf_request;
 
 struct eviction_policy {
 	union {

src/metadata/metadata_io.c

@@ -215,9 +215,11 @@ static int metadata_io_restart_req(struct ocf_request *req)
 
 	/* Fill with the latest metadata. */
 	if (m_req->req.rw == OCF_WRITE) {
-		ocf_metadata_start_shared_access(&cache->metadata.lock);
+		ocf_metadata_start_shared_access(&cache->metadata.lock,
+				m_req->page % OCF_NUM_GLOBAL_META_LOCKS);
 		metadata_io_req_fill(m_req);
-		ocf_metadata_end_shared_access(&cache->metadata.lock);
+		ocf_metadata_end_shared_access(&cache->metadata.lock,
+				 m_req->page % OCF_NUM_GLOBAL_META_LOCKS);
 	}
 
 	io = ocf_new_cache_io(cache, req->io_queue,

src/metadata/metadata_structs.h

@@ -43,9 +43,13 @@ struct ocf_cache_line_settings {
 	uint64_t sector_end;
 };
 
+
+#define OCF_METADATA_GLOBAL_LOCK_IDX_BITS 2
+#define OCF_NUM_GLOBAL_META_LOCKS (1 << (OCF_METADATA_GLOBAL_LOCK_IDX_BITS))
+
 struct ocf_metadata_lock
 {
-	env_rwsem global; /*!< global metadata lock (GML) */
+	env_rwsem global[OCF_NUM_GLOBAL_META_LOCKS]; /*!< global metadata lock (GML) */
 	env_rwlock status; /*!< Fast lock for status bits */
 	env_spinlock eviction[OCF_NUM_EVICTION_LISTS]; /*!< Fast lock for eviction policy */
 	env_rwsem *hash; /*!< Hash bucket locks */

src/mngt/ocf_mngt_cache.c

@@ -2344,11 +2344,11 @@ ocf_promotion_t ocf_mngt_cache_promotion_get_policy(ocf_cache_t cache)
 {
 	ocf_promotion_t result;
 
-	ocf_metadata_start_shared_access(&cache->metadata.lock);
+	ocf_metadata_start_shared_access(&cache->metadata.lock, 0);
 
 	result = cache->conf_meta->promotion_policy_type;
 
-	ocf_metadata_end_shared_access(&cache->metadata.lock);
+	ocf_metadata_end_shared_access(&cache->metadata.lock, 0);
 
 	return result;
 }
@@ -2358,11 +2358,11 @@ int ocf_mngt_cache_promotion_get_param(ocf_cache_t cache, ocf_promotion_t type,
 {
 	int result;
 
-	ocf_metadata_start_shared_access(&cache->metadata.lock);
+	ocf_metadata_start_shared_access(&cache->metadata.lock, 0);
 
 	result = ocf_promotion_get_param(cache, type, param_id, param_value);
 
-	ocf_metadata_end_shared_access(&cache->metadata.lock);
+	ocf_metadata_end_shared_access(&cache->metadata.lock, 0);
 
 	return result;
 }

src/mngt/ocf_mngt_common.c

@@ -59,10 +59,13 @@ void cache_mngt_core_deinit_attached_meta(ocf_core_t core)
 	ocf_core_id_t iter_core_id;
 	ocf_cache_line_t curr_cline, prev_cline;
 	uint32_t hash, num_hash = cache->device->hash_table_entries;
+	unsigned lock_idx;
 
 	for (hash = 0; hash < num_hash;) {
 		prev_cline = cache->device->collision_table_entries;
-		ocf_hb_id_prot_lock_wr(&cache->metadata.lock, hash);
+
+		lock_idx = ocf_metadata_concurrency_next_idx(cache->mngt_queue);
+		ocf_hb_id_prot_lock_wr(&cache->metadata.lock, lock_idx, hash);
 
 		curr_cline = ocf_metadata_get_hash(cache, hash);
 		while (curr_cline != cache->device->collision_table_entries) {
@@ -91,7 +94,7 @@ void cache_mngt_core_deinit_attached_meta(ocf_core_t core)
 			else
 				curr_cline = ocf_metadata_get_hash(cache, hash);
 		}
-		ocf_hb_id_prot_unlock_wr(&cache->metadata.lock, hash);
+		ocf_hb_id_prot_unlock_wr(&cache->metadata.lock, lock_idx, hash);
 
 		/* Check whether all the cachelines from the hash bucket were sparsed */
 		if (curr_cline == cache->device->collision_table_entries)

src/ocf_queue_priv.h

@@ -18,6 +18,9 @@ struct ocf_queue {
 	struct list_head io_list;
 	env_spinlock io_list_lock;
 
+	/* per-queue free running global metadata lock index */
+	unsigned lock_idx;
+
 	/* Tracing reference counter */
 	env_atomic64 trace_ref_cntr;
 

src/ocf_request.c

@@ -6,7 +6,7 @@
 #include "ocf/ocf.h"
 #include "ocf_request.h"
 #include "ocf_cache_priv.h"
-#include "ocf_queue_priv.h"
+#include "concurrency/ocf_metadata_concurrency.h"
 #include "utils/utils_cache_line.h"
 
 #define OCF_UTILS_RQ_DEBUG 0
@@ -205,6 +205,8 @@ struct ocf_request *ocf_req_new(ocf_queue_t queue, ocf_core_t core,
 	req->discard.nr_sects = BYTES_TO_SECTORS(bytes);
 	req->discard.handled = 0;
 
+	req->lock_idx = ocf_metadata_concurrency_next_idx(queue);
+
 	return req;
 }
 

src/ocf_request.h

@@ -9,6 +9,7 @@
 #include "ocf_env.h"
 #include "ocf_io_priv.h"
 #include "engine/cache_engine.h"
+#include "metadata/metadata_structs.h"
 
 struct ocf_req_allocator;
 
@@ -190,6 +191,9 @@ struct ocf_request {
 	uint8_t part_evict : 1;
 	/* !< Some cachelines from request's partition must be evicted */
 
+	uint8_t lock_idx : OCF_METADATA_GLOBAL_LOCK_IDX_BITS;
+	/* !< Selected global metadata read lock */
+
 	log_sid_t sid;
 	/*!< Tracing sequence ID */