Adding collision table locks

Signed-off-by: Adam Rutkowski <adam.j.rutkowski@intel.com>

src/concurrency/ocf_cache_line_concurrency.c

@@ -71,6 +71,8 @@ int ocf_cache_line_concurrency_init(struct ocf_cache *cache)
 	int error = 0;
 	struct ocf_cache_line_concurrency *c;
 	char name[ALLOCATOR_NAME_MAX];
+	ocf_cache_line_t line_entries = ocf_metadata_collision_table_entries(
+						cache);
 
 	ENV_BUG_ON(cache->device->concurrency.cache_line);
 
@@ -85,8 +87,8 @@ int ocf_cache_line_concurrency_init(struct ocf_cache *cache)
 	cache->device->concurrency.cache_line = c;
 
 	OCF_REALLOC_INIT(&c->access, &c->access_limit);
-	OCF_REALLOC_CP(&c->access, sizeof(c->access[0]),
+	OCF_REALLOC_CP(&c->access, sizeof(c->access[0]), line_entries,
-			cache->device->collision_table_entries, &c->access_limit);
+		&c->access_limit);
 
 	if (!c->access) {
 		error = __LINE__;

src/concurrency/ocf_metadata_concurrency.c

@@ -22,19 +22,44 @@ void ocf_metadata_concurrency_deinit(struct ocf_metadata_lock *metadata_lock)
 
 int ocf_metadata_concurrency_attached_init(
 		struct ocf_metadata_lock *metadata_lock, ocf_cache_t cache,
-		uint64_t hash_table_entries)
+		uint64_t hash_table_entries, uint32_t colision_table_pages)
 {
 	uint64_t i;
+	int err = 0;
 
-	metadata_lock->cache = cache;
-	metadata_lock->num_hash_entries = hash_table_entries;
 	metadata_lock->hash = env_vzalloc(sizeof(env_rwsem) *
 			hash_table_entries);
-	if (!metadata_lock->hash)
+	metadata_lock->collision_pages = env_vzalloc(sizeof(env_rwsem) *
+			colision_table_pages);
+	if (!metadata_lock->hash ||
+			!metadata_lock->collision_pages) {
+		env_vfree(metadata_lock->hash);
+		env_vfree(metadata_lock->collision_pages);
+		metadata_lock->hash = NULL;
+		metadata_lock->collision_pages = NULL;
 		return -OCF_ERR_NO_MEM;
+	}
 
 	for (i = 0; i < hash_table_entries; i++)
 		env_rwsem_init(&metadata_lock->hash[i]);
+	for (i = 0; i < colision_table_pages; i++) {
+		err = env_rwsem_init(&metadata_lock->collision_pages[i]);
+		if (err)
+			break;
+	}
+
+	if (err) {
+		while (i--)
+			env_rwsem_destroy(&metadata_lock->collision_pages[i]);
+		env_vfree(metadata_lock->collision_pages);
+		metadata_lock->collision_pages = NULL;
+		ocf_metadata_concurrency_attached_deinit(metadata_lock);
+		return err;
+	}
+
+	metadata_lock->cache = cache;
+	metadata_lock->num_hash_entries = hash_table_entries;
+	metadata_lock->num_collision_pages = colision_table_pages;
 
 	return 0;
 }
@@ -44,10 +69,21 @@ void ocf_metadata_concurrency_attached_deinit(
 {
 	uint64_t i;
 
+	if (metadata_lock->hash) {
 		for (i = 0; i < metadata_lock->num_hash_entries; i++)
 			env_rwsem_destroy(&metadata_lock->hash[i]);
-
 		env_vfree(metadata_lock->hash);
+		metadata_lock->hash = NULL;
+		metadata_lock->num_hash_entries = 0;
+	}
+
+	if (metadata_lock->collision_pages) {
+		for (i = 0; i < metadata_lock->num_collision_pages; i++)
+			env_rwsem_destroy(&metadata_lock->collision_pages[i]);
+		env_vfree(metadata_lock->collision_pages);
+		metadata_lock->collision_pages = NULL;
+		metadata_lock->num_collision_pages = 0;
+	}
 }
 
 void ocf_metadata_start_exclusive_access(
@@ -266,3 +302,27 @@ void ocf_req_hash_unlock_wr(struct ocf_request *req)
 	}
 	ocf_metadata_end_shared_access(&req->cache->metadata.lock);
 }
+
+void ocf_collision_start_shared_access(struct ocf_metadata_lock *metadata_lock,
+		uint32_t page)
+{
+	env_rwsem_down_read(&metadata_lock->collision_pages[page]);
+}
+
+void ocf_collision_end_shared_access(struct ocf_metadata_lock *metadata_lock,
+		uint32_t page)
+{
+	env_rwsem_up_read(&metadata_lock->collision_pages[page]);
+}
+
+void ocf_collision_start_exclusive_access(struct ocf_metadata_lock *metadata_lock,
+		uint32_t page)
+{
+	env_rwsem_down_write(&metadata_lock->collision_pages[page]);
+}
+
+void ocf_collision_end_exclusive_access(struct ocf_metadata_lock *metadata_lock,
+		uint32_t page)
+{
+	env_rwsem_up_write(&metadata_lock->collision_pages[page]);
+}

src/concurrency/ocf_metadata_concurrency.h

@@ -16,7 +16,7 @@ void ocf_metadata_concurrency_deinit(struct ocf_metadata_lock *metadata_lock);
 
 int ocf_metadata_concurrency_attached_init(
 		struct ocf_metadata_lock *metadata_lock, ocf_cache_t cache,
-		uint64_t hash_table_entries);
+		uint64_t hash_table_entries, uint32_t colision_table_pages);
 
 void ocf_metadata_concurrency_attached_deinit(
 		struct ocf_metadata_lock *metadata_lock);
@@ -111,4 +111,13 @@ void ocf_req_hash_lock_wr(struct ocf_request *req);
 void ocf_req_hash_unlock_wr(struct ocf_request *req);
 void ocf_req_hash_lock_upgrade(struct ocf_request *req);
 
+/* collision table page lock interface */
+void ocf_collision_start_shared_access(struct ocf_metadata_lock *metadata_lock,
+		uint32_t page);
+void ocf_collision_end_shared_access(struct ocf_metadata_lock *metadata_lock,
+		uint32_t page);
+void ocf_collision_start_exclusive_access(struct ocf_metadata_lock *metadata_lock,
+		uint32_t page);
+void ocf_collision_end_exclusive_access(struct ocf_metadata_lock *metadata_lock,
+		uint32_t page);
 #endif

src/metadata/metadata_collision.h

@@ -105,4 +105,16 @@ void ocf_metadata_add_to_collision(struct ocf_cache *cache,
 void ocf_metadata_remove_from_collision(struct ocf_cache *cache,
 		ocf_cache_line_t line, ocf_part_id_t part_id);
 
+static inline void ocf_metadata_start_collision_shared_access(
+		struct ocf_cache *cache, ocf_cache_line_t line)
+{
+	cache->metadata.iface.start_collision_shared_access(cache, line);
+}
+
+static inline void ocf_metadata_end_collision_shared_access(
+		struct ocf_cache *cache, ocf_cache_line_t line)
+{
+	cache->metadata.iface.end_collision_shared_access(cache, line);
+}
+
 #endif /* METADATA_COLLISION_H_ */

src/metadata/metadata_hash.c

@@ -414,6 +414,8 @@ static void ocf_metadata_hash_deinit_variable_size(struct ocf_cache *cache)
 
 	OCF_DEBUG_TRACE(cache);
 
+	ocf_metadata_concurrency_attached_deinit(&cache->metadata.lock);
+
 	/*
 	 * De initialize RAW types
 	 */
@@ -982,7 +984,9 @@ finalize:
 		 * Hash De-Init also contains RAW deinitialization
 		 */
 		ocf_metadata_hash_deinit_variable_size(cache);
-	} else {
+		return result;
+	}
+
 	cache->device->runtime_meta = METADATA_MEM_POOL(ctrl,
 			metadata_segment_sb_runtime);
 
@@ -1003,7 +1007,6 @@ finalize:
 
 	ocf_cache_log(cache, log_info, "Metadata capacity: %llu MiB\n",
 			(uint64_t)ocf_metadata_size_of(cache) / MiB);
-	}
 
 	/*
 	 * Self test of metadata
@@ -1029,7 +1032,18 @@ finalize:
 				"OCF metadata self-test ERROR\n");
 	}
 
+	result = ocf_metadata_concurrency_attached_init(&cache->metadata.lock,
+			cache, ctrl->raw_desc[metadata_segment_hash].entries,
+			(uint32_t)ctrl->raw_desc[metadata_segment_collision].
+			ssd_pages);
+	if (result) {
+		ocf_cache_log(cache, log_err, "Failed to initialize attached "
+				"metadata concurrency\n");
+		ocf_metadata_hash_deinit_variable_size(cache);
 		return  result;
+	}
+
+	return 0;
 }
 
 static inline void _ocf_init_collision_entry(struct ocf_cache *cache,
@@ -2533,6 +2547,30 @@ static void ocf_metadata_hash_get_collision_info(
 	}
 }
 
+void ocf_metadata_hash_start_collision_shared_access(struct ocf_cache *cache,
+		ocf_cache_line_t line)
+{
+	struct ocf_metadata_hash_ctrl *ctrl =
+		(struct ocf_metadata_hash_ctrl *) cache->metadata.iface_priv;
+	struct ocf_metadata_raw *raw =
+			&ctrl->raw_desc[metadata_segment_collision];
+	uint32_t page = ocf_metadata_raw_page(raw, line);
+
+	ocf_collision_start_shared_access(&cache->metadata.lock, page);
+}
+
+void ocf_metadata_hash_end_collision_shared_access(struct ocf_cache *cache,
+		ocf_cache_line_t line)
+{
+	struct ocf_metadata_hash_ctrl *ctrl =
+		(struct ocf_metadata_hash_ctrl *) cache->metadata.iface_priv;
+	struct ocf_metadata_raw *raw =
+			&ctrl->raw_desc[metadata_segment_collision];
+	uint32_t page = ocf_metadata_raw_page(raw, line);
+
+	ocf_collision_start_shared_access(&cache->metadata.lock, page);
+}
+
 /*******************************************************************************
  *  Partition
  ******************************************************************************/
@@ -2682,6 +2720,10 @@ static const struct ocf_metadata_iface metadata_hash_iface = {
 	.set_collision_info = ocf_metadata_hash_set_collision_info,
 	.set_collision_next = ocf_metadata_hash_set_collision_next,
 	.set_collision_prev = ocf_metadata_hash_set_collision_prev,
+	.start_collision_shared_access =
+			ocf_metadata_hash_start_collision_shared_access,
+	.end_collision_shared_access =
+			ocf_metadata_hash_end_collision_shared_access,
 
 	/*
 	 * Partition Info

src/metadata/metadata_raw.c

@@ -149,6 +149,16 @@ static uint32_t _raw_ram_checksum(ocf_cache_t cache,
 	return crc;
 }
 
+/*
+ * RAM Implementation - Entry page number
+ */
+uint32_t _raw_ram_page(struct ocf_metadata_raw *raw, uint32_t entry)
+{
+	ENV_BUG_ON(entry >= raw->entries);
+
+	return _RAW_RAM_PAGE(raw, entry);
+}
+
 /*
  * RAM Implementation - Get entry
  */
@@ -541,6 +551,7 @@ static const struct raw_iface IRAW[metadata_raw_type_max] = {
 		.size_of		= _raw_ram_size_of,
 		.size_on_ssd		= _raw_ram_size_on_ssd,
 		.checksum		= _raw_ram_checksum,
+		.page			= _raw_ram_page,
 		.get			= _raw_ram_get,
 		.set			= _raw_ram_set,
 		.access			= _raw_ram_access,
@@ -555,6 +566,7 @@ static const struct raw_iface IRAW[metadata_raw_type_max] = {
 		.size_of		= raw_dynamic_size_of,
 		.size_on_ssd		= raw_dynamic_size_on_ssd,
 		.checksum		= raw_dynamic_checksum,
+		.page			= raw_dynamic_page,
 		.get			= raw_dynamic_get,
 		.set			= raw_dynamic_set,
 		.access			= raw_dynamic_access,
@@ -569,6 +581,7 @@ static const struct raw_iface IRAW[metadata_raw_type_max] = {
 		.size_of		= _raw_ram_size_of,
 		.size_on_ssd		= raw_volatile_size_on_ssd,
 		.checksum		= raw_volatile_checksum,
+		.page			= _raw_ram_page,
 		.get			= _raw_ram_get,
 		.set			= _raw_ram_set,
 		.access			= _raw_ram_access,
@@ -583,6 +596,7 @@ static const struct raw_iface IRAW[metadata_raw_type_max] = {
 		.size_of		= _raw_ram_size_of,
 		.size_on_ssd		= _raw_ram_size_on_ssd,
 		.checksum		= _raw_ram_checksum,
+		.page			= _raw_ram_page,
 		.get			= _raw_ram_get,
 		.set			= _raw_ram_set,
 		.access			= _raw_ram_access,

src/metadata/metadata_raw.h

@@ -102,6 +102,7 @@ struct raw_iface {
 	uint32_t (*checksum)(ocf_cache_t cache,
 			struct ocf_metadata_raw *raw);
 
+	uint32_t (*page)(struct ocf_metadata_raw *raw, uint32_t entry);
 
 	int (*get)(ocf_cache_t cache, struct ocf_metadata_raw *raw,
 			uint32_t entry, void *data);
@@ -183,6 +184,19 @@ static inline uint32_t ocf_metadata_raw_checksum(struct ocf_cache* cache,
 	return raw->iface->checksum(cache, raw);
 }
 
+/**
+ * @brief Calculate entry page index
+ *
+ * @param raw - RAW descriptor
+ * @param entry - Entry number
+ * @return Page index
+ */
+static inline uint32_t ocf_metadata_raw_page(struct ocf_metadata_raw* raw,
+		uint32_t entry)
+{
+	return raw->iface->page(raw, entry);
+}
+
 /**
  * @brief Get specified element of metadata
  *

src/metadata/metadata_raw_dynamic.c

@@ -219,6 +219,16 @@ uint32_t raw_dynamic_checksum(ocf_cache_t cache,
 	return crc;
 }
 
+/*
+ * RAM DYNAMIC Implementation - Entry page number
+ */
+uint32_t raw_dynamic_page(struct ocf_metadata_raw *raw, uint32_t entry)
+{
+	ENV_BUG_ON(entry >= raw->entries);
+
+	return _RAW_DYNAMIC_PAGE(raw, entry);
+}
+
 /*
 * RAM DYNAMIC Implementation - Get
 */

src/metadata/metadata_raw_dynamic.h

@@ -40,6 +40,11 @@ uint32_t raw_dynamic_size_on_ssd(struct ocf_metadata_raw *raw);
 uint32_t raw_dynamic_checksum(ocf_cache_t cache,
 		struct ocf_metadata_raw *raw);
 
+/*
+ * RAM DYNAMIC Implementation - Entry page number
+ */
+uint32_t raw_dynamic_page(struct ocf_metadata_raw *raw, uint32_t entry);
+
 /*
  * RAW DYNAMIC - Get specified entry
  */

src/metadata/metadata_structs.h

@@ -360,6 +360,12 @@ struct ocf_metadata_iface {
 	void (*set_collision_prev)(struct ocf_cache *cache,
 				ocf_cache_line_t line, ocf_cache_line_t prev);
 
+	void (*start_collision_shared_access)(struct ocf_cache *cache,
+			ocf_cache_line_t line);
+
+	void (*end_collision_shared_access)(struct ocf_cache *cache,
+			ocf_cache_line_t line);
+
 	void (*get_partition_info)(struct ocf_cache *cache,
 			ocf_cache_line_t line, ocf_part_id_t *part_id,
 			ocf_cache_line_t *next_line,
@@ -434,7 +440,9 @@ struct ocf_metadata_lock
 	env_rwlock status; /*!< Fast lock for status bits */
 	env_spinlock eviction; /*!< Fast lock for eviction policy */
 	env_rwsem *hash; /*!< Hash bucket locks */
+	env_rwsem *collision_pages; /*!< Collision table page locks */
 	uint32_t num_hash_entries;  /*!< Hash bucket count */
+	uint32_t num_collision_pages; /*!< Collision table page count */
 	ocf_cache_t cache;  /*!< Parent cache object */
 };
 

src/mngt/ocf_mngt_cache.c

@@ -980,7 +980,8 @@ static void _ocf_mngt_attach_prepare_metadata(ocf_pipeline_t pipeline,
 	context->flags.attached_metadata_inited = true;
 
 	if (ocf_metadata_concurrency_attached_init(&cache->metadata.lock,
-			cache, cache->device->hash_table_entries)) {
+			cache, cache->device->hash_table_entries,
+			ocf_metadata_get_num_collision_pages(cache))) {
 		ocf_cache_log(cache, log_err, "Failed to initialize attached "
 				"metadata concurrency\n");
 		OCF_PL_FINISH_RET(context->pipeline, -OCF_ERR_START_CACHE_FAIL);