Merge pull request #811 from mmichal10/req-rewrq

Request improvements

inc/ocf_queue.h

@@ -153,12 +153,12 @@ void *ocf_queue_get_priv(ocf_queue_t q);
 uint32_t ocf_queue_pending_io(ocf_queue_t q);
 
 /**
- * @brief Get cache instance to which I/O queue belongs
+ * @brief Return if queue is management queue
  *
- * @param[in] q I/O queue
+ * @param[in] queue - queue object
  *
- * @retval Cache instance
+ * @retval true - if management queue, otherwise false
  */
-ocf_cache_t ocf_queue_get_cache(ocf_queue_t q);
+bool ocf_queue_is_mngt(ocf_queue_t queue);
 
 #endif

src/metadata/metadata_io.c

@@ -156,7 +156,7 @@ int metadata_io_read_i_atomic(ocf_cache_t cache, ocf_queue_t queue, void *priv,
 	if (!context)
 		return -OCF_ERR_NO_MEM;
 
-	context->req = ocf_req_new(queue, NULL, 0, 0, 0);
+	context->req = ocf_req_new_mngt(cache, queue);
 	if (!context->req) {
 		env_vfree(context);
 		return -OCF_ERR_NO_MEM;

src/metadata/metadata_raw_dynamic.c

@@ -493,7 +493,7 @@ void raw_dynamic_load_all(ocf_cache_t cache, struct ocf_metadata_raw *raw,
 		goto err_zpage;
 	}
 
-	context->req = ocf_req_new(cache->mngt_queue, NULL, 0, 0, 0);
+	context->req = ocf_req_new_mngt(cache, cache->mngt_queue);
 	if (!context->req) {
 		result = -OCF_ERR_NO_MEM;
 		goto err_req;

src/mngt/ocf_mngt_cache.c

@@ -857,6 +857,8 @@ static int _ocf_mngt_init_new_cache(struct ocf_cache_mngt_init_params *params)
 	/* start with freezed metadata ref counter to indicate detached device*/
 	ocf_refcnt_freeze(&cache->refcnt.metadata);
 
+	ocf_refcnt_init(&cache->refcnt.d2c);
+
 	env_atomic_set(&(cache->last_access_ms),
 			env_ticks_to_msecs(env_get_tick_count()));
 

src/mngt/ocf_mngt_flush.c

@@ -432,7 +432,7 @@ static void _ocf_mngt_flush_container(
 	fc->end = end;
 	fc->context = context;
 
-	req = ocf_req_new(cache->mngt_queue, NULL, 0, 0, 0);
+	req = ocf_req_new_mngt(cache, cache->mngt_queue);
 	if (!req) {
 		error = OCF_ERR_NO_MEM;
 		goto finish;

src/ocf_cache_priv.h

@@ -85,6 +85,8 @@ struct ocf_cache {
 		/* # of requests accessing attached metadata, excluding
 		 * management reqs */
 		struct ocf_refcnt metadata __attribute__((aligned(64)));
+		/* # of requests in d2c mode */
+		struct ocf_refcnt d2c;
 	} refcnt;
 
 	struct {

src/ocf_core.c

@@ -154,7 +154,7 @@ static uint64_t _calc_dirty_for(uint64_t dirty_since)
 	return dirty_since ? (current_time - dirty_since) : 0;
 }
 
-static inline struct ocf_request *ocf_io_to_req(struct ocf_io *io)
+struct ocf_request *ocf_io_to_req(struct ocf_io *io)
 {
 	struct ocf_io_internal *ioi;
 
@@ -489,9 +489,10 @@ static void *ocf_core_io_allocator_new(ocf_io_allocator_t allocator,
 		ocf_volume_t volume, ocf_queue_t queue,
 		uint64_t addr, uint32_t bytes, uint32_t dir)
 {
+	ocf_core_t core = ocf_volume_to_core(volume);
 	struct ocf_request *req;
 
-	req = ocf_req_new(queue, NULL, addr, bytes, dir);
+	req = ocf_req_new(queue, core, addr, bytes, dir);
 	if (!req)
 		return NULL;
 

src/ocf_core_priv.h

@@ -1,5 +1,6 @@
 /*
  * Copyright(c) 2012-2021 Intel Corporation
+ * Copyright(c) 2024 Huawei Technologies
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -105,4 +106,6 @@ ocf_core_id_t ocf_core_get_id(ocf_core_t core);
 
 int ocf_core_volume_type_init(ocf_ctx_t ctx);
 
+struct ocf_request *ocf_io_to_req(struct ocf_io *io);
+
 #endif /* __OCF_CORE_PRIV_H__ */

src/ocf_queue.c

@@ -126,6 +126,11 @@ int ocf_queue_create_mngt(ocf_cache_t cache, ocf_queue_t *queue,
 	return 0;
 }
 
+bool ocf_queue_is_mngt(ocf_queue_t queue)
+{
+	return queue == queue->cache->mngt_queue;
+}
+
 void ocf_queue_get(ocf_queue_t queue)
 {
 	OCF_CHECK_NULL(queue);
@@ -144,7 +149,7 @@ void ocf_queue_put(ocf_queue_t queue)
 		return;
 
 	queue->ops->stop(queue);
-	if (queue != queue->cache->mngt_queue) {
+	if (!ocf_queue_is_mngt(queue)) {
 		env_spinlock_lock_irqsave(&cache->io_queues_lock, flags);
 		list_del(&queue->list);
 		env_spinlock_unlock_irqrestore(&cache->io_queues_lock, flags);
@@ -247,12 +252,6 @@ uint32_t ocf_queue_pending_io(ocf_queue_t q)
 	return env_atomic_read(&q->io_no);
 }
 
-ocf_cache_t ocf_queue_get_cache(ocf_queue_t q)
-{
-	OCF_CHECK_NULL(q);
-	return q->cache;
-}
-
 void ocf_queue_push_req(struct ocf_request *req, uint flags)
 {
 	ocf_cache_t cache = req->cache;

src/ocf_request.c

@@ -1,5 +1,6 @@
 /*
  * Copyright(c) 2012-2022 Intel Corporation
+ * Copyright(c) 2024 Huawei Technologies
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -7,6 +8,7 @@
 #include "ocf_request.h"
 #include "ocf_cache_priv.h"
 #include "concurrency/ocf_metadata_concurrency.h"
+#include "engine/engine_common.h"
 #include "utils/utils_cache_line.h"
 
 #define OCF_UTILS_RQ_DEBUG 0
@@ -34,9 +36,8 @@ enum ocf_req_size {
 	ocf_req_size_128,
 };
 
-static inline size_t ocf_req_sizeof_map(struct ocf_request *req)
+static inline size_t ocf_req_sizeof_map(uint32_t lines)
 {
-	uint32_t lines = req->core_line_count;
 	size_t size = (lines * sizeof(struct ocf_map_info));
 
 	ENV_BUG_ON(lines == 0);
@@ -80,14 +81,128 @@ void ocf_req_allocator_deinit(struct ocf_ctx *ocf_ctx)
 	ocf_ctx->resources.req = NULL;
 }
 
+static inline void ocf_req_init(struct ocf_request *req, ocf_cache_t cache,
+		ocf_queue_t queue, ocf_core_t core,
+		uint64_t addr, uint32_t bytes, int rw)
+{
+	req->io_queue = queue;
+
+	req->core = core;
+	req->cache = cache;
+
+	env_atomic_set(&req->ref_count, 1);
+
+	req->byte_position = addr;
+	req->byte_length = bytes;
+	req->rw = rw;
+}
+
+struct ocf_request *ocf_req_new_mngt(ocf_cache_t cache, ocf_queue_t queue)
+{
+	struct ocf_request *req;
+
+	req = env_zalloc(sizeof(*req), ENV_MEM_NORMAL);
+	if (unlikely(!req))
+		return NULL;
+
+	ocf_queue_get(queue);
+
+	ocf_req_init(req, cache, queue, NULL, 0, 0, 0);
+
+	req->is_mngt = true;
+
+	return req;
+}
+
+struct ocf_request *ocf_req_new_cleaner(ocf_cache_t cache, ocf_queue_t queue,
+		uint32_t count)
+{
+	struct ocf_request *req;
+	bool map_allocated = true, is_mngt = false;
+
+	if (!ocf_refcnt_inc(&cache->refcnt.metadata))
+		return NULL;
+
+	if (unlikely(ocf_queue_is_mngt(queue))) {
+		req = env_zalloc(sizeof(*req) + ocf_req_sizeof_map(count) +
+				ocf_req_sizeof_alock_status(count),
+				ENV_MEM_NORMAL);
+		is_mngt = true;
+	} else {
+		req = env_mpool_new(cache->owner->resources.req, count);
+		if (!req) {
+			map_allocated = false;
+			req = env_mpool_new(cache->owner->resources.req, 1);
+		}
+	}
+
+	if (!req) {
+		ocf_refcnt_dec(&cache->refcnt.metadata);
+		return NULL;
+	}
+	req->is_mngt = is_mngt;
+
+	ocf_queue_get(queue);
+
+	ocf_req_init(req, cache, queue, NULL, 0, 0, OCF_READ);
+
+	if (map_allocated) {
+		req->map = req->__map;
+		req->alock_status = (uint8_t*)&req->__map[count];
+		req->alloc_core_line_count = count;
+	} else {
+		req->alloc_core_line_count = 1;
+	}
+	req->core_line_count = count;
+	req->lock_idx = ocf_metadata_concurrency_next_idx(queue);
+	req->cleaner = true;
+
+	if (ocf_req_alloc_map(req)) {
+		ocf_req_put(req);
+		req = NULL;
+	}
+	return req;
+}
+
+static inline struct ocf_request *ocf_req_new_d2c(ocf_queue_t queue,
+		ocf_core_t core, uint64_t addr, uint32_t bytes, int rw)
+{
+	ocf_cache_t cache = ocf_core_get_cache(core);
+	struct ocf_request *req;
+
+	req = env_mpool_new(cache->owner->resources.req, 1);
+	if (unlikely(!req))
+		        return NULL;
+
+	ocf_req_init(req, cache, queue, core, addr, bytes, rw);
+
+	req->d2c = true;
+	return req;
+}
+
 struct ocf_request *ocf_req_new(ocf_queue_t queue, ocf_core_t core,
 		uint64_t addr, uint32_t bytes, int rw)
 {
 	uint64_t core_line_first, core_line_last, core_line_count;
-	ocf_cache_t cache = queue->cache;
+	ocf_cache_t cache = ocf_core_get_cache(core);
 	struct ocf_request *req;
 	bool map_allocated = true;
 
+	ENV_BUG_ON(ocf_queue_is_mngt(queue));
+
+	ocf_queue_get(queue);
+
+	if (!ocf_refcnt_inc(&cache->refcnt.metadata)) {
+		if (!ocf_refcnt_inc(&cache->refcnt.d2c))
+			ENV_BUG();
+		req = ocf_req_new_d2c(queue, core, addr, bytes, rw);
+		if (unlikely(!req)) {
+			ocf_queue_put(queue);
+			return NULL;
+		}
+		return req;
+	}
+
 	if (likely(bytes)) {
 		core_line_first = ocf_bytes_2_lines(cache, addr);
 		core_line_last = ocf_bytes_2_lines(cache, addr + bytes - 1);
@@ -104,8 +219,11 @@ struct ocf_request *ocf_req_new(ocf_queue_t queue, ocf_core_t core,
 		req = env_mpool_new(cache->owner->resources.req, 1);
 	}
 
-	if (unlikely(!req))
+	if (unlikely(!req)) {
+		ocf_refcnt_dec(&cache->refcnt.metadata);
+		ocf_queue_put(queue);
 		return NULL;
+	}
 
 	if (map_allocated) {
 		req->map = req->__map;
@@ -115,32 +233,69 @@ struct ocf_request *ocf_req_new(ocf_queue_t queue, ocf_core_t core,
 		req->alloc_core_line_count = 1;
 	}
 
-
 	OCF_DEBUG_TRACE(cache);
 
-	ocf_queue_get(queue);
+	ocf_req_init(req, cache, queue, NULL, addr, bytes, rw);
-	req->io_queue = queue;
 
-	req->core = core;
-	req->cache = cache;
-
-	req->d2c = (queue != cache->mngt_queue) && !ocf_refcnt_inc(
-			&cache->refcnt.metadata);
-
-	env_atomic_set(&req->ref_count, 1);
-
-	req->byte_position = addr;
-	req->byte_length = bytes;
 	req->core_line_first = core_line_first;
 	req->core_line_last = core_line_last;
 	req->core_line_count = core_line_count;
-	req->rw = rw;
-	req->part_id = PARTITION_DEFAULT;
 
 	req->discard.sector = BYTES_TO_SECTORS(addr);
 	req->discard.nr_sects = BYTES_TO_SECTORS(bytes);
 	req->discard.handled = 0;
 
+	req->part_id = PARTITION_DEFAULT;
+
+	req->lock_idx = ocf_metadata_concurrency_next_idx(queue);
+
+	return req;
+}
+
+struct ocf_request *ocf_req_new_cache(ocf_cache_t cache, ocf_queue_t queue,
+		uint64_t addr, uint32_t bytes, int rw)
+{
+	uint64_t core_line_first, core_line_last, core_line_count;
+	struct ocf_request *req;
+	bool map_allocated = true;
+
+	ENV_BUG_ON(ocf_queue_is_mngt(queue));
+
+	if (!ocf_refcnt_inc(&cache->refcnt.metadata))
+		return NULL;
+
+	ocf_queue_get(queue);
+
+	if (likely(bytes)) {
+		core_line_first = ocf_bytes_2_lines(cache, addr);
+		core_line_last = ocf_bytes_2_lines(cache, addr + bytes - 1);
+		core_line_count = core_line_last - core_line_first + 1;
+	} else {
+		core_line_count = 1;
+	}
+
+	req = env_mpool_new(cache->owner->resources.req, core_line_count);
+	if (!req) {
+		map_allocated = false;
+		req = env_mpool_new(cache->owner->resources.req, 1);
+	}
+
+	if (unlikely(!req)) {
+		ocf_refcnt_dec(&cache->refcnt.metadata);
+		ocf_queue_put(queue);
+		return NULL;
+	}
+
+	if (map_allocated) {
+		req->map = req->__map;
+		req->alock_status = (uint8_t *)&req->__map[core_line_count];
+		req->alloc_core_line_count = core_line_count;
+	} else {
+		req->alloc_core_line_count = 1;
+	}
+
+	ocf_req_init(req, cache, queue, NULL, addr, bytes, rw);
+
 	req->lock_idx = ocf_metadata_concurrency_next_idx(queue);
 
 	return req;
@@ -148,10 +303,12 @@ struct ocf_request *ocf_req_new(ocf_queue_t queue, ocf_core_t core,
 
 int ocf_req_alloc_map(struct ocf_request *req)
 {
+	uint32_t lines = req->core_line_count;
+
 	if (req->map)
 		return 0;
 
-	req->map = env_zalloc(ocf_req_sizeof_map(req) +
+	req->map = env_zalloc(ocf_req_sizeof_map(lines) +
 			ocf_req_sizeof_alock_status(req->core_line_count),
 			ENV_MEM_NOIO);
 	if (!req->map) {
@@ -159,7 +316,7 @@ int ocf_req_alloc_map(struct ocf_request *req)
 		return -OCF_ERR_NO_MEM;
 	}
 
-	req->alock_status = &((uint8_t*)req->map)[ocf_req_sizeof_map(req)];
+	req->alock_status = &((uint8_t*)req->map)[ocf_req_sizeof_map(lines)];
 
 	return 0;
 }
@@ -229,14 +386,19 @@ void ocf_req_put(struct ocf_request *req)
 
 	OCF_DEBUG_TRACE(req->cache);
 
-	if (!req->d2c && req->io_queue != req->cache->mngt_queue)
+	if (req->d2c)
+		ocf_refcnt_dec(&req->cache->refcnt.d2c);
+	else if (!req->is_mngt || req->cleaner)
 		ocf_refcnt_dec(&req->cache->refcnt.metadata);
 
+	if (unlikely(req->is_mngt)) {
+		env_free(req);
+	} else {
 		if (req->map != req->__map)
 			env_free(req->map);
-
 		env_mpool_del(req->cache->owner->resources.req, req,
 				req->alloc_core_line_count);
+	}
 
 	ocf_queue_put(queue);
 }

src/ocf_request.h

@@ -198,6 +198,9 @@ struct ocf_request {
 	uint8_t d2c : 1;
 	/**!< request affects metadata cachelines (is not direct-to-core) */
 
+	uint8_t cleaner : 1;
+	/**!< request allocated by cleaner */
+
 	uint8_t dirty : 1;
 	/**!< indicates that request produces dirty data */
 
@@ -228,6 +231,9 @@ struct ocf_request {
 	uint8_t is_deferred : 1;
 	/* !< request handling was deferred and eventually resumed */
 
+	uint8_t is_mngt : 1;
+	/* !< It's a management path request */
+
 	ocf_req_cache_mode_t cache_mode;
 
 	uint64_t timestamp;
@@ -275,6 +281,26 @@ int ocf_req_allocator_init(struct ocf_ctx *ocf_ctx);
  */
 void ocf_req_allocator_deinit(struct ocf_ctx *ocf_ctx);
 
+/**
+ * @brief Allocate new OCF request for the management path
+ *
+ * @param queue - I/O queue handle
+ *
+ * @return new OCF request
+ */
+struct ocf_request *ocf_req_new_mngt(ocf_cache_t cache, ocf_queue_t queue);
+
+/**
+ * @brief Allocate new OCF request for cleaner
+ *
+ * @param queue - I/O queue handle
+ * @param count - Number of map entries
+ *
+ * @return new OCF request
+ */
+struct ocf_request *ocf_req_new_cleaner(ocf_cache_t cache, ocf_queue_t queue,
+		uint32_t count);
+
 /**
  * @brief Allocate new OCF request
  *
@@ -289,6 +315,20 @@ void ocf_req_allocator_deinit(struct ocf_ctx *ocf_ctx);
 struct ocf_request *ocf_req_new(ocf_queue_t queue, ocf_core_t core,
 		uint64_t addr, uint32_t bytes, int rw);
 
+/**
+ * @brief Allocate new OCF request for cache IO
+ *
+ * @param cache - OCF cache instance
+ * @param queue - I/O queue handle
+ * @param addr - LBA of request
+ * @param bytes - number of bytes of request
+ * @param rw - Read or Write
+ *
+ * @return new OCF request
+ */
+struct ocf_request *ocf_req_new_cache(ocf_cache_t cache, ocf_queue_t queue,
+		uint64_t addr, uint32_t bytes, int rw);
+
 /**
  * @brief Allocate OCF request map
  *

src/utils/utils_cleaner.c

@@ -40,10 +40,10 @@
 static struct ocf_request *_ocf_cleaner_alloc_req(struct ocf_cache *cache,
 		uint32_t count, const struct ocf_cleaner_attribs *attribs)
 {
-	struct ocf_request *req = ocf_req_new_extended(attribs->io_queue, NULL,
+	struct ocf_request *req;
-			0, count * ocf_line_size(cache), OCF_READ);
 	int ret;
 
+	req = ocf_req_new_cleaner(cache, attribs->io_queue, count);
 	if (!req)
 		return NULL;
 

src/utils/utils_parallelize.c

@@ -105,8 +105,7 @@ int ocf_parallelize_create(ocf_parallelize_t *parallelize,
 		} else {
 			queue = cache->mngt_queue;
 		}
-		tmp_parallelize->reqs[i] = ocf_req_new(queue,
+		tmp_parallelize->reqs[i] = ocf_req_new_mngt(cache, queue);
-				NULL, 0, 0, 0);
 		if (!tmp_parallelize->reqs[i]) {
 			result = -OCF_ERR_NO_MEM;
 			goto err_reqs;

src/utils/utils_pipeline.c

@@ -87,7 +87,7 @@ int ocf_pipeline_create(ocf_pipeline_t *pipeline, ocf_cache_t cache,
 		tmp_pipeline->priv = (void *)priv;
 	}
 
-	req = ocf_req_new(cache->mngt_queue, NULL, 0, 0, 0);
+	req = ocf_req_new_mngt(cache, cache->mngt_queue);
 	if (!req) {
 		env_vfree(tmp_pipeline);
 		return -OCF_ERR_NO_MEM;