volume: Introduce general IO forward mechanism

Allow the core volume IOs to be forwarded directly to backend volumes to
avoid unnecessary allocations.

Signed-off-by: Robert Baldyga <robert.baldyga@huawei.com>
Signed-off-by: Michal Mielewczyk <michal.mielewczyk@huawei.com>

inc/ocf_io.h

@@ -1,5 +1,6 @@
 /*
  * Copyright(c) 2012-2021 Intel Corporation
+ * Copyright(c) 2024 Huawei Technologies
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -235,4 +236,68 @@ void ocf_io_handle(struct ocf_io *io, void *opaque);
  */
 ocf_volume_t ocf_io_get_volume(struct ocf_io *io);
 
+/**
+ * @brief Get the original OCF IO associated with forward token
+ *
+ * @param[in] token Forward token
+ */
+struct ocf_io *ocf_forward_get_io(ocf_forward_token_t token);
+
+/**
+ * @brief Forward io to another subvolume
+ *
+ * Forwarding automatically increases forwarded io refcount, so at some
+ * point additional ocf_forward_end() needs to be called to balance it.
+ *
+ * @param[in] token Forward token
+ * @param[in] volume Volume to which IO is being submitted
+ * @param[in] token Token representing IO to be forwarded
+ * @param[in] dir Direction OCF_READ/OCF_WRITE
+ * @param[in] addr Address to which IO is being submitted
+ * @param[in] bytes Length of the IO
+ * @param[in] offset Offset within the IO data
+ */
+void ocf_forward_io(ocf_volume_t volume, ocf_forward_token_t token,
+		int dir, uint64_t addr, uint64_t bytes, uint64_t offset);
+
+/**
+ * @brief Forward flush to another subvolume
+ *
+ * Forwarding automatically increases forwarded io refcount, so at some
+ * point additional ocf_forward_end() needs to be called to balance it.
+ *
+ * @param[in] volume Volume to which IO is being submitted
+ * @param[in] token Token representing IO to be forwarded
+ */
+void ocf_forward_flush(ocf_volume_t volume, ocf_forward_token_t token);
+
+/**
+ * @brief Forward discard to another subvolume
+ *
+ * Forwarding automatically increases forwarded io refcount, so at some
+ * point additional ocf_forward_end() needs to be called to balance it.
+ *
+ * @param[in] volume Volume to which IO is being submitted
+ * @param[in] token Token representing IO to be forwarded
+ * @param[in] addr Address to which IO is being submitted
+ * @param[in] bytes Length of the IO
+ */
+void ocf_forward_discard(ocf_volume_t volume, ocf_forward_token_t token,
+		uint64_t addr, uint64_t bytes);
+
+/**
+ * @brief Increment forwarded io refcount
+ *
+ * @param[in] token Forward token
+ */
+void ocf_forward_get(ocf_forward_token_t token);
+
+/**
+ * @brief Complete the forwarded io
+ *
+ * @param[in] token Forward token to be completed
+ * @param[in] error Completion status code
+ */
+void ocf_forward_end(ocf_forward_token_t token, int error);
+
 #endif /* __OCF_IO_H__ */

inc/ocf_types.h

@@ -1,5 +1,6 @@
 /*
  * Copyright(c) 2012-2021 Intel Corporation
+ * Copyright(c) 2024 Huawei Technologies
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -72,6 +73,17 @@ typedef struct ocf_volume_uuid *ocf_uuid_t;
  */
 typedef void ctx_data_t;
 
+/**
+ * @brief IO forward token
+ *
+ * The token is associated with IO that is being forwarded. It allows
+ * OCF to keep track of which IO has been forwarded where. It also has
+ * refcount which can be increased/decreased on each forward level, so
+ * that there is no need to introduce additional counters if at some
+ * level the forward needs to be splitted into several sub-forwards.
+ */
+typedef uint64_t ocf_forward_token_t;
+
 /**
  * @brief handle to I/O queue
  */

inc/ocf_volume.h

@@ -1,5 +1,6 @@
 /*
  * Copyright(c) 2012-2022 Intel Corporation
+ * Copyright(c) 2024 Huawei Technologies
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -82,6 +83,39 @@ struct ocf_volume_ops {
 	 */
 	void (*submit_write_zeroes)(struct ocf_io *io);
 
+	/**
+	 * @brief Forward the original io directly to the volume
+	 *
+	 * @param[in] volume Volume to which IO is being submitted
+	 * @param[in] token Token representing IO to be forwarded
+	 * @param[in] dir Direction OCF_READ/OCF_WRITE
+	 * @param[in] addr Address to which IO is being submitted
+	 * @param[in] bytes Length of the IO
+	 * @param[in] offset Offset within the IO data
+	 */
+	void (*forward_io)(ocf_volume_t volume, ocf_forward_token_t token,
+			int dir, uint64_t addr, uint64_t bytes,
+			uint64_t offset);
+
+	/**
+	 * @brief Forward the original flush io directly to the volume
+	 *
+	 * @param[in] volume Volume to which IO is being submitted
+	 * @param[in] token Token representing IO to be forwarded
+	 */
+	void (*forward_flush)(ocf_volume_t volume, ocf_forward_token_t token);
+
+	/**
+	 * @brief Forward the original discard io directly to the volume
+	 *
+	 * @param[in] volume Volume to which IO is being submitted
+	 * @param[in] token Token representing IO to be forwarded
+	 * @param[in] addr Address to which IO is being submitted
+	 * @param[in] bytes Length of the IO
+	 */
+	void (*forward_discard)(ocf_volume_t volume, ocf_forward_token_t token,
+			uint64_t addr, uint64_t bytes);
+
 	/**
 	 * @brief Volume initialization callback, called when volume object
 	 *        is being initialized

src/engine/engine_io.c

@@ -0,0 +1,162 @@
+/*
+ * Copyright(c) 2024 Huawei Technologies
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include "ocf/ocf.h"
+#include "engine_io.h"
+#include "engine_common.h"
+#include "../ocf_priv.h"
+#include "../ocf_cache_priv.h"
+#include "../ocf_volume_priv.h"
+#include "../ocf_request.h"
+#include "../utils/utils_cache_line.h"
+
+void ocf_engine_forward_cache_io(struct ocf_request *req, int dir,
+		uint64_t offset, uint64_t size, ocf_req_end_t callback)
+{
+	ocf_cache_t cache = req->cache;
+	uint32_t seek = req->byte_position % ocf_line_size(cache);
+	uint32_t first_cl = ocf_bytes_2_lines(cache, offset + seek);
+	uint64_t addr;
+
+	req->cache_forward_end = callback;
+
+	addr = cache->device->metadata_offset;
+	addr += req->map[first_cl].coll_idx * ocf_line_size(cache);
+	addr += (offset + seek) % ocf_line_size(cache);
+
+	ocf_core_stats_cache_block_update(req->core, req->part_id,
+			dir, req->byte_length);
+
+	ocf_req_forward_cache_io(req, dir, addr, size,
+			req->offset + offset);
+}
+
+void ocf_engine_forward_cache_io_req(struct ocf_request *req, int dir,
+		ocf_req_end_t callback)
+{
+	ocf_cache_t cache = req->cache;
+	uint64_t addr, bytes, total_bytes = 0, addr_next = 0;
+	uint32_t i;
+
+	req->cache_forward_end = callback;
+
+	if (ocf_engine_is_sequential(req)) {
+		addr = cache->device->metadata_offset;
+		addr += req->map[0].coll_idx * ocf_line_size(cache);
+		addr += req->byte_position % ocf_line_size(cache);
+
+		ocf_core_stats_cache_block_update(req->core, req->part_id,
+				dir, req->byte_length);
+
+		ocf_req_forward_cache_io(req, dir, addr, req->byte_length,
+				req->offset);
+		return;
+	}
+
+	ocf_req_forward_cache_get(req);
+	for (i = 0; i < req->core_line_count; i++) {
+		if (addr_next) {
+			addr = addr_next;
+		} else {
+			addr  = req->map[i].coll_idx;
+			addr *= ocf_line_size(cache);
+			addr += cache->device->metadata_offset;
+		}
+		bytes = ocf_line_size(cache);
+
+		if (i == 0) {
+			uint64_t seek = (req->byte_position) %
+					ocf_line_size(cache);
+
+			addr += seek;
+			bytes -= seek;
+		}
+
+		for (; i < (req->core_line_count - 1); i++) {
+			addr_next = req->map[i + 1].coll_idx;
+			addr_next *= ocf_line_size(cache);
+			addr_next += cache->device->metadata_offset;
+
+			if (addr_next != (addr + bytes))
+				break;
+
+			bytes += ocf_line_size(cache);
+		}
+
+		if (i == (req->core_line_count - 1)) {
+			uint64_t skip = (ocf_line_size(cache) -
+				((req->byte_position + req->byte_length) %
+				ocf_line_size(cache))) % ocf_line_size(cache);
+
+			bytes -= skip;
+		}
+
+		bytes = OCF_MIN(bytes, req->byte_length - total_bytes);
+		ENV_BUG_ON(bytes == 0);
+
+		ocf_core_stats_cache_block_update(req->core, req->part_id,
+				dir, bytes);
+
+		ocf_req_forward_cache_io(req, dir, addr, bytes,
+				req->offset + total_bytes);
+
+		total_bytes += bytes;
+	}
+
+	ENV_BUG_ON(total_bytes != req->byte_length);
+
+	ocf_req_forward_cache_put(req);
+}
+
+void ocf_engine_forward_cache_flush_req(struct ocf_request *req,
+		ocf_req_end_t callback)
+{
+	req->cache_forward_end = callback;
+
+	ocf_req_forward_cache_flush(req);
+}
+
+void ocf_engine_forward_cache_discard_req(struct ocf_request *req,
+		ocf_req_end_t callback)
+{
+	req->cache_forward_end = callback;
+
+	ocf_req_forward_cache_discard(req, req->byte_position,
+			req->byte_length);
+}
+
+void ocf_engine_forward_core_io_req(struct ocf_request *req,
+		ocf_req_end_t callback)
+{
+	ocf_core_stats_core_block_update(req->core, req->part_id, req->rw,
+			req->byte_length);
+
+	req->core_forward_end = callback;
+
+	ocf_req_forward_core_io(req, req->rw, req->byte_position,
+			req->byte_length, req->offset);
+}
+
+void ocf_engine_forward_core_flush_req(struct ocf_request *req,
+		ocf_req_end_t callback)
+{
+	ocf_core_stats_core_block_update(req->core, req->part_id, req->rw,
+			req->byte_length);
+
+	req->core_forward_end = callback;
+
+	ocf_req_forward_core_flush(req);
+}
+
+void ocf_engine_forward_core_discard_req(struct ocf_request *req,
+		ocf_req_end_t callback)
+{
+	ocf_core_stats_core_block_update(req->core, req->part_id, req->rw,
+			req->byte_length);
+
+	req->core_forward_end = callback;
+
+	ocf_req_forward_core_discard(req, req->byte_position, req->byte_length);
+}

src/engine/engine_io.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright(c) 2024 Huawei Technologies
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ENGINE_IO_H_
+#define ENGINE_IO_H_
+
+#include "../ocf_request.h"
+
+void ocf_engine_forward_cache_io(struct ocf_request *req, int dir,
+		uint64_t offset, uint64_t size, ocf_req_end_t callback);
+
+void ocf_engine_forward_cache_io_req(struct ocf_request *req, int dir,
+		ocf_req_end_t callback);
+
+void ocf_engine_forward_cache_flush_req(struct ocf_request *req,
+		ocf_req_end_t callback);
+
+void ocf_engine_forward_cache_discard_req(struct ocf_request *req,
+		ocf_req_end_t callback);
+
+void ocf_engine_forward_core_io_req(struct ocf_request *req,
+		ocf_req_end_t callback);
+
+void ocf_engine_forward_core_flush_req(struct ocf_request *req,
+		ocf_req_end_t callback);
+
+void ocf_engine_forward_core_discard_req(struct ocf_request *req,
+		ocf_req_end_t callback);
+
+#endif /* ENGINE_IO_H_ */

src/ocf_def_priv.h

@@ -1,5 +1,6 @@
 /*
  * Copyright(c) 2012-2021 Intel Corporation
+ * Copyright(c) 2024 Huawei Technologies
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -56,4 +57,8 @@ ocf_rotate_right(unsigned long long bits, unsigned shift, unsigned width)
 		((1ULL << width) - 1);
 }
 
+struct ocf_request;
+
+typedef void (*ocf_req_end_t)(struct ocf_request *req, int error);
+
 #endif

src/ocf_request.c

@@ -431,3 +431,130 @@ void ocf_req_hash(struct ocf_request *req)
 				ocf_core_get_id(req->core));
 	}
 }
+
+void ocf_req_forward_cache_io(struct ocf_request *req, int dir, uint64_t addr,
+		uint64_t bytes, uint64_t offset)
+{
+	ocf_volume_t volume = ocf_cache_get_volume(req->cache);
+	ocf_forward_token_t token = ocf_req_to_cache_forward_token(req);
+
+	req->cache_error = 0;
+
+	ocf_req_forward_cache_get(req);
+	ocf_volume_forward_io(volume, token, dir, addr, bytes, offset);
+}
+
+void ocf_req_forward_cache_flush(struct ocf_request *req)
+{
+	ocf_volume_t volume = ocf_cache_get_volume(req->cache);
+	ocf_forward_token_t token = ocf_req_to_cache_forward_token(req);
+
+	req->cache_error = 0;
+
+	ocf_req_forward_cache_get(req);
+	ocf_volume_forward_flush(volume, token);
+}
+
+void ocf_req_forward_cache_discard(struct ocf_request *req, uint64_t addr,
+		uint64_t bytes)
+{
+	ocf_volume_t volume = ocf_cache_get_volume(req->cache);
+	ocf_forward_token_t token = ocf_req_to_cache_forward_token(req);
+
+	req->cache_error = 0;
+
+	ocf_req_forward_cache_get(req);
+	ocf_volume_forward_discard(volume, token, addr, bytes);
+}
+
+void ocf_req_forward_core_io(struct ocf_request *req, int dir, uint64_t addr,
+		uint64_t bytes, uint64_t offset)
+{
+	ocf_volume_t volume = ocf_core_get_volume(req->core);
+	ocf_forward_token_t token = ocf_req_to_core_forward_token(req);
+
+	req->core_error = 0;
+
+	ocf_req_forward_core_get(req);
+	ocf_volume_forward_io(volume, token, dir, addr, bytes, offset);
+}
+
+void ocf_req_forward_core_flush(struct ocf_request *req)
+{
+	ocf_volume_t volume = ocf_core_get_volume(req->core);
+	ocf_forward_token_t token = ocf_req_to_core_forward_token(req);
+
+	req->core_error = 0;
+
+	ocf_req_forward_core_get(req);
+	ocf_volume_forward_flush(volume, token);
+}
+
+void ocf_req_forward_core_discard(struct ocf_request *req, uint64_t addr,
+		uint64_t bytes)
+{
+	ocf_volume_t volume = ocf_core_get_volume(req->core);
+	ocf_forward_token_t token = ocf_req_to_core_forward_token(req);
+
+	req->core_error = 0;
+
+	ocf_req_forward_core_get(req);
+	ocf_volume_forward_discard(volume, token, addr, bytes);
+}
+
+struct ocf_io *ocf_forward_get_io(ocf_forward_token_t token)
+{
+	struct ocf_request *req = (struct ocf_request *)(token & ~1);
+
+	return &req->ioi.io;
+}
+
+static inline void _ocf_forward_get(ocf_forward_token_t token)
+{
+	struct ocf_request *req = (struct ocf_request *)(token & ~1);
+
+	if (token & 1)
+		ocf_req_forward_cache_get(req);
+	else
+		ocf_req_forward_core_get(req);
+}
+
+void ocf_forward_get(ocf_forward_token_t token)
+{
+	_ocf_forward_get(token);
+}
+
+void ocf_forward_io(ocf_volume_t volume, ocf_forward_token_t token,
+		int dir, uint64_t addr, uint64_t bytes, uint64_t offset)
+{
+	_ocf_forward_get(token);
+	ocf_volume_forward_io(volume, token, dir, addr, bytes, offset);
+}
+
+void ocf_forward_flush(ocf_volume_t volume, ocf_forward_token_t token)
+{
+	_ocf_forward_get(token);
+	ocf_volume_forward_flush(volume, token);
+}
+
+void ocf_forward_discard(ocf_volume_t volume, ocf_forward_token_t token,
+		uint64_t addr, uint64_t bytes)
+{
+	_ocf_forward_get(token);
+	ocf_volume_forward_discard(volume, token, addr, bytes);
+}
+
+void ocf_forward_end(ocf_forward_token_t token, int error)
+{
+	struct ocf_request *req = ocf_req_forward_token_to_req(token);
+
+	req->error |= error;
+
+	if (token & 1) {
+		req->cache_error = req->cache_error ?: error;
+		ocf_req_forward_cache_put(req);
+	} else {
+		req->core_error = req->core_error ?: error;
+		ocf_req_forward_core_put(req);
+	}
+}

src/ocf_request.h

@@ -9,6 +9,7 @@
 
 #include "ocf_env.h"
 #include "ocf_io_priv.h"
+#include "ocf_def_priv.h"
 #include "metadata/metadata_structs.h"
 
 typedef enum {
@@ -128,6 +129,11 @@ struct ocf_request {
 	struct ocf_io_internal ioi;
 	/*!< OCF IO associated with request */
 
+	ocf_req_end_t cache_forward_end;
+	ocf_req_end_t core_forward_end;
+	env_atomic cache_remaining;
+	env_atomic core_remaining;
+
 	env_atomic ref_count;
 	/*!< Reference usage count, once OCF request reaches zero it
 	 * will be de-initialed. Get/Put method are intended to modify
@@ -260,7 +266,7 @@ struct ocf_request {
 	struct ocf_req_info info;
 	/*!< Detailed request info */
 
-	void (*complete)(struct ocf_request *ocf_req, int error);
+	ocf_req_end_t complete;
 	/*!< Request completion function */
 
 	struct ocf_req_discard_info discard;
@@ -276,8 +282,6 @@ struct ocf_request {
 	struct ocf_map_info __map[0];
 };
 
-typedef void (*ocf_req_end_t)(struct ocf_request *req, int error);
-
 /**
  * @brief Initialize OCF request allocation utility
  *
@@ -512,4 +516,57 @@ static inline bool ocf_req_is_4k(uint64_t addr, uint32_t bytes)
 	return !((addr % PAGE_SIZE) || (bytes % PAGE_SIZE));
 }
 
+static inline void ocf_req_forward_cache_get(struct ocf_request *req)
+{
+	env_atomic_inc(&req->cache_remaining);
+}
+
+static inline void ocf_req_forward_cache_put(struct ocf_request *req)
+{
+	if (env_atomic_dec_return(&req->cache_remaining) == 0)
+		req->cache_forward_end(req, req->cache_error);
+}
+
+static inline void ocf_req_forward_core_get(struct ocf_request *req)
+{
+	env_atomic_inc(&req->core_remaining);
+}
+
+static inline void ocf_req_forward_core_put(struct ocf_request *req)
+{
+	if (env_atomic_dec_return(&req->core_remaining) == 0)
+		req->core_forward_end(req, req->core_error);
+}
+
+static inline ocf_forward_token_t ocf_req_to_cache_forward_token(struct ocf_request *req)
+{
+	return (ocf_forward_token_t)req | 1;
+}
+
+static inline ocf_forward_token_t ocf_req_to_core_forward_token(struct ocf_request *req)
+{
+	return (ocf_forward_token_t)req;
+}
+
+static inline struct ocf_request *ocf_req_forward_token_to_req(ocf_forward_token_t token)
+{
+	return (struct ocf_request *)(token & ~1);
+}
+
+void ocf_req_forward_cache_io(struct ocf_request *req, int dir, uint64_t addr,
+		uint64_t bytes, uint64_t offset);
+
+void ocf_req_forward_cache_flush(struct ocf_request *req);
+
+void ocf_req_forward_cache_discard(struct ocf_request *req, uint64_t addr,
+		uint64_t bytes);
+
+void ocf_req_forward_core_io(struct ocf_request *req, int dir, uint64_t addr,
+		uint64_t bytes, uint64_t offset);
+
+void ocf_req_forward_core_flush(struct ocf_request *req);
+
+void ocf_req_forward_core_discard(struct ocf_request *req, uint64_t addr,
+		uint64_t bytes);
+
 #endif /* __OCF_REQUEST_H__ */

src/ocf_volume.c

@@ -1,11 +1,13 @@
 /*
  * Copyright(c) 2012-2022 Intel Corporation
+ * Copyright(c) 2024 Huawei Technologies
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
 #include "ocf/ocf.h"
 #include "ocf_priv.h"
 #include "ocf_volume_priv.h"
+#include "ocf_request.h"
 #include "ocf_io_priv.h"
 #include "ocf_env.h"
 
@@ -328,6 +330,46 @@ void ocf_volume_submit_discard(struct ocf_io *io)
 	volume->type->properties->ops.submit_discard(io);
 }
 
+void ocf_volume_forward_io(ocf_volume_t volume, ocf_forward_token_t token,
+		int dir, uint64_t addr, uint64_t bytes, uint64_t offset)
+{
+	ENV_BUG_ON(!volume->type->properties->ops.forward_io);
+
+	if (!volume->opened) {
+		ocf_forward_end(token, -OCF_ERR_IO);
+		return;
+	}
+
+	volume->type->properties->ops.forward_io(volume, token,
+			dir, addr, bytes, offset);
+}
+
+void ocf_volume_forward_flush(ocf_volume_t volume, ocf_forward_token_t token)
+{
+	ENV_BUG_ON(!volume->type->properties->ops.forward_flush);
+
+	if (!volume->opened) {
+		ocf_forward_end(token, -OCF_ERR_IO);
+		return;
+	}
+
+	volume->type->properties->ops.forward_flush(volume, token);
+}
+
+void ocf_volume_forward_discard(ocf_volume_t volume, ocf_forward_token_t token,
+		uint64_t addr, uint64_t bytes)
+{
+	ENV_BUG_ON(!volume->type->properties->ops.forward_discard);
+
+	if (!volume->opened) {
+		ocf_forward_end(token, -OCF_ERR_IO);
+		return;
+	}
+
+	volume->type->properties->ops.forward_discard(volume, token,
+			addr, bytes);
+}
+
 int ocf_volume_open(ocf_volume_t volume, void *volume_params)
 {
 	int ret;

src/ocf_volume_priv.h

@@ -1,5 +1,6 @@
 /*
  * Copyright(c) 2012-2021 Intel Corporation
+ * Copyright(c) 2024 Huawei Technologies
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -46,6 +47,14 @@ void ocf_volume_move(ocf_volume_t volume, ocf_volume_t from);
 void ocf_volume_set_uuid(ocf_volume_t volume,
 		const struct ocf_volume_uuid *uuid);
 
+void ocf_volume_forward_io(ocf_volume_t volume, ocf_forward_token_t token,
+		int dir, uint64_t addr, uint64_t bytes, uint64_t offset);
+
+void ocf_volume_forward_flush(ocf_volume_t volume, ocf_forward_token_t token);
+
+void ocf_volume_forward_discard(ocf_volume_t volume, ocf_forward_token_t token,
+		uint64_t addr, uint64_t bytes);
+
 static inline void ocf_volume_submit_metadata(struct ocf_io *io)
 {
 	ocf_volume_t volume = ocf_io_get_volume(io);