From 7e73de0d511c70583ea082d154cc447c4178c0bb Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Tue, 22 Aug 2023 21:11:25 +0200 Subject: [PATCH] volume: Introduce general IO forward mechanism Allow the core volume IOs to be forwarded directly to backend volumes to avoid unnecessary allocations. Signed-off-by: Robert Baldyga Signed-off-by: Michal Mielewczyk --- inc/ocf_io.h | 65 +++++++++++++++++ inc/ocf_types.h | 12 +++ inc/ocf_volume.h | 34 +++++++++ src/engine/engine_io.c | 162 +++++++++++++++++++++++++++++++++++++++++ src/engine/engine_io.h | 32 ++++++++ src/ocf_def_priv.h | 5 ++ src/ocf_request.c | 127 ++++++++++++++++++++++++++++++++ src/ocf_request.h | 63 +++++++++++++++- src/ocf_volume.c | 42 +++++++++++ src/ocf_volume_priv.h | 9 +++ 10 files changed, 548 insertions(+), 3 deletions(-) create mode 100644 src/engine/engine_io.c create mode 100644 src/engine/engine_io.h diff --git a/inc/ocf_io.h b/inc/ocf_io.h index f5e69b4..e7ff2b5 100644 --- a/inc/ocf_io.h +++ b/inc/ocf_io.h @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -235,4 +236,68 @@ void ocf_io_handle(struct ocf_io *io, void *opaque); */ ocf_volume_t ocf_io_get_volume(struct ocf_io *io); +/** + * @brief Get the original OCF IO associated with forward token + * + * @param[in] token Forward token + */ +struct ocf_io *ocf_forward_get_io(ocf_forward_token_t token); + +/** + * @brief Forward io to another subvolume + * + * Forwarding automatically increases forwarded io refcount, so at some + * point additional ocf_forward_end() needs to be called to balance it. + * + * @param[in] token Forward token + * @param[in] volume Volume to which IO is being submitted + * @param[in] token Token representing IO to be forwarded + * @param[in] dir Direction OCF_READ/OCF_WRITE + * @param[in] addr Address to which IO is being submitted + * @param[in] bytes Length of the IO + * @param[in] offset Offset within the IO data + */ +void ocf_forward_io(ocf_volume_t volume, ocf_forward_token_t token, + int dir, uint64_t addr, uint64_t bytes, uint64_t offset); + +/** + * @brief Forward flush to another subvolume + * + * Forwarding automatically increases forwarded io refcount, so at some + * point additional ocf_forward_end() needs to be called to balance it. + * + * @param[in] volume Volume to which IO is being submitted + * @param[in] token Token representing IO to be forwarded + */ +void ocf_forward_flush(ocf_volume_t volume, ocf_forward_token_t token); + +/** + * @brief Forward discard to another subvolume + * + * Forwarding automatically increases forwarded io refcount, so at some + * point additional ocf_forward_end() needs to be called to balance it. + * + * @param[in] volume Volume to which IO is being submitted + * @param[in] token Token representing IO to be forwarded + * @param[in] addr Address to which IO is being submitted + * @param[in] bytes Length of the IO + */ +void ocf_forward_discard(ocf_volume_t volume, ocf_forward_token_t token, + uint64_t addr, uint64_t bytes); + +/** + * @brief Increment forwarded io refcount + * + * @param[in] token Forward token + */ +void ocf_forward_get(ocf_forward_token_t token); + +/** + * @brief Complete the forwarded io + * + * @param[in] token Forward token to be completed + * @param[in] error Completion status code + */ +void ocf_forward_end(ocf_forward_token_t token, int error); + #endif /* __OCF_IO_H__ */ diff --git a/inc/ocf_types.h b/inc/ocf_types.h index 1dca3d7..fe6b415 100644 --- a/inc/ocf_types.h +++ b/inc/ocf_types.h @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -72,6 +73,17 @@ typedef struct ocf_volume_uuid *ocf_uuid_t; */ typedef void ctx_data_t; +/** + * @brief IO forward token + * + * The token is associated with IO that is being forwarded. It allows + * OCF to keep track of which IO has been forwarded where. It also has + * refcount which can be increased/decreased on each forward level, so + * that there is no need to introduce additional counters if at some + * level the forward needs to be splitted into several sub-forwards. + */ +typedef uint64_t ocf_forward_token_t; + /** * @brief handle to I/O queue */ diff --git a/inc/ocf_volume.h b/inc/ocf_volume.h index f6b26f6..7a2f9a2 100644 --- a/inc/ocf_volume.h +++ b/inc/ocf_volume.h @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -82,6 +83,39 @@ struct ocf_volume_ops { */ void (*submit_write_zeroes)(struct ocf_io *io); + /** + * @brief Forward the original io directly to the volume + * + * @param[in] volume Volume to which IO is being submitted + * @param[in] token Token representing IO to be forwarded + * @param[in] dir Direction OCF_READ/OCF_WRITE + * @param[in] addr Address to which IO is being submitted + * @param[in] bytes Length of the IO + * @param[in] offset Offset within the IO data + */ + void (*forward_io)(ocf_volume_t volume, ocf_forward_token_t token, + int dir, uint64_t addr, uint64_t bytes, + uint64_t offset); + + /** + * @brief Forward the original flush io directly to the volume + * + * @param[in] volume Volume to which IO is being submitted + * @param[in] token Token representing IO to be forwarded + */ + void (*forward_flush)(ocf_volume_t volume, ocf_forward_token_t token); + + /** + * @brief Forward the original discard io directly to the volume + * + * @param[in] volume Volume to which IO is being submitted + * @param[in] token Token representing IO to be forwarded + * @param[in] addr Address to which IO is being submitted + * @param[in] bytes Length of the IO + */ + void (*forward_discard)(ocf_volume_t volume, ocf_forward_token_t token, + uint64_t addr, uint64_t bytes); + /** * @brief Volume initialization callback, called when volume object * is being initialized diff --git a/src/engine/engine_io.c b/src/engine/engine_io.c new file mode 100644 index 0000000..c742cd3 --- /dev/null +++ b/src/engine/engine_io.c @@ -0,0 +1,162 @@ +/* + * Copyright(c) 2024 Huawei Technologies + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "ocf/ocf.h" +#include "engine_io.h" +#include "engine_common.h" +#include "../ocf_priv.h" +#include "../ocf_cache_priv.h" +#include "../ocf_volume_priv.h" +#include "../ocf_request.h" +#include "../utils/utils_cache_line.h" + +void ocf_engine_forward_cache_io(struct ocf_request *req, int dir, + uint64_t offset, uint64_t size, ocf_req_end_t callback) +{ + ocf_cache_t cache = req->cache; + uint32_t seek = req->byte_position % ocf_line_size(cache); + uint32_t first_cl = ocf_bytes_2_lines(cache, offset + seek); + uint64_t addr; + + req->cache_forward_end = callback; + + addr = cache->device->metadata_offset; + addr += req->map[first_cl].coll_idx * ocf_line_size(cache); + addr += (offset + seek) % ocf_line_size(cache); + + ocf_core_stats_cache_block_update(req->core, req->part_id, + dir, req->byte_length); + + ocf_req_forward_cache_io(req, dir, addr, size, + req->offset + offset); +} + +void ocf_engine_forward_cache_io_req(struct ocf_request *req, int dir, + ocf_req_end_t callback) +{ + ocf_cache_t cache = req->cache; + uint64_t addr, bytes, total_bytes = 0, addr_next = 0; + uint32_t i; + + req->cache_forward_end = callback; + + if (ocf_engine_is_sequential(req)) { + addr = cache->device->metadata_offset; + addr += req->map[0].coll_idx * ocf_line_size(cache); + addr += req->byte_position % ocf_line_size(cache); + + ocf_core_stats_cache_block_update(req->core, req->part_id, + dir, req->byte_length); + + ocf_req_forward_cache_io(req, dir, addr, req->byte_length, + req->offset); + return; + } + + ocf_req_forward_cache_get(req); + for (i = 0; i < req->core_line_count; i++) { + if (addr_next) { + addr = addr_next; + } else { + addr = req->map[i].coll_idx; + addr *= ocf_line_size(cache); + addr += cache->device->metadata_offset; + } + bytes = ocf_line_size(cache); + + if (i == 0) { + uint64_t seek = (req->byte_position) % + ocf_line_size(cache); + + addr += seek; + bytes -= seek; + } + + for (; i < (req->core_line_count - 1); i++) { + addr_next = req->map[i + 1].coll_idx; + addr_next *= ocf_line_size(cache); + addr_next += cache->device->metadata_offset; + + if (addr_next != (addr + bytes)) + break; + + bytes += ocf_line_size(cache); + } + + if (i == (req->core_line_count - 1)) { + uint64_t skip = (ocf_line_size(cache) - + ((req->byte_position + req->byte_length) % + ocf_line_size(cache))) % ocf_line_size(cache); + + bytes -= skip; + } + + bytes = OCF_MIN(bytes, req->byte_length - total_bytes); + ENV_BUG_ON(bytes == 0); + + ocf_core_stats_cache_block_update(req->core, req->part_id, + dir, bytes); + + ocf_req_forward_cache_io(req, dir, addr, bytes, + req->offset + total_bytes); + + total_bytes += bytes; + } + + ENV_BUG_ON(total_bytes != req->byte_length); + + ocf_req_forward_cache_put(req); +} + +void ocf_engine_forward_cache_flush_req(struct ocf_request *req, + ocf_req_end_t callback) +{ + req->cache_forward_end = callback; + + ocf_req_forward_cache_flush(req); +} + +void ocf_engine_forward_cache_discard_req(struct ocf_request *req, + ocf_req_end_t callback) +{ + req->cache_forward_end = callback; + + ocf_req_forward_cache_discard(req, req->byte_position, + req->byte_length); +} + +void ocf_engine_forward_core_io_req(struct ocf_request *req, + ocf_req_end_t callback) +{ + ocf_core_stats_core_block_update(req->core, req->part_id, req->rw, + req->byte_length); + + req->core_forward_end = callback; + + ocf_req_forward_core_io(req, req->rw, req->byte_position, + req->byte_length, req->offset); +} + +void ocf_engine_forward_core_flush_req(struct ocf_request *req, + ocf_req_end_t callback) +{ + ocf_core_stats_core_block_update(req->core, req->part_id, req->rw, + req->byte_length); + + req->core_forward_end = callback; + + ocf_req_forward_core_flush(req); +} + +void ocf_engine_forward_core_discard_req(struct ocf_request *req, + ocf_req_end_t callback) +{ + ocf_core_stats_core_block_update(req->core, req->part_id, req->rw, + req->byte_length); + + req->core_forward_end = callback; + + ocf_req_forward_core_discard(req, req->byte_position, req->byte_length); +} diff --git a/src/engine/engine_io.h b/src/engine/engine_io.h new file mode 100644 index 0000000..16a2a33 --- /dev/null +++ b/src/engine/engine_io.h @@ -0,0 +1,32 @@ +/* + * Copyright(c) 2024 Huawei Technologies + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ENGINE_IO_H_ +#define ENGINE_IO_H_ + +#include "../ocf_request.h" + +void ocf_engine_forward_cache_io(struct ocf_request *req, int dir, + uint64_t offset, uint64_t size, ocf_req_end_t callback); + +void ocf_engine_forward_cache_io_req(struct ocf_request *req, int dir, + ocf_req_end_t callback); + +void ocf_engine_forward_cache_flush_req(struct ocf_request *req, + ocf_req_end_t callback); + +void ocf_engine_forward_cache_discard_req(struct ocf_request *req, + ocf_req_end_t callback); + +void ocf_engine_forward_core_io_req(struct ocf_request *req, + ocf_req_end_t callback); + +void ocf_engine_forward_core_flush_req(struct ocf_request *req, + ocf_req_end_t callback); + +void ocf_engine_forward_core_discard_req(struct ocf_request *req, + ocf_req_end_t callback); + +#endif /* ENGINE_IO_H_ */ diff --git a/src/ocf_def_priv.h b/src/ocf_def_priv.h index 98ba8ce..4236ce5 100644 --- a/src/ocf_def_priv.h +++ b/src/ocf_def_priv.h @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -56,4 +57,8 @@ ocf_rotate_right(unsigned long long bits, unsigned shift, unsigned width) ((1ULL << width) - 1); } +struct ocf_request; + +typedef void (*ocf_req_end_t)(struct ocf_request *req, int error); + #endif diff --git a/src/ocf_request.c b/src/ocf_request.c index 71ba2da..82c75f8 100644 --- a/src/ocf_request.c +++ b/src/ocf_request.c @@ -431,3 +431,130 @@ void ocf_req_hash(struct ocf_request *req) ocf_core_get_id(req->core)); } } + +void ocf_req_forward_cache_io(struct ocf_request *req, int dir, uint64_t addr, + uint64_t bytes, uint64_t offset) +{ + ocf_volume_t volume = ocf_cache_get_volume(req->cache); + ocf_forward_token_t token = ocf_req_to_cache_forward_token(req); + + req->cache_error = 0; + + ocf_req_forward_cache_get(req); + ocf_volume_forward_io(volume, token, dir, addr, bytes, offset); +} + +void ocf_req_forward_cache_flush(struct ocf_request *req) +{ + ocf_volume_t volume = ocf_cache_get_volume(req->cache); + ocf_forward_token_t token = ocf_req_to_cache_forward_token(req); + + req->cache_error = 0; + + ocf_req_forward_cache_get(req); + ocf_volume_forward_flush(volume, token); +} + +void ocf_req_forward_cache_discard(struct ocf_request *req, uint64_t addr, + uint64_t bytes) +{ + ocf_volume_t volume = ocf_cache_get_volume(req->cache); + ocf_forward_token_t token = ocf_req_to_cache_forward_token(req); + + req->cache_error = 0; + + ocf_req_forward_cache_get(req); + ocf_volume_forward_discard(volume, token, addr, bytes); +} + +void ocf_req_forward_core_io(struct ocf_request *req, int dir, uint64_t addr, + uint64_t bytes, uint64_t offset) +{ + ocf_volume_t volume = ocf_core_get_volume(req->core); + ocf_forward_token_t token = ocf_req_to_core_forward_token(req); + + req->core_error = 0; + + ocf_req_forward_core_get(req); + ocf_volume_forward_io(volume, token, dir, addr, bytes, offset); +} + +void ocf_req_forward_core_flush(struct ocf_request *req) +{ + ocf_volume_t volume = ocf_core_get_volume(req->core); + ocf_forward_token_t token = ocf_req_to_core_forward_token(req); + + req->core_error = 0; + + ocf_req_forward_core_get(req); + ocf_volume_forward_flush(volume, token); +} + +void ocf_req_forward_core_discard(struct ocf_request *req, uint64_t addr, + uint64_t bytes) +{ + ocf_volume_t volume = ocf_core_get_volume(req->core); + ocf_forward_token_t token = ocf_req_to_core_forward_token(req); + + req->core_error = 0; + + ocf_req_forward_core_get(req); + ocf_volume_forward_discard(volume, token, addr, bytes); +} + +struct ocf_io *ocf_forward_get_io(ocf_forward_token_t token) +{ + struct ocf_request *req = (struct ocf_request *)(token & ~1); + + return &req->ioi.io; +} + +static inline void _ocf_forward_get(ocf_forward_token_t token) +{ + struct ocf_request *req = (struct ocf_request *)(token & ~1); + + if (token & 1) + ocf_req_forward_cache_get(req); + else + ocf_req_forward_core_get(req); +} + +void ocf_forward_get(ocf_forward_token_t token) +{ + _ocf_forward_get(token); +} + +void ocf_forward_io(ocf_volume_t volume, ocf_forward_token_t token, + int dir, uint64_t addr, uint64_t bytes, uint64_t offset) +{ + _ocf_forward_get(token); + ocf_volume_forward_io(volume, token, dir, addr, bytes, offset); +} + +void ocf_forward_flush(ocf_volume_t volume, ocf_forward_token_t token) +{ + _ocf_forward_get(token); + ocf_volume_forward_flush(volume, token); +} + +void ocf_forward_discard(ocf_volume_t volume, ocf_forward_token_t token, + uint64_t addr, uint64_t bytes) +{ + _ocf_forward_get(token); + ocf_volume_forward_discard(volume, token, addr, bytes); +} + +void ocf_forward_end(ocf_forward_token_t token, int error) +{ + struct ocf_request *req = ocf_req_forward_token_to_req(token); + + req->error |= error; + + if (token & 1) { + req->cache_error = req->cache_error ?: error; + ocf_req_forward_cache_put(req); + } else { + req->core_error = req->core_error ?: error; + ocf_req_forward_core_put(req); + } +} diff --git a/src/ocf_request.h b/src/ocf_request.h index 2ce0f53..6fd5c2e 100644 --- a/src/ocf_request.h +++ b/src/ocf_request.h @@ -9,6 +9,7 @@ #include "ocf_env.h" #include "ocf_io_priv.h" +#include "ocf_def_priv.h" #include "metadata/metadata_structs.h" typedef enum { @@ -128,6 +129,11 @@ struct ocf_request { struct ocf_io_internal ioi; /*!< OCF IO associated with request */ + ocf_req_end_t cache_forward_end; + ocf_req_end_t core_forward_end; + env_atomic cache_remaining; + env_atomic core_remaining; + env_atomic ref_count; /*!< Reference usage count, once OCF request reaches zero it * will be de-initialed. Get/Put method are intended to modify @@ -260,7 +266,7 @@ struct ocf_request { struct ocf_req_info info; /*!< Detailed request info */ - void (*complete)(struct ocf_request *ocf_req, int error); + ocf_req_end_t complete; /*!< Request completion function */ struct ocf_req_discard_info discard; @@ -276,8 +282,6 @@ struct ocf_request { struct ocf_map_info __map[0]; }; -typedef void (*ocf_req_end_t)(struct ocf_request *req, int error); - /** * @brief Initialize OCF request allocation utility * @@ -512,4 +516,57 @@ static inline bool ocf_req_is_4k(uint64_t addr, uint32_t bytes) return !((addr % PAGE_SIZE) || (bytes % PAGE_SIZE)); } +static inline void ocf_req_forward_cache_get(struct ocf_request *req) +{ + env_atomic_inc(&req->cache_remaining); +} + +static inline void ocf_req_forward_cache_put(struct ocf_request *req) +{ + if (env_atomic_dec_return(&req->cache_remaining) == 0) + req->cache_forward_end(req, req->cache_error); +} + +static inline void ocf_req_forward_core_get(struct ocf_request *req) +{ + env_atomic_inc(&req->core_remaining); +} + +static inline void ocf_req_forward_core_put(struct ocf_request *req) +{ + if (env_atomic_dec_return(&req->core_remaining) == 0) + req->core_forward_end(req, req->core_error); +} + +static inline ocf_forward_token_t ocf_req_to_cache_forward_token(struct ocf_request *req) +{ + return (ocf_forward_token_t)req | 1; +} + +static inline ocf_forward_token_t ocf_req_to_core_forward_token(struct ocf_request *req) +{ + return (ocf_forward_token_t)req; +} + +static inline struct ocf_request *ocf_req_forward_token_to_req(ocf_forward_token_t token) +{ + return (struct ocf_request *)(token & ~1); +} + +void ocf_req_forward_cache_io(struct ocf_request *req, int dir, uint64_t addr, + uint64_t bytes, uint64_t offset); + +void ocf_req_forward_cache_flush(struct ocf_request *req); + +void ocf_req_forward_cache_discard(struct ocf_request *req, uint64_t addr, + uint64_t bytes); + +void ocf_req_forward_core_io(struct ocf_request *req, int dir, uint64_t addr, + uint64_t bytes, uint64_t offset); + +void ocf_req_forward_core_flush(struct ocf_request *req); + +void ocf_req_forward_core_discard(struct ocf_request *req, uint64_t addr, + uint64_t bytes); + #endif /* __OCF_REQUEST_H__ */ diff --git a/src/ocf_volume.c b/src/ocf_volume.c index bf64ec2..143fd03 100644 --- a/src/ocf_volume.c +++ b/src/ocf_volume.c @@ -1,11 +1,13 @@ /* * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ #include "ocf/ocf.h" #include "ocf_priv.h" #include "ocf_volume_priv.h" +#include "ocf_request.h" #include "ocf_io_priv.h" #include "ocf_env.h" @@ -328,6 +330,46 @@ void ocf_volume_submit_discard(struct ocf_io *io) volume->type->properties->ops.submit_discard(io); } +void ocf_volume_forward_io(ocf_volume_t volume, ocf_forward_token_t token, + int dir, uint64_t addr, uint64_t bytes, uint64_t offset) +{ + ENV_BUG_ON(!volume->type->properties->ops.forward_io); + + if (!volume->opened) { + ocf_forward_end(token, -OCF_ERR_IO); + return; + } + + volume->type->properties->ops.forward_io(volume, token, + dir, addr, bytes, offset); +} + +void ocf_volume_forward_flush(ocf_volume_t volume, ocf_forward_token_t token) +{ + ENV_BUG_ON(!volume->type->properties->ops.forward_flush); + + if (!volume->opened) { + ocf_forward_end(token, -OCF_ERR_IO); + return; + } + + volume->type->properties->ops.forward_flush(volume, token); +} + +void ocf_volume_forward_discard(ocf_volume_t volume, ocf_forward_token_t token, + uint64_t addr, uint64_t bytes) +{ + ENV_BUG_ON(!volume->type->properties->ops.forward_discard); + + if (!volume->opened) { + ocf_forward_end(token, -OCF_ERR_IO); + return; + } + + volume->type->properties->ops.forward_discard(volume, token, + addr, bytes); +} + int ocf_volume_open(ocf_volume_t volume, void *volume_params) { int ret; diff --git a/src/ocf_volume_priv.h b/src/ocf_volume_priv.h index 08b70e0..d5942d9 100644 --- a/src/ocf_volume_priv.h +++ b/src/ocf_volume_priv.h @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -46,6 +47,14 @@ void ocf_volume_move(ocf_volume_t volume, ocf_volume_t from); void ocf_volume_set_uuid(ocf_volume_t volume, const struct ocf_volume_uuid *uuid); +void ocf_volume_forward_io(ocf_volume_t volume, ocf_forward_token_t token, + int dir, uint64_t addr, uint64_t bytes, uint64_t offset); + +void ocf_volume_forward_flush(ocf_volume_t volume, ocf_forward_token_t token); + +void ocf_volume_forward_discard(ocf_volume_t volume, ocf_forward_token_t token, + uint64_t addr, uint64_t bytes); + static inline void ocf_volume_submit_metadata(struct ocf_io *io) { ocf_volume_t volume = ocf_io_get_volume(io);