diff --git a/inc/ocf_core.h b/inc/ocf_core.h index e372369..a919590 100644 --- a/inc/ocf_core.h +++ b/inc/ocf_core.h @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -151,7 +152,7 @@ ocf_core_state_t ocf_core_get_state(ocf_core_t core); * * @param[in] io IO to be submitted */ -static inline void ocf_core_submit_io(struct ocf_io *io) +static inline void ocf_core_submit_io(ocf_io_t io) { ocf_volume_submit_io(io); } @@ -161,7 +162,7 @@ static inline void ocf_core_submit_io(struct ocf_io *io) * * @param[in] io IO to be submitted */ -static inline void ocf_core_submit_flush(struct ocf_io *io) +static inline void ocf_core_submit_flush(ocf_io_t io) { ocf_volume_submit_flush(io); } @@ -171,7 +172,7 @@ static inline void ocf_core_submit_flush(struct ocf_io *io) * * @param[in] io IO to be submitted */ -static inline void ocf_core_submit_discard(struct ocf_io *io) +static inline void ocf_core_submit_discard(ocf_io_t io) { ocf_volume_submit_discard(io); } diff --git a/inc/ocf_io.h b/inc/ocf_io.h index 9e7617b..89a82cf 100644 --- a/inc/ocf_io.h +++ b/inc/ocf_io.h @@ -15,8 +15,6 @@ * @brief OCF IO definitions */ -struct ocf_io; - /** * @brief OCF IO start * @@ -24,7 +22,7 @@ struct ocf_io; * * @param[in] io OCF IO being started */ -typedef void (*ocf_start_io_t)(struct ocf_io *io); +typedef void (*ocf_start_io_t)(ocf_io_t io); /** * @brief OCF IO handle @@ -33,7 +31,7 @@ typedef void (*ocf_start_io_t)(struct ocf_io *io); * * @param[in] io OCF IO to handle */ -typedef void (*ocf_handle_io_t)(struct ocf_io *io, void *opaque); +typedef void (*ocf_handle_io_t)(ocf_io_t io, void *opaque); /** * @brief OCF IO completion @@ -41,69 +39,11 @@ typedef void (*ocf_handle_io_t)(struct ocf_io *io, void *opaque); * @note Completion function for OCF IO * * @param[in] io OCF IO being completed + * @param[in] priv1 Completion priv 1 + * @param[in] priv2 Completion priv 2 * @param[in] error Completion status code */ -typedef void (*ocf_end_io_t)(struct ocf_io *io, int error); - -/** - * @brief OCF IO main structure - */ -struct ocf_io { - /** - * @brief OCF IO destination address - */ - uint64_t addr; - - /** - * @brief OCF IO flags - */ - uint64_t flags; - - /** - * @brief OCF IO size in bytes - */ - uint32_t bytes; - - /** - * @brief OCF IO destination class - */ - uint32_t io_class; - - /** - * @brief OCF IO direction - */ - uint32_t dir; - - /** - * @brief Queue handle - */ - ocf_queue_t io_queue; - - /** - * @brief OCF IO start function - */ - ocf_start_io_t start; - - /** - * @brief OCF IO private 1 - */ - void *priv1; - - /** - * @brief OCF IO private 2 - */ - void *priv2; - - /** - * @brief OCF IO handle function - */ - ocf_handle_io_t handle; - - /** - * @brief OCF IO completion function - */ - ocf_end_io_t end; -}; +typedef void (*ocf_end_io_t)(ocf_io_t io, void *priv1, void *priv2, int error); /** * @brief Increase reference counter in OCF IO @@ -112,7 +52,7 @@ struct ocf_io { * * @param[in] io OCF IO */ -void ocf_io_get(struct ocf_io *io); +void ocf_io_get(ocf_io_t io); /** * @brief Decrease reference counter in OCF IO @@ -121,7 +61,7 @@ void ocf_io_get(struct ocf_io *io); * * @param[in] io OCF IO */ -void ocf_io_put(struct ocf_io *io); +void ocf_io_put(ocf_io_t io); /** * @brief Set OCF IO completion function @@ -130,13 +70,8 @@ void ocf_io_put(struct ocf_io *io); * @param[in] context Context for completion function * @param[in] fn Completion function */ -static inline void ocf_io_set_cmpl(struct ocf_io *io, void *context, - void *context2, ocf_end_io_t fn) -{ - io->priv1 = context; - io->priv2 = context2; - io->end = fn; -} +void ocf_io_set_cmpl(ocf_io_t io, void *context, + void *context2, ocf_end_io_t fn); /** * @brief Set OCF IO start function @@ -144,10 +79,7 @@ static inline void ocf_io_set_cmpl(struct ocf_io *io, void *context, * @param[in] io OCF IO * @param[in] fn Start callback function */ -static inline void ocf_io_set_start(struct ocf_io *io, ocf_start_io_t fn) -{ - io->start = fn; -} +void ocf_io_set_start(ocf_io_t io, ocf_start_io_t fn); /** * @brief Set OCF IO handle function @@ -155,10 +87,7 @@ static inline void ocf_io_set_start(struct ocf_io *io, ocf_start_io_t fn) * @param[in] io OCF IO * @param[in] fn Handle callback function */ -static inline void ocf_io_set_handle(struct ocf_io *io, ocf_handle_io_t fn) -{ - io->handle = fn; -} +void ocf_io_set_handle(ocf_io_t io, ocf_handle_io_t fn); /** * @brief Set up data vector in OCF IO @@ -170,7 +99,7 @@ static inline void ocf_io_set_handle(struct ocf_io *io, ocf_handle_io_t fn) * @retval 0 Data set up successfully * @retval Non-zero Data set up failure */ -int ocf_io_set_data(struct ocf_io *io, ctx_data_t *data, uint32_t offset); +int ocf_io_set_data(ocf_io_t io, ctx_data_t *data, uint32_t offset); /** * @brief Get data vector from OCF IO @@ -179,7 +108,7 @@ int ocf_io_set_data(struct ocf_io *io, ctx_data_t *data, uint32_t offset); * * @return Data vector from IO */ -ctx_data_t *ocf_io_get_data(struct ocf_io *io); +ctx_data_t *ocf_io_get_data(ocf_io_t io); /** * @brief Get offset within the data from OCF IO @@ -188,7 +117,7 @@ ctx_data_t *ocf_io_get_data(struct ocf_io *io); * * @return Offset within data */ -uint32_t ocf_io_get_offset(struct ocf_io *io); +uint32_t ocf_io_get_offset(ocf_io_t io); /** * @brief Handle IO in cache engine @@ -196,14 +125,14 @@ uint32_t ocf_io_get_offset(struct ocf_io *io); * @param[in] io OCF IO to be handled * @param[in] opaque OCF opaque */ -void ocf_io_handle(struct ocf_io *io, void *opaque); +void ocf_io_handle(ocf_io_t io, void *opaque); /** * @brief Get volume associated with io * * @param[in] io OCF IO to be handled */ -ocf_volume_t ocf_io_get_volume(struct ocf_io *io); +ocf_volume_t ocf_io_get_volume(ocf_io_t io); /** * @brief Get the data to be submitted diff --git a/inc/ocf_types.h b/inc/ocf_types.h index fe6b415..64d419d 100644 --- a/inc/ocf_types.h +++ b/inc/ocf_types.h @@ -73,6 +73,12 @@ typedef struct ocf_volume_uuid *ocf_uuid_t; */ typedef void ctx_data_t; +struct ocf_request; +/** + * @brief handle to io + */ +typedef struct ocf_request *ocf_io_t; + /** * @brief IO forward token * diff --git a/inc/ocf_volume.h b/inc/ocf_volume.h index 9a57c10..385ce6f 100644 --- a/inc/ocf_volume.h +++ b/inc/ocf_volume.h @@ -17,8 +17,6 @@ #include "ocf/ocf_err.h" #include "ocf/ocf_io.h" -struct ocf_io; - /** * @brief OCF volume UUID maximum allowed size */ @@ -52,28 +50,28 @@ struct ocf_volume_ops { * * @param[in] io IO to be submitted */ - void (*submit_io)(struct ocf_io *io); + void (*submit_io)(ocf_io_t io); /** * @brief Submit IO with flush command * * @param[in] io IO to be submitted */ - void (*submit_flush)(struct ocf_io *io); + void (*submit_flush)(ocf_io_t io); /** * @brief Submit IO with metadata * * @param[in] io IO to be submitted */ - void (*submit_metadata)(struct ocf_io *io); + void (*submit_metadata)(ocf_io_t io); /** * @brief Submit IO with discard command * * @param[in] io IO to be submitted */ - void (*submit_discard)(struct ocf_io *io); + void (*submit_discard)(ocf_io_t io); /** * @brief Submit operation to write zeroes to target address (including @@ -81,7 +79,7 @@ struct ocf_volume_ops { * * @param[in] io IO description (addr, size) */ - void (*submit_write_zeroes)(struct ocf_io *io); + void (*submit_write_zeroes)(ocf_io_t io); /** * @brief Forward the original io directly to the volume @@ -351,7 +349,7 @@ int ocf_volume_is_atomic(ocf_volume_t volume); * * @return ocf_io on success atomic, otherwise NULL */ -struct ocf_io *ocf_volume_new_io(ocf_volume_t volume, ocf_queue_t queue, +ocf_io_t ocf_volume_new_io(ocf_volume_t volume, ocf_queue_t queue, uint64_t addr, uint32_t bytes, uint32_t dir, uint32_t io_class, uint64_t flags); @@ -361,21 +359,21 @@ struct ocf_io *ocf_volume_new_io(ocf_volume_t volume, ocf_queue_t queue, * * @param[in] io IO */ -void ocf_volume_submit_io(struct ocf_io *io); +void ocf_volume_submit_io(ocf_io_t io); /** * @brief Submit flush to volume * * @param[in] io IO */ -void ocf_volume_submit_flush(struct ocf_io *io); +void ocf_volume_submit_flush(ocf_io_t io); /** * @brief Submit discard to volume * * @param[in] io IO */ -void ocf_volume_submit_discard(struct ocf_io *io); +void ocf_volume_submit_discard(ocf_io_t io); /** * @brief Open volume diff --git a/src/metadata/metadata_io.c b/src/metadata/metadata_io.c index 991715a..27524d8 100644 --- a/src/metadata/metadata_io.c +++ b/src/metadata/metadata_io.c @@ -426,7 +426,7 @@ static int metadata_io_i_asynch(ocf_cache_t cache, ocf_queue_t queue, int dir, m_req->req.rw = dir; m_req->req.map = LIST_POISON1; m_req->req.alock_status = (uint8_t*)&m_req->alock_status; - m_req->req.ioi.io.flags = flags; + m_req->req.io.flags = flags; /* If req_count == io_count and count is not multiple of * max_count, for last we can allocate data smaller that diff --git a/src/metadata/metadata_passive_update.c b/src/metadata/metadata_passive_update.c index d29e77b..30197e5 100644 --- a/src/metadata/metadata_passive_update.c +++ b/src/metadata/metadata_passive_update.c @@ -28,11 +28,10 @@ static int passive_io_resume(struct ocf_request *req) { struct ocf_request *master = req->master_io_req; - struct ocf_io *io = &master->ioi.io; ocf_cache_t cache = req->cache; struct ocf_metadata_ctrl *ctrl = cache->metadata.priv; - uint64_t io_start_page = BYTES_TO_PAGES(io->addr); - uint64_t io_pages_count = BYTES_TO_PAGES(io->bytes); + uint64_t io_start_page = BYTES_TO_PAGES(master->io.addr); + uint64_t io_pages_count = BYTES_TO_PAGES(master->io.bytes); uint64_t io_end_page = io_start_page + io_pages_count - 1; enum ocf_metadata_segment_id update_segments[] = { metadata_segment_sb_config, @@ -78,13 +77,12 @@ int ocf_metadata_passive_update(struct ocf_request *master) { ocf_cache_t cache = master->cache; struct ocf_metadata_ctrl *ctrl = cache->metadata.priv; - struct ocf_io *io = &master->ioi.io; - uint64_t io_start_page = BYTES_TO_PAGES(io->addr); - uint64_t io_end_page = io_start_page + BYTES_TO_PAGES(io->bytes); + uint64_t io_start_page = BYTES_TO_PAGES(master->io.addr); + uint64_t io_end_page = io_start_page + BYTES_TO_PAGES(master->io.bytes); struct ocf_request *req; int lock = 0; - if (io->dir == OCF_READ) { + if (master->io.dir == OCF_READ) { master->complete(master, 0); return 0; } @@ -94,14 +92,14 @@ int ocf_metadata_passive_update(struct ocf_request *master) return 0; } - if (io->addr % PAGE_SIZE || io->bytes % PAGE_SIZE) { + if (master->io.addr % PAGE_SIZE || master->io.bytes % PAGE_SIZE) { ocf_cache_log(cache, log_warn, "Metadata update not aligned to page size!\n"); master->complete(master, -OCF_ERR_INVAL); return -OCF_ERR_INVAL; } - if (io->bytes > MAX_PASSIVE_IO_SIZE) { + if (master->io.bytes > MAX_PASSIVE_IO_SIZE) { //FIXME handle greater IOs ocf_cache_log(cache, log_warn, "IO size exceedes max supported size!\n"); @@ -115,7 +113,7 @@ int ocf_metadata_passive_update(struct ocf_request *master) return -OCF_ERR_NO_MEM; } - req->io_queue = io->io_queue;; + req->io_queue = master->io.io_queue;; req->info.internal = true; req->engine_handler = passive_io_resume; req->rw = OCF_WRITE; diff --git a/src/metadata/metadata_raw.c b/src/metadata/metadata_raw.c index 0a32f0a..2811409 100644 --- a/src/metadata/metadata_raw.c +++ b/src/metadata/metadata_raw.c @@ -621,7 +621,7 @@ static int _raw_ram_flush_do_asynch(ocf_cache_t cache, result |= metadata_io_write_i_asynch(cache, req->io_queue, ctx, raw->ssd_pages_offset + start_page, count, - req->ioi.io.flags, + req->io.flags, _raw_ram_flush_do_asynch_fill, _raw_ram_flush_do_asynch_io_complete, raw->mio_conc); diff --git a/src/ocf_cache.c b/src/ocf_cache.c index 0df4cb0..33aeda1 100644 --- a/src/ocf_cache.c +++ b/src/ocf_cache.c @@ -298,7 +298,7 @@ static void ocf_cache_volume_io_complete_generic(struct ocf_request *req, ocf_cache_t cache = req->cache; ocf_refcnt_dec(&cache->refcnt.metadata); - ocf_io_end(&req->ioi.io, error); + ocf_io_end_func(req, error); } static void ocf_cache_io_complete(struct ocf_request *req, int error) @@ -312,21 +312,21 @@ static void ocf_cache_io_complete(struct ocf_request *req, int error) return; ocf_refcnt_dec(&cache->refcnt.metadata); - ocf_io_end(&req->ioi.io, error); + ocf_io_end_func(req, req->error); } -static void ocf_cache_volume_submit_io(struct ocf_io *io) +static void ocf_cache_volume_submit_io(ocf_io_t io) { struct ocf_request *req = ocf_io_to_req(io); ocf_cache_t cache = req->cache; int result; if (!ocf_refcnt_inc(&cache->refcnt.metadata)) { - ocf_io_end(io, -OCF_ERR_IO); + ocf_io_end_func(io, -OCF_ERR_IO); return; } if (unlikely(!ocf_cache_is_standby(cache))) { - ocf_io_end(io, -OCF_ERR_CACHE_NOT_STANDBY); + ocf_io_end_func(io, -OCF_ERR_CACHE_NOT_STANDBY); return; } @@ -347,17 +347,17 @@ static void ocf_cache_volume_submit_io(struct ocf_io *io) ocf_cache_io_complete(req, 0); } -static void ocf_cache_volume_submit_flush(struct ocf_io *io) +static void ocf_cache_volume_submit_flush(ocf_io_t io) { struct ocf_request *req = ocf_io_to_req(io); ocf_cache_t cache = req->cache; if (!ocf_refcnt_inc(&cache->refcnt.metadata)) { - ocf_io_end(io, -OCF_ERR_IO); + ocf_io_end_func(io, -OCF_ERR_IO); return; } if (unlikely(!ocf_cache_is_standby(cache))) { - ocf_io_end(io, -OCF_ERR_CACHE_NOT_STANDBY); + ocf_io_end_func(io, -OCF_ERR_CACHE_NOT_STANDBY); return; } @@ -366,17 +366,17 @@ static void ocf_cache_volume_submit_flush(struct ocf_io *io) } -static void ocf_cache_volume_submit_discard(struct ocf_io *io) +static void ocf_cache_volume_submit_discard(ocf_io_t io) { struct ocf_request *req = ocf_io_to_req(io); ocf_cache_t cache = req->cache; if (!ocf_refcnt_inc(&cache->refcnt.metadata)) { - ocf_io_end(io, -OCF_ERR_IO); + ocf_io_end_func(io, -OCF_ERR_IO); return; } if (unlikely(!ocf_cache_is_standby(cache))) { - ocf_io_end(io, -OCF_ERR_CACHE_NOT_STANDBY); + ocf_io_end_func(io, -OCF_ERR_CACHE_NOT_STANDBY); return; } @@ -453,20 +453,15 @@ static void *ocf_cache_io_allocator_new(ocf_io_allocator_t allocator, ocf_volume_t volume, ocf_queue_t queue, uint64_t addr, uint32_t bytes, uint32_t dir) { - struct ocf_request *req; + ocf_cache_t cache = ocf_volume_to_cache(volume); - req = ocf_req_new(queue, NULL, addr, bytes, dir); - if (!req) - return NULL; - - return &req->ioi; + return ocf_req_new_cache(cache, queue, addr, bytes, dir); } static void ocf_cache_io_allocator_del(ocf_io_allocator_t allocator, void *obj) { - struct ocf_request *req; + struct ocf_request *req = obj; - req = container_of(obj, struct ocf_request, ioi); ocf_req_put(req); } diff --git a/src/ocf_core.c b/src/ocf_core.c index 3d4cc5b..c4e3415 100644 --- a/src/ocf_core.c +++ b/src/ocf_core.c @@ -155,12 +155,9 @@ static uint64_t _calc_dirty_for(uint64_t dirty_since) return dirty_since ? (current_time - dirty_since) : 0; } -struct ocf_request *ocf_io_to_req(struct ocf_io *io) +struct ocf_request *ocf_io_to_req(ocf_io_t io) { - struct ocf_io_internal *ioi; - - ioi = container_of(io, struct ocf_io_internal, io); - return container_of(ioi, struct ocf_request, ioi); + return io; } static inline ocf_core_t ocf_volume_to_core(ocf_volume_t volume) @@ -179,30 +176,31 @@ static inline void dec_counter_if_req_was_dirty(struct ocf_request *req) ocf_refcnt_dec(&req->cache->refcnt.dirty); } -static inline int ocf_core_validate_io(struct ocf_io *io) +static inline int ocf_core_validate_io(ocf_io_t io) { + struct ocf_request *req = ocf_io_to_req(io); ocf_volume_t volume = ocf_io_get_volume(io); ocf_core_t core = ocf_volume_to_core(volume); - if (io->addr + io->bytes > ocf_volume_get_length(volume)) + if (req->io.addr + req->io.bytes > ocf_volume_get_length(volume)) return -OCF_ERR_INVAL; - if (io->io_class >= OCF_USER_IO_CLASS_MAX) + if (req->io.io_class >= OCF_USER_IO_CLASS_MAX) return -OCF_ERR_INVAL; - if (io->dir != OCF_READ && io->dir != OCF_WRITE) + if (req->io.dir != OCF_READ && req->io.dir != OCF_WRITE) return -OCF_ERR_INVAL; - if (!io->io_queue) + if (!req->io.io_queue) return -OCF_ERR_INVAL; - if (!io->end) + if (!req->io.end) return -OCF_ERR_INVAL; /* Core volume I/O must not be queued on management queue - this would * break I/O accounting code, resulting in use-after-free type of errors * after cache detach, core remove etc. */ - if (io->io_queue == ocf_core_get_cache(core)->mngt_queue) + if (req->io.io_queue == ocf_core_get_cache(core)->mngt_queue) return -OCF_ERR_INVAL; return 0; @@ -211,12 +209,12 @@ static inline int ocf_core_validate_io(struct ocf_io *io) static void ocf_req_complete(struct ocf_request *req, int error) { /* Complete IO */ - ocf_io_end(&req->ioi.io, error); + ocf_io_end_func(req, error); dec_counter_if_req_was_dirty(req); /* Invalidate OCF IO, it is not valid after completion */ - ocf_io_put(&req->ioi.io); + ocf_io_put(req); } static inline ocf_req_cache_mode_t _ocf_core_req_resolve_fast_mode( @@ -236,8 +234,7 @@ static inline ocf_req_cache_mode_t _ocf_core_req_resolve_fast_mode( return ocf_req_cache_mode_fast; } -static int ocf_core_submit_io_fast(struct ocf_io *io, struct ocf_request *req, - ocf_cache_t cache) +static int ocf_core_submit_io_fast(struct ocf_request *req, ocf_cache_t cache) { ocf_req_cache_mode_t original_mode, resolved_mode; int ret; @@ -259,9 +256,9 @@ static int ocf_core_submit_io_fast(struct ocf_io *io, struct ocf_request *req, return ret; } -static void ocf_core_volume_submit_io(struct ocf_io *io) +static void ocf_core_volume_submit_io(ocf_io_t io) { - struct ocf_request *req; + struct ocf_request *req = ocf_io_to_req(io); ocf_core_t core; ocf_cache_t cache; int ret; @@ -270,20 +267,18 @@ static void ocf_core_volume_submit_io(struct ocf_io *io) ret = ocf_core_validate_io(io); if (ret < 0) { - ocf_io_end(io, ret); + ocf_io_end_func(io, ret); return; } - req = ocf_io_to_req(io); - core = ocf_volume_to_core(ocf_io_get_volume(io)); + core = req->core; cache = ocf_core_get_cache(core); if (unlikely(ocf_cache_is_standby(cache))) { - ocf_io_end(io, -OCF_ERR_CACHE_STANDBY); + ocf_io_end_func(io, -OCF_ERR_CACHE_STANDBY); return; } - req->core = core; req->complete = ocf_req_complete; ocf_io_get(io); @@ -298,16 +293,17 @@ static void ocf_core_volume_submit_io(struct ocf_io *io) if (ret) goto err; - req->part_id = ocf_user_part_class2id(cache, io->io_class); + req->part_id = ocf_user_part_class2id(cache, req->io.io_class); ocf_resolve_effective_cache_mode(cache, core, req); ocf_core_update_stats(core, io); - /* Prevent race condition */ + /* In case of fastpath prevent completing the requets before updating + * sequential cutoff info */ ocf_req_get(req); - if (ocf_core_submit_io_fast(io, req, cache) == OCF_FAST_PATH_YES) { + if (ocf_core_submit_io_fast(req, cache) == OCF_FAST_PATH_YES) { ocf_core_seq_cutoff_update(core, req); ocf_req_put(req); return; @@ -326,14 +322,13 @@ static void ocf_core_volume_submit_io(struct ocf_io *io) return; err: - ocf_io_end(io, ret); + ocf_io_end_func(io, ret); ocf_io_put(io); } -static void ocf_core_volume_submit_flush(struct ocf_io *io) +static void ocf_core_volume_submit_flush(ocf_io_t io) { - struct ocf_request *req; - ocf_core_t core; + struct ocf_request *req = ocf_io_to_req(io); ocf_cache_t cache; int ret; @@ -341,20 +336,17 @@ static void ocf_core_volume_submit_flush(struct ocf_io *io) ret = ocf_core_validate_io(io); if (ret < 0) { - ocf_io_end(io, ret); + ocf_io_end_func(io, ret); return; } - req = ocf_io_to_req(io); - core = ocf_volume_to_core(ocf_io_get_volume(io)); - cache = ocf_core_get_cache(core); + cache = ocf_core_get_cache(req->core); if (unlikely(ocf_cache_is_standby(cache))) { - ocf_io_end(io, -OCF_ERR_CACHE_STANDBY); + ocf_io_end_func(io, -OCF_ERR_CACHE_STANDBY); return; } - req->core = core; req->complete = ocf_req_complete; ocf_io_get(io); @@ -367,36 +359,32 @@ static void ocf_core_volume_submit_flush(struct ocf_io *io) ocf_engine_hndl_flush_req(req); } -static void ocf_core_volume_submit_discard(struct ocf_io *io) +static void ocf_core_volume_submit_discard(ocf_io_t io) { - struct ocf_request *req; - ocf_core_t core; + struct ocf_request *req = ocf_io_to_req(io); ocf_cache_t cache; int ret; OCF_CHECK_NULL(io); - if (io->bytes == 0) { - ocf_io_end(io, -OCF_ERR_INVAL); + if (req->io.bytes == 0) { + ocf_io_end_func(io, -OCF_ERR_INVAL); return; } ret = ocf_core_validate_io(io); if (ret < 0) { - ocf_io_end(io, ret); + ocf_io_end_func(io, ret); return; } - req = ocf_io_to_req(io); - core = ocf_volume_to_core(ocf_io_get_volume(io)); - cache = ocf_core_get_cache(core); + cache = ocf_core_get_cache(req->core); if (unlikely(ocf_cache_is_standby(cache))) { - ocf_io_end(io, -OCF_ERR_CACHE_STANDBY); + ocf_io_end_func(io, -OCF_ERR_CACHE_STANDBY); return; } - req->core = core; req->complete = ocf_req_complete; ocf_io_get(io); @@ -408,8 +396,7 @@ static void ocf_core_volume_submit_discard(struct ocf_io *io) ret = ocf_req_alloc_map_discard(req); if (ret) { - ocf_io_end(io, -OCF_ERR_NO_MEM); - ocf_io_put(io); + ocf_io_end_func(io, -OCF_ERR_NO_MEM); return; } @@ -488,14 +475,15 @@ static void *ocf_core_io_allocator_new(ocf_io_allocator_t allocator, if (!req) return NULL; - return &req->ioi; + req->core = ocf_volume_to_core(volume); + + return req; } static void ocf_core_io_allocator_del(ocf_io_allocator_t allocator, void *obj) { - struct ocf_request *req; + struct ocf_request *req = obj; - req = container_of(obj, struct ocf_request, ioi); ocf_req_put(req); } diff --git a/src/ocf_core_priv.h b/src/ocf_core_priv.h index da66e08..02b82d8 100644 --- a/src/ocf_core_priv.h +++ b/src/ocf_core_priv.h @@ -106,6 +106,6 @@ ocf_core_id_t ocf_core_get_id(ocf_core_t core); int ocf_core_volume_type_init(ocf_ctx_t ctx); -struct ocf_request *ocf_io_to_req(struct ocf_io *io); +struct ocf_request *ocf_io_to_req(ocf_io_t io); #endif /* __OCF_CORE_PRIV_H__ */ diff --git a/src/ocf_io.c b/src/ocf_io.c index 11bc99d..d7510e1 100644 --- a/src/ocf_io.c +++ b/src/ocf_io.c @@ -58,7 +58,7 @@ ocf_io_allocator_type_t ocf_io_allocator_get_type_default(void) * IO internal API */ -struct ocf_io *ocf_io_new(ocf_volume_t volume, ocf_queue_t queue, +ocf_io_t ocf_io_new(ocf_volume_t volume, ocf_queue_t queue, uint64_t addr, uint32_t bytes, uint32_t dir, uint32_t io_class, uint64_t flags) { @@ -78,24 +78,23 @@ struct ocf_io *ocf_io_new(ocf_volume_t volume, ocf_queue_t queue, return NULL; } - req->ioi.meta.volume = volume; - env_atomic_set(&req->ioi.meta.ref_count, 1); + env_atomic_set(&req->io.ref_count, 1); + req->io.volume = volume; + req->io.io_queue = queue; + req->io.addr = addr; + req->io.bytes = bytes; + req->io.dir = dir; + req->io.io_class = io_class; + req->io.flags = flags; - req->ioi.io.io_queue = queue; - req->ioi.io.addr = addr; - req->ioi.io.bytes = bytes; - req->ioi.io.dir = dir; - req->ioi.io.io_class = io_class; - req->ioi.io.flags = flags; - - return &req->ioi.io; + return req; } /* * IO external API */ -int ocf_io_set_data(struct ocf_io *io, ctx_data_t *data, uint32_t offset) +int ocf_io_set_data(ocf_io_t io, ctx_data_t *data, uint32_t offset) { struct ocf_request *req = ocf_io_to_req(io); @@ -105,46 +104,69 @@ int ocf_io_set_data(struct ocf_io *io, ctx_data_t *data, uint32_t offset) return 0; } -ctx_data_t *ocf_io_get_data(struct ocf_io *io) +ctx_data_t *ocf_io_get_data(ocf_io_t io) { struct ocf_request *req = ocf_io_to_req(io); return req->data; } -uint32_t ocf_io_get_offset(struct ocf_io *io) +uint32_t ocf_io_get_offset(ocf_io_t io) { struct ocf_request *req = ocf_io_to_req(io); return req->offset; } -void ocf_io_get(struct ocf_io *io) +void ocf_io_get(ocf_io_t io) { struct ocf_request *req = ocf_io_to_req(io); - env_atomic_inc_return(&req->ioi.meta.ref_count); + env_atomic_inc_return(&req->io.ref_count); } -void ocf_io_put(struct ocf_io *io) +void ocf_io_put(ocf_io_t io) { struct ocf_request *req = ocf_io_to_req(io); struct ocf_volume *volume; - if (env_atomic_dec_return(&req->ioi.meta.ref_count)) + if (env_atomic_dec_return(&req->io.ref_count)) return; - /* Hold volume reference to avoid use after free of req */ - volume = req->ioi.meta.volume; + volume = req->io.volume; ocf_io_allocator_del(&volume->type->allocator, (void *)req); ocf_refcnt_dec(&volume->refcnt); } -ocf_volume_t ocf_io_get_volume(struct ocf_io *io) +ocf_volume_t ocf_io_get_volume(ocf_io_t io) { struct ocf_request *req = ocf_io_to_req(io); - return req->ioi.meta.volume; + return req->io.volume; +} + +void ocf_io_set_cmpl(ocf_io_t io, void *context, + void *context2, ocf_end_io_t fn) +{ + struct ocf_request *req = ocf_io_to_req(io); + + req->io.priv1 = context; + req->io.priv2 = context2; + req->io.end = fn; +} + +void ocf_io_set_start(ocf_io_t io, ocf_start_io_t fn) +{ + struct ocf_request *req = ocf_io_to_req(io); + + req->io.start = fn; +} + +void ocf_io_set_handle(ocf_io_t io, ocf_handle_io_t fn) +{ + struct ocf_request *req = ocf_io_to_req(io); + + req->io.handle = fn; } diff --git a/src/ocf_io_priv.h b/src/ocf_io_priv.h index 2c01aed..647248e 100644 --- a/src/ocf_io_priv.h +++ b/src/ocf_io_priv.h @@ -10,17 +10,6 @@ #include "ocf/ocf.h" #include "utils/utils_io_allocator.h" -struct ocf_io_meta { - ocf_volume_t volume; - env_atomic ref_count; - struct ocf_request *req; -}; - -struct ocf_io_internal { - struct ocf_io_meta meta; - struct ocf_io io; -}; - int ocf_io_allocator_default_init(ocf_io_allocator_t allocator, const char *name); @@ -32,16 +21,10 @@ void *ocf_io_allocator_default_new(ocf_io_allocator_t allocator, void ocf_io_allocator_default_del(ocf_io_allocator_t allocator, void *obj); -struct ocf_io *ocf_io_new(ocf_volume_t volume, ocf_queue_t queue, +ocf_io_t ocf_io_new(ocf_volume_t volume, ocf_queue_t queue, uint64_t addr, uint32_t bytes, uint32_t dir, uint32_t io_class, uint64_t flags); - -static inline void ocf_io_end(struct ocf_io *io, int error) -{ - if (io->end) - io->end(io, error); - -} +void ocf_io_end_func(ocf_io_t io, int error); #endif /* __OCF_IO_PRIV_H__ */ diff --git a/src/ocf_queue.c b/src/ocf_queue.c index c77524a..5a7447d 100644 --- a/src/ocf_queue.c +++ b/src/ocf_queue.c @@ -160,9 +160,10 @@ void ocf_queue_put(ocf_queue_t queue) env_free(queue); } -void ocf_io_handle(struct ocf_io *io, void *opaque) +/* TODO: Remove opaque. It's not longer needed. */ +void ocf_io_handle(ocf_io_t io, void *opaque) { - struct ocf_request *req = opaque; + struct ocf_request *req = ocf_io_to_req(io); OCF_CHECK_NULL(req); @@ -215,10 +216,10 @@ void ocf_queue_run_single(ocf_queue_t q) if (!io_req) return; - if (io_req->ioi.io.handle) - io_req->ioi.io.handle(&io_req->ioi.io, io_req); + if (io_req->io.handle) + io_req->io.handle(io_req, io_req); else - ocf_io_handle(&io_req->ioi.io, io_req); + ocf_io_handle(io_req, io_req); } void ocf_queue_run(ocf_queue_t q) diff --git a/src/ocf_request.c b/src/ocf_request.c index 8902675..0ed6bac 100644 --- a/src/ocf_request.c +++ b/src/ocf_request.c @@ -432,6 +432,14 @@ void ocf_req_hash(struct ocf_request *req) } } +void ocf_io_end_func(ocf_io_t io, int error) +{ + struct ocf_request *req = ocf_io_to_req(io); + + if (req->io.end) + req->io.end(io, req->io.priv1, req->io.priv2, error); +} + void ocf_req_forward_volume_io(struct ocf_request *req, ocf_volume_t volume, int dir, uint64_t addr, uint64_t bytes, uint64_t offset) { @@ -581,14 +589,14 @@ uint8_t ocf_forward_get_io_class(ocf_forward_token_t token) { struct ocf_request *req = (struct ocf_request *)(token & ~1); - return req->ioi.io.io_class; + return req->io.io_class; } uint64_t ocf_forward_get_flags(ocf_forward_token_t token) { struct ocf_request *req = (struct ocf_request *)(token & ~1); - return (token & 1) ? 0 : req->ioi.io.flags; + return (token & 1) ? 0 : req->io.flags; } static inline void _ocf_forward_get(ocf_forward_token_t token) diff --git a/src/ocf_request.h b/src/ocf_request.h index 206718d..70d9868 100644 --- a/src/ocf_request.h +++ b/src/ocf_request.h @@ -122,12 +122,79 @@ struct ocf_req_discard_info { struct ocf_request; typedef int (*ocf_req_cb)(struct ocf_request *req); +struct ocf_request_io { + /** + * @brief OCF IO destination address + */ + uint64_t addr; + + /** + * @brief OCF IO flags + */ + uint64_t flags; + + /** + * @brief OCF IO size in bytes + */ + uint32_t bytes; + + /** + * @brief OCF IO destination class + */ + uint8_t io_class; + + /** + * @brief OCF IO direction + */ + uint8_t dir:1; + + /** + * @brief OCF IO reference count + */ + env_atomic ref_count; + + /** + * @brief Front volume handle + */ + ocf_volume_t volume; + + /** + * @brief Queue handle + */ + ocf_queue_t io_queue; + + /** + * @brief OCF IO start function + */ + ocf_start_io_t start; + + /** + * @brief OCF IO private 1 + */ + void *priv1; + + /** + * @brief OCF IO private 2 + */ + void *priv2; + + /** + * @brief OCF IO handle function + */ + ocf_handle_io_t handle; + + /** + * @brief OCF IO completion function + */ + ocf_end_io_t end; +}; + /** * @brief OCF IO request */ struct ocf_request { - struct ocf_io_internal ioi; - /*!< OCF IO associated with request */ + /* This struct is temporary. It will be consolidated with ocf_request */ + struct ocf_request_io io; union { ocf_req_end_t cache_forward_end; diff --git a/src/ocf_stats.c b/src/ocf_stats.c index 8e28ecf..4d0ecea 100644 --- a/src/ocf_stats.c +++ b/src/ocf_stats.c @@ -420,8 +420,9 @@ static int to_packet_idx(uint32_t len) return IO_PACKET_SIZE; } -void ocf_core_update_stats(ocf_core_t core, struct ocf_io *io) +void ocf_core_update_stats(ocf_core_t core, ocf_io_t io) { + struct ocf_request *req = ocf_io_to_req(io); struct ocf_counters_debug *stats; int idx; @@ -430,14 +431,14 @@ void ocf_core_update_stats(ocf_core_t core, struct ocf_io *io) stats = &core->counters->debug_stats; - idx = to_packet_idx(io->bytes); - if (io->dir == OCF_WRITE) + idx = to_packet_idx(req->io.bytes); + if (req->io.dir == OCF_WRITE) env_atomic64_inc(&stats->write_size[idx]); else env_atomic64_inc(&stats->read_size[idx]); - idx = to_align_idx(io->addr); - if (io->dir == OCF_WRITE) + idx = to_align_idx(req->io.addr); + if (req->io.dir == OCF_WRITE) env_atomic64_inc(&stats->write_align[idx]); else env_atomic64_inc(&stats->read_align[idx]); @@ -445,6 +446,6 @@ void ocf_core_update_stats(ocf_core_t core, struct ocf_io *io) #else -void ocf_core_update_stats(ocf_core_t core, struct ocf_io *io) {} +void ocf_core_update_stats(ocf_core_t core, ocf_io_t io) {} #endif diff --git a/src/ocf_stats_priv.h b/src/ocf_stats_priv.h index 03daf4f..5b78d3c 100644 --- a/src/ocf_stats_priv.h +++ b/src/ocf_stats_priv.h @@ -247,6 +247,6 @@ int ocf_core_get_stats(ocf_core_t core, struct ocf_stats_core *stats); * @param[in] core to which request pertains * @param[in] io request for which stats are being updated */ -void ocf_core_update_stats(ocf_core_t core, struct ocf_io *io); +void ocf_core_update_stats(ocf_core_t core, ocf_io_t io); #endif diff --git a/src/ocf_volume.c b/src/ocf_volume.c index b19c01d..580c59f 100644 --- a/src/ocf_volume.c +++ b/src/ocf_volume.c @@ -274,7 +274,7 @@ int ocf_volume_is_atomic(ocf_volume_t volume) return volume->type->properties->caps.atomic_writes; } -struct ocf_io *ocf_volume_new_io(ocf_volume_t volume, ocf_queue_t queue, +ocf_io_t ocf_volume_new_io(ocf_volume_t volume, ocf_queue_t queue, uint64_t addr, uint32_t bytes, uint32_t dir, uint32_t io_class, uint64_t flags) { @@ -283,16 +283,16 @@ struct ocf_io *ocf_volume_new_io(ocf_volume_t volume, ocf_queue_t queue, static void ocf_volume_req_forward_complete(struct ocf_request *req, int error) { - ocf_io_end(&req->ioi.io, error); + ocf_io_end_func(req, error); } -void ocf_volume_submit_io(struct ocf_io *io) +void ocf_volume_submit_io(ocf_io_t io) { struct ocf_request *req = ocf_io_to_req(io); ocf_volume_t volume = ocf_io_get_volume(io); if (!volume->opened) { - io->end(io, -OCF_ERR_IO); + ocf_io_end_func(io, -OCF_ERR_IO); return; } @@ -300,18 +300,18 @@ void ocf_volume_submit_io(struct ocf_io *io) volume->type->properties->ops.submit_io(io); } else { req->volume_forward_end = ocf_volume_req_forward_complete; - ocf_req_forward_volume_io(req, volume, io->dir, io->addr, - io->bytes, req->offset); + ocf_req_forward_volume_io(req, volume, req->io.dir, req->io.addr, + req->io.bytes, req->offset); } } -void ocf_volume_submit_flush(struct ocf_io *io) +void ocf_volume_submit_flush(ocf_io_t io) { struct ocf_request *req = ocf_io_to_req(io); ocf_volume_t volume = ocf_io_get_volume(io); if (!volume->opened) { - io->end(io, -OCF_ERR_IO); + ocf_io_end_func(io, -OCF_ERR_IO); return; } @@ -323,13 +323,13 @@ void ocf_volume_submit_flush(struct ocf_io *io) } } -void ocf_volume_submit_discard(struct ocf_io *io) +void ocf_volume_submit_discard(ocf_io_t io) { struct ocf_request *req = ocf_io_to_req(io); ocf_volume_t volume = ocf_io_get_volume(io); if (!volume->opened) { - io->end(io, -OCF_ERR_IO); + ocf_io_end_func(io, -OCF_ERR_IO); return; } @@ -338,7 +338,7 @@ void ocf_volume_submit_discard(struct ocf_io *io) } else { req->volume_forward_end = ocf_volume_req_forward_complete; ocf_req_forward_volume_discard(req, volume, - io->addr, io->bytes); + req->io.addr, req->io.bytes); } } diff --git a/src/ocf_volume_priv.h b/src/ocf_volume_priv.h index 1800518..c327976 100644 --- a/src/ocf_volume_priv.h +++ b/src/ocf_volume_priv.h @@ -65,7 +65,7 @@ void ocf_volume_forward_io_simple(ocf_volume_t volume, ocf_forward_token_t token, int dir, uint64_t addr, uint64_t bytes); -static inline void ocf_volume_submit_metadata(struct ocf_io *io) +static inline void ocf_volume_submit_metadata(ocf_io_t io) { ocf_volume_t volume = ocf_io_get_volume(io); @@ -74,7 +74,7 @@ static inline void ocf_volume_submit_metadata(struct ocf_io *io) volume->type->properties->ops.submit_metadata(io); } -static inline void ocf_volume_submit_write_zeroes(struct ocf_io *io) +static inline void ocf_volume_submit_write_zeroes(ocf_io_t io) { ocf_volume_t volume = ocf_io_get_volume(io); diff --git a/src/utils/utils_io.h b/src/utils/utils_io.h index 58f2ab0..742c0dc 100644 --- a/src/utils/utils_io.h +++ b/src/utils/utils_io.h @@ -23,7 +23,7 @@ void ocf_submit_cache_write_zeros(ocf_cache_t cache, uint64_t addr, void ocf_submit_cache_page(ocf_cache_t cache, uint64_t addr, int dir, void *buffer, ocf_submit_end_t cmpl, void *priv); -static inline struct ocf_io *ocf_new_cache_io(ocf_cache_t cache, +static inline ocf_io_t ocf_new_cache_io(ocf_cache_t cache, ocf_queue_t queue, uint64_t addr, uint32_t bytes, uint32_t dir, uint32_t io_class, uint64_t flags) @@ -32,7 +32,7 @@ static inline struct ocf_io *ocf_new_cache_io(ocf_cache_t cache, addr, bytes, dir, io_class, flags); } -static inline struct ocf_io *ocf_new_core_io(ocf_core_t core, +static inline ocf_io_t ocf_new_core_io(ocf_core_t core, ocf_queue_t queue, uint64_t addr, uint32_t bytes, uint32_t dir, uint32_t io_class, uint64_t flags) {