From ec4bea4fc01fec6adee43d7dfede220bd9201722 Mon Sep 17 00:00:00 2001 From: Kozlowski Mateusz Date: Wed, 21 Jul 2021 08:22:33 +0200 Subject: [PATCH 1/3] Add missing ocf_io_put to error path in ocf_core_volume_submit_io Signed-off-by: Kozlowski Mateusz --- src/ocf_core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ocf_core.c b/src/ocf_core.c index a7c7ea3..d9b33b5 100644 --- a/src/ocf_core.c +++ b/src/ocf_core.c @@ -269,6 +269,7 @@ void ocf_core_volume_submit_io(struct ocf_io *io) if (ret) { dec_counter_if_req_was_dirty(req); ocf_io_end(io, ret); + ocf_io_put(io); } } From af1f3d73c2e2a6d40ac420370a5b259572462588 Mon Sep 17 00:00:00 2001 From: Kozlowski Mateusz Date: Mon, 19 Jul 2021 15:10:11 +0200 Subject: [PATCH 2/3] Add back fastpath Signed-off-by: Kozlowski Mateusz --- inc/ocf_core.h | 11 ---- src/ocf_core.c | 155 +++++++++++++++++++++------------------------- src/ocf_request.c | 9 +++ src/ocf_request.h | 7 +++ 4 files changed, 85 insertions(+), 97 deletions(-) diff --git a/inc/ocf_core.h b/inc/ocf_core.h index 4c75c55..17526ce 100644 --- a/inc/ocf_core.h +++ b/inc/ocf_core.h @@ -172,17 +172,6 @@ static inline void ocf_core_submit_io(struct ocf_io *io) ocf_volume_submit_io(io); } -/** - * @brief Fast path for submitting IO. If possible, request is processed - * immediately without adding to internal request queue - * - * @param[in] io IO to be submitted - * - * @retval 0 IO has been submitted successfully - * @retval Non-zero Fast submit failed. Try to submit IO with ocf_core_submit_io() - */ -int ocf_core_submit_io_fast(struct ocf_io *io); - /** * @brief Submit ocf_io with flush command * diff --git a/src/ocf_core.c b/src/ocf_core.c index d9b33b5..3d3f551 100644 --- a/src/ocf_core.c +++ b/src/ocf_core.c @@ -221,6 +221,66 @@ static void ocf_req_complete(struct ocf_request *req, int error) ocf_io_put(&req->ioi.io); } +static int ocf_core_submit_io_fast(struct ocf_io *io, struct ocf_request *req, + ocf_core_t core, ocf_cache_t cache) +{ + struct ocf_event_io trace_event; + ocf_req_cache_mode_t original_cache_mode; + int fast; + int ret; + + if (req->d2c) { + return -OCF_ERR_IO; + } + + ret = ocf_req_alloc_map(req); + if (ret) { + return -OCF_ERR_IO; + } + + original_cache_mode = req->cache_mode; + + switch (req->cache_mode) { + case ocf_req_cache_mode_pt: + ret = -OCF_ERR_IO; + goto map_allocated; + case ocf_req_cache_mode_wb: + case ocf_req_cache_mode_wo: + req->cache_mode = ocf_req_cache_mode_fast; + break; + default: + if (cache->use_submit_io_fast) + break; + + if (io->dir == OCF_WRITE) + { + ret = -OCF_ERR_IO; + goto map_allocated; + } + + req->cache_mode = ocf_req_cache_mode_fast; + } + + if (cache->trace.trace_callback) { + if (io->dir == OCF_WRITE) + ocf_trace_prep_io_event(&trace_event, req, ocf_event_operation_wr); + else if (io->dir == OCF_READ) + ocf_trace_prep_io_event(&trace_event, req, ocf_event_operation_rd); + } + + fast = ocf_engine_hndl_fast_req(req); + if (fast != OCF_FAST_PATH_NO) { + ocf_trace_push(io->io_queue, &trace_event, sizeof(trace_event)); + return 0; + } + + req->cache_mode = original_cache_mode; + ret = -OCF_ERR_IO; +map_allocated: + ocf_req_dealloc_map(req); + return ret; +} + void ocf_core_volume_submit_io(struct ocf_io *io) { struct ocf_request *req; @@ -254,17 +314,22 @@ void ocf_core_volume_submit_io(struct ocf_io *io) ocf_resolve_effective_cache_mode(cache, core, req); - ocf_core_seq_cutoff_update(core, req); - ocf_core_update_stats(core, io); + ocf_io_get(io); + + if (!ocf_core_submit_io_fast(io, req, core, cache)) { + ocf_core_seq_cutoff_update(core, req); + return; + } + + ocf_core_seq_cutoff_update(core, req); + if (io->dir == OCF_WRITE) ocf_trace_io(req, ocf_event_operation_wr); else if (io->dir == OCF_READ) ocf_trace_io(req, ocf_event_operation_rd); - ocf_io_get(io); - ret = ocf_engine_hndl_req(req); if (ret) { dec_counter_if_req_was_dirty(req); @@ -273,88 +338,6 @@ void ocf_core_volume_submit_io(struct ocf_io *io) } } -int ocf_core_submit_io_fast(struct ocf_io *io) -{ - struct ocf_request *req; - struct ocf_event_io trace_event; - ocf_core_t core; - ocf_cache_t cache; - int fast; - int ret; - - OCF_CHECK_NULL(io); - - ret = ocf_core_validate_io(io); - if (ret < 0) - return ret; - - req = ocf_io_to_req(io); - core = ocf_volume_to_core(ocf_io_get_volume(io)); - cache = ocf_core_get_cache(core); - - if (unlikely(!env_bit_test(ocf_cache_state_running, - &cache->cache_state))) { - ocf_io_end(io, -OCF_ERR_CACHE_NOT_AVAIL); - return 0; - } - - if (req->d2c) { - dec_counter_if_req_was_dirty(req); - return -OCF_ERR_IO; - } - - ret = ocf_req_alloc_map(req); - if (ret) { - ocf_io_end(io, -OCF_ERR_NO_MEM); - return 0; - } - - req->core = core; - req->complete = ocf_req_complete; - req->part_id = ocf_user_part_class2id(cache, io->io_class); - - ocf_resolve_effective_cache_mode(cache, core, req); - - switch (req->cache_mode) { - case ocf_req_cache_mode_pt: - return -OCF_ERR_IO; - case ocf_req_cache_mode_wb: - case ocf_req_cache_mode_wo: - req->cache_mode = ocf_req_cache_mode_fast; - break; - default: - if (cache->use_submit_io_fast) - break; - if (io->dir == OCF_WRITE) - return -OCF_ERR_IO; - - req->cache_mode = ocf_req_cache_mode_fast; - } - - ocf_core_update_stats(core, io); - - if (cache->trace.trace_callback) { - if (io->dir == OCF_WRITE) - ocf_trace_prep_io_event(&trace_event, req, ocf_event_operation_wr); - else if (io->dir == OCF_READ) - ocf_trace_prep_io_event(&trace_event, req, ocf_event_operation_rd); - } - - ocf_io_get(io); - - fast = ocf_engine_hndl_fast_req(req); - if (fast != OCF_FAST_PATH_NO) { - ocf_trace_push(io->io_queue, &trace_event, sizeof(trace_event)); - ocf_core_seq_cutoff_update(core, req); - return 0; - } - - dec_counter_if_req_was_dirty(req); - - ocf_io_put(io); - return -OCF_ERR_IO; -} - static void ocf_core_volume_submit_flush(struct ocf_io *io) { struct ocf_request *req; diff --git a/src/ocf_request.c b/src/ocf_request.c index 5b55245..2217ad2 100644 --- a/src/ocf_request.c +++ b/src/ocf_request.c @@ -153,6 +153,15 @@ int ocf_req_alloc_map(struct ocf_request *req) return 0; } +void ocf_req_dealloc_map(struct ocf_request *req) +{ + if (req->map && req->map != req->__map) + { + env_free(req->map); + req->map = NULL; + } +} + int ocf_req_alloc_map_discard(struct ocf_request *req) { ENV_BUILD_BUG_ON(MAX_TRIM_RQ_SIZE / ocf_cache_line_size_4 * diff --git a/src/ocf_request.h b/src/ocf_request.h index a624487..4a512d1 100644 --- a/src/ocf_request.h +++ b/src/ocf_request.h @@ -271,6 +271,13 @@ struct ocf_request *ocf_req_new(ocf_queue_t queue, ocf_core_t core, */ int ocf_req_alloc_map(struct ocf_request *req); +/** + * @brief Deallocate OCF request map, if it wasn't allocated from mpool + * + * @param req OCF request + */ +void ocf_req_dealloc_map(struct ocf_request *req); + /** * @brief Allocate OCF request map for discard request * From bd7a89c819b6032b12c83cfe4c9f0df48ddc93d8 Mon Sep 17 00:00:00 2001 From: Kozlowski Mateusz Date: Tue, 20 Jul 2021 11:55:19 +0200 Subject: [PATCH 3/3] Single map alloc location Signed-off-by: Kozlowski Mateusz --- src/engine/cache_engine.c | 5 ----- src/ocf_core.c | 26 ++++++++++---------------- src/ocf_request.c | 9 --------- src/ocf_request.h | 7 ------- 4 files changed, 10 insertions(+), 37 deletions(-) diff --git a/src/engine/cache_engine.c b/src/engine/cache_engine.c index c52ee53..1dac735 100644 --- a/src/engine/cache_engine.c +++ b/src/engine/cache_engine.c @@ -144,11 +144,6 @@ struct ocf_request *ocf_engine_pop_req(ocf_queue_t q) OCF_CHECK_NULL(req); - if (ocf_req_alloc_map(req)) { - req->complete(req, req->error); - return NULL; - } - return req; } diff --git a/src/ocf_core.c b/src/ocf_core.c index 3d3f551..38a3884 100644 --- a/src/ocf_core.c +++ b/src/ocf_core.c @@ -227,23 +227,16 @@ static int ocf_core_submit_io_fast(struct ocf_io *io, struct ocf_request *req, struct ocf_event_io trace_event; ocf_req_cache_mode_t original_cache_mode; int fast; - int ret; if (req->d2c) { return -OCF_ERR_IO; } - ret = ocf_req_alloc_map(req); - if (ret) { - return -OCF_ERR_IO; - } - original_cache_mode = req->cache_mode; switch (req->cache_mode) { case ocf_req_cache_mode_pt: - ret = -OCF_ERR_IO; - goto map_allocated; + return -OCF_ERR_IO; case ocf_req_cache_mode_wb: case ocf_req_cache_mode_wo: req->cache_mode = ocf_req_cache_mode_fast; @@ -253,10 +246,7 @@ static int ocf_core_submit_io_fast(struct ocf_io *io, struct ocf_request *req, break; if (io->dir == OCF_WRITE) - { - ret = -OCF_ERR_IO; - goto map_allocated; - } + return -OCF_ERR_IO; req->cache_mode = ocf_req_cache_mode_fast; } @@ -275,10 +265,7 @@ static int ocf_core_submit_io_fast(struct ocf_io *io, struct ocf_request *req, } req->cache_mode = original_cache_mode; - ret = -OCF_ERR_IO; -map_allocated: - ocf_req_dealloc_map(req); - return ret; + return -OCF_ERR_IO; } void ocf_core_volume_submit_io(struct ocf_io *io) @@ -308,6 +295,12 @@ void ocf_core_volume_submit_io(struct ocf_io *io) return; } + ret = ocf_req_alloc_map(req); + if (ret) { + ocf_io_end(io, ret); + return; + } + req->part_id = ocf_user_part_class2id(cache, io->io_class); req->core = core; req->complete = ocf_req_complete; @@ -323,6 +316,7 @@ void ocf_core_volume_submit_io(struct ocf_io *io) return; } + ocf_req_clear_map(req); ocf_core_seq_cutoff_update(core, req); if (io->dir == OCF_WRITE) diff --git a/src/ocf_request.c b/src/ocf_request.c index 2217ad2..5b55245 100644 --- a/src/ocf_request.c +++ b/src/ocf_request.c @@ -153,15 +153,6 @@ int ocf_req_alloc_map(struct ocf_request *req) return 0; } -void ocf_req_dealloc_map(struct ocf_request *req) -{ - if (req->map && req->map != req->__map) - { - env_free(req->map); - req->map = NULL; - } -} - int ocf_req_alloc_map_discard(struct ocf_request *req) { ENV_BUILD_BUG_ON(MAX_TRIM_RQ_SIZE / ocf_cache_line_size_4 * diff --git a/src/ocf_request.h b/src/ocf_request.h index 4a512d1..a624487 100644 --- a/src/ocf_request.h +++ b/src/ocf_request.h @@ -271,13 +271,6 @@ struct ocf_request *ocf_req_new(ocf_queue_t queue, ocf_core_t core, */ int ocf_req_alloc_map(struct ocf_request *req); -/** - * @brief Deallocate OCF request map, if it wasn't allocated from mpool - * - * @param req OCF request - */ -void ocf_req_dealloc_map(struct ocf_request *req); - /** * @brief Allocate OCF request map for discard request *