From de32a9649a0b18d58b37eef1e89c7201e3d3d1c5 Mon Sep 17 00:00:00 2001 From: Ian Levine Date: Thu, 12 Oct 2023 14:41:15 +0300 Subject: [PATCH 1/4] Rename ocf_engine_cb to ocf_req_cb and move it from engine_common.h to ocf_request.h Signed-off-by: Ian Levine Signed-off-by: Robert Baldyga --- src/engine/cache_engine.c | 6 +++--- src/engine/cache_engine.h | 26 +++----------------------- src/engine/engine_common.c | 6 +++--- src/engine/engine_common.h | 3 ++- src/ocf_request.h | 28 ++++++++++++++++++++++++++-- 5 files changed, 37 insertions(+), 32 deletions(-) diff --git a/src/engine/cache_engine.c b/src/engine/cache_engine.c index c2465a9..6881b4c 100644 --- a/src/engine/cache_engine.c +++ b/src/engine/cache_engine.c @@ -138,7 +138,7 @@ const char *ocf_get_io_iface_name(ocf_req_cache_mode_t cache_mode) return cache_mode_io_if_map[cache_mode]->name; } -static ocf_engine_cb ocf_io_if_type_to_engine_cb( +static ocf_req_cb ocf_io_if_type_to_engine_cb( enum ocf_io_if_type io_if_type, int rw) { if (unlikely(io_if_type == OCF_IO_MAX_IF || @@ -149,7 +149,7 @@ static ocf_engine_cb ocf_io_if_type_to_engine_cb( return IO_IFS[io_if_type].cbs[rw]; } -static ocf_engine_cb ocf_cache_mode_to_engine_cb( +static ocf_req_cb ocf_cache_mode_to_engine_cb( ocf_req_cache_mode_t req_cache_mode, int rw) { if (req_cache_mode == ocf_req_cache_mode_max) @@ -271,7 +271,7 @@ int ocf_engine_hndl_req(struct ocf_request *req) int ocf_engine_hndl_fast_req(struct ocf_request *req) { - ocf_engine_cb engine_cb; + ocf_req_cb engine_cb; int ret; engine_cb = ocf_cache_mode_to_engine_cb(req->cache_mode, req->rw); diff --git a/src/engine/cache_engine.h b/src/engine/cache_engine.h index 6e211de..88c6da4 100644 --- a/src/engine/cache_engine.h +++ b/src/engine/cache_engine.h @@ -7,42 +7,22 @@ #ifndef __CACHE_ENGINE_H_ #define __CACHE_ENGINE_H_ +#include "../ocf_request.h" + struct ocf_thread_priv; -struct ocf_request; #define LOOKUP_HIT 5 #define LOOKUP_MISS 6 #define LOOKUP_REMAPPED 8 -typedef enum { - /* modes inherited from user API */ - ocf_req_cache_mode_wt = ocf_cache_mode_wt, - ocf_req_cache_mode_wb = ocf_cache_mode_wb, - ocf_req_cache_mode_wa = ocf_cache_mode_wa, - ocf_req_cache_mode_pt = ocf_cache_mode_pt, - ocf_req_cache_mode_wi = ocf_cache_mode_wi, - ocf_req_cache_mode_wo = ocf_cache_mode_wo, - - /* internal modes */ - ocf_req_cache_mode_fast, - /*!< Fast path */ - ocf_req_cache_mode_d2c, - /*!< Direct to Core - pass through to core without - touching cacheline metadata */ - - ocf_req_cache_mode_max, -} ocf_req_cache_mode_t; - static inline ocf_req_cache_mode_t ocf_cache_mode_to_req_cache_mode( ocf_cache_mode_t mode) { return (ocf_req_cache_mode_t)mode; } -typedef int (*ocf_engine_cb)(struct ocf_request *req); - struct ocf_io_if { - ocf_engine_cb cbs[2]; /* READ and WRITE */ + ocf_req_cb cbs[2]; /* READ and WRITE */ const char *name; }; diff --git a/src/engine/engine_common.c b/src/engine/engine_common.c index 24643f6..fab760c 100644 --- a/src/engine/engine_common.c +++ b/src/engine/engine_common.c @@ -1,6 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation - * Copyright(c) 2024 Huawei Technologies Co., Ltd. + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -645,11 +645,11 @@ void ocf_engine_push_req_front(struct ocf_request *req, bool allow_sync) } void ocf_engine_push_req_front_cb(struct ocf_request *req, - ocf_engine_cb engine_cb, + ocf_req_cb req_cb, bool allow_sync) { req->error = 0; /* Please explain why!!! */ - req->engine_handler = engine_cb; + req->engine_handler = req_cb; ocf_engine_push_req_front(req, allow_sync); } diff --git a/src/engine/engine_common.h b/src/engine/engine_common.h index 6f9c73f..fa30f2a 100644 --- a/src/engine/engine_common.h +++ b/src/engine/engine_common.h @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -309,7 +310,7 @@ void ocf_engine_push_req_front(struct ocf_request *req, from push function in caller context */ void ocf_engine_push_req_front_cb(struct ocf_request *req, - ocf_engine_cb engine_cb, + ocf_req_cb req_cb, bool allow_sync); void inc_fallback_pt_error_counter(ocf_cache_t cache); diff --git a/src/ocf_request.h b/src/ocf_request.h index 5e05139..5357176 100644 --- a/src/ocf_request.h +++ b/src/ocf_request.h @@ -9,9 +9,27 @@ #include "ocf_env.h" #include "ocf_io_priv.h" -#include "engine/cache_engine.h" #include "metadata/metadata_structs.h" +typedef enum { + /* modes inherited from user API */ + ocf_req_cache_mode_wt = ocf_cache_mode_wt, + ocf_req_cache_mode_wb = ocf_cache_mode_wb, + ocf_req_cache_mode_wa = ocf_cache_mode_wa, + ocf_req_cache_mode_pt = ocf_cache_mode_pt, + ocf_req_cache_mode_wi = ocf_cache_mode_wi, + ocf_req_cache_mode_wo = ocf_cache_mode_wo, + + /* internal modes */ + ocf_req_cache_mode_fast, + /*!< Fast path */ + ocf_req_cache_mode_d2c, + /*!< Direct to Core - pass through to core without + touching cacheline metadata */ + + ocf_req_cache_mode_max, +} ocf_req_cache_mode_t; + struct ocf_req_allocator; struct ocf_req_info { @@ -94,6 +112,12 @@ struct ocf_req_discard_info { /*!< Number of processed sector during discard operation */ }; +/** + * @brief OCF IO engine handler callback + */ +struct ocf_request; +typedef int (*ocf_req_cb)(struct ocf_request *req); + /** * @brief OCF IO request */ @@ -129,7 +153,7 @@ struct ocf_request { ocf_core_t core; /*!< Handle to core instance */ - ocf_engine_cb engine_handler; + ocf_req_cb engine_handler; /*!< IO engine handler */ void *priv; From 038126e9abc99cfa7c8e037214d4d19ad4385e00 Mon Sep 17 00:00:00 2001 From: Ian Levine Date: Thu, 12 Oct 2023 15:00:06 +0300 Subject: [PATCH 2/4] Move and rename ocf_engine_push_req_* from engine_common to ocf_queue_push_req_* in ocf_queue Signed-off-by: Ian Levine Signed-off-by: Robert Baldyga --- src/engine/cache_engine.c | 4 +- src/engine/engine_bf.c | 3 +- src/engine/engine_common.c | 73 +----------------------- src/engine/engine_common.h | 32 ----------- src/engine/engine_discard.c | 6 +- src/engine/engine_fast.c | 3 +- src/engine/engine_inv.c | 3 +- src/engine/engine_pt.c | 6 +- src/engine/engine_pt.h | 3 +- src/engine/engine_rd.c | 3 +- src/engine/engine_wb.c | 3 +- src/engine/engine_wi.c | 6 +- src/engine/engine_wo.c | 2 +- src/engine/engine_wt.c | 3 +- src/engine/engine_zero.c | 5 +- src/metadata/metadata_io.c | 8 +-- src/metadata/metadata_passive_update.c | 3 +- src/metadata/metadata_raw_dynamic.c | 6 +- src/mngt/ocf_mngt_flush.c | 4 +- src/ocf_queue.c | 79 ++++++++++++++++++++++++++ src/ocf_queue_priv.h | 46 +++++++++++++++ src/utils/utils_cleaner.c | 12 ++-- src/utils/utils_parallelize.c | 4 +- src/utils/utils_pipeline.c | 8 +-- 24 files changed, 179 insertions(+), 146 deletions(-) diff --git a/src/engine/cache_engine.c b/src/engine/cache_engine.c index 6881b4c..014ba49 100644 --- a/src/engine/cache_engine.c +++ b/src/engine/cache_engine.c @@ -264,7 +264,7 @@ int ocf_engine_hndl_req(struct ocf_request *req) * to into OCF workers */ - ocf_engine_push_req_back(req, true); + ocf_queue_push_req_back(req, true); return 0; } @@ -313,7 +313,7 @@ void ocf_engine_hndl_ops_req(struct ocf_request *req) ocf_io_if_type_to_engine_cb(OCF_IO_D2C_IF, req->rw) : ocf_io_if_type_to_engine_cb(OCF_IO_OPS_IF, req->rw); - ocf_engine_push_req_back(req, true); + ocf_queue_push_req_back(req, true); } bool ocf_req_cache_mode_has_lazy_write(ocf_req_cache_mode_t mode) diff --git a/src/engine/engine_bf.c b/src/engine/engine_bf.c index 60d2cea..ab6bc6c 100644 --- a/src/engine/engine_bf.c +++ b/src/engine/engine_bf.c @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -94,5 +95,5 @@ static int _ocf_backfill_do(struct ocf_request *req) void ocf_engine_backfill(struct ocf_request *req) { backfill_queue_inc_block(req->cache); - ocf_engine_push_req_front_cb(req, _ocf_backfill_do, true); + ocf_queue_push_req_front_cb(req, _ocf_backfill_do, true); } diff --git a/src/engine/engine_common.c b/src/engine/engine_common.c index fab760c..4f21c33 100644 --- a/src/engine/engine_common.c +++ b/src/engine/engine_common.c @@ -381,7 +381,7 @@ static void _ocf_engine_clean_end(void *private_data, int error) } else { req->info.dirty_any = 0; req->info.dirty_all = 0; - ocf_engine_push_req_front(req, true); + ocf_queue_push_req_front(req, true); } } @@ -584,75 +584,6 @@ void ocf_engine_update_request_stats(struct ocf_request *req) req->info.hit_no, req->core_line_count); } -void ocf_engine_push_req_back(struct ocf_request *req, bool allow_sync) -{ - ocf_cache_t cache = req->cache; - ocf_queue_t q = NULL; - unsigned long lock_flags = 0; - - INIT_LIST_HEAD(&req->list); - - ENV_BUG_ON(!req->io_queue); - q = req->io_queue; - - if (!req->info.internal) { - env_atomic_set(&cache->last_access_ms, - env_ticks_to_msecs(env_get_tick_count())); - } - - env_spinlock_lock_irqsave(&q->io_list_lock, lock_flags); - - list_add_tail(&req->list, &q->io_list); - env_atomic_inc(&q->io_no); - - env_spinlock_unlock_irqrestore(&q->io_list_lock, lock_flags); - - /* NOTE: do not dereference @req past this line, it might - * be picked up by concurrent io thread and deallocated - * at this point */ - - ocf_queue_kick(q, allow_sync); -} - -void ocf_engine_push_req_front(struct ocf_request *req, bool allow_sync) -{ - ocf_cache_t cache = req->cache; - ocf_queue_t q = NULL; - unsigned long lock_flags = 0; - - ENV_BUG_ON(!req->io_queue); - INIT_LIST_HEAD(&req->list); - - q = req->io_queue; - - if (!req->info.internal) { - env_atomic_set(&cache->last_access_ms, - env_ticks_to_msecs(env_get_tick_count())); - } - - env_spinlock_lock_irqsave(&q->io_list_lock, lock_flags); - - list_add(&req->list, &q->io_list); - env_atomic_inc(&q->io_no); - - env_spinlock_unlock_irqrestore(&q->io_list_lock, lock_flags); - - /* NOTE: do not dereference @req past this line, it might - * be picked up by concurrent io thread and deallocated - * at this point */ - - ocf_queue_kick(q, allow_sync); -} - -void ocf_engine_push_req_front_cb(struct ocf_request *req, - ocf_req_cb req_cb, - bool allow_sync) -{ - req->error = 0; /* Please explain why!!! */ - req->engine_handler = req_cb; - ocf_engine_push_req_front(req, allow_sync); -} - void inc_fallback_pt_error_counter(ocf_cache_t cache) { ENV_BUG_ON(env_atomic_read(&cache->fallback_pt_error_counter) < 0); @@ -712,5 +643,5 @@ void ocf_engine_on_resume(struct ocf_request *req) OCF_DEBUG_RQ(req, "On resume"); - ocf_engine_push_req_front_cb(req, _ocf_engine_refresh, false); + ocf_queue_push_req_front_cb(req, _ocf_engine_refresh, false); } diff --git a/src/engine/engine_common.h b/src/engine/engine_common.h index fa30f2a..6c6946e 100644 --- a/src/engine/engine_common.h +++ b/src/engine/engine_common.h @@ -281,38 +281,6 @@ void ocf_engine_update_block_stats(struct ocf_request *req); */ void ocf_engine_update_request_stats(struct ocf_request *req); -/** - * @brief Push front OCF request to the OCF thread worker queue - * - * @param req OCF request - * @param allow_sync caller allows for request from queue to be ran immediately - from push function in caller context - */ -void ocf_engine_push_req_back(struct ocf_request *req, - bool allow_sync); - -/** - * @brief Push back OCF request to the OCF thread worker queue - * - * @param req OCF request - * @param allow_sync caller allows for request from queue to be ran immediately - from push function in caller context - */ -void ocf_engine_push_req_front(struct ocf_request *req, - bool allow_sync); - -/** - * @brief Set interface and push from request to the OCF thread worker queue - * - * @param req OCF request - * @param engine_cb IO engine handler callback - * @param allow_sync caller allows for request from queue to be ran immediately - from push function in caller context - */ -void ocf_engine_push_req_front_cb(struct ocf_request *req, - ocf_req_cb req_cb, - bool allow_sync); - void inc_fallback_pt_error_counter(ocf_cache_t cache); void ocf_engine_on_resume(struct ocf_request *req); diff --git a/src/engine/engine_discard.c b/src/engine/engine_discard.c index 38532de..9298099 100644 --- a/src/engine/engine_discard.c +++ b/src/engine/engine_discard.c @@ -74,7 +74,7 @@ static void _ocf_discard_cache_flush_complete(struct ocf_io *io, int error) } req->engine_handler = _ocf_discard_core; - ocf_engine_push_req_front(req, true); + ocf_queue_push_req_front(req, true); ocf_io_put(io); } @@ -111,7 +111,7 @@ static void _ocf_discard_finish_step(struct ocf_request *req) else req->engine_handler = _ocf_discard_core; - ocf_engine_push_req_front(req, true); + ocf_queue_push_req_front(req, true); } static void _ocf_discard_step_complete(struct ocf_request *req, int error) @@ -182,7 +182,7 @@ static int _ocf_discard_step_do(struct ocf_request *req) static void _ocf_discard_on_resume(struct ocf_request *req) { OCF_DEBUG_RQ(req, "On resume"); - ocf_engine_push_req_front(req, true); + ocf_queue_push_req_front(req, true); } static int _ocf_discard_step(struct ocf_request *req) diff --git a/src/engine/engine_fast.c b/src/engine/engine_fast.c index fc7e608..5676e0b 100644 --- a/src/engine/engine_fast.c +++ b/src/engine/engine_fast.c @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -45,7 +46,7 @@ static void _ocf_read_fast_complete(struct ocf_request *req, int error) if (req->error) { OCF_DEBUG_RQ(req, "ERROR"); - ocf_engine_push_req_front_pt(req); + ocf_queue_push_req_front_pt(req); } else { ocf_req_unlock(ocf_cache_line_concurrency(req->cache), req); diff --git a/src/engine/engine_inv.c b/src/engine/engine_inv.c index 4b60b2b..fa84d66 100644 --- a/src/engine/engine_inv.c +++ b/src/engine/engine_inv.c @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -62,5 +63,5 @@ static int _ocf_invalidate_do(struct ocf_request *req) void ocf_engine_invalidate(struct ocf_request *req) { - ocf_engine_push_req_front_cb(req, _ocf_invalidate_do, true); + ocf_queue_push_req_front_cb(req, _ocf_invalidate_do, true); } diff --git a/src/engine/engine_pt.c b/src/engine/engine_pt.c index 6d3c53d..4223160 100644 --- a/src/engine/engine_pt.c +++ b/src/engine/engine_pt.c @@ -1,6 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation - * Copyright(c) 2023 Huawei Technologies + * Copyright(c) 2023-2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ #include "ocf/ocf.h" @@ -165,8 +165,8 @@ int ocf_read_pt(struct ocf_request *req) return 0; } -void ocf_engine_push_req_front_pt(struct ocf_request *req) +void ocf_queue_push_req_front_pt(struct ocf_request *req) { - ocf_engine_push_req_front_cb(req, ocf_read_pt_do, true); + ocf_queue_push_req_front_cb(req, ocf_read_pt_do, true); } diff --git a/src/engine/engine_pt.h b/src/engine/engine_pt.h index 59da6ce..01e34d1 100644 --- a/src/engine/engine_pt.h +++ b/src/engine/engine_pt.h @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -10,6 +11,6 @@ int ocf_read_pt(struct ocf_request *req); int ocf_read_pt_do(struct ocf_request *req); -void ocf_engine_push_req_front_pt(struct ocf_request *req); +void ocf_queue_push_req_front_pt(struct ocf_request *req); #endif /* ENGINE_OFF_H_ */ diff --git a/src/engine/engine_rd.c b/src/engine/engine_rd.c index 52fd4d3..2d3f201 100644 --- a/src/engine/engine_rd.c +++ b/src/engine/engine_rd.c @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -41,7 +42,7 @@ static void _ocf_read_generic_hit_complete(struct ocf_request *req, int error) OCF_DEBUG_RQ(req, "HIT completion"); if (req->error) { - ocf_engine_push_req_front_pt(req); + ocf_queue_push_req_front_pt(req); } else { ocf_req_unlock(c, req); diff --git a/src/engine/engine_wb.c b/src/engine/engine_wb.c index 9343360..4012f32 100644 --- a/src/engine/engine_wb.c +++ b/src/engine/engine_wb.c @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -102,7 +103,7 @@ static void _ocf_write_wb_complete(struct ocf_request *req, int error) ocf_engine_invalidate(req); } else { - ocf_engine_push_req_front_cb(req, + ocf_queue_push_req_front_cb(req, ocf_write_wb_do_flush_metadata, true); } } diff --git a/src/engine/engine_wi.c b/src/engine/engine_wi.c index 6db17a4..aed03bf 100644 --- a/src/engine/engine_wi.c +++ b/src/engine/engine_wi.c @@ -56,7 +56,7 @@ static void _ocf_write_wi_io_flush_metadata(struct ocf_request *req, int error) if (!req->error && !req->wi_second_pass && ocf_engine_is_miss(req)) { /* need another pass */ - ocf_engine_push_req_front_cb(req, _ocf_write_wi_next_pass, + ocf_queue_push_req_front_cb(req, _ocf_write_wi_next_pass, true); return; } @@ -123,7 +123,7 @@ static void _ocf_write_wi_core_complete(struct ocf_request *req, int error) ocf_req_put(req); } else { - ocf_engine_push_req_front_cb(req, + ocf_queue_push_req_front_cb(req, ocf_write_wi_update_and_flush_metadata, true); } } @@ -155,7 +155,7 @@ static int _ocf_write_wi_core_write(struct ocf_request *req) static void _ocf_write_wi_on_resume(struct ocf_request *req) { OCF_DEBUG_RQ(req, "On resume"); - ocf_engine_push_req_front(req, true); + ocf_queue_push_req_front(req, true); } int ocf_write_wi(struct ocf_request *req) diff --git a/src/engine/engine_wo.c b/src/engine/engine_wo.c index 1b6afee..4104f55 100644 --- a/src/engine/engine_wo.c +++ b/src/engine/engine_wo.c @@ -171,7 +171,7 @@ static void _ocf_read_wo_core_complete(struct ocf_request *req, int error) } req->engine_handler = ocf_read_wo_cache_do; - ocf_engine_push_req_front(req, true); + ocf_queue_push_req_front(req, true); } static int ocf_read_wo_do(struct ocf_request *req) diff --git a/src/engine/engine_wt.c b/src/engine/engine_wt.c index 95a2321..13e2edc 100644 --- a/src/engine/engine_wt.c +++ b/src/engine/engine_wt.c @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -112,7 +113,7 @@ static void _ocf_write_wt_req_complete(struct ocf_request *req) if (req->info.dirty_any) { /* Some of the request's cachelines changed its state to clean */ - ocf_engine_push_req_front_cb(req, + ocf_queue_push_req_front_cb(req, ocf_write_wt_do_flush_metadata, true); } else { ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); diff --git a/src/engine/engine_zero.c b/src/engine/engine_zero.c index 5d6352b..49b5843 100644 --- a/src/engine/engine_zero.c +++ b/src/engine/engine_zero.c @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -50,7 +51,7 @@ static void _ocf_zero_io_flush_metadata(struct ocf_request *req, int error) if (env_atomic_dec_return(&req->req_remaining)) return; - ocf_engine_push_req_front_cb(req, ocf_zero_purge, true); + ocf_queue_push_req_front_cb(req, ocf_zero_purge, true); } static inline void ocf_zero_map_info(struct ocf_request *req) @@ -148,7 +149,7 @@ void ocf_engine_zero_line(struct ocf_request *req) if (lock >= 0) { ENV_BUG_ON(lock != OCF_LOCK_ACQUIRED); - ocf_engine_push_req_front_cb(req, _ocf_zero_do, true); + ocf_queue_push_req_front_cb(req, _ocf_zero_do, true); } else { OCF_DEBUG_RQ(req, "LOCK ERROR %d", lock); req->complete(req, lock); diff --git a/src/metadata/metadata_io.c b/src/metadata/metadata_io.c index 65a4778..87a774b 100644 --- a/src/metadata/metadata_io.c +++ b/src/metadata/metadata_io.c @@ -93,7 +93,7 @@ static void metadata_io_read_i_atomic_step_end(struct ocf_io *io, int error) context->curr_offset += context->curr_count; if (context->count > 0) - ocf_engine_push_req_front(context->req, true); + ocf_queue_push_req_front(context->req, true); else metadata_io_read_i_atomic_complete(context, 0); } @@ -181,7 +181,7 @@ int metadata_io_read_i_atomic(ocf_cache_t cache, ocf_queue_t queue, void *priv, context->compl_hndl = compl_hndl; context->priv = priv; - ocf_engine_push_req_front(context->req, true); + ocf_queue_push_req_front(context->req, true); return 0; } @@ -269,7 +269,7 @@ static void metadata_io_req_finalize(struct metadata_io_request *m_req) static void metadata_io_page_lock_acquired(struct ocf_request *req) { - ocf_engine_push_req_front(req, true); + ocf_queue_push_req_front(req, true); } static int metadata_io_restart_req(struct ocf_request *req) @@ -401,7 +401,7 @@ void metadata_io_req_complete(struct metadata_io_request *m_req) } m_req->req.engine_handler = metadata_io_restart_req; - ocf_engine_push_req_front(&m_req->req, true); + ocf_queue_push_req_front(&m_req->req, true); } /* diff --git a/src/metadata/metadata_passive_update.c b/src/metadata/metadata_passive_update.c index 8e84a6a..709738e 100644 --- a/src/metadata/metadata_passive_update.c +++ b/src/metadata/metadata_passive_update.c @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -70,7 +71,7 @@ static int passive_io_resume(struct ocf_request *req) static void passive_io_page_lock_acquired(struct ocf_request *req) { - ocf_engine_push_req_front(req, true); + ocf_queue_push_req_front(req, true); } int ocf_metadata_passive_update(ocf_cache_t cache, struct ocf_io *io, diff --git a/src/metadata/metadata_raw_dynamic.c b/src/metadata/metadata_raw_dynamic.c index fc06418..d25fad9 100644 --- a/src/metadata/metadata_raw_dynamic.c +++ b/src/metadata/metadata_raw_dynamic.c @@ -389,7 +389,7 @@ static void raw_dynamic_load_all_read_end(struct ocf_io *io, int error) } context->req->engine_handler = raw_dynamic_load_all_update; - ocf_engine_push_req_front(context->req, true); + ocf_queue_push_req_front(context->req, true); } static int raw_dynamic_load_all_read(struct ocf_request *req) @@ -455,7 +455,7 @@ static int raw_dynamic_load_all_update(struct ocf_request *req) } context->req->engine_handler = raw_dynamic_load_all_read; - ocf_engine_push_req_front(context->req, true); + ocf_queue_push_req_front(context->req, true); return 0; } @@ -501,7 +501,7 @@ void raw_dynamic_load_all(ocf_cache_t cache, struct ocf_metadata_raw *raw, context->req->priv = context; context->req->engine_handler = raw_dynamic_load_all_read; - ocf_engine_push_req_front(context->req, true); + ocf_queue_push_req_front(context->req, true); return; err_req: diff --git a/src/mngt/ocf_mngt_flush.c b/src/mngt/ocf_mngt_flush.c index 2cfc565..4473b59 100644 --- a/src/mngt/ocf_mngt_flush.c +++ b/src/mngt/ocf_mngt_flush.c @@ -402,7 +402,7 @@ static void _ocf_mngt_flush_portion_end(void *private_data, int error) return; } - ocf_engine_push_req_back(fc->req, false); + ocf_queue_push_req_back(fc->req, false); } @@ -452,7 +452,7 @@ static void _ocf_mngt_flush_container( fc->ticks1 = 0; fc->ticks2 = UINT_MAX; - ocf_engine_push_req_back(fc->req, true); + ocf_queue_push_req_back(fc->req, true); return; finish: diff --git a/src/ocf_queue.c b/src/ocf_queue.c index 120ed75..22ae93e 100644 --- a/src/ocf_queue.c +++ b/src/ocf_queue.c @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ #include "ocf/ocf.h" @@ -141,3 +142,81 @@ ocf_cache_t ocf_queue_get_cache(ocf_queue_t q) OCF_CHECK_NULL(q); return q->cache; } + +void ocf_queue_push_req_back(struct ocf_request *req, bool allow_sync) +{ + ocf_cache_t cache = req->cache; + ocf_queue_t q = NULL; + unsigned long lock_flags = 0; + + INIT_LIST_HEAD(&req->list); + + ENV_BUG_ON(!req->io_queue); + q = req->io_queue; + + if (!req->info.internal) { + env_atomic_set(&cache->last_access_ms, + env_ticks_to_msecs(env_get_tick_count())); + } + + env_spinlock_lock_irqsave(&q->io_list_lock, lock_flags); + + list_add_tail(&req->list, &q->io_list); + env_atomic_inc(&q->io_no); + + env_spinlock_unlock_irqrestore(&q->io_list_lock, lock_flags); + + /* NOTE: do not dereference @req past this line, it might + * be picked up by concurrent io thread and deallocated + * at this point */ + + ocf_queue_kick(q, allow_sync); +} + +void ocf_queue_push_req_front(struct ocf_request *req, bool allow_sync) +{ + ocf_cache_t cache = req->cache; + ocf_queue_t q = NULL; + unsigned long lock_flags = 0; + + ENV_BUG_ON(!req->io_queue); + INIT_LIST_HEAD(&req->list); + + q = req->io_queue; + + if (!req->info.internal) { + env_atomic_set(&cache->last_access_ms, + env_ticks_to_msecs(env_get_tick_count())); + } + + env_spinlock_lock_irqsave(&q->io_list_lock, lock_flags); + + list_add(&req->list, &q->io_list); + env_atomic_inc(&q->io_no); + + env_spinlock_unlock_irqrestore(&q->io_list_lock, lock_flags); + + /* NOTE: do not dereference @req past this line, it might + * be picked up by concurrent io thread and deallocated + * at this point */ + + ocf_queue_kick(q, allow_sync); +} + +void ocf_queue_push_req_front_cb(struct ocf_request *req, + ocf_req_cb req_cb, + bool allow_sync) +{ + req->error = 0; /* Please explain why!!! */ + req->engine_handler = req_cb; + ocf_queue_push_req_front(req, allow_sync); +} + +void ocf_queue_push_req_back_cb(struct ocf_request *req, + ocf_req_cb req_cb, + bool allow_sync) +{ + req->error = 0; /* Please explain why!!! */ + req->engine_handler = req_cb; + ocf_queue_push_req_back(req, allow_sync); +} diff --git a/src/ocf_queue_priv.h b/src/ocf_queue_priv.h index 50160c3..cebfe16 100644 --- a/src/ocf_queue_priv.h +++ b/src/ocf_queue_priv.h @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -7,6 +8,7 @@ #define OCF_QUEUE_PRIV_H_ #include "ocf_env.h" +#include "ocf_request.h" struct ocf_queue { ocf_cache_t cache; @@ -46,4 +48,48 @@ static inline void ocf_queue_kick(ocf_queue_t queue, bool allow_sync) queue->ops->kick(queue); } +/** + * @brief Push front OCF request to the OCF thread worker queue + * + * @param req OCF request + * @param allow_sync caller allows for request from queue to be ran immediately + from push function in caller context + */ +void ocf_queue_push_req_back(struct ocf_request *req, + bool allow_sync); + +/** + * @brief Push back OCF request to the OCF thread worker queue + * + * @param req OCF request + * @param allow_sync caller allows for request from queue to be ran immediately + from push function in caller context + */ +void ocf_queue_push_req_front(struct ocf_request *req, + bool allow_sync); + +/** + * @brief Set interface and push from request to the OCF thread worker queue front + * + * @param req OCF request + * @param engine_cb IO engine handler callback + * @param allow_sync caller allows for request from queue to be ran immediately + from push function in caller context + */ +void ocf_queue_push_req_front_cb(struct ocf_request *req, + ocf_req_cb req_cb, + bool allow_sync); + +/** + * @brief Set interface and push from request to the OCF thread worker queue back + * + * @param req OCF request + * @param engine_cb IO engine handler callback + * @param allow_sync caller allows for request from queue to be ran immediately + from push function in caller context + */ +void ocf_queue_push_req_back_cb(struct ocf_request *req, + ocf_req_cb req_cb, + bool allow_sync); + #endif diff --git a/src/utils/utils_cleaner.c b/src/utils/utils_cleaner.c index 65112e5..ee97eba 100644 --- a/src/utils/utils_cleaner.c +++ b/src/utils/utils_cleaner.c @@ -217,7 +217,7 @@ static void _ocf_cleaner_complete_req(struct ocf_request *req) if (master->complete_queue) { ocf_req_get(master); - ocf_engine_push_req_front_cb(master, + ocf_queue_push_req_front_cb(master, _ocf_cleaner_complete, true); } else { /* Only master contains completion function and priv */ @@ -232,7 +232,7 @@ static void _ocf_cleaner_complete_req(struct ocf_request *req) static void _ocf_cleaner_on_resume(struct ocf_request *req) { OCF_DEBUG_TRACE(req->cache); - ocf_engine_push_req_front(req, true); + ocf_queue_push_req_front(req, true); } /* @@ -336,7 +336,7 @@ static void _ocf_cleaner_metadata_io_end(struct ocf_request *req, int error) OCF_DEBUG_MSG(req->cache, "Metadata flush finished"); req->engine_handler = _ocf_cleaner_fire_flush_cache; - ocf_engine_push_req_front(req, true); + ocf_queue_push_req_front(req, true); } static int _ocf_cleaner_update_metadata(struct ocf_request *req) @@ -415,7 +415,7 @@ static void _ocf_cleaner_flush_cores_io_end(struct ocf_map_info *map, * All core writes done, switch to post cleaning activities */ req->engine_handler = _ocf_cleaner_update_metadata; - ocf_engine_push_req_front(req, true); + ocf_queue_push_req_front(req, true); } static void _ocf_cleaner_flush_cores_io_cmpl(struct ocf_io *io, int error) @@ -487,7 +487,7 @@ static void _ocf_cleaner_core_io_end(struct ocf_request *req) * Move processing to thread, where IO will be (and can be) submitted */ req->engine_handler = _ocf_cleaner_fire_flush_cores; - ocf_engine_push_req_front(req, true); + ocf_queue_push_req_front(req, true); } static void _ocf_cleaner_core_io_cmpl(struct ocf_io *io, int error) @@ -645,7 +645,7 @@ static void _ocf_cleaner_cache_io_end(struct ocf_request *req) * Move processing to thread, where IO will be (and can be) submitted */ req->engine_handler = _ocf_cleaner_fire_core; - ocf_engine_push_req_front(req, true); + ocf_queue_push_req_front(req, true); OCF_DEBUG_MSG(req->cache, "Cache reads finished"); } diff --git a/src/utils/utils_parallelize.c b/src/utils/utils_parallelize.c index a7dd383..91d6233 100644 --- a/src/utils/utils_parallelize.c +++ b/src/utils/utils_parallelize.c @@ -1,6 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation - * Copyright(c) 2023 Huawei Technologies + * Copyright(c) 2023-2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -142,5 +142,5 @@ void ocf_parallelize_run(ocf_parallelize_t parallelize) int i; for (i = 0; i < parallelize->shards_cnt; i++) - ocf_engine_push_req_front(parallelize->reqs[i], false); + ocf_queue_push_req_front(parallelize->reqs[i], false); } diff --git a/src/utils/utils_pipeline.c b/src/utils/utils_pipeline.c index c2a263a..e9428fe 100644 --- a/src/utils/utils_pipeline.c +++ b/src/utils/utils_pipeline.c @@ -1,6 +1,6 @@ /* * Copyright(c) 2019-2022 Intel Corporation - * Copyright(c) 2023 Huawei Technologies + * Copyright(c) 2023-2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ @@ -47,7 +47,7 @@ static int _ocf_pipeline_run_step(struct ocf_request *req) if (step->pred(pipeline, pipeline->priv, &step->arg)) { step->hndl(pipeline, pipeline->priv, &step->arg); return 0; - } + } continue; case ocf_pipeline_step_foreach: arg = &step->args[pipeline->next_arg++]; @@ -126,12 +126,12 @@ void *ocf_pipeline_get_priv(ocf_pipeline_t pipeline) void ocf_pipeline_next(ocf_pipeline_t pipeline) { - ocf_engine_push_req_front(pipeline->req, false); + ocf_queue_push_req_front(pipeline->req, false); } void ocf_pipeline_finish(ocf_pipeline_t pipeline, int error) { pipeline->finish = true; pipeline->error = error; - ocf_engine_push_req_front(pipeline->req, false); + ocf_queue_push_req_front(pipeline->req, false); } From 4f2d5c22d6e095fe36ca100a4fa42e3244748a44 Mon Sep 17 00:00:00 2001 From: Ian Levine Date: Thu, 12 Oct 2023 15:07:17 +0300 Subject: [PATCH 3/4] Move and rename ocf_engine_pop_req from cache_engine to ocf_queue_pop_req in ocf_queue Signed-off-by: Ian Levine Signed-off-by: Robert Baldyga --- src/engine/cache_engine.c | 31 ------------------------------- src/engine/cache_engine.h | 2 -- src/ocf_queue.c | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 32 insertions(+), 34 deletions(-) diff --git a/src/engine/cache_engine.c b/src/engine/cache_engine.c index 014ba49..e731a77 100644 --- a/src/engine/cache_engine.c +++ b/src/engine/cache_engine.c @@ -158,37 +158,6 @@ static ocf_req_cb ocf_cache_mode_to_engine_cb( return cache_mode_io_if_map[req_cache_mode]->cbs[rw]; } -struct ocf_request *ocf_engine_pop_req(ocf_queue_t q) -{ - unsigned long lock_flags = 0; - struct ocf_request *req; - - OCF_CHECK_NULL(q); - - /* LOCK */ - env_spinlock_lock_irqsave(&q->io_list_lock, lock_flags); - - if (list_empty(&q->io_list)) { - /* No items on the list */ - env_spinlock_unlock_irqrestore(&q->io_list_lock, - lock_flags); - return NULL; - } - - /* Get the first request and remove it from the list */ - req = list_first_entry(&q->io_list, struct ocf_request, list); - - env_atomic_dec(&q->io_no); - list_del(&req->list); - - /* UNLOCK */ - env_spinlock_unlock_irqrestore(&q->io_list_lock, lock_flags); - - OCF_CHECK_NULL(req); - - return req; -} - bool ocf_fallback_pt_is_on(ocf_cache_t cache) { ENV_BUG_ON(env_atomic_read(&cache->fallback_pt_error_counter) < 0); diff --git a/src/engine/cache_engine.h b/src/engine/cache_engine.h index 88c6da4..bb61e79 100644 --- a/src/engine/cache_engine.h +++ b/src/engine/cache_engine.h @@ -36,8 +36,6 @@ bool ocf_req_cache_mode_has_lazy_write(ocf_req_cache_mode_t mode); bool ocf_fallback_pt_is_on(ocf_cache_t cache); -struct ocf_request *ocf_engine_pop_req(struct ocf_queue *q); - int ocf_engine_hndl_req(struct ocf_request *req); #define OCF_FAST_PATH_YES 7 diff --git a/src/ocf_queue.c b/src/ocf_queue.c index 22ae93e..409b7b0 100644 --- a/src/ocf_queue.c +++ b/src/ocf_queue.c @@ -89,13 +89,44 @@ void ocf_io_handle(struct ocf_io *io, void *opaque) req->engine_handler(req); } +static struct ocf_request *ocf_queue_pop_req(ocf_queue_t q) +{ + unsigned long lock_flags = 0; + struct ocf_request *req; + + OCF_CHECK_NULL(q); + + /* LOCK */ + env_spinlock_lock_irqsave(&q->io_list_lock, lock_flags); + + if (list_empty(&q->io_list)) { + /* No items on the list */ + env_spinlock_unlock_irqrestore(&q->io_list_lock, + lock_flags); + return NULL; + } + + /* Get the first request and remove it from the list */ + req = list_first_entry(&q->io_list, struct ocf_request, list); + + env_atomic_dec(&q->io_no); + list_del(&req->list); + + /* UNLOCK */ + env_spinlock_unlock_irqrestore(&q->io_list_lock, lock_flags); + + OCF_CHECK_NULL(req); + + return req; +} + void ocf_queue_run_single(ocf_queue_t q) { struct ocf_request *io_req = NULL; OCF_CHECK_NULL(q); - io_req = ocf_engine_pop_req(q); + io_req = ocf_queue_pop_req(q); if (!io_req) return; From ac1b6b774add7a4fe4da5279bcbb4ebd00284fb1 Mon Sep 17 00:00:00 2001 From: Ian Levine Date: Thu, 12 Oct 2023 22:19:18 +0300 Subject: [PATCH 4/4] Added a priority queue for the request instead of push front Now the request can be pushed to a high priority queue (instead of ocf_queue_push_req_front) and to a low priority queue (instead of ocf_queue_push_req_back). Both functions were merged into one function (ocf_queue_push_req) and instead of the allow_sync parameter there is now a flags parameter that can be an OR combination of OCF_QUEUE_ALLOW_SYNC and OCF_QUEUE_PRIO_HIGH Signed-off-by: Ian Levine Signed-off-by: Robert Baldyga --- src/engine/cache_engine.c | 4 +- src/engine/engine_bf.c | 3 +- src/engine/engine_common.c | 5 +- src/engine/engine_discard.c | 6 +-- src/engine/engine_fast.c | 2 +- src/engine/engine_inv.c | 3 +- src/engine/engine_pt.c | 5 +- src/engine/engine_pt.h | 2 +- src/engine/engine_rd.c | 2 +- src/engine/engine_wb.c | 4 +- src/engine/engine_wi.c | 11 +++-- src/engine/engine_wo.c | 2 +- src/engine/engine_wt.c | 4 +- src/engine/engine_zero.c | 6 ++- src/metadata/metadata_io.c | 11 +++-- src/metadata/metadata_passive_update.c | 2 +- src/metadata/metadata_raw_dynamic.c | 9 ++-- src/mngt/ocf_mngt_flush.c | 4 +- src/ocf_queue.c | 67 +++++++------------------- src/ocf_queue_priv.h | 49 ++++++------------- src/utils/utils_cleaner.c | 14 +++--- src/utils/utils_parallelize.c | 2 +- src/utils/utils_pipeline.c | 4 +- 23 files changed, 90 insertions(+), 131 deletions(-) diff --git a/src/engine/cache_engine.c b/src/engine/cache_engine.c index e731a77..1ffde74 100644 --- a/src/engine/cache_engine.c +++ b/src/engine/cache_engine.c @@ -233,7 +233,7 @@ int ocf_engine_hndl_req(struct ocf_request *req) * to into OCF workers */ - ocf_queue_push_req_back(req, true); + ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC); return 0; } @@ -282,7 +282,7 @@ void ocf_engine_hndl_ops_req(struct ocf_request *req) ocf_io_if_type_to_engine_cb(OCF_IO_D2C_IF, req->rw) : ocf_io_if_type_to_engine_cb(OCF_IO_OPS_IF, req->rw); - ocf_queue_push_req_back(req, true); + ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC); } bool ocf_req_cache_mode_has_lazy_write(ocf_req_cache_mode_t mode) diff --git a/src/engine/engine_bf.c b/src/engine/engine_bf.c index ab6bc6c..f6435ca 100644 --- a/src/engine/engine_bf.c +++ b/src/engine/engine_bf.c @@ -95,5 +95,6 @@ static int _ocf_backfill_do(struct ocf_request *req) void ocf_engine_backfill(struct ocf_request *req) { backfill_queue_inc_block(req->cache); - ocf_queue_push_req_front_cb(req, _ocf_backfill_do, true); + ocf_queue_push_req_cb(req, _ocf_backfill_do, + OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } diff --git a/src/engine/engine_common.c b/src/engine/engine_common.c index 4f21c33..49d3dab 100644 --- a/src/engine/engine_common.c +++ b/src/engine/engine_common.c @@ -381,7 +381,8 @@ static void _ocf_engine_clean_end(void *private_data, int error) } else { req->info.dirty_any = 0; req->info.dirty_all = 0; - ocf_queue_push_req_front(req, true); + ocf_queue_push_req(req, + OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } } @@ -643,5 +644,5 @@ void ocf_engine_on_resume(struct ocf_request *req) OCF_DEBUG_RQ(req, "On resume"); - ocf_queue_push_req_front_cb(req, _ocf_engine_refresh, false); + ocf_queue_push_req_cb(req, _ocf_engine_refresh, OCF_QUEUE_PRIO_HIGH); } diff --git a/src/engine/engine_discard.c b/src/engine/engine_discard.c index 9298099..b5e975b 100644 --- a/src/engine/engine_discard.c +++ b/src/engine/engine_discard.c @@ -74,7 +74,7 @@ static void _ocf_discard_cache_flush_complete(struct ocf_io *io, int error) } req->engine_handler = _ocf_discard_core; - ocf_queue_push_req_front(req, true); + ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); ocf_io_put(io); } @@ -111,7 +111,7 @@ static void _ocf_discard_finish_step(struct ocf_request *req) else req->engine_handler = _ocf_discard_core; - ocf_queue_push_req_front(req, true); + ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } static void _ocf_discard_step_complete(struct ocf_request *req, int error) @@ -182,7 +182,7 @@ static int _ocf_discard_step_do(struct ocf_request *req) static void _ocf_discard_on_resume(struct ocf_request *req) { OCF_DEBUG_RQ(req, "On resume"); - ocf_queue_push_req_front(req, true); + ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } static int _ocf_discard_step(struct ocf_request *req) diff --git a/src/engine/engine_fast.c b/src/engine/engine_fast.c index 5676e0b..5c2f4aa 100644 --- a/src/engine/engine_fast.c +++ b/src/engine/engine_fast.c @@ -46,7 +46,7 @@ static void _ocf_read_fast_complete(struct ocf_request *req, int error) if (req->error) { OCF_DEBUG_RQ(req, "ERROR"); - ocf_queue_push_req_front_pt(req); + ocf_queue_push_req_pt(req); } else { ocf_req_unlock(ocf_cache_line_concurrency(req->cache), req); diff --git a/src/engine/engine_inv.c b/src/engine/engine_inv.c index fa84d66..d4c5e9e 100644 --- a/src/engine/engine_inv.c +++ b/src/engine/engine_inv.c @@ -63,5 +63,6 @@ static int _ocf_invalidate_do(struct ocf_request *req) void ocf_engine_invalidate(struct ocf_request *req) { - ocf_queue_push_req_front_cb(req, _ocf_invalidate_do, true); + ocf_queue_push_req_cb(req, _ocf_invalidate_do, + OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } diff --git a/src/engine/engine_pt.c b/src/engine/engine_pt.c index 4223160..dbb2d5e 100644 --- a/src/engine/engine_pt.c +++ b/src/engine/engine_pt.c @@ -165,8 +165,9 @@ int ocf_read_pt(struct ocf_request *req) return 0; } -void ocf_queue_push_req_front_pt(struct ocf_request *req) +void ocf_queue_push_req_pt(struct ocf_request *req) { - ocf_queue_push_req_front_cb(req, ocf_read_pt_do, true); + ocf_queue_push_req_cb(req, ocf_read_pt_do, + OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } diff --git a/src/engine/engine_pt.h b/src/engine/engine_pt.h index 01e34d1..9bea3bb 100644 --- a/src/engine/engine_pt.h +++ b/src/engine/engine_pt.h @@ -11,6 +11,6 @@ int ocf_read_pt(struct ocf_request *req); int ocf_read_pt_do(struct ocf_request *req); -void ocf_queue_push_req_front_pt(struct ocf_request *req); +void ocf_queue_push_req_pt(struct ocf_request *req); #endif /* ENGINE_OFF_H_ */ diff --git a/src/engine/engine_rd.c b/src/engine/engine_rd.c index 2d3f201..3cf67c7 100644 --- a/src/engine/engine_rd.c +++ b/src/engine/engine_rd.c @@ -42,7 +42,7 @@ static void _ocf_read_generic_hit_complete(struct ocf_request *req, int error) OCF_DEBUG_RQ(req, "HIT completion"); if (req->error) { - ocf_queue_push_req_front_pt(req); + ocf_queue_push_req_pt(req); } else { ocf_req_unlock(c, req); diff --git a/src/engine/engine_wb.c b/src/engine/engine_wb.c index 4012f32..d419d50 100644 --- a/src/engine/engine_wb.c +++ b/src/engine/engine_wb.c @@ -103,8 +103,8 @@ static void _ocf_write_wb_complete(struct ocf_request *req, int error) ocf_engine_invalidate(req); } else { - ocf_queue_push_req_front_cb(req, - ocf_write_wb_do_flush_metadata, true); + ocf_queue_push_req_cb(req, ocf_write_wb_do_flush_metadata, + OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } } diff --git a/src/engine/engine_wi.c b/src/engine/engine_wi.c index aed03bf..0adf0fb 100644 --- a/src/engine/engine_wi.c +++ b/src/engine/engine_wi.c @@ -56,8 +56,8 @@ static void _ocf_write_wi_io_flush_metadata(struct ocf_request *req, int error) if (!req->error && !req->wi_second_pass && ocf_engine_is_miss(req)) { /* need another pass */ - ocf_queue_push_req_front_cb(req, _ocf_write_wi_next_pass, - true); + ocf_queue_push_req_cb(req, _ocf_write_wi_next_pass, + OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); return; } @@ -123,8 +123,9 @@ static void _ocf_write_wi_core_complete(struct ocf_request *req, int error) ocf_req_put(req); } else { - ocf_queue_push_req_front_cb(req, - ocf_write_wi_update_and_flush_metadata, true); + ocf_queue_push_req_cb(req, + ocf_write_wi_update_and_flush_metadata, + OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } } @@ -155,7 +156,7 @@ static int _ocf_write_wi_core_write(struct ocf_request *req) static void _ocf_write_wi_on_resume(struct ocf_request *req) { OCF_DEBUG_RQ(req, "On resume"); - ocf_queue_push_req_front(req, true); + ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } int ocf_write_wi(struct ocf_request *req) diff --git a/src/engine/engine_wo.c b/src/engine/engine_wo.c index 4104f55..9414bda 100644 --- a/src/engine/engine_wo.c +++ b/src/engine/engine_wo.c @@ -171,7 +171,7 @@ static void _ocf_read_wo_core_complete(struct ocf_request *req, int error) } req->engine_handler = ocf_read_wo_cache_do; - ocf_queue_push_req_front(req, true); + ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } static int ocf_read_wo_do(struct ocf_request *req) diff --git a/src/engine/engine_wt.c b/src/engine/engine_wt.c index 13e2edc..4dcce9b 100644 --- a/src/engine/engine_wt.c +++ b/src/engine/engine_wt.c @@ -113,8 +113,8 @@ static void _ocf_write_wt_req_complete(struct ocf_request *req) if (req->info.dirty_any) { /* Some of the request's cachelines changed its state to clean */ - ocf_queue_push_req_front_cb(req, - ocf_write_wt_do_flush_metadata, true); + ocf_queue_push_req_cb(req, ocf_write_wt_do_flush_metadata, + OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } else { ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); diff --git a/src/engine/engine_zero.c b/src/engine/engine_zero.c index 49b5843..cdd9b85 100644 --- a/src/engine/engine_zero.c +++ b/src/engine/engine_zero.c @@ -51,7 +51,8 @@ static void _ocf_zero_io_flush_metadata(struct ocf_request *req, int error) if (env_atomic_dec_return(&req->req_remaining)) return; - ocf_queue_push_req_front_cb(req, ocf_zero_purge, true); + ocf_queue_push_req_cb(req, ocf_zero_purge, + OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } static inline void ocf_zero_map_info(struct ocf_request *req) @@ -149,7 +150,8 @@ void ocf_engine_zero_line(struct ocf_request *req) if (lock >= 0) { ENV_BUG_ON(lock != OCF_LOCK_ACQUIRED); - ocf_queue_push_req_front_cb(req, _ocf_zero_do, true); + ocf_queue_push_req_cb(req, _ocf_zero_do, + OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } else { OCF_DEBUG_RQ(req, "LOCK ERROR %d", lock); req->complete(req, lock); diff --git a/src/metadata/metadata_io.c b/src/metadata/metadata_io.c index 87a774b..9be1d2f 100644 --- a/src/metadata/metadata_io.c +++ b/src/metadata/metadata_io.c @@ -93,7 +93,8 @@ static void metadata_io_read_i_atomic_step_end(struct ocf_io *io, int error) context->curr_offset += context->curr_count; if (context->count > 0) - ocf_queue_push_req_front(context->req, true); + ocf_queue_push_req(context->req, + OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); else metadata_io_read_i_atomic_complete(context, 0); } @@ -181,7 +182,8 @@ int metadata_io_read_i_atomic(ocf_cache_t cache, ocf_queue_t queue, void *priv, context->compl_hndl = compl_hndl; context->priv = priv; - ocf_queue_push_req_front(context->req, true); + ocf_queue_push_req(context->req, + OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); return 0; } @@ -269,7 +271,7 @@ static void metadata_io_req_finalize(struct metadata_io_request *m_req) static void metadata_io_page_lock_acquired(struct ocf_request *req) { - ocf_queue_push_req_front(req, true); + ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } static int metadata_io_restart_req(struct ocf_request *req) @@ -401,7 +403,8 @@ void metadata_io_req_complete(struct metadata_io_request *m_req) } m_req->req.engine_handler = metadata_io_restart_req; - ocf_queue_push_req_front(&m_req->req, true); + ocf_queue_push_req(&m_req->req, + OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } /* diff --git a/src/metadata/metadata_passive_update.c b/src/metadata/metadata_passive_update.c index 709738e..c77fec4 100644 --- a/src/metadata/metadata_passive_update.c +++ b/src/metadata/metadata_passive_update.c @@ -71,7 +71,7 @@ static int passive_io_resume(struct ocf_request *req) static void passive_io_page_lock_acquired(struct ocf_request *req) { - ocf_queue_push_req_front(req, true); + ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } int ocf_metadata_passive_update(ocf_cache_t cache, struct ocf_io *io, diff --git a/src/metadata/metadata_raw_dynamic.c b/src/metadata/metadata_raw_dynamic.c index d25fad9..d426ff9 100644 --- a/src/metadata/metadata_raw_dynamic.c +++ b/src/metadata/metadata_raw_dynamic.c @@ -389,7 +389,8 @@ static void raw_dynamic_load_all_read_end(struct ocf_io *io, int error) } context->req->engine_handler = raw_dynamic_load_all_update; - ocf_queue_push_req_front(context->req, true); + ocf_queue_push_req(context->req, + OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } static int raw_dynamic_load_all_read(struct ocf_request *req) @@ -455,7 +456,8 @@ static int raw_dynamic_load_all_update(struct ocf_request *req) } context->req->engine_handler = raw_dynamic_load_all_read; - ocf_queue_push_req_front(context->req, true); + ocf_queue_push_req(context->req, + OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); return 0; } @@ -501,7 +503,8 @@ void raw_dynamic_load_all(ocf_cache_t cache, struct ocf_metadata_raw *raw, context->req->priv = context; context->req->engine_handler = raw_dynamic_load_all_read; - ocf_queue_push_req_front(context->req, true); + ocf_queue_push_req(context->req, + OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); return; err_req: diff --git a/src/mngt/ocf_mngt_flush.c b/src/mngt/ocf_mngt_flush.c index 4473b59..cccffc4 100644 --- a/src/mngt/ocf_mngt_flush.c +++ b/src/mngt/ocf_mngt_flush.c @@ -402,7 +402,7 @@ static void _ocf_mngt_flush_portion_end(void *private_data, int error) return; } - ocf_queue_push_req_back(fc->req, false); + ocf_queue_push_req(fc->req, 0); } @@ -452,7 +452,7 @@ static void _ocf_mngt_flush_container( fc->ticks1 = 0; fc->ticks2 = UINT_MAX; - ocf_queue_push_req_back(fc->req, true); + ocf_queue_push_req(fc->req, OCF_QUEUE_ALLOW_SYNC); return; finish: diff --git a/src/ocf_queue.c b/src/ocf_queue.c index 409b7b0..71203c9 100644 --- a/src/ocf_queue.c +++ b/src/ocf_queue.c @@ -40,7 +40,8 @@ int ocf_queue_create(ocf_cache_t cache, ocf_queue_t *queue, return result; } - INIT_LIST_HEAD(&tmp_queue->io_list); + INIT_LIST_HEAD(&tmp_queue->io_list_high); + INIT_LIST_HEAD(&tmp_queue->io_list_low); env_atomic_set(&tmp_queue->ref_count, 1); tmp_queue->cache = cache; tmp_queue->ops = ops; @@ -93,21 +94,25 @@ static struct ocf_request *ocf_queue_pop_req(ocf_queue_t q) { unsigned long lock_flags = 0; struct ocf_request *req; + struct list_head *io_list; OCF_CHECK_NULL(q); /* LOCK */ env_spinlock_lock_irqsave(&q->io_list_lock, lock_flags); - if (list_empty(&q->io_list)) { - /* No items on the list */ + if (!list_empty(&q->io_list_high)) { + io_list = &q->io_list_high; + } else if (!list_empty(&q->io_list_low)) { + io_list = &q->io_list_low; + } else { /* No items on the list */ env_spinlock_unlock_irqrestore(&q->io_list_lock, lock_flags); return NULL; } /* Get the first request and remove it from the list */ - req = list_first_entry(&q->io_list, struct ocf_request, list); + req = list_first_entry(io_list, struct ocf_request, list); env_atomic_dec(&q->io_no); list_del(&req->list); @@ -174,11 +179,12 @@ ocf_cache_t ocf_queue_get_cache(ocf_queue_t q) return q->cache; } -void ocf_queue_push_req_back(struct ocf_request *req, bool allow_sync) +void ocf_queue_push_req(struct ocf_request *req, uint flags) { ocf_cache_t cache = req->cache; ocf_queue_t q = NULL; unsigned long lock_flags = 0; + struct list_head *io_list; INIT_LIST_HEAD(&req->list); @@ -192,7 +198,8 @@ void ocf_queue_push_req_back(struct ocf_request *req, bool allow_sync) env_spinlock_lock_irqsave(&q->io_list_lock, lock_flags); - list_add_tail(&req->list, &q->io_list); + io_list = (flags & OCF_QUEUE_PRIO_HIGH) ? &q->io_list_high : &q->io_list_low; + list_add_tail(&req->list, io_list); env_atomic_inc(&q->io_no); env_spinlock_unlock_irqrestore(&q->io_list_lock, lock_flags); @@ -201,53 +208,13 @@ void ocf_queue_push_req_back(struct ocf_request *req, bool allow_sync) * be picked up by concurrent io thread and deallocated * at this point */ - ocf_queue_kick(q, allow_sync); + ocf_queue_kick(q, (bool)(flags & OCF_QUEUE_ALLOW_SYNC)); } -void ocf_queue_push_req_front(struct ocf_request *req, bool allow_sync) -{ - ocf_cache_t cache = req->cache; - ocf_queue_t q = NULL; - unsigned long lock_flags = 0; - - ENV_BUG_ON(!req->io_queue); - INIT_LIST_HEAD(&req->list); - - q = req->io_queue; - - if (!req->info.internal) { - env_atomic_set(&cache->last_access_ms, - env_ticks_to_msecs(env_get_tick_count())); - } - - env_spinlock_lock_irqsave(&q->io_list_lock, lock_flags); - - list_add(&req->list, &q->io_list); - env_atomic_inc(&q->io_no); - - env_spinlock_unlock_irqrestore(&q->io_list_lock, lock_flags); - - /* NOTE: do not dereference @req past this line, it might - * be picked up by concurrent io thread and deallocated - * at this point */ - - ocf_queue_kick(q, allow_sync); -} - -void ocf_queue_push_req_front_cb(struct ocf_request *req, - ocf_req_cb req_cb, - bool allow_sync) +void ocf_queue_push_req_cb(struct ocf_request *req, + ocf_req_cb req_cb, uint flags) { req->error = 0; /* Please explain why!!! */ req->engine_handler = req_cb; - ocf_queue_push_req_front(req, allow_sync); -} - -void ocf_queue_push_req_back_cb(struct ocf_request *req, - ocf_req_cb req_cb, - bool allow_sync) -{ - req->error = 0; /* Please explain why!!! */ - req->engine_handler = req_cb; - ocf_queue_push_req_back(req, allow_sync); + ocf_queue_push_req(req, flags); } diff --git a/src/ocf_queue_priv.h b/src/ocf_queue_priv.h index cebfe16..8cf4725 100644 --- a/src/ocf_queue_priv.h +++ b/src/ocf_queue_priv.h @@ -10,12 +10,17 @@ #include "ocf_env.h" #include "ocf_request.h" +/* ocf_queue_push_req flags */ +#define OCF_QUEUE_ALLOW_SYNC 0x01 /* Request can run immediately in caller context */ +#define OCF_QUEUE_PRIO_HIGH 0x02 /* Push to the high priority queue */ + struct ocf_queue { ocf_cache_t cache; void *priv; - struct list_head io_list; + struct list_head io_list_high; + struct list_head io_list_low; /* per-queue free running global metadata lock index */ unsigned lock_idx; @@ -49,47 +54,21 @@ static inline void ocf_queue_kick(ocf_queue_t queue, bool allow_sync) } /** - * @brief Push front OCF request to the OCF thread worker queue + * @brief Push OCF request to the OCF thread worker queue * * @param req OCF request - * @param allow_sync caller allows for request from queue to be ran immediately - from push function in caller context + * @param flags See ocf_queue_push_req flags above */ -void ocf_queue_push_req_back(struct ocf_request *req, - bool allow_sync); +void ocf_queue_push_req(struct ocf_request *req, uint flags); /** - * @brief Push back OCF request to the OCF thread worker queue + * @brief Set interface and push from request to the OCF thread worker queue * * @param req OCF request - * @param allow_sync caller allows for request from queue to be ran immediately - from push function in caller context + * @param req_cb IO engine handler callback + * @param flags See ocf_queue_push_req flags above */ -void ocf_queue_push_req_front(struct ocf_request *req, - bool allow_sync); - -/** - * @brief Set interface and push from request to the OCF thread worker queue front - * - * @param req OCF request - * @param engine_cb IO engine handler callback - * @param allow_sync caller allows for request from queue to be ran immediately - from push function in caller context - */ -void ocf_queue_push_req_front_cb(struct ocf_request *req, - ocf_req_cb req_cb, - bool allow_sync); - -/** - * @brief Set interface and push from request to the OCF thread worker queue back - * - * @param req OCF request - * @param engine_cb IO engine handler callback - * @param allow_sync caller allows for request from queue to be ran immediately - from push function in caller context - */ -void ocf_queue_push_req_back_cb(struct ocf_request *req, - ocf_req_cb req_cb, - bool allow_sync); +void ocf_queue_push_req_cb(struct ocf_request *req, + ocf_req_cb req_cb, uint flags); #endif diff --git a/src/utils/utils_cleaner.c b/src/utils/utils_cleaner.c index ee97eba..210086d 100644 --- a/src/utils/utils_cleaner.c +++ b/src/utils/utils_cleaner.c @@ -217,8 +217,8 @@ static void _ocf_cleaner_complete_req(struct ocf_request *req) if (master->complete_queue) { ocf_req_get(master); - ocf_queue_push_req_front_cb(master, - _ocf_cleaner_complete, true); + ocf_queue_push_req_cb(master, _ocf_cleaner_complete, + OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } else { /* Only master contains completion function and priv */ cmpl = master->master_io_req; @@ -232,7 +232,7 @@ static void _ocf_cleaner_complete_req(struct ocf_request *req) static void _ocf_cleaner_on_resume(struct ocf_request *req) { OCF_DEBUG_TRACE(req->cache); - ocf_queue_push_req_front(req, true); + ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } /* @@ -336,7 +336,7 @@ static void _ocf_cleaner_metadata_io_end(struct ocf_request *req, int error) OCF_DEBUG_MSG(req->cache, "Metadata flush finished"); req->engine_handler = _ocf_cleaner_fire_flush_cache; - ocf_queue_push_req_front(req, true); + ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } static int _ocf_cleaner_update_metadata(struct ocf_request *req) @@ -415,7 +415,7 @@ static void _ocf_cleaner_flush_cores_io_end(struct ocf_map_info *map, * All core writes done, switch to post cleaning activities */ req->engine_handler = _ocf_cleaner_update_metadata; - ocf_queue_push_req_front(req, true); + ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } static void _ocf_cleaner_flush_cores_io_cmpl(struct ocf_io *io, int error) @@ -487,7 +487,7 @@ static void _ocf_cleaner_core_io_end(struct ocf_request *req) * Move processing to thread, where IO will be (and can be) submitted */ req->engine_handler = _ocf_cleaner_fire_flush_cores; - ocf_queue_push_req_front(req, true); + ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } static void _ocf_cleaner_core_io_cmpl(struct ocf_io *io, int error) @@ -645,7 +645,7 @@ static void _ocf_cleaner_cache_io_end(struct ocf_request *req) * Move processing to thread, where IO will be (and can be) submitted */ req->engine_handler = _ocf_cleaner_fire_core; - ocf_queue_push_req_front(req, true); + ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); OCF_DEBUG_MSG(req->cache, "Cache reads finished"); } diff --git a/src/utils/utils_parallelize.c b/src/utils/utils_parallelize.c index 91d6233..43983bd 100644 --- a/src/utils/utils_parallelize.c +++ b/src/utils/utils_parallelize.c @@ -142,5 +142,5 @@ void ocf_parallelize_run(ocf_parallelize_t parallelize) int i; for (i = 0; i < parallelize->shards_cnt; i++) - ocf_queue_push_req_front(parallelize->reqs[i], false); + ocf_queue_push_req(parallelize->reqs[i], OCF_QUEUE_PRIO_HIGH); } diff --git a/src/utils/utils_pipeline.c b/src/utils/utils_pipeline.c index e9428fe..2e17616 100644 --- a/src/utils/utils_pipeline.c +++ b/src/utils/utils_pipeline.c @@ -126,12 +126,12 @@ void *ocf_pipeline_get_priv(ocf_pipeline_t pipeline) void ocf_pipeline_next(ocf_pipeline_t pipeline) { - ocf_queue_push_req_front(pipeline->req, false); + ocf_queue_push_req(pipeline->req, OCF_QUEUE_PRIO_HIGH); } void ocf_pipeline_finish(ocf_pipeline_t pipeline, int error) { pipeline->finish = true; pipeline->error = error; - ocf_queue_push_req_front(pipeline->req, false); + ocf_queue_push_req(pipeline->req, OCF_QUEUE_PRIO_HIGH); }