From 1ed707361f2e90bac42eae8ec31e7dcdf3545da0 Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Sun, 3 Sep 2023 20:27:27 +0200 Subject: [PATCH] Modify engines to use forward API Signed-off-by: Robert Baldyga Signed-off-by: Rafal Stefanowski Signed-off-by: Michal Mielewczyk --- src/engine/cache_engine.c | 57 +++++++++++++++++++------------- src/engine/engine_bf.c | 28 +++------------- src/engine/engine_d2c.c | 62 ++++++++++++++++++++++++++++------- src/engine/engine_d2c.h | 7 +++- src/engine/engine_discard.c | 65 ++++++------------------------------- src/engine/engine_discard.h | 3 +- src/engine/engine_fast.c | 23 ++++--------- src/engine/engine_flush.c | 12 +++---- src/engine/engine_inv.c | 18 +++------- src/engine/engine_pt.c | 16 +++------ src/engine/engine_rd.c | 37 +++++---------------- src/engine/engine_wa.c | 3 +- src/engine/engine_wb.c | 37 +++++---------------- src/engine/engine_wi.c | 35 ++++++-------------- src/engine/engine_wo.c | 27 +++++++-------- src/engine/engine_wt.c | 29 +++++------------ src/engine/engine_zero.c | 26 +++++++-------- src/ocf_cache.c | 4 +++ 18 files changed, 191 insertions(+), 298 deletions(-) diff --git a/src/engine/cache_engine.c b/src/engine/cache_engine.c index 52158fe..6a32db8 100644 --- a/src/engine/cache_engine.c +++ b/src/engine/cache_engine.c @@ -43,6 +43,8 @@ enum ocf_io_if_type { OCF_IO_FLUSH_IF, OCF_IO_DISCARD_IF, OCF_IO_D2C_IF, + OCF_IO_D2C_FLUSH_IF, + OCF_IO_D2C_DISCARD_IF, OCF_IO_PRIV_MAX_IF, }; @@ -96,26 +98,40 @@ static const struct ocf_io_if IO_IFS[OCF_IO_PRIV_MAX_IF] = { }, .name = "Fast", }, - [OCF_IO_DISCARD_IF] = { - .cbs = { - [OCF_READ] = ocf_discard, - [OCF_WRITE] = ocf_discard, - }, - .name = "Discard", - }, - [OCF_IO_D2C_IF] = { - .cbs = { - [OCF_READ] = ocf_io_d2c, - [OCF_WRITE] = ocf_io_d2c, - }, - .name = "Direct to core", - }, [OCF_IO_FLUSH_IF] = { .cbs = { [OCF_READ] = ocf_engine_flush, [OCF_WRITE] = ocf_engine_flush, }, - .name = "Ops engine", + .name = "Flush", + }, + [OCF_IO_DISCARD_IF] = { + .cbs = { + [OCF_READ] = ocf_engine_discard, + [OCF_WRITE] = ocf_engine_discard, + }, + .name = "Discard", + }, + [OCF_IO_D2C_IF] = { + .cbs = { + [OCF_READ] = ocf_d2c_io, + [OCF_WRITE] = ocf_d2c_io, + }, + .name = "Direct to core io", + }, + [OCF_IO_D2C_FLUSH_IF] = { + .cbs = { + [OCF_READ] = ocf_d2c_flush, + [OCF_WRITE] = ocf_d2c_flush, + }, + .name = "Direct to core flush", + }, + [OCF_IO_D2C_DISCARD_IF] = { + .cbs = { + [OCF_READ] = ocf_d2c_discard, + [OCF_WRITE] = ocf_d2c_discard, + }, + .name = "Direct to core discard", }, }; @@ -257,17 +273,12 @@ int ocf_engine_hndl_fast_req(struct ocf_request *req) return ret; } -static void ocf_engine_hndl_2dc_req(struct ocf_request *req) -{ - IO_IFS[OCF_IO_D2C_IF].cbs[req->rw](req); -} - void ocf_engine_hndl_discard_req(struct ocf_request *req) { ocf_req_get(req); if (req->d2c) { - ocf_engine_hndl_2dc_req(req); + IO_IFS[OCF_IO_D2C_DISCARD_IF].cbs[req->rw](req); return; } @@ -279,8 +290,8 @@ void ocf_engine_hndl_flush_req(struct ocf_request *req) ocf_req_get(req); req->engine_handler = (req->d2c) ? - ocf_io_if_type_to_engine_cb(OCF_IO_D2C_IF, req->rw) : - ocf_io_if_type_to_engine_cb(OCF_IO_FLUSH_IF, req->rw); + IO_IFS[OCF_IO_D2C_FLUSH_IF].cbs[req->rw] : + IO_IFS[OCF_IO_FLUSH_IF].cbs[req->rw]; ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC); } diff --git a/src/engine/engine_bf.c b/src/engine/engine_bf.c index 82f9005..2d80d78 100644 --- a/src/engine/engine_bf.c +++ b/src/engine/engine_bf.c @@ -10,10 +10,11 @@ #include "engine_bf.h" #include "engine_inv.h" #include "engine_common.h" +#include "engine_io.h" #include "cache_engine.h" #include "../ocf_request.h" -#include "../utils/utils_io.h" #include "../concurrency/ocf_concurrency.h" +#include "../utils/utils_io.h" #define OCF_ENGINE_DEBUG_IO_NAME "bf" #include "engine_debug.h" @@ -43,19 +44,9 @@ static void _ocf_backfill_complete(struct ocf_request *req, int error) struct ocf_cache *cache = req->cache; if (error) { - req->error = error; ocf_core_stats_cache_error_update(req->core, OCF_WRITE); - } - - if (req->error) inc_fallback_pt_error_counter(req->cache); - - /* Handle callback-caller race to let only one of the two complete the - * request. Also, complete original request only if this is the last - * sub-request to complete - */ - if (env_atomic_dec_return(&req->req_remaining)) - return; + } backfill_queue_dec_unblock(req->cache); @@ -67,7 +58,7 @@ static void _ocf_backfill_complete(struct ocf_request *req, int error) req->data = NULL; } - if (req->error) { + if (error) { ocf_engine_invalidate(req); } else { ocf_req_unlock(ocf_cache_line_concurrency(cache), req); @@ -79,22 +70,13 @@ static void _ocf_backfill_complete(struct ocf_request *req, int error) static int _ocf_backfill_do(struct ocf_request *req) { - unsigned int reqs_to_issue; - req->data = req->cp_data; if (unlikely(req->data == NULL)) { - env_atomic_set(&req->req_remaining, 1); _ocf_backfill_complete(req, -OCF_ERR_NO_MEM); return 0; } - reqs_to_issue = ocf_engine_io_count(req); - - /* There will be #reqs_to_issue completions */ - env_atomic_set(&req->req_remaining, reqs_to_issue); - - ocf_submit_cache_reqs(req->cache, req, OCF_WRITE, 0, req->byte_length, - reqs_to_issue, _ocf_backfill_complete); + ocf_engine_forward_cache_io_req(req, OCF_WRITE, _ocf_backfill_complete); return 0; } diff --git a/src/engine/engine_d2c.c b/src/engine/engine_d2c.c index 02cd21e..a692f9b 100644 --- a/src/engine/engine_d2c.c +++ b/src/engine/engine_d2c.c @@ -7,9 +7,9 @@ #include "../ocf_cache_priv.h" #include "engine_d2c.h" #include "engine_common.h" +#include "engine_io.h" #include "cache_engine.h" #include "../ocf_request.h" -#include "../utils/utils_io.h" #include "../metadata/metadata.h" #define OCF_ENGINE_DEBUG_IO_NAME "d2c" @@ -17,31 +17,72 @@ static void _ocf_d2c_completion(struct ocf_request *req, int error) { - req->error = error; - OCF_DEBUG_RQ(req, "Completion"); - if (req->error) + if (error) ocf_core_stats_core_error_update(req->core, req->rw); /* Complete request */ - req->complete(req, req->error); + req->complete(req, error); /* Release OCF request */ ocf_req_put(req); } -int ocf_io_d2c(struct ocf_request *req) +int ocf_d2c_io(struct ocf_request *req) { - ocf_core_t core = req->core; - OCF_DEBUG_TRACE(req->cache); - /* Get OCF request - increase reference counter */ ocf_req_get(req); - ocf_submit_volume_req(&core->volume, req, _ocf_d2c_completion); + ocf_engine_forward_core_io_req(req, _ocf_d2c_completion); + + ocf_engine_update_block_stats(req); + + ocf_core_stats_pt_block_update(req->core, req->part_id, req->rw, + req->byte_length); + + ocf_core_stats_request_pt_update(req->core, req->part_id, req->rw, + req->info.hit_no, req->core_line_count); + + /* Put OCF request - decrease reference counter */ + ocf_req_put(req); + + return 0; +} + +int ocf_d2c_flush(struct ocf_request *req) +{ + OCF_DEBUG_TRACE(req->cache); + + /* Get OCF request - increase reference counter */ + ocf_req_get(req); + + ocf_engine_forward_core_flush_req(req, _ocf_d2c_completion); + + ocf_engine_update_block_stats(req); + + ocf_core_stats_pt_block_update(req->core, req->part_id, req->rw, + req->byte_length); + + ocf_core_stats_request_pt_update(req->core, req->part_id, req->rw, + req->info.hit_no, req->core_line_count); + + /* Put OCF request - decrease reference counter */ + ocf_req_put(req); + + return 0; +} + +int ocf_d2c_discard(struct ocf_request *req) +{ + OCF_DEBUG_TRACE(req->cache); + + /* Get OCF request - increase reference counter */ + ocf_req_get(req); + + ocf_engine_forward_core_discard_req(req, _ocf_d2c_completion); ocf_engine_update_block_stats(req); @@ -55,5 +96,4 @@ int ocf_io_d2c(struct ocf_request *req) ocf_req_put(req); return 0; - } diff --git a/src/engine/engine_d2c.h b/src/engine/engine_d2c.h index 3bd69ef..f41ba57 100644 --- a/src/engine/engine_d2c.h +++ b/src/engine/engine_d2c.h @@ -1,11 +1,16 @@ /* * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ #ifndef ENGINE_2DC_H_ #define ENGINE_2DC_H_ -int ocf_io_d2c(struct ocf_request *req); +int ocf_d2c_io(struct ocf_request *req); + +int ocf_d2c_flush(struct ocf_request *req); + +int ocf_d2c_discard(struct ocf_request *req); #endif /* ENGINE_2DC_H_ */ diff --git a/src/engine/engine_discard.c b/src/engine/engine_discard.c index b270de9..02372e8 100644 --- a/src/engine/engine_discard.c +++ b/src/engine/engine_discard.c @@ -8,10 +8,10 @@ #include "cache_engine.h" #include "engine_common.h" #include "engine_discard.h" +#include "engine_io.h" #include "../metadata/metadata.h" #include "../ocf_request.h" #include "../utils/utils_io.h" -#include "../utils/utils_cache_line.h" #include "../concurrency/ocf_concurrency.h" #define OCF_ENGINE_DEBUG 0 @@ -25,75 +25,33 @@ static void _ocf_discard_complete_req(struct ocf_request *req, int error) ocf_req_put(req); } -static void _ocf_discard_core_complete(struct ocf_io *io, int error) -{ - struct ocf_request *req = io->priv1; - - OCF_DEBUG_RQ(req, "Core DISCARD Completion"); - - _ocf_discard_complete_req(req, error); - - ocf_io_put(io); -} static int _ocf_discard_core(struct ocf_request *req) { - struct ocf_io *io; - int err; + req->byte_position = SECTORS_TO_BYTES(req->discard.sector); + req->byte_length = SECTORS_TO_BYTES(req->discard.nr_sects); - io = ocf_volume_new_io(&req->core->volume, req->io_queue, - SECTORS_TO_BYTES(req->discard.sector), - SECTORS_TO_BYTES(req->discard.nr_sects), - OCF_WRITE, 0, 0); - if (!io) { - _ocf_discard_complete_req(req, -OCF_ERR_NO_MEM); - return -OCF_ERR_NO_MEM; - } - - ocf_io_set_cmpl(io, req, NULL, _ocf_discard_core_complete); - err = ocf_io_set_data(io, req->data, req->offset); - if (err) { - _ocf_discard_core_complete(io, err); - return err; - } - - ocf_volume_submit_discard(io); + ocf_engine_forward_core_discard_req(req, _ocf_discard_complete_req); return 0; } -static void _ocf_discard_cache_flush_complete(struct ocf_io *io, int error) +static void _ocf_discard_cache_flush_complete(struct ocf_request *req, int error) { - struct ocf_request *req = io->priv1; - if (error) { ocf_metadata_error(req->cache); _ocf_discard_complete_req(req, error); - ocf_io_put(io); return; } req->engine_handler = _ocf_discard_core; ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); - - ocf_io_put(io); } static int _ocf_discard_flush_cache(struct ocf_request *req) { - struct ocf_io *io; - - io = ocf_volume_new_io(&req->cache->device->volume, req->io_queue, - 0, 0, OCF_WRITE, 0, 0); - if (!io) { - ocf_metadata_error(req->cache); - _ocf_discard_complete_req(req, -OCF_ERR_NO_MEM); - return -OCF_ERR_NO_MEM; - } - - ocf_io_set_cmpl(io, req, NULL, _ocf_discard_cache_flush_complete); - - ocf_volume_submit_flush(io); + ocf_engine_forward_cache_flush_req(req, + _ocf_discard_cache_flush_complete); return 0; } @@ -116,9 +74,6 @@ static void _ocf_discard_finish_step(struct ocf_request *req) static void _ocf_discard_step_complete(struct ocf_request *req, int error) { - if (error) - req->error |= error; - if (env_atomic_dec_return(&req->req_remaining)) return; @@ -127,9 +82,9 @@ static void _ocf_discard_step_complete(struct ocf_request *req, int error) /* Release WRITE lock of request */ ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); - if (req->error) { + if (error) { ocf_metadata_error(req->cache); - _ocf_discard_complete_req(req, req->error); + _ocf_discard_complete_req(req, error); return; } @@ -242,7 +197,7 @@ static int _ocf_discard_step(struct ocf_request *req) return 0; } -int ocf_discard(struct ocf_request *req) +int ocf_engine_discard(struct ocf_request *req) { OCF_DEBUG_TRACE(req->cache); diff --git a/src/engine/engine_discard.h b/src/engine/engine_discard.h index d04676b..ec63355 100644 --- a/src/engine/engine_discard.h +++ b/src/engine/engine_discard.h @@ -1,11 +1,12 @@ /* * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ #ifndef __ENGINE_DISCARD_H__ #define __ENGINE_DISCARD_H__ -int ocf_discard(struct ocf_request *req); +int ocf_engine_discard(struct ocf_request *req); #endif diff --git a/src/engine/engine_fast.c b/src/engine/engine_fast.c index 5c2f4aa..b35788e 100644 --- a/src/engine/engine_fast.c +++ b/src/engine/engine_fast.c @@ -8,11 +8,11 @@ #include "../ocf_cache_priv.h" #include "engine_fast.h" #include "engine_common.h" +#include "engine_io.h" #include "engine_pt.h" #include "engine_wb.h" #include "../ocf_request.h" #include "../utils/utils_user_part.h" -#include "../utils/utils_io.h" #include "../concurrency/ocf_concurrency.h" #include "../metadata/metadata.h" @@ -31,19 +31,11 @@ static void _ocf_read_fast_complete(struct ocf_request *req, int error) { - if (error) { - req->error |= error; - ocf_core_stats_cache_error_update(req->core, OCF_READ); - } - - if (env_atomic_dec_return(&req->req_remaining)) { - /* Not all requests finished */ - return; - } - OCF_DEBUG_RQ(req, "HIT completion"); - if (req->error) { + if (error) { + ocf_core_stats_cache_error_update(req->core, OCF_READ); + OCF_DEBUG_RQ(req, "ERROR"); ocf_queue_push_req_pt(req); @@ -51,7 +43,7 @@ static void _ocf_read_fast_complete(struct ocf_request *req, int error) ocf_req_unlock(ocf_cache_line_concurrency(req->cache), req); /* Complete request */ - req->complete(req, req->error); + req->complete(req, error); /* Free the request at the last point of the completion path */ ocf_req_put(req); @@ -86,10 +78,7 @@ static int _ocf_read_fast_do(struct ocf_request *req) /* Submit IO */ OCF_DEBUG_RQ(req, "Submit"); - env_atomic_set(&req->req_remaining, ocf_engine_io_count(req)); - ocf_submit_cache_reqs(req->cache, req, OCF_READ, 0, req->byte_length, - ocf_engine_io_count(req), _ocf_read_fast_complete); - + ocf_engine_forward_cache_io_req(req, OCF_READ, _ocf_read_fast_complete); /* Update statistics */ ocf_engine_update_request_stats(req); diff --git a/src/engine/engine_flush.c b/src/engine/engine_flush.c index af6cf45..c0013e4 100644 --- a/src/engine/engine_flush.c +++ b/src/engine/engine_flush.c @@ -6,10 +6,10 @@ #include "ocf/ocf.h" #include "../ocf_cache_priv.h" #include "engine_common.h" +#include "engine_io.h" #include "cache_engine.h" #include "engine_flush.h" #include "../ocf_request.h" -#include "../utils/utils_io.h" #define OCF_ENGINE_DEBUG_IO_NAME "flush" #include "engine_debug.h" @@ -17,7 +17,7 @@ static void _ocf_engine_flush_complete(struct ocf_request *req, int error) { if (error) - req->error |= error; + req->error = req->error ?: error; if (env_atomic_dec_return(&req->req_remaining)) return; @@ -45,17 +45,13 @@ int ocf_engine_flush(struct ocf_request *req) env_atomic_set(&req->req_remaining, 2); /* Submit operation into core device */ - ocf_submit_volume_req(&req->core->volume, req, - _ocf_engine_flush_complete); - + ocf_engine_forward_core_flush_req(req, _ocf_engine_flush_complete); /* submit flush to cache device */ - ocf_submit_cache_flush(req, _ocf_engine_flush_complete); + ocf_engine_forward_cache_flush_req(req, _ocf_engine_flush_complete); /* Put OCF request - decrease reference counter */ ocf_req_put(req); return 0; } - - diff --git a/src/engine/engine_inv.c b/src/engine/engine_inv.c index d4c5e9e..87d41ab 100644 --- a/src/engine/engine_inv.c +++ b/src/engine/engine_inv.c @@ -19,18 +19,12 @@ static void _ocf_invalidate_req(struct ocf_request *req, int error) { - if (error) { - req->error = error; - ocf_core_stats_cache_error_update(req->core, OCF_WRITE); - } - - if (env_atomic_dec_return(&req->req_remaining)) - return; - OCF_DEBUG_RQ(req, "Completion"); - if (req->error) + if (error) { + ocf_core_stats_cache_error_update(req->core, OCF_WRITE); ocf_engine_error(req, true, "Failed to flush metadata to cache"); + } ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); @@ -48,16 +42,14 @@ static int _ocf_invalidate_do(struct ocf_request *req) ocf_purge_map_info(req); ocf_hb_req_prot_unlock_wr(req); - env_atomic_inc(&req->req_remaining); - if (ocf_volume_is_atomic(&cache->device->volume) && req->info.flush_metadata) { /* Metadata flush IO */ ocf_metadata_flush_do_asynch(cache, req, _ocf_invalidate_req); + } else { + _ocf_invalidate_req(req, 0); } - _ocf_invalidate_req(req, 0); - return 0; } diff --git a/src/engine/engine_pt.c b/src/engine/engine_pt.c index eec4afc..f570e79 100644 --- a/src/engine/engine_pt.c +++ b/src/engine/engine_pt.c @@ -8,9 +8,9 @@ #include "engine_pt.h" #include "engine_rd.h" #include "engine_common.h" +#include "engine_io.h" #include "cache_engine.h" #include "../ocf_request.h" -#include "../utils/utils_io.h" #include "../utils/utils_user_part.h" #include "../metadata/metadata.h" #include "../concurrency/ocf_concurrency.h" @@ -20,19 +20,13 @@ static void _ocf_read_pt_complete(struct ocf_request *req, int error) { - if (error) - req->error |= error; - - if (env_atomic_dec_return(&req->req_remaining)) - return; - OCF_DEBUG_RQ(req, "Completion"); - if (req->error) + if (error) ocf_core_stats_core_error_update(req->core, OCF_READ); /* Complete request */ - req->complete(req, req->error); + req->complete(req, error); ocf_req_unlock(ocf_cache_line_concurrency(req->cache), req); @@ -42,12 +36,10 @@ static void _ocf_read_pt_complete(struct ocf_request *req, int error) static inline void _ocf_read_pt_submit(struct ocf_request *req) { - env_atomic_set(&req->req_remaining, 1); /* Core device IO */ - OCF_DEBUG_RQ(req, "Submit"); /* Core read */ - ocf_submit_volume_req(&req->core->volume, req, _ocf_read_pt_complete); + ocf_engine_forward_core_io_req(req, _ocf_read_pt_complete); } int ocf_read_pt_do(struct ocf_request *req) diff --git a/src/engine/engine_rd.c b/src/engine/engine_rd.c index 8b3ff73..6c6cd92 100644 --- a/src/engine/engine_rd.c +++ b/src/engine/engine_rd.c @@ -11,9 +11,9 @@ #include "engine_inv.h" #include "engine_bf.h" #include "engine_common.h" +#include "engine_io.h" #include "cache_engine.h" #include "../concurrency/ocf_concurrency.h" -#include "../utils/utils_io.h" #include "../ocf_request.h" #include "../utils/utils_cache_line.h" #include "../utils/utils_user_part.h" @@ -31,19 +31,12 @@ static void _ocf_read_generic_hit_complete(struct ocf_request *req, int error) OCF_DEBUG_RQ(req, "HIT completion"); if (error) { - req->error |= error; ocf_core_stats_cache_error_update(req->core, OCF_READ); inc_fallback_pt_error_counter(req->cache); - } - - if (env_atomic_dec_return(&req->req_remaining) > 0) - return; - - if (req->error) { ocf_queue_push_req_pt(req); } else { ocf_req_unlock(c, req); - req->complete(req, req->error); + req->complete(req, error); ocf_req_put(req); } } @@ -54,14 +47,9 @@ static void _ocf_read_generic_miss_complete(struct ocf_request *req, int error) OCF_DEBUG_RQ(req, "MISS completion"); - if (error) - req->error = error; - - if (env_atomic_dec_return(&req->req_remaining) > 0) - return; - - if (req->error) { - req->complete(req, req->error); + if (error) { + /* --- Do not backfill --- */ + req->complete(req, error); ocf_core_stats_core_error_update(req->core, OCF_READ); @@ -80,18 +68,15 @@ static void _ocf_read_generic_miss_complete(struct ocf_request *req, int error) req->byte_length); } - /* Complete request */ - req->complete(req, req->error); + req->complete(req, error); ocf_engine_backfill(req); } void ocf_read_generic_submit_hit(struct ocf_request *req) { - env_atomic_set(&req->req_remaining, ocf_engine_io_count(req)); - - ocf_submit_cache_reqs(req->cache, req, OCF_READ, 0, req->byte_length, - ocf_engine_io_count(req), _ocf_read_generic_hit_complete); + ocf_engine_forward_cache_io_req(req, OCF_READ, + _ocf_read_generic_hit_complete); } static inline void _ocf_read_generic_submit_miss(struct ocf_request *req) @@ -99,8 +84,6 @@ static inline void _ocf_read_generic_submit_miss(struct ocf_request *req) struct ocf_cache *cache = req->cache; int ret; - env_atomic_set(&req->req_remaining, 1); - req->cp_data = ctx_data_alloc(cache->owner, BYTES_TO_PAGES(req->byte_length)); if (!req->cp_data) { @@ -119,9 +102,7 @@ static inline void _ocf_read_generic_submit_miss(struct ocf_request *req) } err_alloc: - /* Submit read request to core device. */ - ocf_submit_volume_req(&req->core->volume, req, - _ocf_read_generic_miss_complete); + ocf_engine_forward_core_io_req(req, _ocf_read_generic_miss_complete); } static int _ocf_read_generic_do(struct ocf_request *req) diff --git a/src/engine/engine_wa.c b/src/engine/engine_wa.c index cb72d0e..c781318 100644 --- a/src/engine/engine_wa.c +++ b/src/engine/engine_wa.c @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2022 Intel Corporation + * Copyright(c) 2024 Huawei Technologies * SPDX-License-Identifier: BSD-3-Clause */ #include "ocf/ocf.h" @@ -8,9 +9,9 @@ #include "engine_wt.h" #include "engine_wi.h" #include "engine_common.h" +#include "engine_io.h" #include "cache_engine.h" #include "../ocf_request.h" -#include "../utils/utils_io.h" #include "../metadata/metadata.h" #define OCF_ENGINE_DEBUG_IO_NAME "wa" diff --git a/src/engine/engine_wb.c b/src/engine/engine_wb.c index d419d50..1860de4 100644 --- a/src/engine/engine_wb.c +++ b/src/engine/engine_wb.c @@ -8,12 +8,12 @@ #include "../ocf_cache_priv.h" #include "cache_engine.h" #include "engine_common.h" +#include "engine_io.h" #include "engine_wb.h" #include "engine_wi.h" #include "engine_inv.h" #include "../metadata/metadata.h" #include "../ocf_request.h" -#include "../utils/utils_io.h" #include "../utils/utils_cache_line.h" #include "../utils/utils_request.h" #include "../utils/utils_user_part.h" @@ -50,17 +50,11 @@ static void _ocf_write_wb_update_bits(struct ocf_request *req) static void _ocf_write_wb_io_flush_metadata(struct ocf_request *req, int error) { if (error) - req->error = error; - - if (env_atomic_dec_return(&req->req_remaining)) - return; - - if (req->error) ocf_engine_error(req, true, "Failed to write data to cache"); ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); - req->complete(req, req->error); + req->complete(req, error); ocf_req_put(req); } @@ -69,37 +63,29 @@ static int ocf_write_wb_do_flush_metadata(struct ocf_request *req) { struct ocf_cache *cache = req->cache; - env_atomic_set(&req->req_remaining, 1); /* One core IO */ - _ocf_write_wb_update_bits(req); if (req->info.flush_metadata) { OCF_DEBUG_RQ(req, "Flush metadata"); ocf_metadata_flush_do_asynch(cache, req, _ocf_write_wb_io_flush_metadata); + } else { + _ocf_write_wb_io_flush_metadata(req, 0); } - _ocf_write_wb_io_flush_metadata(req, 0); - return 0; } static void _ocf_write_wb_complete(struct ocf_request *req, int error) { - if (error) { - ocf_core_stats_cache_error_update(req->core, OCF_WRITE); - req->error |= error; - } - - if (env_atomic_dec_return(&req->req_remaining)) - return; - OCF_DEBUG_RQ(req, "Completion"); - if (req->error) { + if (error) { + ocf_core_stats_cache_error_update(req->core, OCF_WRITE); + ocf_engine_error(req, true, "Failed to write data to cache"); - req->complete(req, req->error); + req->complete(req, error); ocf_engine_invalidate(req); } else { @@ -111,10 +97,6 @@ static void _ocf_write_wb_complete(struct ocf_request *req, int error) static inline void _ocf_write_wb_submit(struct ocf_request *req) { - struct ocf_cache *cache = req->cache; - - env_atomic_set(&req->req_remaining, ocf_engine_io_count(req)); - /* * 1. Submit data * 2. Wait for completion of data @@ -137,8 +119,7 @@ static inline void _ocf_write_wb_submit(struct ocf_request *req) OCF_DEBUG_RQ(req, "Submit Data"); /* Data IO */ - ocf_submit_cache_reqs(cache, req, OCF_WRITE, 0, req->byte_length, - ocf_engine_io_count(req), _ocf_write_wb_complete); + ocf_engine_forward_cache_io_req(req, OCF_WRITE, _ocf_write_wb_complete); } int ocf_write_wb_do(struct ocf_request *req) diff --git a/src/engine/engine_wi.c b/src/engine/engine_wi.c index 9a6d6f1..5257af1 100644 --- a/src/engine/engine_wi.c +++ b/src/engine/engine_wi.c @@ -8,10 +8,10 @@ #include "../ocf_cache_priv.h" #include "engine_wi.h" #include "engine_common.h" +#include "engine_io.h" #include "../concurrency/ocf_concurrency.h" #include "../ocf_request.h" #include "../utils/utils_cache_line.h" -#include "../utils/utils_io.h" #include "../metadata/metadata.h" #define OCF_ENGINE_DEBUG_IO_NAME "wi" @@ -22,7 +22,7 @@ static int _ocf_write_wi_next_pass(struct ocf_request *req) ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); if (req->wi_second_pass) { - req->complete(req, req->error); + req->complete(req, 0); ocf_req_put(req); return 0; @@ -48,25 +48,19 @@ static void _ocf_write_wi_io_flush_metadata(struct ocf_request *req, int error) { if (error) { ocf_core_stats_cache_error_update(req->core, OCF_WRITE); - req->error |= error; + ocf_engine_error(req, true, "Failed to write data to cache"); } - if (env_atomic_dec_return(&req->req_remaining)) - return; - - if (!req->error && !req->wi_second_pass && ocf_engine_is_miss(req)) { + if (!error && !req->wi_second_pass && ocf_engine_is_miss(req)) { /* need another pass */ ocf_queue_push_req_cb(req, _ocf_write_wi_next_pass, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); return; } - if (req->error) - ocf_engine_error(req, true, "Failed to write data to cache"); - ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); - req->complete(req, req->error); + req->complete(req, error); ocf_req_put(req); } @@ -105,20 +99,14 @@ static int ocf_write_wi_update_and_flush_metadata(struct ocf_request *req) static void _ocf_write_wi_core_complete(struct ocf_request *req, int error) { - if (error) { - req->error = error; - ocf_core_stats_core_error_update(req->core, OCF_WRITE); - } - - if (env_atomic_dec_return(&req->req_remaining)) - return; - OCF_DEBUG_RQ(req, "Completion"); - if (req->error) { + if (error) { + ocf_core_stats_core_error_update(req->core, OCF_WRITE); + ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); - req->complete(req, req->error); + req->complete(req, error); ocf_req_put(req); } else { @@ -146,13 +134,10 @@ static int _ocf_write_wi_core_write(struct ocf_request *req) return 0; } - env_atomic_set(&req->req_remaining, 1); /* One core IO */ - OCF_DEBUG_RQ(req, "Submit"); /* Submit write IO to the core */ - ocf_submit_volume_req(&req->core->volume, req, - _ocf_write_wi_core_complete); + ocf_engine_forward_core_io_req(req, _ocf_write_wi_core_complete); /* Update statistics */ ocf_engine_update_block_stats(req); diff --git a/src/engine/engine_wo.c b/src/engine/engine_wo.c index ea2c4ed..11bce5b 100644 --- a/src/engine/engine_wo.c +++ b/src/engine/engine_wo.c @@ -9,10 +9,10 @@ #include "../ocf_cache_priv.h" #include "cache_engine.h" #include "engine_common.h" +#include "engine_io.h" #include "engine_rd.h" #include "engine_pt.h" #include "../metadata/metadata.h" -#include "../utils/utils_io.h" #include "../utils/utils_cache_line.h" #include "../utils/utils_user_part.h" #include "../concurrency/ocf_concurrency.h" @@ -23,13 +23,10 @@ static void ocf_read_wo_cache_complete(struct ocf_request *req, int error) { if (error) { + req->error = req->error ?: error; ocf_core_stats_cache_error_update(req->core, OCF_READ); - req->error |= error; } - if (env_atomic_dec_return(&req->req_remaining)) - return; - OCF_DEBUG_RQ(req, "Completion"); if (req->error) @@ -48,8 +45,7 @@ static void ocf_read_wo_cache_io(struct ocf_request *req, uint64_t offset, uint64_t size) { OCF_DEBUG_RQ(req, "Submit cache"); - env_atomic_inc(&req->req_remaining); - ocf_submit_cache_reqs(req->cache, req, OCF_READ, offset, size, 1, + ocf_engine_forward_cache_io(req, OCF_READ, offset, size, ocf_read_wo_cache_complete); } @@ -66,7 +62,9 @@ static int ocf_read_wo_cache_do(struct ocf_request *req) uint64_t offset = 0; uint64_t increment = 0; - env_atomic_set(&req->req_remaining, 1); + req->cache_forward_end = ocf_read_wo_cache_complete; + + ocf_req_forward_cache_get(req); for (line = 0; line < req->core_line_count; ++line) { entry = &req->map[line]; @@ -147,23 +145,21 @@ static int ocf_read_wo_cache_do(struct ocf_request *req) if (io) ocf_read_wo_cache_io(req, io_start, offset - io_start); - ocf_read_wo_cache_complete(req, 0); + ocf_req_forward_cache_put(req); return 0; } static void _ocf_read_wo_core_complete(struct ocf_request *req, int error) { - if (error) { - req->error |= error; + if (error) ocf_core_stats_core_error_update(req->core, OCF_READ); - } /* if all mapped cachelines are clean, the data we've read from core * is valid and we can complete the request */ - if (!req->info.dirty_any || req->error) { + if (error || !req->info.dirty_any) { OCF_DEBUG_RQ(req, "Completion"); - req->complete(req, req->error); + req->complete(req, error); ocf_req_unlock_rd(ocf_cache_line_concurrency(req->cache), req); ocf_req_put(req); return; @@ -191,8 +187,7 @@ static int ocf_read_wo_do(struct ocf_request *req) } else { OCF_DEBUG_RQ(req, "Submit core"); - ocf_submit_volume_req(&req->core->volume, req, - _ocf_read_wo_core_complete); + ocf_engine_forward_core_io_req(req, _ocf_read_wo_core_complete); } ocf_engine_update_request_stats(req); diff --git a/src/engine/engine_wt.c b/src/engine/engine_wt.c index b4a8d7f..303dc07 100644 --- a/src/engine/engine_wt.c +++ b/src/engine/engine_wt.c @@ -10,8 +10,8 @@ #include "engine_wi.h" #include "engine_inv.h" #include "engine_common.h" +#include "engine_io.h" #include "../ocf_request.h" -#include "../utils/utils_io.h" #include "../utils/utils_cache_line.h" #include "../utils/utils_user_part.h" #include "../metadata/metadata.h" @@ -58,12 +58,6 @@ static void _ocf_write_wt_do_flush_metadata_compl(struct ocf_request *req, int error) { if (error) - req->error = error; - - if (env_atomic_dec_return(&req->req_remaining)) - return; - - if (req->error) ocf_engine_error(req, true, "Failed to write data to cache"); ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); @@ -77,8 +71,6 @@ static int ocf_write_wt_do_flush_metadata(struct ocf_request *req) { struct ocf_cache *cache = req->cache; - env_atomic_set(&req->req_remaining, 1); - _ocf_write_wt_update_bits(req); if (req->info.flush_metadata) { @@ -86,10 +78,10 @@ static int ocf_write_wt_do_flush_metadata(struct ocf_request *req) ocf_metadata_flush_do_asynch(cache, req, _ocf_write_wt_do_flush_metadata_compl); + } else { + _ocf_write_wt_do_flush_metadata_compl(req, 0); } - _ocf_write_wt_do_flush_metadata_compl(req, 0); - return 0; } @@ -136,8 +128,8 @@ static void _ocf_write_wt_cache_complete(struct ocf_request *req, int error) static void _ocf_write_wt_core_complete(struct ocf_request *req, int error) { if (error) { - req->error = error; req->info.core_error = 1; + req->error = error; ocf_core_stats_core_error_update(req->core, OCF_WRITE); } @@ -146,22 +138,17 @@ static void _ocf_write_wt_core_complete(struct ocf_request *req, int error) static inline void _ocf_write_wt_submit(struct ocf_request *req) { - struct ocf_cache *cache = req->cache; - /* Submit IOs */ OCF_DEBUG_RQ(req, "Submit"); - /* Calculate how many IOs need to be submited */ - env_atomic_set(&req->req_remaining, ocf_engine_io_count(req)); /* Cache IO */ - env_atomic_inc(&req->req_remaining); /* Core device IO */ + env_atomic_set(&req->req_remaining, 2); /* cache IO + core IO */ /* To cache */ - ocf_submit_cache_reqs(cache, req, OCF_WRITE, 0, req->byte_length, - ocf_engine_io_count(req), _ocf_write_wt_cache_complete); + ocf_engine_forward_cache_io_req(req, OCF_WRITE, + _ocf_write_wt_cache_complete); /* To core */ - ocf_submit_volume_req(&req->core->volume, req, - _ocf_write_wt_core_complete); + ocf_engine_forward_core_io_req(req, _ocf_write_wt_core_complete); } static int _ocf_write_wt_do(struct ocf_request *req) diff --git a/src/engine/engine_zero.c b/src/engine/engine_zero.c index cdd9b85..ec98f45 100644 --- a/src/engine/engine_zero.c +++ b/src/engine/engine_zero.c @@ -19,23 +19,17 @@ static int ocf_zero_purge(struct ocf_request *req) { - if (req->error) { - ocf_engine_error(req, true, "Failed to discard data on cache"); - } else { - /* There are mapped cache line, need to remove them */ + /* There are mapped cache line, need to remove them */ - ocf_hb_req_prot_lock_wr(req); /*- Metadata WR access ---------------*/ + ocf_hb_req_prot_lock_wr(req); /*- Metadata WR access ---------------*/ - /* Remove mapped cache lines from metadata */ - ocf_purge_map_info(req); + /* Remove mapped cache lines from metadata */ + ocf_purge_map_info(req); - ocf_hb_req_prot_unlock_wr(req); /*- END Metadata WR access ---------*/ - } + ocf_hb_req_prot_unlock_wr(req); /*- END Metadata WR access ---------*/ ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); - - req->complete(req, req->error); - + req->complete(req, 0); ocf_req_put(req); return 0; @@ -45,11 +39,13 @@ static void _ocf_zero_io_flush_metadata(struct ocf_request *req, int error) { if (error) { ocf_core_stats_cache_error_update(req->core, OCF_WRITE); - req->error = error; - } + ocf_engine_error(req, true, "Failed to discard data on cache"); - if (env_atomic_dec_return(&req->req_remaining)) + ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); + req->complete(req, error); + ocf_req_put(req); return; + } ocf_queue_push_req_cb(req, ocf_zero_purge, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); diff --git a/src/ocf_cache.c b/src/ocf_cache.c index 00c6cee..9e4d7be 100644 --- a/src/ocf_cache.c +++ b/src/ocf_cache.c @@ -320,9 +320,13 @@ static void ocf_cache_io_complete(struct ocf_io *io, int error) { struct ocf_cache_volume_io_priv *priv; ocf_cache_t cache; + struct ocf_request *req = ocf_io_to_req(io); cache = ocf_volume_to_cache(ocf_io_get_volume(io)); + if (error) + req->error = req->error ?: error; + priv = ocf_io_get_priv(io); env_atomic_cmpxchg(&priv->error, 0, error);