Modify engines to use forward API

Signed-off-by: Robert Baldyga <robert.baldyga@huawei.com>
Signed-off-by: Rafal Stefanowski <rafal.stefanowski@huawei.com>
Signed-off-by: Michal Mielewczyk <michal.mielewczyk@huawei.com>
This commit is contained in:
Robert Baldyga 2023-09-03 20:27:27 +02:00 committed by Michal Mielewczyk
parent e667c10b4a
commit 1ed707361f
18 changed files with 191 additions and 298 deletions

View File

@ -43,6 +43,8 @@ enum ocf_io_if_type {
OCF_IO_FLUSH_IF, OCF_IO_FLUSH_IF,
OCF_IO_DISCARD_IF, OCF_IO_DISCARD_IF,
OCF_IO_D2C_IF, OCF_IO_D2C_IF,
OCF_IO_D2C_FLUSH_IF,
OCF_IO_D2C_DISCARD_IF,
OCF_IO_PRIV_MAX_IF, OCF_IO_PRIV_MAX_IF,
}; };
@ -96,26 +98,40 @@ static const struct ocf_io_if IO_IFS[OCF_IO_PRIV_MAX_IF] = {
}, },
.name = "Fast", .name = "Fast",
}, },
[OCF_IO_DISCARD_IF] = {
.cbs = {
[OCF_READ] = ocf_discard,
[OCF_WRITE] = ocf_discard,
},
.name = "Discard",
},
[OCF_IO_D2C_IF] = {
.cbs = {
[OCF_READ] = ocf_io_d2c,
[OCF_WRITE] = ocf_io_d2c,
},
.name = "Direct to core",
},
[OCF_IO_FLUSH_IF] = { [OCF_IO_FLUSH_IF] = {
.cbs = { .cbs = {
[OCF_READ] = ocf_engine_flush, [OCF_READ] = ocf_engine_flush,
[OCF_WRITE] = ocf_engine_flush, [OCF_WRITE] = ocf_engine_flush,
}, },
.name = "Ops engine", .name = "Flush",
},
[OCF_IO_DISCARD_IF] = {
.cbs = {
[OCF_READ] = ocf_engine_discard,
[OCF_WRITE] = ocf_engine_discard,
},
.name = "Discard",
},
[OCF_IO_D2C_IF] = {
.cbs = {
[OCF_READ] = ocf_d2c_io,
[OCF_WRITE] = ocf_d2c_io,
},
.name = "Direct to core io",
},
[OCF_IO_D2C_FLUSH_IF] = {
.cbs = {
[OCF_READ] = ocf_d2c_flush,
[OCF_WRITE] = ocf_d2c_flush,
},
.name = "Direct to core flush",
},
[OCF_IO_D2C_DISCARD_IF] = {
.cbs = {
[OCF_READ] = ocf_d2c_discard,
[OCF_WRITE] = ocf_d2c_discard,
},
.name = "Direct to core discard",
}, },
}; };
@ -257,17 +273,12 @@ int ocf_engine_hndl_fast_req(struct ocf_request *req)
return ret; return ret;
} }
static void ocf_engine_hndl_2dc_req(struct ocf_request *req)
{
IO_IFS[OCF_IO_D2C_IF].cbs[req->rw](req);
}
void ocf_engine_hndl_discard_req(struct ocf_request *req) void ocf_engine_hndl_discard_req(struct ocf_request *req)
{ {
ocf_req_get(req); ocf_req_get(req);
if (req->d2c) { if (req->d2c) {
ocf_engine_hndl_2dc_req(req); IO_IFS[OCF_IO_D2C_DISCARD_IF].cbs[req->rw](req);
return; return;
} }
@ -279,8 +290,8 @@ void ocf_engine_hndl_flush_req(struct ocf_request *req)
ocf_req_get(req); ocf_req_get(req);
req->engine_handler = (req->d2c) ? req->engine_handler = (req->d2c) ?
ocf_io_if_type_to_engine_cb(OCF_IO_D2C_IF, req->rw) : IO_IFS[OCF_IO_D2C_FLUSH_IF].cbs[req->rw] :
ocf_io_if_type_to_engine_cb(OCF_IO_FLUSH_IF, req->rw); IO_IFS[OCF_IO_FLUSH_IF].cbs[req->rw];
ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC); ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC);
} }

View File

@ -10,10 +10,11 @@
#include "engine_bf.h" #include "engine_bf.h"
#include "engine_inv.h" #include "engine_inv.h"
#include "engine_common.h" #include "engine_common.h"
#include "engine_io.h"
#include "cache_engine.h" #include "cache_engine.h"
#include "../ocf_request.h" #include "../ocf_request.h"
#include "../utils/utils_io.h"
#include "../concurrency/ocf_concurrency.h" #include "../concurrency/ocf_concurrency.h"
#include "../utils/utils_io.h"
#define OCF_ENGINE_DEBUG_IO_NAME "bf" #define OCF_ENGINE_DEBUG_IO_NAME "bf"
#include "engine_debug.h" #include "engine_debug.h"
@ -43,19 +44,9 @@ static void _ocf_backfill_complete(struct ocf_request *req, int error)
struct ocf_cache *cache = req->cache; struct ocf_cache *cache = req->cache;
if (error) { if (error) {
req->error = error;
ocf_core_stats_cache_error_update(req->core, OCF_WRITE); ocf_core_stats_cache_error_update(req->core, OCF_WRITE);
}
if (req->error)
inc_fallback_pt_error_counter(req->cache); inc_fallback_pt_error_counter(req->cache);
}
/* Handle callback-caller race to let only one of the two complete the
* request. Also, complete original request only if this is the last
* sub-request to complete
*/
if (env_atomic_dec_return(&req->req_remaining))
return;
backfill_queue_dec_unblock(req->cache); backfill_queue_dec_unblock(req->cache);
@ -67,7 +58,7 @@ static void _ocf_backfill_complete(struct ocf_request *req, int error)
req->data = NULL; req->data = NULL;
} }
if (req->error) { if (error) {
ocf_engine_invalidate(req); ocf_engine_invalidate(req);
} else { } else {
ocf_req_unlock(ocf_cache_line_concurrency(cache), req); ocf_req_unlock(ocf_cache_line_concurrency(cache), req);
@ -79,22 +70,13 @@ static void _ocf_backfill_complete(struct ocf_request *req, int error)
static int _ocf_backfill_do(struct ocf_request *req) static int _ocf_backfill_do(struct ocf_request *req)
{ {
unsigned int reqs_to_issue;
req->data = req->cp_data; req->data = req->cp_data;
if (unlikely(req->data == NULL)) { if (unlikely(req->data == NULL)) {
env_atomic_set(&req->req_remaining, 1);
_ocf_backfill_complete(req, -OCF_ERR_NO_MEM); _ocf_backfill_complete(req, -OCF_ERR_NO_MEM);
return 0; return 0;
} }
reqs_to_issue = ocf_engine_io_count(req); ocf_engine_forward_cache_io_req(req, OCF_WRITE, _ocf_backfill_complete);
/* There will be #reqs_to_issue completions */
env_atomic_set(&req->req_remaining, reqs_to_issue);
ocf_submit_cache_reqs(req->cache, req, OCF_WRITE, 0, req->byte_length,
reqs_to_issue, _ocf_backfill_complete);
return 0; return 0;
} }

View File

@ -7,9 +7,9 @@
#include "../ocf_cache_priv.h" #include "../ocf_cache_priv.h"
#include "engine_d2c.h" #include "engine_d2c.h"
#include "engine_common.h" #include "engine_common.h"
#include "engine_io.h"
#include "cache_engine.h" #include "cache_engine.h"
#include "../ocf_request.h" #include "../ocf_request.h"
#include "../utils/utils_io.h"
#include "../metadata/metadata.h" #include "../metadata/metadata.h"
#define OCF_ENGINE_DEBUG_IO_NAME "d2c" #define OCF_ENGINE_DEBUG_IO_NAME "d2c"
@ -17,31 +17,72 @@
static void _ocf_d2c_completion(struct ocf_request *req, int error) static void _ocf_d2c_completion(struct ocf_request *req, int error)
{ {
req->error = error;
OCF_DEBUG_RQ(req, "Completion"); OCF_DEBUG_RQ(req, "Completion");
if (req->error) if (error)
ocf_core_stats_core_error_update(req->core, req->rw); ocf_core_stats_core_error_update(req->core, req->rw);
/* Complete request */ /* Complete request */
req->complete(req, req->error); req->complete(req, error);
/* Release OCF request */ /* Release OCF request */
ocf_req_put(req); ocf_req_put(req);
} }
int ocf_io_d2c(struct ocf_request *req) int ocf_d2c_io(struct ocf_request *req)
{ {
ocf_core_t core = req->core;
OCF_DEBUG_TRACE(req->cache); OCF_DEBUG_TRACE(req->cache);
/* Get OCF request - increase reference counter */ /* Get OCF request - increase reference counter */
ocf_req_get(req); ocf_req_get(req);
ocf_submit_volume_req(&core->volume, req, _ocf_d2c_completion); ocf_engine_forward_core_io_req(req, _ocf_d2c_completion);
ocf_engine_update_block_stats(req);
ocf_core_stats_pt_block_update(req->core, req->part_id, req->rw,
req->byte_length);
ocf_core_stats_request_pt_update(req->core, req->part_id, req->rw,
req->info.hit_no, req->core_line_count);
/* Put OCF request - decrease reference counter */
ocf_req_put(req);
return 0;
}
int ocf_d2c_flush(struct ocf_request *req)
{
OCF_DEBUG_TRACE(req->cache);
/* Get OCF request - increase reference counter */
ocf_req_get(req);
ocf_engine_forward_core_flush_req(req, _ocf_d2c_completion);
ocf_engine_update_block_stats(req);
ocf_core_stats_pt_block_update(req->core, req->part_id, req->rw,
req->byte_length);
ocf_core_stats_request_pt_update(req->core, req->part_id, req->rw,
req->info.hit_no, req->core_line_count);
/* Put OCF request - decrease reference counter */
ocf_req_put(req);
return 0;
}
int ocf_d2c_discard(struct ocf_request *req)
{
OCF_DEBUG_TRACE(req->cache);
/* Get OCF request - increase reference counter */
ocf_req_get(req);
ocf_engine_forward_core_discard_req(req, _ocf_d2c_completion);
ocf_engine_update_block_stats(req); ocf_engine_update_block_stats(req);
@ -55,5 +96,4 @@ int ocf_io_d2c(struct ocf_request *req)
ocf_req_put(req); ocf_req_put(req);
return 0; return 0;
} }

View File

@ -1,11 +1,16 @@
/* /*
* Copyright(c) 2012-2021 Intel Corporation * Copyright(c) 2012-2021 Intel Corporation
* Copyright(c) 2024 Huawei Technologies
* SPDX-License-Identifier: BSD-3-Clause * SPDX-License-Identifier: BSD-3-Clause
*/ */
#ifndef ENGINE_2DC_H_ #ifndef ENGINE_2DC_H_
#define ENGINE_2DC_H_ #define ENGINE_2DC_H_
int ocf_io_d2c(struct ocf_request *req); int ocf_d2c_io(struct ocf_request *req);
int ocf_d2c_flush(struct ocf_request *req);
int ocf_d2c_discard(struct ocf_request *req);
#endif /* ENGINE_2DC_H_ */ #endif /* ENGINE_2DC_H_ */

View File

@ -8,10 +8,10 @@
#include "cache_engine.h" #include "cache_engine.h"
#include "engine_common.h" #include "engine_common.h"
#include "engine_discard.h" #include "engine_discard.h"
#include "engine_io.h"
#include "../metadata/metadata.h" #include "../metadata/metadata.h"
#include "../ocf_request.h" #include "../ocf_request.h"
#include "../utils/utils_io.h" #include "../utils/utils_io.h"
#include "../utils/utils_cache_line.h"
#include "../concurrency/ocf_concurrency.h" #include "../concurrency/ocf_concurrency.h"
#define OCF_ENGINE_DEBUG 0 #define OCF_ENGINE_DEBUG 0
@ -25,75 +25,33 @@ static void _ocf_discard_complete_req(struct ocf_request *req, int error)
ocf_req_put(req); ocf_req_put(req);
} }
static void _ocf_discard_core_complete(struct ocf_io *io, int error)
{
struct ocf_request *req = io->priv1;
OCF_DEBUG_RQ(req, "Core DISCARD Completion");
_ocf_discard_complete_req(req, error);
ocf_io_put(io);
}
static int _ocf_discard_core(struct ocf_request *req) static int _ocf_discard_core(struct ocf_request *req)
{ {
struct ocf_io *io; req->byte_position = SECTORS_TO_BYTES(req->discard.sector);
int err; req->byte_length = SECTORS_TO_BYTES(req->discard.nr_sects);
io = ocf_volume_new_io(&req->core->volume, req->io_queue, ocf_engine_forward_core_discard_req(req, _ocf_discard_complete_req);
SECTORS_TO_BYTES(req->discard.sector),
SECTORS_TO_BYTES(req->discard.nr_sects),
OCF_WRITE, 0, 0);
if (!io) {
_ocf_discard_complete_req(req, -OCF_ERR_NO_MEM);
return -OCF_ERR_NO_MEM;
}
ocf_io_set_cmpl(io, req, NULL, _ocf_discard_core_complete);
err = ocf_io_set_data(io, req->data, req->offset);
if (err) {
_ocf_discard_core_complete(io, err);
return err;
}
ocf_volume_submit_discard(io);
return 0; return 0;
} }
static void _ocf_discard_cache_flush_complete(struct ocf_io *io, int error) static void _ocf_discard_cache_flush_complete(struct ocf_request *req, int error)
{ {
struct ocf_request *req = io->priv1;
if (error) { if (error) {
ocf_metadata_error(req->cache); ocf_metadata_error(req->cache);
_ocf_discard_complete_req(req, error); _ocf_discard_complete_req(req, error);
ocf_io_put(io);
return; return;
} }
req->engine_handler = _ocf_discard_core; req->engine_handler = _ocf_discard_core;
ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH);
ocf_io_put(io);
} }
static int _ocf_discard_flush_cache(struct ocf_request *req) static int _ocf_discard_flush_cache(struct ocf_request *req)
{ {
struct ocf_io *io; ocf_engine_forward_cache_flush_req(req,
_ocf_discard_cache_flush_complete);
io = ocf_volume_new_io(&req->cache->device->volume, req->io_queue,
0, 0, OCF_WRITE, 0, 0);
if (!io) {
ocf_metadata_error(req->cache);
_ocf_discard_complete_req(req, -OCF_ERR_NO_MEM);
return -OCF_ERR_NO_MEM;
}
ocf_io_set_cmpl(io, req, NULL, _ocf_discard_cache_flush_complete);
ocf_volume_submit_flush(io);
return 0; return 0;
} }
@ -116,9 +74,6 @@ static void _ocf_discard_finish_step(struct ocf_request *req)
static void _ocf_discard_step_complete(struct ocf_request *req, int error) static void _ocf_discard_step_complete(struct ocf_request *req, int error)
{ {
if (error)
req->error |= error;
if (env_atomic_dec_return(&req->req_remaining)) if (env_atomic_dec_return(&req->req_remaining))
return; return;
@ -127,9 +82,9 @@ static void _ocf_discard_step_complete(struct ocf_request *req, int error)
/* Release WRITE lock of request */ /* Release WRITE lock of request */
ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req);
if (req->error) { if (error) {
ocf_metadata_error(req->cache); ocf_metadata_error(req->cache);
_ocf_discard_complete_req(req, req->error); _ocf_discard_complete_req(req, error);
return; return;
} }
@ -242,7 +197,7 @@ static int _ocf_discard_step(struct ocf_request *req)
return 0; return 0;
} }
int ocf_discard(struct ocf_request *req) int ocf_engine_discard(struct ocf_request *req)
{ {
OCF_DEBUG_TRACE(req->cache); OCF_DEBUG_TRACE(req->cache);

View File

@ -1,11 +1,12 @@
/* /*
* Copyright(c) 2012-2021 Intel Corporation * Copyright(c) 2012-2021 Intel Corporation
* Copyright(c) 2024 Huawei Technologies
* SPDX-License-Identifier: BSD-3-Clause * SPDX-License-Identifier: BSD-3-Clause
*/ */
#ifndef __ENGINE_DISCARD_H__ #ifndef __ENGINE_DISCARD_H__
#define __ENGINE_DISCARD_H__ #define __ENGINE_DISCARD_H__
int ocf_discard(struct ocf_request *req); int ocf_engine_discard(struct ocf_request *req);
#endif #endif

View File

@ -8,11 +8,11 @@
#include "../ocf_cache_priv.h" #include "../ocf_cache_priv.h"
#include "engine_fast.h" #include "engine_fast.h"
#include "engine_common.h" #include "engine_common.h"
#include "engine_io.h"
#include "engine_pt.h" #include "engine_pt.h"
#include "engine_wb.h" #include "engine_wb.h"
#include "../ocf_request.h" #include "../ocf_request.h"
#include "../utils/utils_user_part.h" #include "../utils/utils_user_part.h"
#include "../utils/utils_io.h"
#include "../concurrency/ocf_concurrency.h" #include "../concurrency/ocf_concurrency.h"
#include "../metadata/metadata.h" #include "../metadata/metadata.h"
@ -31,19 +31,11 @@
static void _ocf_read_fast_complete(struct ocf_request *req, int error) static void _ocf_read_fast_complete(struct ocf_request *req, int error)
{ {
if (error) {
req->error |= error;
ocf_core_stats_cache_error_update(req->core, OCF_READ);
}
if (env_atomic_dec_return(&req->req_remaining)) {
/* Not all requests finished */
return;
}
OCF_DEBUG_RQ(req, "HIT completion"); OCF_DEBUG_RQ(req, "HIT completion");
if (req->error) { if (error) {
ocf_core_stats_cache_error_update(req->core, OCF_READ);
OCF_DEBUG_RQ(req, "ERROR"); OCF_DEBUG_RQ(req, "ERROR");
ocf_queue_push_req_pt(req); ocf_queue_push_req_pt(req);
@ -51,7 +43,7 @@ static void _ocf_read_fast_complete(struct ocf_request *req, int error)
ocf_req_unlock(ocf_cache_line_concurrency(req->cache), req); ocf_req_unlock(ocf_cache_line_concurrency(req->cache), req);
/* Complete request */ /* Complete request */
req->complete(req, req->error); req->complete(req, error);
/* Free the request at the last point of the completion path */ /* Free the request at the last point of the completion path */
ocf_req_put(req); ocf_req_put(req);
@ -86,10 +78,7 @@ static int _ocf_read_fast_do(struct ocf_request *req)
/* Submit IO */ /* Submit IO */
OCF_DEBUG_RQ(req, "Submit"); OCF_DEBUG_RQ(req, "Submit");
env_atomic_set(&req->req_remaining, ocf_engine_io_count(req)); ocf_engine_forward_cache_io_req(req, OCF_READ, _ocf_read_fast_complete);
ocf_submit_cache_reqs(req->cache, req, OCF_READ, 0, req->byte_length,
ocf_engine_io_count(req), _ocf_read_fast_complete);
/* Update statistics */ /* Update statistics */
ocf_engine_update_request_stats(req); ocf_engine_update_request_stats(req);

View File

@ -6,10 +6,10 @@
#include "ocf/ocf.h" #include "ocf/ocf.h"
#include "../ocf_cache_priv.h" #include "../ocf_cache_priv.h"
#include "engine_common.h" #include "engine_common.h"
#include "engine_io.h"
#include "cache_engine.h" #include "cache_engine.h"
#include "engine_flush.h" #include "engine_flush.h"
#include "../ocf_request.h" #include "../ocf_request.h"
#include "../utils/utils_io.h"
#define OCF_ENGINE_DEBUG_IO_NAME "flush" #define OCF_ENGINE_DEBUG_IO_NAME "flush"
#include "engine_debug.h" #include "engine_debug.h"
@ -17,7 +17,7 @@
static void _ocf_engine_flush_complete(struct ocf_request *req, int error) static void _ocf_engine_flush_complete(struct ocf_request *req, int error)
{ {
if (error) if (error)
req->error |= error; req->error = req->error ?: error;
if (env_atomic_dec_return(&req->req_remaining)) if (env_atomic_dec_return(&req->req_remaining))
return; return;
@ -45,17 +45,13 @@ int ocf_engine_flush(struct ocf_request *req)
env_atomic_set(&req->req_remaining, 2); env_atomic_set(&req->req_remaining, 2);
/* Submit operation into core device */ /* Submit operation into core device */
ocf_submit_volume_req(&req->core->volume, req, ocf_engine_forward_core_flush_req(req, _ocf_engine_flush_complete);
_ocf_engine_flush_complete);
/* submit flush to cache device */ /* submit flush to cache device */
ocf_submit_cache_flush(req, _ocf_engine_flush_complete); ocf_engine_forward_cache_flush_req(req, _ocf_engine_flush_complete);
/* Put OCF request - decrease reference counter */ /* Put OCF request - decrease reference counter */
ocf_req_put(req); ocf_req_put(req);
return 0; return 0;
} }

View File

@ -19,18 +19,12 @@
static void _ocf_invalidate_req(struct ocf_request *req, int error) static void _ocf_invalidate_req(struct ocf_request *req, int error)
{ {
if (error) {
req->error = error;
ocf_core_stats_cache_error_update(req->core, OCF_WRITE);
}
if (env_atomic_dec_return(&req->req_remaining))
return;
OCF_DEBUG_RQ(req, "Completion"); OCF_DEBUG_RQ(req, "Completion");
if (req->error) if (error) {
ocf_core_stats_cache_error_update(req->core, OCF_WRITE);
ocf_engine_error(req, true, "Failed to flush metadata to cache"); ocf_engine_error(req, true, "Failed to flush metadata to cache");
}
ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req);
@ -48,15 +42,13 @@ static int _ocf_invalidate_do(struct ocf_request *req)
ocf_purge_map_info(req); ocf_purge_map_info(req);
ocf_hb_req_prot_unlock_wr(req); ocf_hb_req_prot_unlock_wr(req);
env_atomic_inc(&req->req_remaining);
if (ocf_volume_is_atomic(&cache->device->volume) && if (ocf_volume_is_atomic(&cache->device->volume) &&
req->info.flush_metadata) { req->info.flush_metadata) {
/* Metadata flush IO */ /* Metadata flush IO */
ocf_metadata_flush_do_asynch(cache, req, _ocf_invalidate_req); ocf_metadata_flush_do_asynch(cache, req, _ocf_invalidate_req);
} } else {
_ocf_invalidate_req(req, 0); _ocf_invalidate_req(req, 0);
}
return 0; return 0;
} }

View File

@ -8,9 +8,9 @@
#include "engine_pt.h" #include "engine_pt.h"
#include "engine_rd.h" #include "engine_rd.h"
#include "engine_common.h" #include "engine_common.h"
#include "engine_io.h"
#include "cache_engine.h" #include "cache_engine.h"
#include "../ocf_request.h" #include "../ocf_request.h"
#include "../utils/utils_io.h"
#include "../utils/utils_user_part.h" #include "../utils/utils_user_part.h"
#include "../metadata/metadata.h" #include "../metadata/metadata.h"
#include "../concurrency/ocf_concurrency.h" #include "../concurrency/ocf_concurrency.h"
@ -20,19 +20,13 @@
static void _ocf_read_pt_complete(struct ocf_request *req, int error) static void _ocf_read_pt_complete(struct ocf_request *req, int error)
{ {
if (error)
req->error |= error;
if (env_atomic_dec_return(&req->req_remaining))
return;
OCF_DEBUG_RQ(req, "Completion"); OCF_DEBUG_RQ(req, "Completion");
if (req->error) if (error)
ocf_core_stats_core_error_update(req->core, OCF_READ); ocf_core_stats_core_error_update(req->core, OCF_READ);
/* Complete request */ /* Complete request */
req->complete(req, req->error); req->complete(req, error);
ocf_req_unlock(ocf_cache_line_concurrency(req->cache), req); ocf_req_unlock(ocf_cache_line_concurrency(req->cache), req);
@ -42,12 +36,10 @@ static void _ocf_read_pt_complete(struct ocf_request *req, int error)
static inline void _ocf_read_pt_submit(struct ocf_request *req) static inline void _ocf_read_pt_submit(struct ocf_request *req)
{ {
env_atomic_set(&req->req_remaining, 1); /* Core device IO */
OCF_DEBUG_RQ(req, "Submit"); OCF_DEBUG_RQ(req, "Submit");
/* Core read */ /* Core read */
ocf_submit_volume_req(&req->core->volume, req, _ocf_read_pt_complete); ocf_engine_forward_core_io_req(req, _ocf_read_pt_complete);
} }
int ocf_read_pt_do(struct ocf_request *req) int ocf_read_pt_do(struct ocf_request *req)

View File

@ -11,9 +11,9 @@
#include "engine_inv.h" #include "engine_inv.h"
#include "engine_bf.h" #include "engine_bf.h"
#include "engine_common.h" #include "engine_common.h"
#include "engine_io.h"
#include "cache_engine.h" #include "cache_engine.h"
#include "../concurrency/ocf_concurrency.h" #include "../concurrency/ocf_concurrency.h"
#include "../utils/utils_io.h"
#include "../ocf_request.h" #include "../ocf_request.h"
#include "../utils/utils_cache_line.h" #include "../utils/utils_cache_line.h"
#include "../utils/utils_user_part.h" #include "../utils/utils_user_part.h"
@ -31,19 +31,12 @@ static void _ocf_read_generic_hit_complete(struct ocf_request *req, int error)
OCF_DEBUG_RQ(req, "HIT completion"); OCF_DEBUG_RQ(req, "HIT completion");
if (error) { if (error) {
req->error |= error;
ocf_core_stats_cache_error_update(req->core, OCF_READ); ocf_core_stats_cache_error_update(req->core, OCF_READ);
inc_fallback_pt_error_counter(req->cache); inc_fallback_pt_error_counter(req->cache);
}
if (env_atomic_dec_return(&req->req_remaining) > 0)
return;
if (req->error) {
ocf_queue_push_req_pt(req); ocf_queue_push_req_pt(req);
} else { } else {
ocf_req_unlock(c, req); ocf_req_unlock(c, req);
req->complete(req, req->error); req->complete(req, error);
ocf_req_put(req); ocf_req_put(req);
} }
} }
@ -54,14 +47,9 @@ static void _ocf_read_generic_miss_complete(struct ocf_request *req, int error)
OCF_DEBUG_RQ(req, "MISS completion"); OCF_DEBUG_RQ(req, "MISS completion");
if (error) if (error) {
req->error = error; /* --- Do not backfill --- */
req->complete(req, error);
if (env_atomic_dec_return(&req->req_remaining) > 0)
return;
if (req->error) {
req->complete(req, req->error);
ocf_core_stats_core_error_update(req->core, OCF_READ); ocf_core_stats_core_error_update(req->core, OCF_READ);
@ -80,18 +68,15 @@ static void _ocf_read_generic_miss_complete(struct ocf_request *req, int error)
req->byte_length); req->byte_length);
} }
/* Complete request */ req->complete(req, error);
req->complete(req, req->error);
ocf_engine_backfill(req); ocf_engine_backfill(req);
} }
void ocf_read_generic_submit_hit(struct ocf_request *req) void ocf_read_generic_submit_hit(struct ocf_request *req)
{ {
env_atomic_set(&req->req_remaining, ocf_engine_io_count(req)); ocf_engine_forward_cache_io_req(req, OCF_READ,
_ocf_read_generic_hit_complete);
ocf_submit_cache_reqs(req->cache, req, OCF_READ, 0, req->byte_length,
ocf_engine_io_count(req), _ocf_read_generic_hit_complete);
} }
static inline void _ocf_read_generic_submit_miss(struct ocf_request *req) static inline void _ocf_read_generic_submit_miss(struct ocf_request *req)
@ -99,8 +84,6 @@ static inline void _ocf_read_generic_submit_miss(struct ocf_request *req)
struct ocf_cache *cache = req->cache; struct ocf_cache *cache = req->cache;
int ret; int ret;
env_atomic_set(&req->req_remaining, 1);
req->cp_data = ctx_data_alloc(cache->owner, req->cp_data = ctx_data_alloc(cache->owner,
BYTES_TO_PAGES(req->byte_length)); BYTES_TO_PAGES(req->byte_length));
if (!req->cp_data) { if (!req->cp_data) {
@ -119,9 +102,7 @@ static inline void _ocf_read_generic_submit_miss(struct ocf_request *req)
} }
err_alloc: err_alloc:
/* Submit read request to core device. */ ocf_engine_forward_core_io_req(req, _ocf_read_generic_miss_complete);
ocf_submit_volume_req(&req->core->volume, req,
_ocf_read_generic_miss_complete);
} }
static int _ocf_read_generic_do(struct ocf_request *req) static int _ocf_read_generic_do(struct ocf_request *req)

View File

@ -1,5 +1,6 @@
/* /*
* Copyright(c) 2012-2022 Intel Corporation * Copyright(c) 2012-2022 Intel Corporation
* Copyright(c) 2024 Huawei Technologies
* SPDX-License-Identifier: BSD-3-Clause * SPDX-License-Identifier: BSD-3-Clause
*/ */
#include "ocf/ocf.h" #include "ocf/ocf.h"
@ -8,9 +9,9 @@
#include "engine_wt.h" #include "engine_wt.h"
#include "engine_wi.h" #include "engine_wi.h"
#include "engine_common.h" #include "engine_common.h"
#include "engine_io.h"
#include "cache_engine.h" #include "cache_engine.h"
#include "../ocf_request.h" #include "../ocf_request.h"
#include "../utils/utils_io.h"
#include "../metadata/metadata.h" #include "../metadata/metadata.h"
#define OCF_ENGINE_DEBUG_IO_NAME "wa" #define OCF_ENGINE_DEBUG_IO_NAME "wa"

View File

@ -8,12 +8,12 @@
#include "../ocf_cache_priv.h" #include "../ocf_cache_priv.h"
#include "cache_engine.h" #include "cache_engine.h"
#include "engine_common.h" #include "engine_common.h"
#include "engine_io.h"
#include "engine_wb.h" #include "engine_wb.h"
#include "engine_wi.h" #include "engine_wi.h"
#include "engine_inv.h" #include "engine_inv.h"
#include "../metadata/metadata.h" #include "../metadata/metadata.h"
#include "../ocf_request.h" #include "../ocf_request.h"
#include "../utils/utils_io.h"
#include "../utils/utils_cache_line.h" #include "../utils/utils_cache_line.h"
#include "../utils/utils_request.h" #include "../utils/utils_request.h"
#include "../utils/utils_user_part.h" #include "../utils/utils_user_part.h"
@ -50,17 +50,11 @@ static void _ocf_write_wb_update_bits(struct ocf_request *req)
static void _ocf_write_wb_io_flush_metadata(struct ocf_request *req, int error) static void _ocf_write_wb_io_flush_metadata(struct ocf_request *req, int error)
{ {
if (error) if (error)
req->error = error;
if (env_atomic_dec_return(&req->req_remaining))
return;
if (req->error)
ocf_engine_error(req, true, "Failed to write data to cache"); ocf_engine_error(req, true, "Failed to write data to cache");
ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req);
req->complete(req, req->error); req->complete(req, error);
ocf_req_put(req); ocf_req_put(req);
} }
@ -69,37 +63,29 @@ static int ocf_write_wb_do_flush_metadata(struct ocf_request *req)
{ {
struct ocf_cache *cache = req->cache; struct ocf_cache *cache = req->cache;
env_atomic_set(&req->req_remaining, 1); /* One core IO */
_ocf_write_wb_update_bits(req); _ocf_write_wb_update_bits(req);
if (req->info.flush_metadata) { if (req->info.flush_metadata) {
OCF_DEBUG_RQ(req, "Flush metadata"); OCF_DEBUG_RQ(req, "Flush metadata");
ocf_metadata_flush_do_asynch(cache, req, ocf_metadata_flush_do_asynch(cache, req,
_ocf_write_wb_io_flush_metadata); _ocf_write_wb_io_flush_metadata);
} } else {
_ocf_write_wb_io_flush_metadata(req, 0); _ocf_write_wb_io_flush_metadata(req, 0);
}
return 0; return 0;
} }
static void _ocf_write_wb_complete(struct ocf_request *req, int error) static void _ocf_write_wb_complete(struct ocf_request *req, int error)
{ {
if (error) {
ocf_core_stats_cache_error_update(req->core, OCF_WRITE);
req->error |= error;
}
if (env_atomic_dec_return(&req->req_remaining))
return;
OCF_DEBUG_RQ(req, "Completion"); OCF_DEBUG_RQ(req, "Completion");
if (req->error) { if (error) {
ocf_core_stats_cache_error_update(req->core, OCF_WRITE);
ocf_engine_error(req, true, "Failed to write data to cache"); ocf_engine_error(req, true, "Failed to write data to cache");
req->complete(req, req->error); req->complete(req, error);
ocf_engine_invalidate(req); ocf_engine_invalidate(req);
} else { } else {
@ -111,10 +97,6 @@ static void _ocf_write_wb_complete(struct ocf_request *req, int error)
static inline void _ocf_write_wb_submit(struct ocf_request *req) static inline void _ocf_write_wb_submit(struct ocf_request *req)
{ {
struct ocf_cache *cache = req->cache;
env_atomic_set(&req->req_remaining, ocf_engine_io_count(req));
/* /*
* 1. Submit data * 1. Submit data
* 2. Wait for completion of data * 2. Wait for completion of data
@ -137,8 +119,7 @@ static inline void _ocf_write_wb_submit(struct ocf_request *req)
OCF_DEBUG_RQ(req, "Submit Data"); OCF_DEBUG_RQ(req, "Submit Data");
/* Data IO */ /* Data IO */
ocf_submit_cache_reqs(cache, req, OCF_WRITE, 0, req->byte_length, ocf_engine_forward_cache_io_req(req, OCF_WRITE, _ocf_write_wb_complete);
ocf_engine_io_count(req), _ocf_write_wb_complete);
} }
int ocf_write_wb_do(struct ocf_request *req) int ocf_write_wb_do(struct ocf_request *req)

View File

@ -8,10 +8,10 @@
#include "../ocf_cache_priv.h" #include "../ocf_cache_priv.h"
#include "engine_wi.h" #include "engine_wi.h"
#include "engine_common.h" #include "engine_common.h"
#include "engine_io.h"
#include "../concurrency/ocf_concurrency.h" #include "../concurrency/ocf_concurrency.h"
#include "../ocf_request.h" #include "../ocf_request.h"
#include "../utils/utils_cache_line.h" #include "../utils/utils_cache_line.h"
#include "../utils/utils_io.h"
#include "../metadata/metadata.h" #include "../metadata/metadata.h"
#define OCF_ENGINE_DEBUG_IO_NAME "wi" #define OCF_ENGINE_DEBUG_IO_NAME "wi"
@ -22,7 +22,7 @@ static int _ocf_write_wi_next_pass(struct ocf_request *req)
ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req);
if (req->wi_second_pass) { if (req->wi_second_pass) {
req->complete(req, req->error); req->complete(req, 0);
ocf_req_put(req); ocf_req_put(req);
return 0; return 0;
@ -48,25 +48,19 @@ static void _ocf_write_wi_io_flush_metadata(struct ocf_request *req, int error)
{ {
if (error) { if (error) {
ocf_core_stats_cache_error_update(req->core, OCF_WRITE); ocf_core_stats_cache_error_update(req->core, OCF_WRITE);
req->error |= error; ocf_engine_error(req, true, "Failed to write data to cache");
} }
if (env_atomic_dec_return(&req->req_remaining)) if (!error && !req->wi_second_pass && ocf_engine_is_miss(req)) {
return;
if (!req->error && !req->wi_second_pass && ocf_engine_is_miss(req)) {
/* need another pass */ /* need another pass */
ocf_queue_push_req_cb(req, _ocf_write_wi_next_pass, ocf_queue_push_req_cb(req, _ocf_write_wi_next_pass,
OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH);
return; return;
} }
if (req->error)
ocf_engine_error(req, true, "Failed to write data to cache");
ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req);
req->complete(req, req->error); req->complete(req, error);
ocf_req_put(req); ocf_req_put(req);
} }
@ -105,20 +99,14 @@ static int ocf_write_wi_update_and_flush_metadata(struct ocf_request *req)
static void _ocf_write_wi_core_complete(struct ocf_request *req, int error) static void _ocf_write_wi_core_complete(struct ocf_request *req, int error)
{ {
if (error) {
req->error = error;
ocf_core_stats_core_error_update(req->core, OCF_WRITE);
}
if (env_atomic_dec_return(&req->req_remaining))
return;
OCF_DEBUG_RQ(req, "Completion"); OCF_DEBUG_RQ(req, "Completion");
if (req->error) { if (error) {
ocf_core_stats_core_error_update(req->core, OCF_WRITE);
ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req);
req->complete(req, req->error); req->complete(req, error);
ocf_req_put(req); ocf_req_put(req);
} else { } else {
@ -146,13 +134,10 @@ static int _ocf_write_wi_core_write(struct ocf_request *req)
return 0; return 0;
} }
env_atomic_set(&req->req_remaining, 1); /* One core IO */
OCF_DEBUG_RQ(req, "Submit"); OCF_DEBUG_RQ(req, "Submit");
/* Submit write IO to the core */ /* Submit write IO to the core */
ocf_submit_volume_req(&req->core->volume, req, ocf_engine_forward_core_io_req(req, _ocf_write_wi_core_complete);
_ocf_write_wi_core_complete);
/* Update statistics */ /* Update statistics */
ocf_engine_update_block_stats(req); ocf_engine_update_block_stats(req);

View File

@ -9,10 +9,10 @@
#include "../ocf_cache_priv.h" #include "../ocf_cache_priv.h"
#include "cache_engine.h" #include "cache_engine.h"
#include "engine_common.h" #include "engine_common.h"
#include "engine_io.h"
#include "engine_rd.h" #include "engine_rd.h"
#include "engine_pt.h" #include "engine_pt.h"
#include "../metadata/metadata.h" #include "../metadata/metadata.h"
#include "../utils/utils_io.h"
#include "../utils/utils_cache_line.h" #include "../utils/utils_cache_line.h"
#include "../utils/utils_user_part.h" #include "../utils/utils_user_part.h"
#include "../concurrency/ocf_concurrency.h" #include "../concurrency/ocf_concurrency.h"
@ -23,13 +23,10 @@
static void ocf_read_wo_cache_complete(struct ocf_request *req, int error) static void ocf_read_wo_cache_complete(struct ocf_request *req, int error)
{ {
if (error) { if (error) {
req->error = req->error ?: error;
ocf_core_stats_cache_error_update(req->core, OCF_READ); ocf_core_stats_cache_error_update(req->core, OCF_READ);
req->error |= error;
} }
if (env_atomic_dec_return(&req->req_remaining))
return;
OCF_DEBUG_RQ(req, "Completion"); OCF_DEBUG_RQ(req, "Completion");
if (req->error) if (req->error)
@ -48,8 +45,7 @@ static void ocf_read_wo_cache_io(struct ocf_request *req, uint64_t offset,
uint64_t size) uint64_t size)
{ {
OCF_DEBUG_RQ(req, "Submit cache"); OCF_DEBUG_RQ(req, "Submit cache");
env_atomic_inc(&req->req_remaining); ocf_engine_forward_cache_io(req, OCF_READ, offset, size,
ocf_submit_cache_reqs(req->cache, req, OCF_READ, offset, size, 1,
ocf_read_wo_cache_complete); ocf_read_wo_cache_complete);
} }
@ -66,7 +62,9 @@ static int ocf_read_wo_cache_do(struct ocf_request *req)
uint64_t offset = 0; uint64_t offset = 0;
uint64_t increment = 0; uint64_t increment = 0;
env_atomic_set(&req->req_remaining, 1); req->cache_forward_end = ocf_read_wo_cache_complete;
ocf_req_forward_cache_get(req);
for (line = 0; line < req->core_line_count; ++line) { for (line = 0; line < req->core_line_count; ++line) {
entry = &req->map[line]; entry = &req->map[line];
@ -147,23 +145,21 @@ static int ocf_read_wo_cache_do(struct ocf_request *req)
if (io) if (io)
ocf_read_wo_cache_io(req, io_start, offset - io_start); ocf_read_wo_cache_io(req, io_start, offset - io_start);
ocf_read_wo_cache_complete(req, 0); ocf_req_forward_cache_put(req);
return 0; return 0;
} }
static void _ocf_read_wo_core_complete(struct ocf_request *req, int error) static void _ocf_read_wo_core_complete(struct ocf_request *req, int error)
{ {
if (error) { if (error)
req->error |= error;
ocf_core_stats_core_error_update(req->core, OCF_READ); ocf_core_stats_core_error_update(req->core, OCF_READ);
}
/* if all mapped cachelines are clean, the data we've read from core /* if all mapped cachelines are clean, the data we've read from core
* is valid and we can complete the request */ * is valid and we can complete the request */
if (!req->info.dirty_any || req->error) { if (error || !req->info.dirty_any) {
OCF_DEBUG_RQ(req, "Completion"); OCF_DEBUG_RQ(req, "Completion");
req->complete(req, req->error); req->complete(req, error);
ocf_req_unlock_rd(ocf_cache_line_concurrency(req->cache), req); ocf_req_unlock_rd(ocf_cache_line_concurrency(req->cache), req);
ocf_req_put(req); ocf_req_put(req);
return; return;
@ -191,8 +187,7 @@ static int ocf_read_wo_do(struct ocf_request *req)
} else { } else {
OCF_DEBUG_RQ(req, "Submit core"); OCF_DEBUG_RQ(req, "Submit core");
ocf_submit_volume_req(&req->core->volume, req, ocf_engine_forward_core_io_req(req, _ocf_read_wo_core_complete);
_ocf_read_wo_core_complete);
} }
ocf_engine_update_request_stats(req); ocf_engine_update_request_stats(req);

View File

@ -10,8 +10,8 @@
#include "engine_wi.h" #include "engine_wi.h"
#include "engine_inv.h" #include "engine_inv.h"
#include "engine_common.h" #include "engine_common.h"
#include "engine_io.h"
#include "../ocf_request.h" #include "../ocf_request.h"
#include "../utils/utils_io.h"
#include "../utils/utils_cache_line.h" #include "../utils/utils_cache_line.h"
#include "../utils/utils_user_part.h" #include "../utils/utils_user_part.h"
#include "../metadata/metadata.h" #include "../metadata/metadata.h"
@ -58,12 +58,6 @@ static void _ocf_write_wt_do_flush_metadata_compl(struct ocf_request *req,
int error) int error)
{ {
if (error) if (error)
req->error = error;
if (env_atomic_dec_return(&req->req_remaining))
return;
if (req->error)
ocf_engine_error(req, true, "Failed to write data to cache"); ocf_engine_error(req, true, "Failed to write data to cache");
ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req);
@ -77,8 +71,6 @@ static int ocf_write_wt_do_flush_metadata(struct ocf_request *req)
{ {
struct ocf_cache *cache = req->cache; struct ocf_cache *cache = req->cache;
env_atomic_set(&req->req_remaining, 1);
_ocf_write_wt_update_bits(req); _ocf_write_wt_update_bits(req);
if (req->info.flush_metadata) { if (req->info.flush_metadata) {
@ -86,9 +78,9 @@ static int ocf_write_wt_do_flush_metadata(struct ocf_request *req)
ocf_metadata_flush_do_asynch(cache, req, ocf_metadata_flush_do_asynch(cache, req,
_ocf_write_wt_do_flush_metadata_compl); _ocf_write_wt_do_flush_metadata_compl);
} } else {
_ocf_write_wt_do_flush_metadata_compl(req, 0); _ocf_write_wt_do_flush_metadata_compl(req, 0);
}
return 0; return 0;
} }
@ -136,8 +128,8 @@ static void _ocf_write_wt_cache_complete(struct ocf_request *req, int error)
static void _ocf_write_wt_core_complete(struct ocf_request *req, int error) static void _ocf_write_wt_core_complete(struct ocf_request *req, int error)
{ {
if (error) { if (error) {
req->error = error;
req->info.core_error = 1; req->info.core_error = 1;
req->error = error;
ocf_core_stats_core_error_update(req->core, OCF_WRITE); ocf_core_stats_core_error_update(req->core, OCF_WRITE);
} }
@ -146,22 +138,17 @@ static void _ocf_write_wt_core_complete(struct ocf_request *req, int error)
static inline void _ocf_write_wt_submit(struct ocf_request *req) static inline void _ocf_write_wt_submit(struct ocf_request *req)
{ {
struct ocf_cache *cache = req->cache;
/* Submit IOs */ /* Submit IOs */
OCF_DEBUG_RQ(req, "Submit"); OCF_DEBUG_RQ(req, "Submit");
/* Calculate how many IOs need to be submited */ env_atomic_set(&req->req_remaining, 2); /* cache IO + core IO */
env_atomic_set(&req->req_remaining, ocf_engine_io_count(req)); /* Cache IO */
env_atomic_inc(&req->req_remaining); /* Core device IO */
/* To cache */ /* To cache */
ocf_submit_cache_reqs(cache, req, OCF_WRITE, 0, req->byte_length, ocf_engine_forward_cache_io_req(req, OCF_WRITE,
ocf_engine_io_count(req), _ocf_write_wt_cache_complete); _ocf_write_wt_cache_complete);
/* To core */ /* To core */
ocf_submit_volume_req(&req->core->volume, req, ocf_engine_forward_core_io_req(req, _ocf_write_wt_core_complete);
_ocf_write_wt_core_complete);
} }
static int _ocf_write_wt_do(struct ocf_request *req) static int _ocf_write_wt_do(struct ocf_request *req)

View File

@ -19,9 +19,6 @@
static int ocf_zero_purge(struct ocf_request *req) static int ocf_zero_purge(struct ocf_request *req)
{ {
if (req->error) {
ocf_engine_error(req, true, "Failed to discard data on cache");
} else {
/* There are mapped cache line, need to remove them */ /* There are mapped cache line, need to remove them */
ocf_hb_req_prot_lock_wr(req); /*- Metadata WR access ---------------*/ ocf_hb_req_prot_lock_wr(req); /*- Metadata WR access ---------------*/
@ -30,12 +27,9 @@ static int ocf_zero_purge(struct ocf_request *req)
ocf_purge_map_info(req); ocf_purge_map_info(req);
ocf_hb_req_prot_unlock_wr(req); /*- END Metadata WR access ---------*/ ocf_hb_req_prot_unlock_wr(req); /*- END Metadata WR access ---------*/
}
ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req); ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req);
req->complete(req, 0);
req->complete(req, req->error);
ocf_req_put(req); ocf_req_put(req);
return 0; return 0;
@ -45,11 +39,13 @@ static void _ocf_zero_io_flush_metadata(struct ocf_request *req, int error)
{ {
if (error) { if (error) {
ocf_core_stats_cache_error_update(req->core, OCF_WRITE); ocf_core_stats_cache_error_update(req->core, OCF_WRITE);
req->error = error; ocf_engine_error(req, true, "Failed to discard data on cache");
}
if (env_atomic_dec_return(&req->req_remaining)) ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req);
req->complete(req, error);
ocf_req_put(req);
return; return;
}
ocf_queue_push_req_cb(req, ocf_zero_purge, ocf_queue_push_req_cb(req, ocf_zero_purge,
OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH);

View File

@ -320,9 +320,13 @@ static void ocf_cache_io_complete(struct ocf_io *io, int error)
{ {
struct ocf_cache_volume_io_priv *priv; struct ocf_cache_volume_io_priv *priv;
ocf_cache_t cache; ocf_cache_t cache;
struct ocf_request *req = ocf_io_to_req(io);
cache = ocf_volume_to_cache(ocf_io_get_volume(io)); cache = ocf_volume_to_cache(ocf_io_get_volume(io));
if (error)
req->error = req->error ?: error;
priv = ocf_io_get_priv(io); priv = ocf_io_get_priv(io);
env_atomic_cmpxchg(&priv->error, 0, error); env_atomic_cmpxchg(&priv->error, 0, error);