github/ocf
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Modify engines to use forward API

Browse Source 
Signed-off-by: Robert Baldyga <robert.baldyga@huawei.com>
Signed-off-by: Rafal Stefanowski <rafal.stefanowski@huawei.com>
Signed-off-by: Michal Mielewczyk <michal.mielewczyk@huawei.com>
This commit is contained in:
Robert Baldyga 2023-09-03 20:27:27 +02:00 committed by Michal Mielewczyk
parent e667c10b4a
commit 1ed707361f
18 changed files with 191 additions and 298 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
src/engine/cache_engine.c
View File

@ -43,6 +43,8 @@ enum ocf_io_if_type {
OCF_IO_FLUSH_IF,
OCF_IO_DISCARD_IF,
OCF_IO_D2C_IF,
OCF_IO_D2C_FLUSH_IF,
OCF_IO_D2C_DISCARD_IF,
OCF_IO_PRIV_MAX_IF,
};
@ -96,26 +98,40 @@ static const struct ocf_io_if IO_IFS[OCF_IO_PRIV_MAX_IF] = {
},
.name = "Fast",
},
[OCF_IO_DISCARD_IF] = {
.cbs = {
[OCF_READ] = ocf_discard,
[OCF_WRITE] = ocf_discard,
},
.name = "Discard",
},
[OCF_IO_D2C_IF] = {
.cbs = {
[OCF_READ] = ocf_io_d2c,
[OCF_WRITE] = ocf_io_d2c,
},
.name = "Direct to core",
},
[OCF_IO_FLUSH_IF] = {
.cbs = {
[OCF_READ] = ocf_engine_flush,
[OCF_WRITE] = ocf_engine_flush,
},
.name = "Ops engine",
.name = "Flush",
},
[OCF_IO_DISCARD_IF] = {
.cbs = {
[OCF_READ] = ocf_engine_discard,
[OCF_WRITE] = ocf_engine_discard,
},
.name = "Discard",
},
[OCF_IO_D2C_IF] = {
.cbs = {
[OCF_READ] = ocf_d2c_io,
[OCF_WRITE] = ocf_d2c_io,
},
.name = "Direct to core io",
},
[OCF_IO_D2C_FLUSH_IF] = {
.cbs = {
[OCF_READ] = ocf_d2c_flush,
[OCF_WRITE] = ocf_d2c_flush,
},
.name = "Direct to core flush",
},
[OCF_IO_D2C_DISCARD_IF] = {
.cbs = {
[OCF_READ] = ocf_d2c_discard,
[OCF_WRITE] = ocf_d2c_discard,
},
.name = "Direct to core discard",
},
};
@ -257,17 +273,12 @@ int ocf_engine_hndl_fast_req(struct ocf_request *req)
return ret;
}
static void ocf_engine_hndl_2dc_req(struct ocf_request *req)
{
IO_IFS[OCF_IO_D2C_IF].cbs[req->rw](req);
}
void ocf_engine_hndl_discard_req(struct ocf_request *req)
{
ocf_req_get(req);
if (req->d2c) {
ocf_engine_hndl_2dc_req(req);
IO_IFS[OCF_IO_D2C_DISCARD_IF].cbs[req->rw](req);
return;
}
@ -279,8 +290,8 @@ void ocf_engine_hndl_flush_req(struct ocf_request *req)
ocf_req_get(req);
req->engine_handler = (req->d2c) ?
ocf_io_if_type_to_engine_cb(OCF_IO_D2C_IF, req->rw) :
ocf_io_if_type_to_engine_cb(OCF_IO_FLUSH_IF, req->rw);
IO_IFS[OCF_IO_D2C_FLUSH_IF].cbs[req->rw] :
IO_IFS[OCF_IO_FLUSH_IF].cbs[req->rw];
ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC);
}

28
src/engine/engine_bf.c
View File

@ -10,10 +10,11 @@
#include "engine_bf.h"
#include "engine_inv.h"
#include "engine_common.h"
#include "engine_io.h"
#include "cache_engine.h"
#include "../ocf_request.h"
#include "../utils/utils_io.h"
#include "../concurrency/ocf_concurrency.h"
#include "../utils/utils_io.h"
#define OCF_ENGINE_DEBUG_IO_NAME "bf"
#include "engine_debug.h"
@ -43,19 +44,9 @@ static void _ocf_backfill_complete(struct ocf_request *req, int error)
struct ocf_cache *cache = req->cache;
if (error) {
req->error = error;
ocf_core_stats_cache_error_update(req->core, OCF_WRITE);
}
if (req->error)
inc_fallback_pt_error_counter(req->cache);
/* Handle callback-caller race to let only one of the two complete the
* request. Also, complete original request only if this is the last
* sub-request to complete
*/
if (env_atomic_dec_return(&req->req_remaining))
return;
}
backfill_queue_dec_unblock(req->cache);
@ -67,7 +58,7 @@ static void _ocf_backfill_complete(struct ocf_request *req, int error)
req->data = NULL;
}
if (req->error) {
if (error) {
ocf_engine_invalidate(req);
} else {
ocf_req_unlock(ocf_cache_line_concurrency(cache), req);
@ -79,22 +70,13 @@ static void _ocf_backfill_complete(struct ocf_request *req, int error)
static int _ocf_backfill_do(struct ocf_request *req)
{
unsigned int reqs_to_issue;
req->data = req->cp_data;
if (unlikely(req->data == NULL)) {
env_atomic_set(&req->req_remaining, 1);
_ocf_backfill_complete(req, -OCF_ERR_NO_MEM);
return 0;
}
reqs_to_issue = ocf_engine_io_count(req);
/* There will be #reqs_to_issue completions */
env_atomic_set(&req->req_remaining, reqs_to_issue);
ocf_submit_cache_reqs(req->cache, req, OCF_WRITE, 0, req->byte_length,
reqs_to_issue, _ocf_backfill_complete);
ocf_engine_forward_cache_io_req(req, OCF_WRITE, _ocf_backfill_complete);
return 0;
}

62
src/engine/engine_d2c.c
View File

@ -7,9 +7,9 @@
#include "../ocf_cache_priv.h"
#include "engine_d2c.h"
#include "engine_common.h"
#include "engine_io.h"
#include "cache_engine.h"
#include "../ocf_request.h"
#include "../utils/utils_io.h"
#include "../metadata/metadata.h"
#define OCF_ENGINE_DEBUG_IO_NAME "d2c"
@ -17,31 +17,72 @@
static void _ocf_d2c_completion(struct ocf_request *req, int error)
{
req->error = error;
OCF_DEBUG_RQ(req, "Completion");
if (req->error)
if (error)
ocf_core_stats_core_error_update(req->core, req->rw);
/* Complete request */
req->complete(req, req->error);
req->complete(req, error);
/* Release OCF request */
ocf_req_put(req);
}
int ocf_io_d2c(struct ocf_request *req)
int ocf_d2c_io(struct ocf_request *req)
{
ocf_core_t core = req->core;
OCF_DEBUG_TRACE(req->cache);
/* Get OCF request - increase reference counter */
ocf_req_get(req);
ocf_submit_volume_req(&core->volume, req, _ocf_d2c_completion);
ocf_engine_forward_core_io_req(req, _ocf_d2c_completion);
ocf_engine_update_block_stats(req);
ocf_core_stats_pt_block_update(req->core, req->part_id, req->rw,
req->byte_length);
ocf_core_stats_request_pt_update(req->core, req->part_id, req->rw,
req->info.hit_no, req->core_line_count);
/* Put OCF request - decrease reference counter */
ocf_req_put(req);
return 0;
}
int ocf_d2c_flush(struct ocf_request *req)
{
OCF_DEBUG_TRACE(req->cache);
/* Get OCF request - increase reference counter */
ocf_req_get(req);
ocf_engine_forward_core_flush_req(req, _ocf_d2c_completion);
ocf_engine_update_block_stats(req);
ocf_core_stats_pt_block_update(req->core, req->part_id, req->rw,
req->byte_length);
ocf_core_stats_request_pt_update(req->core, req->part_id, req->rw,
req->info.hit_no, req->core_line_count);
/* Put OCF request - decrease reference counter */
ocf_req_put(req);
return 0;
}
int ocf_d2c_discard(struct ocf_request *req)
{
OCF_DEBUG_TRACE(req->cache);
/* Get OCF request - increase reference counter */
ocf_req_get(req);
ocf_engine_forward_core_discard_req(req, _ocf_d2c_completion);
ocf_engine_update_block_stats(req);
@ -55,5 +96,4 @@ int ocf_io_d2c(struct ocf_request *req)
ocf_req_put(req);
return 0;
}

7
src/engine/engine_d2c.h
View File

@ -1,11 +1,16 @@
/*
* Copyright(c) 2012-2021 Intel Corporation
* Copyright(c) 2024 Huawei Technologies
* SPDX-License-Identifier: BSD-3-Clause
*/
#ifndef ENGINE_2DC_H_
#define ENGINE_2DC_H_
int ocf_io_d2c(struct ocf_request *req);
int ocf_d2c_io(struct ocf_request *req);
int ocf_d2c_flush(struct ocf_request *req);
int ocf_d2c_discard(struct ocf_request *req);
#endif /* ENGINE_2DC_H_ */

65
src/engine/engine_discard.c
View File

@ -8,10 +8,10 @@
#include "cache_engine.h"
#include "engine_common.h"
#include "engine_discard.h"
#include "engine_io.h"
#include "../metadata/metadata.h"
#include "../ocf_request.h"
#include "../utils/utils_io.h"
#include "../utils/utils_cache_line.h"
#include "../concurrency/ocf_concurrency.h"
#define OCF_ENGINE_DEBUG 0
@ -25,75 +25,33 @@ static void _ocf_discard_complete_req(struct ocf_request *req, int error)
ocf_req_put(req);
}
static void _ocf_discard_core_complete(struct ocf_io *io, int error)
{
struct ocf_request *req = io->priv1;
OCF_DEBUG_RQ(req, "Core DISCARD Completion");
_ocf_discard_complete_req(req, error);
ocf_io_put(io);
}
static int _ocf_discard_core(struct ocf_request *req)
{
struct ocf_io *io;
int err;
req->byte_position = SECTORS_TO_BYTES(req->discard.sector);
req->byte_length = SECTORS_TO_BYTES(req->discard.nr_sects);
io = ocf_volume_new_io(&req->core->volume, req->io_queue,
SECTORS_TO_BYTES(req->discard.sector),
SECTORS_TO_BYTES(req->discard.nr_sects),
OCF_WRITE, 0, 0);
if (!io) {
_ocf_discard_complete_req(req, -OCF_ERR_NO_MEM);
return -OCF_ERR_NO_MEM;
}
ocf_io_set_cmpl(io, req, NULL, _ocf_discard_core_complete);
err = ocf_io_set_data(io, req->data, req->offset);
if (err) {
_ocf_discard_core_complete(io, err);
return err;
}
ocf_volume_submit_discard(io);
ocf_engine_forward_core_discard_req(req, _ocf_discard_complete_req);
return 0;
}
static void _ocf_discard_cache_flush_complete(struct ocf_io *io, int error)
static void _ocf_discard_cache_flush_complete(struct ocf_request *req, int error)
{
struct ocf_request *req = io->priv1;
if (error) {
ocf_metadata_error(req->cache);
_ocf_discard_complete_req(req, error);
ocf_io_put(io);
return;
}
req->engine_handler = _ocf_discard_core;
ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH);
ocf_io_put(io);
}
static int _ocf_discard_flush_cache(struct ocf_request *req)
{
struct ocf_io *io;
io = ocf_volume_new_io(&req->cache->device->volume, req->io_queue,
0, 0, OCF_WRITE, 0, 0);
if (!io) {
ocf_metadata_error(req->cache);
_ocf_discard_complete_req(req, -OCF_ERR_NO_MEM);
return -OCF_ERR_NO_MEM;
}
ocf_io_set_cmpl(io, req, NULL, _ocf_discard_cache_flush_complete);
ocf_volume_submit_flush(io);
ocf_engine_forward_cache_flush_req(req,
_ocf_discard_cache_flush_complete);
return 0;
}
@ -116,9 +74,6 @@ static void _ocf_discard_finish_step(struct ocf_request *req)
static void _ocf_discard_step_complete(struct ocf_request *req, int error)
{
if (error)
req->error |= error;
if (env_atomic_dec_return(&req->req_remaining))
return;
@ -127,9 +82,9 @@ static void _ocf_discard_step_complete(struct ocf_request *req, int error)
/* Release WRITE lock of request */
ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req);
if (req->error) {
if (error) {
ocf_metadata_error(req->cache);
_ocf_discard_complete_req(req, req->error);
_ocf_discard_complete_req(req, error);
return;
}
@ -242,7 +197,7 @@ static int _ocf_discard_step(struct ocf_request *req)
return 0;
}
int ocf_discard(struct ocf_request *req)
int ocf_engine_discard(struct ocf_request *req)
{
OCF_DEBUG_TRACE(req->cache);

3
src/engine/engine_discard.h
View File

@ -1,11 +1,12 @@
/*
* Copyright(c) 2012-2021 Intel Corporation
* Copyright(c) 2024 Huawei Technologies
* SPDX-License-Identifier: BSD-3-Clause
*/
#ifndef __ENGINE_DISCARD_H__
#define __ENGINE_DISCARD_H__
int ocf_discard(struct ocf_request *req);
int ocf_engine_discard(struct ocf_request *req);
#endif

23
src/engine/engine_fast.c
View File

@ -8,11 +8,11 @@
#include "../ocf_cache_priv.h"
#include "engine_fast.h"
#include "engine_common.h"
#include "engine_io.h"
#include "engine_pt.h"
#include "engine_wb.h"
#include "../ocf_request.h"
#include "../utils/utils_user_part.h"
#include "../utils/utils_io.h"
#include "../concurrency/ocf_concurrency.h"
#include "../metadata/metadata.h"
@ -31,19 +31,11 @@
static void _ocf_read_fast_complete(struct ocf_request *req, int error)
{
if (error) {
req->error |= error;
ocf_core_stats_cache_error_update(req->core, OCF_READ);
}
if (env_atomic_dec_return(&req->req_remaining)) {
/* Not all requests finished */
return;
}
OCF_DEBUG_RQ(req, "HIT completion");
if (req->error) {
if (error) {
ocf_core_stats_cache_error_update(req->core, OCF_READ);
OCF_DEBUG_RQ(req, "ERROR");
ocf_queue_push_req_pt(req);
@ -51,7 +43,7 @@ static void _ocf_read_fast_complete(struct ocf_request *req, int error)
ocf_req_unlock(ocf_cache_line_concurrency(req->cache), req);
/* Complete request */
req->complete(req, req->error);
req->complete(req, error);
/* Free the request at the last point of the completion path */
ocf_req_put(req);
@ -86,10 +78,7 @@ static int _ocf_read_fast_do(struct ocf_request *req)
/* Submit IO */
OCF_DEBUG_RQ(req, "Submit");
env_atomic_set(&req->req_remaining, ocf_engine_io_count(req));
ocf_submit_cache_reqs(req->cache, req, OCF_READ, 0, req->byte_length,
ocf_engine_io_count(req), _ocf_read_fast_complete);
ocf_engine_forward_cache_io_req(req, OCF_READ, _ocf_read_fast_complete);
/* Update statistics */
ocf_engine_update_request_stats(req);

12
src/engine/engine_flush.c
View File

@ -6,10 +6,10 @@
#include "ocf/ocf.h"
#include "../ocf_cache_priv.h"
#include "engine_common.h"
#include "engine_io.h"
#include "cache_engine.h"
#include "engine_flush.h"
#include "../ocf_request.h"
#include "../utils/utils_io.h"
#define OCF_ENGINE_DEBUG_IO_NAME "flush"
#include "engine_debug.h"
@ -17,7 +17,7 @@
static void _ocf_engine_flush_complete(struct ocf_request *req, int error)
{
if (error)
req->error |= error;
req->error = req->error ?: error;
if (env_atomic_dec_return(&req->req_remaining))
return;
@ -45,17 +45,13 @@ int ocf_engine_flush(struct ocf_request *req)
env_atomic_set(&req->req_remaining, 2);
/* Submit operation into core device */
ocf_submit_volume_req(&req->core->volume, req,
_ocf_engine_flush_complete);
ocf_engine_forward_core_flush_req(req, _ocf_engine_flush_complete);
/* submit flush to cache device */
ocf_submit_cache_flush(req, _ocf_engine_flush_complete);
ocf_engine_forward_cache_flush_req(req, _ocf_engine_flush_complete);
/* Put OCF request - decrease reference counter */
ocf_req_put(req);
return 0;
}

18
src/engine/engine_inv.c
View File

@ -19,18 +19,12 @@
static void _ocf_invalidate_req(struct ocf_request *req, int error)
{
if (error) {
req->error = error;
ocf_core_stats_cache_error_update(req->core, OCF_WRITE);
}
if (env_atomic_dec_return(&req->req_remaining))
return;
OCF_DEBUG_RQ(req, "Completion");
if (req->error)
if (error) {
ocf_core_stats_cache_error_update(req->core, OCF_WRITE);
ocf_engine_error(req, true, "Failed to flush metadata to cache");
}
ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req);
@ -48,16 +42,14 @@ static int _ocf_invalidate_do(struct ocf_request *req)
ocf_purge_map_info(req);
ocf_hb_req_prot_unlock_wr(req);
env_atomic_inc(&req->req_remaining);
if (ocf_volume_is_atomic(&cache->device->volume) &&
req->info.flush_metadata) {
/* Metadata flush IO */
ocf_metadata_flush_do_asynch(cache, req, _ocf_invalidate_req);
} else {
_ocf_invalidate_req(req, 0);
}
_ocf_invalidate_req(req, 0);
return 0;
}

16
src/engine/engine_pt.c
View File

@ -8,9 +8,9 @@
#include "engine_pt.h"
#include "engine_rd.h"
#include "engine_common.h"
#include "engine_io.h"
#include "cache_engine.h"
#include "../ocf_request.h"
#include "../utils/utils_io.h"
#include "../utils/utils_user_part.h"
#include "../metadata/metadata.h"
#include "../concurrency/ocf_concurrency.h"
@ -20,19 +20,13 @@
static void _ocf_read_pt_complete(struct ocf_request *req, int error)
{
if (error)
req->error |= error;
if (env_atomic_dec_return(&req->req_remaining))
return;
OCF_DEBUG_RQ(req, "Completion");
if (req->error)
if (error)
ocf_core_stats_core_error_update(req->core, OCF_READ);
/* Complete request */
req->complete(req, req->error);
req->complete(req, error);
ocf_req_unlock(ocf_cache_line_concurrency(req->cache), req);
@ -42,12 +36,10 @@ static void _ocf_read_pt_complete(struct ocf_request *req, int error)
static inline void _ocf_read_pt_submit(struct ocf_request *req)
{
env_atomic_set(&req->req_remaining, 1); /* Core device IO */
OCF_DEBUG_RQ(req, "Submit");
/* Core read */
ocf_submit_volume_req(&req->core->volume, req, _ocf_read_pt_complete);
ocf_engine_forward_core_io_req(req, _ocf_read_pt_complete);
}
int ocf_read_pt_do(struct ocf_request *req)

37
src/engine/engine_rd.c
View File

@ -11,9 +11,9 @@
#include "engine_inv.h"
#include "engine_bf.h"
#include "engine_common.h"
#include "engine_io.h"
#include "cache_engine.h"
#include "../concurrency/ocf_concurrency.h"
#include "../utils/utils_io.h"
#include "../ocf_request.h"
#include "../utils/utils_cache_line.h"
#include "../utils/utils_user_part.h"
@ -31,19 +31,12 @@ static void _ocf_read_generic_hit_complete(struct ocf_request *req, int error)
OCF_DEBUG_RQ(req, "HIT completion");
if (error) {
req->error |= error;
ocf_core_stats_cache_error_update(req->core, OCF_READ);
inc_fallback_pt_error_counter(req->cache);
}
if (env_atomic_dec_return(&req->req_remaining) > 0)
return;
if (req->error) {
ocf_queue_push_req_pt(req);
} else {
ocf_req_unlock(c, req);
req->complete(req, req->error);
req->complete(req, error);
ocf_req_put(req);
}
}
@ -54,14 +47,9 @@ static void _ocf_read_generic_miss_complete(struct ocf_request *req, int error)
OCF_DEBUG_RQ(req, "MISS completion");
if (error)
req->error = error;
if (env_atomic_dec_return(&req->req_remaining) > 0)
return;
if (req->error) {
req->complete(req, req->error);
if (error) {
/* --- Do not backfill --- */
req->complete(req, error);
ocf_core_stats_core_error_update(req->core, OCF_READ);
@ -80,18 +68,15 @@ static void _ocf_read_generic_miss_complete(struct ocf_request *req, int error)
req->byte_length);
}
/* Complete request */
req->complete(req, req->error);
req->complete(req, error);
ocf_engine_backfill(req);
}
void ocf_read_generic_submit_hit(struct ocf_request *req)
{
env_atomic_set(&req->req_remaining, ocf_engine_io_count(req));
ocf_submit_cache_reqs(req->cache, req, OCF_READ, 0, req->byte_length,
ocf_engine_io_count(req), _ocf_read_generic_hit_complete);
ocf_engine_forward_cache_io_req(req, OCF_READ,
_ocf_read_generic_hit_complete);
}
static inline void _ocf_read_generic_submit_miss(struct ocf_request *req)
@ -99,8 +84,6 @@ static inline void _ocf_read_generic_submit_miss(struct ocf_request *req)
struct ocf_cache *cache = req->cache;
int ret;
env_atomic_set(&req->req_remaining, 1);
req->cp_data = ctx_data_alloc(cache->owner,
BYTES_TO_PAGES(req->byte_length));
if (!req->cp_data) {
@ -119,9 +102,7 @@ static inline void _ocf_read_generic_submit_miss(struct ocf_request *req)
}
err_alloc:
/* Submit read request to core device. */
ocf_submit_volume_req(&req->core->volume, req,
_ocf_read_generic_miss_complete);
ocf_engine_forward_core_io_req(req, _ocf_read_generic_miss_complete);
}
static int _ocf_read_generic_do(struct ocf_request *req)

3
src/engine/engine_wa.c
View File

@ -1,5 +1,6 @@
/*
* Copyright(c) 2012-2022 Intel Corporation
* Copyright(c) 2024 Huawei Technologies
* SPDX-License-Identifier: BSD-3-Clause
*/
#include "ocf/ocf.h"
@ -8,9 +9,9 @@
#include "engine_wt.h"
#include "engine_wi.h"
#include "engine_common.h"
#include "engine_io.h"
#include "cache_engine.h"
#include "../ocf_request.h"
#include "../utils/utils_io.h"
#include "../metadata/metadata.h"
#define OCF_ENGINE_DEBUG_IO_NAME "wa"

37
src/engine/engine_wb.c
View File

@ -8,12 +8,12 @@
#include "../ocf_cache_priv.h"
#include "cache_engine.h"
#include "engine_common.h"
#include "engine_io.h"
#include "engine_wb.h"
#include "engine_wi.h"
#include "engine_inv.h"
#include "../metadata/metadata.h"
#include "../ocf_request.h"
#include "../utils/utils_io.h"
#include "../utils/utils_cache_line.h"
#include "../utils/utils_request.h"
#include "../utils/utils_user_part.h"
@ -50,17 +50,11 @@ static void _ocf_write_wb_update_bits(struct ocf_request *req)
static void _ocf_write_wb_io_flush_metadata(struct ocf_request *req, int error)
{
if (error)
req->error = error;
if (env_atomic_dec_return(&req->req_remaining))
return;
if (req->error)
ocf_engine_error(req, true, "Failed to write data to cache");
ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req);
req->complete(req, req->error);
req->complete(req, error);
ocf_req_put(req);
}
@ -69,37 +63,29 @@ static int ocf_write_wb_do_flush_metadata(struct ocf_request *req)
{
struct ocf_cache *cache = req->cache;
env_atomic_set(&req->req_remaining, 1); /* One core IO */
_ocf_write_wb_update_bits(req);
if (req->info.flush_metadata) {
OCF_DEBUG_RQ(req, "Flush metadata");
ocf_metadata_flush_do_asynch(cache, req,
_ocf_write_wb_io_flush_metadata);
} else {
_ocf_write_wb_io_flush_metadata(req, 0);
}
_ocf_write_wb_io_flush_metadata(req, 0);
return 0;
}
static void _ocf_write_wb_complete(struct ocf_request *req, int error)
{
if (error) {
ocf_core_stats_cache_error_update(req->core, OCF_WRITE);
req->error |= error;
}
if (env_atomic_dec_return(&req->req_remaining))
return;
OCF_DEBUG_RQ(req, "Completion");
if (req->error) {
if (error) {
ocf_core_stats_cache_error_update(req->core, OCF_WRITE);
ocf_engine_error(req, true, "Failed to write data to cache");
req->complete(req, req->error);
req->complete(req, error);
ocf_engine_invalidate(req);
} else {
@ -111,10 +97,6 @@ static void _ocf_write_wb_complete(struct ocf_request *req, int error)
static inline void _ocf_write_wb_submit(struct ocf_request *req)
{
struct ocf_cache *cache = req->cache;
env_atomic_set(&req->req_remaining, ocf_engine_io_count(req));
/*
* 1. Submit data
* 2. Wait for completion of data
@ -137,8 +119,7 @@ static inline void _ocf_write_wb_submit(struct ocf_request *req)
OCF_DEBUG_RQ(req, "Submit Data");
/* Data IO */
ocf_submit_cache_reqs(cache, req, OCF_WRITE, 0, req->byte_length,
ocf_engine_io_count(req), _ocf_write_wb_complete);
ocf_engine_forward_cache_io_req(req, OCF_WRITE, _ocf_write_wb_complete);
}
int ocf_write_wb_do(struct ocf_request *req)

35
src/engine/engine_wi.c
View File

@ -8,10 +8,10 @@
#include "../ocf_cache_priv.h"
#include "engine_wi.h"
#include "engine_common.h"
#include "engine_io.h"
#include "../concurrency/ocf_concurrency.h"
#include "../ocf_request.h"
#include "../utils/utils_cache_line.h"
#include "../utils/utils_io.h"
#include "../metadata/metadata.h"
#define OCF_ENGINE_DEBUG_IO_NAME "wi"
@ -22,7 +22,7 @@ static int _ocf_write_wi_next_pass(struct ocf_request *req)
ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req);
if (req->wi_second_pass) {
req->complete(req, req->error);
req->complete(req, 0);
ocf_req_put(req);
return 0;
@ -48,25 +48,19 @@ static void _ocf_write_wi_io_flush_metadata(struct ocf_request *req, int error)
{
if (error) {
ocf_core_stats_cache_error_update(req->core, OCF_WRITE);
req->error |= error;
ocf_engine_error(req, true, "Failed to write data to cache");
}
if (env_atomic_dec_return(&req->req_remaining))
return;
if (!req->error && !req->wi_second_pass && ocf_engine_is_miss(req)) {
if (!error && !req->wi_second_pass && ocf_engine_is_miss(req)) {
/* need another pass */
ocf_queue_push_req_cb(req, _ocf_write_wi_next_pass,
OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH);
return;
}
if (req->error)
ocf_engine_error(req, true, "Failed to write data to cache");
ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req);
req->complete(req, req->error);
req->complete(req, error);
ocf_req_put(req);
}
@ -105,20 +99,14 @@ static int ocf_write_wi_update_and_flush_metadata(struct ocf_request *req)
static void _ocf_write_wi_core_complete(struct ocf_request *req, int error)
{
if (error) {
req->error = error;
ocf_core_stats_core_error_update(req->core, OCF_WRITE);
}
if (env_atomic_dec_return(&req->req_remaining))
return;
OCF_DEBUG_RQ(req, "Completion");
if (req->error) {
if (error) {
ocf_core_stats_core_error_update(req->core, OCF_WRITE);
ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req);
req->complete(req, req->error);
req->complete(req, error);
ocf_req_put(req);
} else {
@ -146,13 +134,10 @@ static int _ocf_write_wi_core_write(struct ocf_request *req)
return 0;
}
env_atomic_set(&req->req_remaining, 1); /* One core IO */
OCF_DEBUG_RQ(req, "Submit");
/* Submit write IO to the core */
ocf_submit_volume_req(&req->core->volume, req,
_ocf_write_wi_core_complete);
ocf_engine_forward_core_io_req(req, _ocf_write_wi_core_complete);
/* Update statistics */
ocf_engine_update_block_stats(req);

27
src/engine/engine_wo.c
View File

@ -9,10 +9,10 @@
#include "../ocf_cache_priv.h"
#include "cache_engine.h"
#include "engine_common.h"
#include "engine_io.h"
#include "engine_rd.h"
#include "engine_pt.h"
#include "../metadata/metadata.h"
#include "../utils/utils_io.h"
#include "../utils/utils_cache_line.h"
#include "../utils/utils_user_part.h"
#include "../concurrency/ocf_concurrency.h"
@ -23,13 +23,10 @@
static void ocf_read_wo_cache_complete(struct ocf_request *req, int error)
{
if (error) {
req->error = req->error ?: error;
ocf_core_stats_cache_error_update(req->core, OCF_READ);
req->error |= error;
}
if (env_atomic_dec_return(&req->req_remaining))
return;
OCF_DEBUG_RQ(req, "Completion");
if (req->error)
@ -48,8 +45,7 @@ static void ocf_read_wo_cache_io(struct ocf_request *req, uint64_t offset,
uint64_t size)
{
OCF_DEBUG_RQ(req, "Submit cache");
env_atomic_inc(&req->req_remaining);
ocf_submit_cache_reqs(req->cache, req, OCF_READ, offset, size, 1,
ocf_engine_forward_cache_io(req, OCF_READ, offset, size,
ocf_read_wo_cache_complete);
}
@ -66,7 +62,9 @@ static int ocf_read_wo_cache_do(struct ocf_request *req)
uint64_t offset = 0;
uint64_t increment = 0;
env_atomic_set(&req->req_remaining, 1);
req->cache_forward_end = ocf_read_wo_cache_complete;
ocf_req_forward_cache_get(req);
for (line = 0; line < req->core_line_count; ++line) {
entry = &req->map[line];
@ -147,23 +145,21 @@ static int ocf_read_wo_cache_do(struct ocf_request *req)
if (io)
ocf_read_wo_cache_io(req, io_start, offset - io_start);
ocf_read_wo_cache_complete(req, 0);
ocf_req_forward_cache_put(req);
return 0;
}
static void _ocf_read_wo_core_complete(struct ocf_request *req, int error)
{
if (error) {
req->error |= error;
if (error)
ocf_core_stats_core_error_update(req->core, OCF_READ);
}
/* if all mapped cachelines are clean, the data we've read from core
* is valid and we can complete the request */
if (!req->info.dirty_any || req->error) {
if (error || !req->info.dirty_any) {
OCF_DEBUG_RQ(req, "Completion");
req->complete(req, req->error);
req->complete(req, error);
ocf_req_unlock_rd(ocf_cache_line_concurrency(req->cache), req);
ocf_req_put(req);
return;
@ -191,8 +187,7 @@ static int ocf_read_wo_do(struct ocf_request *req)
} else {
OCF_DEBUG_RQ(req, "Submit core");
ocf_submit_volume_req(&req->core->volume, req,
_ocf_read_wo_core_complete);
ocf_engine_forward_core_io_req(req, _ocf_read_wo_core_complete);
}
ocf_engine_update_request_stats(req);

29
src/engine/engine_wt.c
View File

@ -10,8 +10,8 @@
#include "engine_wi.h"
#include "engine_inv.h"
#include "engine_common.h"
#include "engine_io.h"
#include "../ocf_request.h"
#include "../utils/utils_io.h"
#include "../utils/utils_cache_line.h"
#include "../utils/utils_user_part.h"
#include "../metadata/metadata.h"
@ -58,12 +58,6 @@ static void _ocf_write_wt_do_flush_metadata_compl(struct ocf_request *req,
int error)
{
if (error)
req->error = error;
if (env_atomic_dec_return(&req->req_remaining))
return;
if (req->error)
ocf_engine_error(req, true, "Failed to write data to cache");
ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req);
@ -77,8 +71,6 @@ static int ocf_write_wt_do_flush_metadata(struct ocf_request *req)
{
struct ocf_cache *cache = req->cache;
env_atomic_set(&req->req_remaining, 1);
_ocf_write_wt_update_bits(req);
if (req->info.flush_metadata) {
@ -86,10 +78,10 @@ static int ocf_write_wt_do_flush_metadata(struct ocf_request *req)
ocf_metadata_flush_do_asynch(cache, req,
_ocf_write_wt_do_flush_metadata_compl);
} else {
_ocf_write_wt_do_flush_metadata_compl(req, 0);
}
_ocf_write_wt_do_flush_metadata_compl(req, 0);
return 0;
}
@ -136,8 +128,8 @@ static void _ocf_write_wt_cache_complete(struct ocf_request *req, int error)
static void _ocf_write_wt_core_complete(struct ocf_request *req, int error)
{
if (error) {
req->error = error;
req->info.core_error = 1;
req->error = error;
ocf_core_stats_core_error_update(req->core, OCF_WRITE);
}
@ -146,22 +138,17 @@ static void _ocf_write_wt_core_complete(struct ocf_request *req, int error)
static inline void _ocf_write_wt_submit(struct ocf_request *req)
{
struct ocf_cache *cache = req->cache;
/* Submit IOs */
OCF_DEBUG_RQ(req, "Submit");
/* Calculate how many IOs need to be submited */
env_atomic_set(&req->req_remaining, ocf_engine_io_count(req)); /* Cache IO */
env_atomic_inc(&req->req_remaining); /* Core device IO */
env_atomic_set(&req->req_remaining, 2); /* cache IO + core IO */
/* To cache */
ocf_submit_cache_reqs(cache, req, OCF_WRITE, 0, req->byte_length,
ocf_engine_io_count(req), _ocf_write_wt_cache_complete);
ocf_engine_forward_cache_io_req(req, OCF_WRITE,
_ocf_write_wt_cache_complete);
/* To core */
ocf_submit_volume_req(&req->core->volume, req,
_ocf_write_wt_core_complete);
ocf_engine_forward_core_io_req(req, _ocf_write_wt_core_complete);
}
static int _ocf_write_wt_do(struct ocf_request *req)

26
src/engine/engine_zero.c
View File

@ -19,23 +19,17 @@
static int ocf_zero_purge(struct ocf_request *req)
{
if (req->error) {
ocf_engine_error(req, true, "Failed to discard data on cache");
} else {
/* There are mapped cache line, need to remove them */
/* There are mapped cache line, need to remove them */
ocf_hb_req_prot_lock_wr(req); /*- Metadata WR access ---------------*/
ocf_hb_req_prot_lock_wr(req); /*- Metadata WR access ---------------*/
/* Remove mapped cache lines from metadata */
ocf_purge_map_info(req);
/* Remove mapped cache lines from metadata */
ocf_purge_map_info(req);
ocf_hb_req_prot_unlock_wr(req); /*- END Metadata WR access ---------*/
}
ocf_hb_req_prot_unlock_wr(req); /*- END Metadata WR access ---------*/
ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req);
req->complete(req, req->error);
req->complete(req, 0);
ocf_req_put(req);
return 0;
@ -45,11 +39,13 @@ static void _ocf_zero_io_flush_metadata(struct ocf_request *req, int error)
{
if (error) {
ocf_core_stats_cache_error_update(req->core, OCF_WRITE);
req->error = error;
}
ocf_engine_error(req, true, "Failed to discard data on cache");
if (env_atomic_dec_return(&req->req_remaining))
ocf_req_unlock_wr(ocf_cache_line_concurrency(req->cache), req);
req->complete(req, error);
ocf_req_put(req);
return;
}
ocf_queue_push_req_cb(req, ocf_zero_purge,
OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH);

4
src/ocf_cache.c
View File

@ -320,9 +320,13 @@ static void ocf_cache_io_complete(struct ocf_io *io, int error)
{
struct ocf_cache_volume_io_priv *priv;
ocf_cache_t cache;
struct ocf_request *req = ocf_io_to_req(io);
cache = ocf_volume_to_cache(ocf_io_get_volume(io));
if (error)
req->error = req->error ?: error;
priv = ocf_io_get_priv(io);
env_atomic_cmpxchg(&priv->error, 0, error);