diff --git a/inc/ocf_def.h b/inc/ocf_def.h index 24503b7..0fa37da 100644 --- a/inc/ocf_def.h +++ b/inc/ocf_def.h @@ -162,6 +162,9 @@ typedef enum { ocf_cache_mode_wi, /*!< Write invalidate cache mode */ + ocf_cache_mode_wo, + /*!< Write-only cache mode */ + ocf_cache_mode_max, /*!< Stopper of cache mode enumerator */ diff --git a/src/engine/cache_engine.c b/src/engine/cache_engine.c index 2cbad43..1e0bdda 100644 --- a/src/engine/cache_engine.c +++ b/src/engine/cache_engine.c @@ -15,6 +15,7 @@ #include "engine_wi.h" #include "engine_wa.h" #include "engine_wb.h" +#include "engine_wo.h" #include "engine_fast.h" #include "engine_discard.h" #include "engine_d2c.h" @@ -32,6 +33,7 @@ enum ocf_io_if_type { OCF_IO_WA_IF, OCF_IO_WI_IF, OCF_IO_PT_IF, + OCF_IO_WO_IF, OCF_IO_MAX_IF, /* Private OCF interfaces */ @@ -68,6 +70,11 @@ static const struct ocf_io_if IO_IFS[OCF_IO_PRIV_MAX_IF] = { .write = ocf_write_wi, .name = "Pass Through", }, + [OCF_IO_WO_IF] = { + .read = ocf_read_wo, + .write = ocf_write_wb, + .name = "Write Only", + }, [OCF_IO_FAST_IF] = { .read = ocf_read_fast, .write = ocf_write_fast, @@ -95,6 +102,7 @@ static const struct ocf_io_if *cache_mode_io_if_map[ocf_req_cache_mode_max] = { [ocf_req_cache_mode_wb] = &IO_IFS[OCF_IO_WB_IF], [ocf_req_cache_mode_wa] = &IO_IFS[OCF_IO_WA_IF], [ocf_req_cache_mode_wi] = &IO_IFS[OCF_IO_WI_IF], + [ocf_req_cache_mode_wo] = &IO_IFS[OCF_IO_WO_IF], [ocf_req_cache_mode_pt] = &IO_IFS[OCF_IO_PT_IF], [ocf_req_cache_mode_fast] = &IO_IFS[OCF_IO_FAST_IF], [ocf_req_cache_mode_d2c] = &IO_IFS[OCF_IO_D2C_IF], diff --git a/src/engine/cache_engine.h b/src/engine/cache_engine.h index 63707b6..d31fe44 100644 --- a/src/engine/cache_engine.h +++ b/src/engine/cache_engine.h @@ -20,6 +20,7 @@ typedef enum { ocf_req_cache_mode_wa = ocf_cache_mode_wa, ocf_req_cache_mode_pt = ocf_cache_mode_pt, ocf_req_cache_mode_wi = ocf_cache_mode_wi, + ocf_req_cache_mode_wo = ocf_cache_mode_wo, /* internal modes */ ocf_req_cache_mode_fast, diff --git a/src/engine/engine_bf.c b/src/engine/engine_bf.c index f045b72..865b793 100644 --- a/src/engine/engine_bf.c +++ b/src/engine/engine_bf.c @@ -84,8 +84,8 @@ static int _ocf_backfill_do(struct ocf_request *req) req->data = req->cp_data; - ocf_submit_cache_reqs(req->cache, req->map, req, OCF_WRITE, reqs_to_issue, - _ocf_backfill_complete); + ocf_submit_cache_reqs(req->cache, req, OCF_WRITE, 0, req->byte_length, + reqs_to_issue, _ocf_backfill_complete); return 0; } diff --git a/src/engine/engine_common.h b/src/engine/engine_common.h index 474a0e7..b581da5 100644 --- a/src/engine/engine_common.h +++ b/src/engine/engine_common.h @@ -7,6 +7,7 @@ #define ENGINE_COMMON_H_ #include "../ocf_request.h" +#include "../utils/utils_cache_line.h" /** * @file engine_common.h @@ -109,6 +110,37 @@ static inline uint32_t ocf_engine_io_count(struct ocf_request *req) return req->info.seq_req ? 1 : req->core_line_count; } +static inline +bool ocf_engine_map_all_sec_dirty(struct ocf_request *req, uint32_t line) +{ + uint8_t start = ocf_map_line_start_sector(req, line); + uint8_t end = ocf_map_line_end_sector(req, line); + + if (req->map[line].status != LOOKUP_HIT) + return false; + + return metadata_test_dirty_all_sec(req->cache, req->map[line].coll_idx, + start, end); +} + +static inline +bool ocf_engine_map_all_sec_clean(struct ocf_request *req, uint32_t line) +{ + uint8_t start = ocf_map_line_start_sector(req, line); + uint8_t end = ocf_map_line_end_sector(req, line); + + if (req->map[line].status != LOOKUP_HIT) + return false; + + if (!metadata_test_valid_sec(req->cache, req->map[line].coll_idx, + start, end)) { + return false; + } + + return !metadata_test_dirty_sec(req->cache, req->map[line].coll_idx, + start, end); +} + /** * @brief Clean request (flush dirty data to the core device) * diff --git a/src/engine/engine_fast.c b/src/engine/engine_fast.c index 0bce983..971e410 100644 --- a/src/engine/engine_fast.c +++ b/src/engine/engine_fast.c @@ -87,7 +87,7 @@ static int _ocf_read_fast_do(struct ocf_request *req) /* Submit IO */ OCF_DEBUG_RQ(req, "Submit"); env_atomic_set(&req->req_remaining, ocf_engine_io_count(req)); - ocf_submit_cache_reqs(req->cache, req->map, req, OCF_READ, + ocf_submit_cache_reqs(req->cache, req, OCF_READ, 0, req->byte_length, ocf_engine_io_count(req), _ocf_read_fast_complete); diff --git a/src/engine/engine_ops.c b/src/engine/engine_ops.c index 4ce0ef6..a56b242 100644 --- a/src/engine/engine_ops.c +++ b/src/engine/engine_ops.c @@ -51,7 +51,7 @@ int ocf_engine_ops(struct ocf_request *req) ocf_submit_volume_req(&req->core->volume, req, _ocf_engine_ops_complete); - ocf_submit_cache_reqs(cache, req->map, req, req->rw, + ocf_submit_cache_reqs(cache, req, req->rw, 0, req->byte_length, 1, _ocf_engine_ops_complete); /* Put OCF request - decrease reference counter */ diff --git a/src/engine/engine_rd.c b/src/engine/engine_rd.c index 97994f3..ed9a5aa 100644 --- a/src/engine/engine_rd.c +++ b/src/engine/engine_rd.c @@ -105,7 +105,7 @@ static inline void _ocf_read_generic_submit_hit(struct ocf_request *req) { env_atomic_set(&req->req_remaining, ocf_engine_io_count(req)); - ocf_submit_cache_reqs(req->cache, req->map, req, OCF_READ, + ocf_submit_cache_reqs(req->cache, req, OCF_READ, 0, req->byte_length, ocf_engine_io_count(req), _ocf_read_generic_hit_complete); } @@ -135,7 +135,7 @@ err_alloc: _ocf_read_generic_miss_complete(req, -OCF_ERR_NO_MEM); } -static int _ocf_read_generic_do(struct ocf_request *req) +int ocf_read_generic_do(struct ocf_request *req) { struct ocf_cache *cache = req->cache; @@ -195,7 +195,7 @@ static int _ocf_read_generic_do(struct ocf_request *req) else _ocf_read_generic_submit_miss(req); - /* Updata statistics */ + /* Update statistics */ ocf_engine_update_request_stats(req); ocf_engine_update_block_stats(req); @@ -206,8 +206,8 @@ static int _ocf_read_generic_do(struct ocf_request *req) } static const struct ocf_io_if _io_if_read_generic_resume = { - .read = _ocf_read_generic_do, - .write = _ocf_read_generic_do, + .read = ocf_read_generic_do, + .write = ocf_read_generic_do, .resume = ocf_engine_on_resume, }; @@ -294,7 +294,7 @@ int ocf_read_generic(struct ocf_request *req) OCF_DEBUG_RQ(req, "NO LOCK"); } else { /* Lock was acquired can perform IO */ - _ocf_read_generic_do(req); + ocf_read_generic_do(req); } } else { OCF_DEBUG_RQ(req, "LOCK ERROR %d", lock); diff --git a/src/engine/engine_rd.h b/src/engine/engine_rd.h index b5cec2b..4881bf3 100644 --- a/src/engine/engine_rd.h +++ b/src/engine/engine_rd.h @@ -8,4 +8,6 @@ int ocf_read_generic(struct ocf_request *req); +int ocf_read_generic_do(struct ocf_request *req); + #endif /* ENGINE_RD_H_ */ diff --git a/src/engine/engine_wb.c b/src/engine/engine_wb.c index 7838415..d7a8c08 100644 --- a/src/engine/engine_wb.c +++ b/src/engine/engine_wb.c @@ -141,7 +141,7 @@ static inline void _ocf_write_wb_submit(struct ocf_request *req) OCF_DEBUG_RQ(req, "Submit Data"); /* Data IO */ - ocf_submit_cache_reqs(cache, req->map, req, OCF_WRITE, + ocf_submit_cache_reqs(cache, req, OCF_WRITE, 0, req->byte_length, ocf_engine_io_count(req), _ocf_write_wb_complete); } diff --git a/src/engine/engine_wo.c b/src/engine/engine_wo.c new file mode 100644 index 0000000..2e3be70 --- /dev/null +++ b/src/engine/engine_wo.c @@ -0,0 +1,258 @@ +/* + * Copyright(c) 2019 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause-Clear + */ + +#include "ocf/ocf.h" +#include "../ocf_cache_priv.h" +#include "cache_engine.h" +#include "engine_common.h" +#include "engine_rd.h" +#include "engine_pt.h" +#include "../metadata/metadata.h" +#include "../utils/utils_io.h" +#include "../utils/utils_cache_line.h" +#include "../utils/utils_part.h" +#include "../concurrency/ocf_concurrency.h" + +#define OCF_ENGINE_DEBUG_IO_NAME "wo" +#include "engine_debug.h" + +static void ocf_read_wo_cache_complete(struct ocf_request *req, int error) +{ + if (error) { + env_atomic_inc(&req->core->counters->cache_errors.read); + req->error |= error; + } + + if (env_atomic_dec_return(&req->req_remaining)) + return; + + OCF_DEBUG_RQ(req, "Completion"); + + if (req->error) + ocf_engine_error(req, true, "Failed to read data from cache"); + + ocf_req_unlock_rd(req); + + /* Complete request */ + req->complete(req, req->error); + + /* Release OCF request */ + ocf_req_put(req); +} + +static void ocf_read_wo_cache_io(struct ocf_request *req, uint64_t offset, + uint64_t size) +{ + OCF_DEBUG_RQ(req, "Submit cache"); + env_atomic_inc(&req->req_remaining); + ocf_submit_cache_reqs(req->cache, req, OCF_READ, offset, size, 1, + ocf_read_wo_cache_complete); +} + +static int ocf_read_wo_cache_do(struct ocf_request *req) +{ + ocf_cache_t cache = req->cache; + uint32_t s, e, i; + uint64_t line; + struct ocf_map_info *entry; + bool dirty = false; + bool io = false; + uint64_t phys_prev, phys_curr = 0; + uint64_t io_start = 0; + uint64_t offset = 0; + uint64_t increment; + + env_atomic_set(&req->req_remaining, 1); + + for (line = 0; line < req->core_line_count; ++line) { + entry = &req->map[line]; + s = ocf_map_line_start_sector(req, line); + e = ocf_map_line_end_sector(req, line); + + /* if cacheline mapping is not sequential, send cache IO to + * previous cacheline(s) */ + phys_prev = phys_curr; + if (entry->status != LOOKUP_MISS) + phys_curr = ocf_metadata_map_lg2phy(cache, + entry->coll_idx); + if (io && phys_prev + 1 != phys_curr) { + ocf_read_wo_cache_io(req, io_start, offset - io_start); + io = false; + } + + /* try to seek directly to the last sector */ + if (entry->status == LOOKUP_MISS || + ocf_engine_map_all_sec_clean(req, line)) { + /* all sectors invalid or clean */ + i = e + 1; + increment = SECTORS_TO_BYTES(e - s + 1); + dirty = false; + } + else if (ocf_engine_map_all_sec_dirty(req, line)) { + /* all sectors dirty */ + i = e + 1; + increment = SECTORS_TO_BYTES(e - s + 1); + dirty = true; + } else { + /* need to iterate through CL sector by sector */ + i = s; + } + + do { + if (i <= e) { + dirty = metadata_test_dirty_one(cache, + entry->coll_idx, i); + increment = 0; + do { + ++i; + increment += SECTORS_TO_BYTES(1); + } while (i <= e && metadata_test_dirty_one( + cache, entry->coll_idx, i) + == dirty); + } + + if (io && !dirty) { + /* end of sequential dirty region */ + ocf_read_wo_cache_io(req, io_start, + offset - io_start); + io = false; + } + + if (!io && dirty) { + /* beginning of sequential dirty region */ + io = true; + io_start = offset; + } + + offset += increment; + } while (i <= e); + } + + if (io) + ocf_read_wo_cache_io(req, io_start, offset - io_start); + + ocf_read_wo_cache_complete(req, 0); + + return 0; +} + +static const struct ocf_io_if _io_if_wo_cache_read = { + .read = ocf_read_wo_cache_do, + .write = ocf_read_wo_cache_do, +}; + +static void _ocf_read_wo_core_complete(struct ocf_request *req, int error) +{ + if (error) { + req->error |= error; + req->info.core_error = 1; + env_atomic_inc(&req->core->counters->core_errors.read); + } + + /* if all mapped cachelines are clean, the data we've read from core + * is valid and we can complete the request */ + if (!req->info.dirty_any || req->error) { + OCF_DEBUG_RQ(req, "Completion"); + req->complete(req, req->error); + ocf_req_unlock_rd(req); + ocf_req_put(req); + return; + } + + req->io_if = &_io_if_wo_cache_read; + ocf_engine_push_req_front(req, true); +} + +int ocf_read_wo_do(struct ocf_request *req) +{ + ocf_cache_t cache = req->cache; + + if (ocf_engine_is_hit(req)) { + /* read hit - just fetch the data from cache using standard read path + */ + ocf_read_generic_do(req); + return 0; + } + + ocf_req_get(req); + + if (req->info.re_part) { + OCF_DEBUG_RQ(req, "Re-Part"); + + OCF_METADATA_LOCK_WR(); + + /* Probably some cache lines are assigned into wrong + * partition. Need to move it to new one + */ + ocf_part_move(req); + + OCF_METADATA_UNLOCK_WR(); + } + + OCF_DEBUG_RQ(req, "Submit core"); + ocf_submit_volume_req(&req->core->volume, req, _ocf_read_wo_core_complete); + + ocf_engine_update_request_stats(req); + ocf_engine_update_block_stats(req); + + ocf_req_put(req); + + return 0; +} + +static const struct ocf_io_if _io_if_wo_resume = { + .read = ocf_read_wo_do, + .write = ocf_read_wo_do, + .resume = ocf_engine_on_resume, +}; + +int ocf_read_wo(struct ocf_request *req) +{ + ocf_cache_t cache = req->cache; + int lock = OCF_LOCK_ACQUIRED; + + OCF_DEBUG_TRACE(req->cache); + + ocf_io_start(req->io); + + /* Get OCF request - increase reference counter */ + ocf_req_get(req); + + /* Set resume call backs */ + req->io_if = &_io_if_wo_resume; + + OCF_METADATA_LOCK_RD(); /*- Metadata RD access -----------------------*/ + + /* Traverse request to check if there are mapped cache lines */ + ocf_engine_traverse(req); + + if (ocf_engine_mapped_count(req)) { + /* There are mapped cache lines, + * lock request for READ access + */ + lock = ocf_req_trylock_rd(req); + } + + OCF_METADATA_UNLOCK_RD(); /*- END Metadata RD access -----------------*/ + + if (lock >= 0) { + if (lock != OCF_LOCK_ACQUIRED) { + /* Lock was not acquired, need to wait for resume */ + OCF_DEBUG_RQ(req, "NO LOCK"); + } else { + /* Lock was acquired can perform IO */ + ocf_read_wo_do(req); + } + } else { + OCF_DEBUG_RQ(req, "LOCK ERROR %d", lock); + req->complete(req, lock); + ocf_req_put(req); + } + + /* Put OCF request - decrease reference counter */ + ocf_req_put(req); + + return 0; +} diff --git a/src/engine/engine_wo.h b/src/engine/engine_wo.h new file mode 100644 index 0000000..2ae7f29 --- /dev/null +++ b/src/engine/engine_wo.h @@ -0,0 +1,11 @@ +/* + * Copyright(c) 2019 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause-Clear + */ + +#ifndef ENGINE_WO_H_ +#define ENGINE_WO_H_ + +int ocf_read_wo(struct ocf_request *req); + +#endif /* ENGINE_WO_H_ */ diff --git a/src/engine/engine_wt.c b/src/engine/engine_wt.c index df85901..a5be349 100644 --- a/src/engine/engine_wt.c +++ b/src/engine/engine_wt.c @@ -87,7 +87,7 @@ static inline void _ocf_write_wt_submit(struct ocf_request *req) } /* To cache */ - ocf_submit_cache_reqs(cache, req->map, req, OCF_WRITE, + ocf_submit_cache_reqs(cache, req, OCF_WRITE, 0, req->byte_length, ocf_engine_io_count(req), _ocf_write_wt_cache_complete); /* To core */ @@ -145,7 +145,7 @@ static int _ocf_write_wt_do(struct ocf_request *req) /* Submit IO */ _ocf_write_wt_submit(req); - /* Updata statistics */ + /* Update statistics */ ocf_engine_update_request_stats(req); ocf_engine_update_block_stats(req); diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index e195608..d95668d 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -1600,6 +1600,7 @@ static const char *_ocf_cache_mode_names[ocf_cache_mode_max] = { [ocf_cache_mode_wa] = "wa", [ocf_cache_mode_pt] = "pt", [ocf_cache_mode_wi] = "wi", + [ocf_cache_mode_wo] = "wo", }; static const char *_ocf_cache_mode_get_name(ocf_cache_mode_t cache_mode) diff --git a/src/ocf_request.h b/src/ocf_request.h index 0ccbc61..44590b2 100644 --- a/src/ocf_request.h +++ b/src/ocf_request.h @@ -144,7 +144,7 @@ struct ocf_request { /*!< Copy of request data */ uint64_t byte_position; - /*!< LBA byte position of request in code domain */ + /*!< LBA byte position of request in core domain */ uint64_t core_line_first; /*! First core line */ diff --git a/src/utils/utils_io.c b/src/utils/utils_io.c index 51210fd..719fd22 100644 --- a/src/utils/utils_io.c +++ b/src/utils/utils_io.c @@ -226,16 +226,22 @@ static void ocf_submit_volume_req_cmpl(struct ocf_io *io, int error) } void ocf_submit_cache_reqs(struct ocf_cache *cache, - struct ocf_map_info *map_info, struct ocf_request *req, int dir, - unsigned int reqs, ocf_req_end_t callback) + struct ocf_request *req, int dir, uint64_t offset, + uint64_t size, unsigned int reqs, ocf_req_end_t callback) { struct ocf_counters_block *cache_stats; uint64_t flags = req->io ? req->io->flags : 0; uint32_t class = req->io ? req->io->io_class : 0; uint64_t addr, bytes, total_bytes = 0; struct ocf_io *io; - uint32_t i; int err; + uint32_t i; + uint32_t entry = ocf_bytes_2_lines(cache, req->byte_position + offset) - + ocf_bytes_2_lines(cache, req->byte_position); + struct ocf_map_info *map_info = &req->map[entry]; + + ENV_BUG_ON(req->byte_length < offset + size); + ENV_BUG_ON(entry + reqs > req->core_line_count); cache_stats = &req->core->counters->cache_blocks; @@ -250,14 +256,14 @@ void ocf_submit_cache_reqs(struct ocf_cache *cache, map_info[0].coll_idx); addr *= ocf_line_size(cache); addr += cache->device->metadata_offset; - addr += (req->byte_position % ocf_line_size(cache)); - bytes = req->byte_length; + addr += ((req->byte_position + offset) % ocf_line_size(cache)); + bytes = size; ocf_io_configure(io, addr, bytes, dir, class, flags); ocf_io_set_queue(io, req->io_queue); ocf_io_set_cmpl(io, req, callback, ocf_submit_volume_req_cmpl); - err = ocf_io_set_data(io, req->data, 0); + err = ocf_io_set_data(io, req->data, offset); if (err) { ocf_io_put(io); callback(req, err); @@ -265,7 +271,7 @@ void ocf_submit_cache_reqs(struct ocf_cache *cache, } ocf_volume_submit_io(io); - total_bytes = req->byte_length; + total_bytes = bytes; goto update_stats; } @@ -288,24 +294,27 @@ void ocf_submit_cache_reqs(struct ocf_cache *cache, bytes = ocf_line_size(cache); if (i == 0) { - uint64_t seek = (req->byte_position % + uint64_t seek = ((req->byte_position + offset) % ocf_line_size(cache)); addr += seek; bytes -= seek; } else if (i == (reqs - 1)) { uint64_t skip = (ocf_line_size(cache) - - ((req->byte_position + req->byte_length) % + ((req->byte_position + offset + size) % ocf_line_size(cache))) % ocf_line_size(cache); bytes -= skip; } + bytes = OCF_MIN(bytes, size - total_bytes); + ENV_BUG_ON(bytes == 0); + ocf_io_configure(io, addr, bytes, dir, class, flags); ocf_io_set_queue(io, req->io_queue); ocf_io_set_cmpl(io, req, callback, ocf_submit_volume_req_cmpl); - err = ocf_io_set_data(io, req->data, total_bytes); + err = ocf_io_set_data(io, req->data, offset + total_bytes); if (err) { ocf_io_put(io); /* Finish all IOs which left with ERROR */ @@ -317,6 +326,8 @@ void ocf_submit_cache_reqs(struct ocf_cache *cache, total_bytes += bytes; } + ENV_BUG_ON(total_bytes != size); + update_stats: if (dir == OCF_WRITE) env_atomic64_add(total_bytes, &cache_stats->write_bytes); diff --git a/src/utils/utils_io.h b/src/utils/utils_io.h index 2a31f16..b1a5d47 100644 --- a/src/utils/utils_io.h +++ b/src/utils/utils_io.h @@ -60,10 +60,9 @@ void ocf_submit_cache_page(ocf_cache_t cache, uint64_t addr, int dir, void ocf_submit_volume_req(ocf_volume_t volume, struct ocf_request *req, ocf_req_end_t callback); - void ocf_submit_cache_reqs(struct ocf_cache *cache, - struct ocf_map_info *map_info, struct ocf_request *req, int dir, - unsigned int reqs, ocf_req_end_t callback); + struct ocf_request *req, int dir, uint64_t offset, + uint64_t size, unsigned int reqs, ocf_req_end_t callback); static inline struct ocf_io *ocf_new_cache_io(struct ocf_cache *cache) {