From 82e8c55f4a33cffd776543df1db84ffa086c3997 Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Mon, 27 May 2019 14:43:08 -0400 Subject: [PATCH] Write-only cache mode Write-only cache mode is similar to writeback, however read operations do not promote data to cache. Reads are mostly serviced by the core device, only dirty sectors are fetched from the cache. Signed-off-by: Adam Rutkowski --- inc/ocf_def.h | 3 + src/engine/cache_engine.c | 8 ++ src/engine/cache_engine.h | 1 + src/engine/engine_bf.c | 4 +- src/engine/engine_common.h | 32 +++++ src/engine/engine_fast.c | 2 +- src/engine/engine_ops.c | 2 +- src/engine/engine_rd.c | 12 +- src/engine/engine_rd.h | 2 + src/engine/engine_wb.c | 2 +- src/engine/engine_wo.c | 258 +++++++++++++++++++++++++++++++++++++ src/engine/engine_wo.h | 11 ++ src/engine/engine_wt.c | 4 +- src/mngt/ocf_mngt_cache.c | 1 + src/ocf_request.h | 2 +- src/utils/utils_io.c | 31 +++-- src/utils/utils_io.h | 5 +- 17 files changed, 353 insertions(+), 27 deletions(-) create mode 100644 src/engine/engine_wo.c create mode 100644 src/engine/engine_wo.h diff --git a/inc/ocf_def.h b/inc/ocf_def.h index 24503b7..0fa37da 100644 --- a/inc/ocf_def.h +++ b/inc/ocf_def.h @@ -162,6 +162,9 @@ typedef enum { ocf_cache_mode_wi, /*!< Write invalidate cache mode */ + ocf_cache_mode_wo, + /*!< Write-only cache mode */ + ocf_cache_mode_max, /*!< Stopper of cache mode enumerator */ diff --git a/src/engine/cache_engine.c b/src/engine/cache_engine.c index 2cbad43..1e0bdda 100644 --- a/src/engine/cache_engine.c +++ b/src/engine/cache_engine.c @@ -15,6 +15,7 @@ #include "engine_wi.h" #include "engine_wa.h" #include "engine_wb.h" +#include "engine_wo.h" #include "engine_fast.h" #include "engine_discard.h" #include "engine_d2c.h" @@ -32,6 +33,7 @@ enum ocf_io_if_type { OCF_IO_WA_IF, OCF_IO_WI_IF, OCF_IO_PT_IF, + OCF_IO_WO_IF, OCF_IO_MAX_IF, /* Private OCF interfaces */ @@ -68,6 +70,11 @@ static const struct ocf_io_if IO_IFS[OCF_IO_PRIV_MAX_IF] = { .write = ocf_write_wi, .name = "Pass Through", }, + [OCF_IO_WO_IF] = { + .read = ocf_read_wo, + .write = ocf_write_wb, + .name = "Write Only", + }, [OCF_IO_FAST_IF] = { .read = ocf_read_fast, .write = ocf_write_fast, @@ -95,6 +102,7 @@ static const struct ocf_io_if *cache_mode_io_if_map[ocf_req_cache_mode_max] = { [ocf_req_cache_mode_wb] = &IO_IFS[OCF_IO_WB_IF], [ocf_req_cache_mode_wa] = &IO_IFS[OCF_IO_WA_IF], [ocf_req_cache_mode_wi] = &IO_IFS[OCF_IO_WI_IF], + [ocf_req_cache_mode_wo] = &IO_IFS[OCF_IO_WO_IF], [ocf_req_cache_mode_pt] = &IO_IFS[OCF_IO_PT_IF], [ocf_req_cache_mode_fast] = &IO_IFS[OCF_IO_FAST_IF], [ocf_req_cache_mode_d2c] = &IO_IFS[OCF_IO_D2C_IF], diff --git a/src/engine/cache_engine.h b/src/engine/cache_engine.h index 63707b6..d31fe44 100644 --- a/src/engine/cache_engine.h +++ b/src/engine/cache_engine.h @@ -20,6 +20,7 @@ typedef enum { ocf_req_cache_mode_wa = ocf_cache_mode_wa, ocf_req_cache_mode_pt = ocf_cache_mode_pt, ocf_req_cache_mode_wi = ocf_cache_mode_wi, + ocf_req_cache_mode_wo = ocf_cache_mode_wo, /* internal modes */ ocf_req_cache_mode_fast, diff --git a/src/engine/engine_bf.c b/src/engine/engine_bf.c index f045b72..865b793 100644 --- a/src/engine/engine_bf.c +++ b/src/engine/engine_bf.c @@ -84,8 +84,8 @@ static int _ocf_backfill_do(struct ocf_request *req) req->data = req->cp_data; - ocf_submit_cache_reqs(req->cache, req->map, req, OCF_WRITE, reqs_to_issue, - _ocf_backfill_complete); + ocf_submit_cache_reqs(req->cache, req, OCF_WRITE, 0, req->byte_length, + reqs_to_issue, _ocf_backfill_complete); return 0; } diff --git a/src/engine/engine_common.h b/src/engine/engine_common.h index 474a0e7..b581da5 100644 --- a/src/engine/engine_common.h +++ b/src/engine/engine_common.h @@ -7,6 +7,7 @@ #define ENGINE_COMMON_H_ #include "../ocf_request.h" +#include "../utils/utils_cache_line.h" /** * @file engine_common.h @@ -109,6 +110,37 @@ static inline uint32_t ocf_engine_io_count(struct ocf_request *req) return req->info.seq_req ? 1 : req->core_line_count; } +static inline +bool ocf_engine_map_all_sec_dirty(struct ocf_request *req, uint32_t line) +{ + uint8_t start = ocf_map_line_start_sector(req, line); + uint8_t end = ocf_map_line_end_sector(req, line); + + if (req->map[line].status != LOOKUP_HIT) + return false; + + return metadata_test_dirty_all_sec(req->cache, req->map[line].coll_idx, + start, end); +} + +static inline +bool ocf_engine_map_all_sec_clean(struct ocf_request *req, uint32_t line) +{ + uint8_t start = ocf_map_line_start_sector(req, line); + uint8_t end = ocf_map_line_end_sector(req, line); + + if (req->map[line].status != LOOKUP_HIT) + return false; + + if (!metadata_test_valid_sec(req->cache, req->map[line].coll_idx, + start, end)) { + return false; + } + + return !metadata_test_dirty_sec(req->cache, req->map[line].coll_idx, + start, end); +} + /** * @brief Clean request (flush dirty data to the core device) * diff --git a/src/engine/engine_fast.c b/src/engine/engine_fast.c index 0bce983..971e410 100644 --- a/src/engine/engine_fast.c +++ b/src/engine/engine_fast.c @@ -87,7 +87,7 @@ static int _ocf_read_fast_do(struct ocf_request *req) /* Submit IO */ OCF_DEBUG_RQ(req, "Submit"); env_atomic_set(&req->req_remaining, ocf_engine_io_count(req)); - ocf_submit_cache_reqs(req->cache, req->map, req, OCF_READ, + ocf_submit_cache_reqs(req->cache, req, OCF_READ, 0, req->byte_length, ocf_engine_io_count(req), _ocf_read_fast_complete); diff --git a/src/engine/engine_ops.c b/src/engine/engine_ops.c index 4ce0ef6..a56b242 100644 --- a/src/engine/engine_ops.c +++ b/src/engine/engine_ops.c @@ -51,7 +51,7 @@ int ocf_engine_ops(struct ocf_request *req) ocf_submit_volume_req(&req->core->volume, req, _ocf_engine_ops_complete); - ocf_submit_cache_reqs(cache, req->map, req, req->rw, + ocf_submit_cache_reqs(cache, req, req->rw, 0, req->byte_length, 1, _ocf_engine_ops_complete); /* Put OCF request - decrease reference counter */ diff --git a/src/engine/engine_rd.c b/src/engine/engine_rd.c index 97994f3..ed9a5aa 100644 --- a/src/engine/engine_rd.c +++ b/src/engine/engine_rd.c @@ -105,7 +105,7 @@ static inline void _ocf_read_generic_submit_hit(struct ocf_request *req) { env_atomic_set(&req->req_remaining, ocf_engine_io_count(req)); - ocf_submit_cache_reqs(req->cache, req->map, req, OCF_READ, + ocf_submit_cache_reqs(req->cache, req, OCF_READ, 0, req->byte_length, ocf_engine_io_count(req), _ocf_read_generic_hit_complete); } @@ -135,7 +135,7 @@ err_alloc: _ocf_read_generic_miss_complete(req, -OCF_ERR_NO_MEM); } -static int _ocf_read_generic_do(struct ocf_request *req) +int ocf_read_generic_do(struct ocf_request *req) { struct ocf_cache *cache = req->cache; @@ -195,7 +195,7 @@ static int _ocf_read_generic_do(struct ocf_request *req) else _ocf_read_generic_submit_miss(req); - /* Updata statistics */ + /* Update statistics */ ocf_engine_update_request_stats(req); ocf_engine_update_block_stats(req); @@ -206,8 +206,8 @@ static int _ocf_read_generic_do(struct ocf_request *req) } static const struct ocf_io_if _io_if_read_generic_resume = { - .read = _ocf_read_generic_do, - .write = _ocf_read_generic_do, + .read = ocf_read_generic_do, + .write = ocf_read_generic_do, .resume = ocf_engine_on_resume, }; @@ -294,7 +294,7 @@ int ocf_read_generic(struct ocf_request *req) OCF_DEBUG_RQ(req, "NO LOCK"); } else { /* Lock was acquired can perform IO */ - _ocf_read_generic_do(req); + ocf_read_generic_do(req); } } else { OCF_DEBUG_RQ(req, "LOCK ERROR %d", lock); diff --git a/src/engine/engine_rd.h b/src/engine/engine_rd.h index b5cec2b..4881bf3 100644 --- a/src/engine/engine_rd.h +++ b/src/engine/engine_rd.h @@ -8,4 +8,6 @@ int ocf_read_generic(struct ocf_request *req); +int ocf_read_generic_do(struct ocf_request *req); + #endif /* ENGINE_RD_H_ */ diff --git a/src/engine/engine_wb.c b/src/engine/engine_wb.c index 7838415..d7a8c08 100644 --- a/src/engine/engine_wb.c +++ b/src/engine/engine_wb.c @@ -141,7 +141,7 @@ static inline void _ocf_write_wb_submit(struct ocf_request *req) OCF_DEBUG_RQ(req, "Submit Data"); /* Data IO */ - ocf_submit_cache_reqs(cache, req->map, req, OCF_WRITE, + ocf_submit_cache_reqs(cache, req, OCF_WRITE, 0, req->byte_length, ocf_engine_io_count(req), _ocf_write_wb_complete); } diff --git a/src/engine/engine_wo.c b/src/engine/engine_wo.c new file mode 100644 index 0000000..2e3be70 --- /dev/null +++ b/src/engine/engine_wo.c @@ -0,0 +1,258 @@ +/* + * Copyright(c) 2019 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause-Clear + */ + +#include "ocf/ocf.h" +#include "../ocf_cache_priv.h" +#include "cache_engine.h" +#include "engine_common.h" +#include "engine_rd.h" +#include "engine_pt.h" +#include "../metadata/metadata.h" +#include "../utils/utils_io.h" +#include "../utils/utils_cache_line.h" +#include "../utils/utils_part.h" +#include "../concurrency/ocf_concurrency.h" + +#define OCF_ENGINE_DEBUG_IO_NAME "wo" +#include "engine_debug.h" + +static void ocf_read_wo_cache_complete(struct ocf_request *req, int error) +{ + if (error) { + env_atomic_inc(&req->core->counters->cache_errors.read); + req->error |= error; + } + + if (env_atomic_dec_return(&req->req_remaining)) + return; + + OCF_DEBUG_RQ(req, "Completion"); + + if (req->error) + ocf_engine_error(req, true, "Failed to read data from cache"); + + ocf_req_unlock_rd(req); + + /* Complete request */ + req->complete(req, req->error); + + /* Release OCF request */ + ocf_req_put(req); +} + +static void ocf_read_wo_cache_io(struct ocf_request *req, uint64_t offset, + uint64_t size) +{ + OCF_DEBUG_RQ(req, "Submit cache"); + env_atomic_inc(&req->req_remaining); + ocf_submit_cache_reqs(req->cache, req, OCF_READ, offset, size, 1, + ocf_read_wo_cache_complete); +} + +static int ocf_read_wo_cache_do(struct ocf_request *req) +{ + ocf_cache_t cache = req->cache; + uint32_t s, e, i; + uint64_t line; + struct ocf_map_info *entry; + bool dirty = false; + bool io = false; + uint64_t phys_prev, phys_curr = 0; + uint64_t io_start = 0; + uint64_t offset = 0; + uint64_t increment; + + env_atomic_set(&req->req_remaining, 1); + + for (line = 0; line < req->core_line_count; ++line) { + entry = &req->map[line]; + s = ocf_map_line_start_sector(req, line); + e = ocf_map_line_end_sector(req, line); + + /* if cacheline mapping is not sequential, send cache IO to + * previous cacheline(s) */ + phys_prev = phys_curr; + if (entry->status != LOOKUP_MISS) + phys_curr = ocf_metadata_map_lg2phy(cache, + entry->coll_idx); + if (io && phys_prev + 1 != phys_curr) { + ocf_read_wo_cache_io(req, io_start, offset - io_start); + io = false; + } + + /* try to seek directly to the last sector */ + if (entry->status == LOOKUP_MISS || + ocf_engine_map_all_sec_clean(req, line)) { + /* all sectors invalid or clean */ + i = e + 1; + increment = SECTORS_TO_BYTES(e - s + 1); + dirty = false; + } + else if (ocf_engine_map_all_sec_dirty(req, line)) { + /* all sectors dirty */ + i = e + 1; + increment = SECTORS_TO_BYTES(e - s + 1); + dirty = true; + } else { + /* need to iterate through CL sector by sector */ + i = s; + } + + do { + if (i <= e) { + dirty = metadata_test_dirty_one(cache, + entry->coll_idx, i); + increment = 0; + do { + ++i; + increment += SECTORS_TO_BYTES(1); + } while (i <= e && metadata_test_dirty_one( + cache, entry->coll_idx, i) + == dirty); + } + + if (io && !dirty) { + /* end of sequential dirty region */ + ocf_read_wo_cache_io(req, io_start, + offset - io_start); + io = false; + } + + if (!io && dirty) { + /* beginning of sequential dirty region */ + io = true; + io_start = offset; + } + + offset += increment; + } while (i <= e); + } + + if (io) + ocf_read_wo_cache_io(req, io_start, offset - io_start); + + ocf_read_wo_cache_complete(req, 0); + + return 0; +} + +static const struct ocf_io_if _io_if_wo_cache_read = { + .read = ocf_read_wo_cache_do, + .write = ocf_read_wo_cache_do, +}; + +static void _ocf_read_wo_core_complete(struct ocf_request *req, int error) +{ + if (error) { + req->error |= error; + req->info.core_error = 1; + env_atomic_inc(&req->core->counters->core_errors.read); + } + + /* if all mapped cachelines are clean, the data we've read from core + * is valid and we can complete the request */ + if (!req->info.dirty_any || req->error) { + OCF_DEBUG_RQ(req, "Completion"); + req->complete(req, req->error); + ocf_req_unlock_rd(req); + ocf_req_put(req); + return; + } + + req->io_if = &_io_if_wo_cache_read; + ocf_engine_push_req_front(req, true); +} + +int ocf_read_wo_do(struct ocf_request *req) +{ + ocf_cache_t cache = req->cache; + + if (ocf_engine_is_hit(req)) { + /* read hit - just fetch the data from cache using standard read path + */ + ocf_read_generic_do(req); + return 0; + } + + ocf_req_get(req); + + if (req->info.re_part) { + OCF_DEBUG_RQ(req, "Re-Part"); + + OCF_METADATA_LOCK_WR(); + + /* Probably some cache lines are assigned into wrong + * partition. Need to move it to new one + */ + ocf_part_move(req); + + OCF_METADATA_UNLOCK_WR(); + } + + OCF_DEBUG_RQ(req, "Submit core"); + ocf_submit_volume_req(&req->core->volume, req, _ocf_read_wo_core_complete); + + ocf_engine_update_request_stats(req); + ocf_engine_update_block_stats(req); + + ocf_req_put(req); + + return 0; +} + +static const struct ocf_io_if _io_if_wo_resume = { + .read = ocf_read_wo_do, + .write = ocf_read_wo_do, + .resume = ocf_engine_on_resume, +}; + +int ocf_read_wo(struct ocf_request *req) +{ + ocf_cache_t cache = req->cache; + int lock = OCF_LOCK_ACQUIRED; + + OCF_DEBUG_TRACE(req->cache); + + ocf_io_start(req->io); + + /* Get OCF request - increase reference counter */ + ocf_req_get(req); + + /* Set resume call backs */ + req->io_if = &_io_if_wo_resume; + + OCF_METADATA_LOCK_RD(); /*- Metadata RD access -----------------------*/ + + /* Traverse request to check if there are mapped cache lines */ + ocf_engine_traverse(req); + + if (ocf_engine_mapped_count(req)) { + /* There are mapped cache lines, + * lock request for READ access + */ + lock = ocf_req_trylock_rd(req); + } + + OCF_METADATA_UNLOCK_RD(); /*- END Metadata RD access -----------------*/ + + if (lock >= 0) { + if (lock != OCF_LOCK_ACQUIRED) { + /* Lock was not acquired, need to wait for resume */ + OCF_DEBUG_RQ(req, "NO LOCK"); + } else { + /* Lock was acquired can perform IO */ + ocf_read_wo_do(req); + } + } else { + OCF_DEBUG_RQ(req, "LOCK ERROR %d", lock); + req->complete(req, lock); + ocf_req_put(req); + } + + /* Put OCF request - decrease reference counter */ + ocf_req_put(req); + + return 0; +} diff --git a/src/engine/engine_wo.h b/src/engine/engine_wo.h new file mode 100644 index 0000000..2ae7f29 --- /dev/null +++ b/src/engine/engine_wo.h @@ -0,0 +1,11 @@ +/* + * Copyright(c) 2019 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause-Clear + */ + +#ifndef ENGINE_WO_H_ +#define ENGINE_WO_H_ + +int ocf_read_wo(struct ocf_request *req); + +#endif /* ENGINE_WO_H_ */ diff --git a/src/engine/engine_wt.c b/src/engine/engine_wt.c index df85901..a5be349 100644 --- a/src/engine/engine_wt.c +++ b/src/engine/engine_wt.c @@ -87,7 +87,7 @@ static inline void _ocf_write_wt_submit(struct ocf_request *req) } /* To cache */ - ocf_submit_cache_reqs(cache, req->map, req, OCF_WRITE, + ocf_submit_cache_reqs(cache, req, OCF_WRITE, 0, req->byte_length, ocf_engine_io_count(req), _ocf_write_wt_cache_complete); /* To core */ @@ -145,7 +145,7 @@ static int _ocf_write_wt_do(struct ocf_request *req) /* Submit IO */ _ocf_write_wt_submit(req); - /* Updata statistics */ + /* Update statistics */ ocf_engine_update_request_stats(req); ocf_engine_update_block_stats(req); diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index e195608..d95668d 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -1600,6 +1600,7 @@ static const char *_ocf_cache_mode_names[ocf_cache_mode_max] = { [ocf_cache_mode_wa] = "wa", [ocf_cache_mode_pt] = "pt", [ocf_cache_mode_wi] = "wi", + [ocf_cache_mode_wo] = "wo", }; static const char *_ocf_cache_mode_get_name(ocf_cache_mode_t cache_mode) diff --git a/src/ocf_request.h b/src/ocf_request.h index 0ccbc61..44590b2 100644 --- a/src/ocf_request.h +++ b/src/ocf_request.h @@ -144,7 +144,7 @@ struct ocf_request { /*!< Copy of request data */ uint64_t byte_position; - /*!< LBA byte position of request in code domain */ + /*!< LBA byte position of request in core domain */ uint64_t core_line_first; /*! First core line */ diff --git a/src/utils/utils_io.c b/src/utils/utils_io.c index 51210fd..719fd22 100644 --- a/src/utils/utils_io.c +++ b/src/utils/utils_io.c @@ -226,16 +226,22 @@ static void ocf_submit_volume_req_cmpl(struct ocf_io *io, int error) } void ocf_submit_cache_reqs(struct ocf_cache *cache, - struct ocf_map_info *map_info, struct ocf_request *req, int dir, - unsigned int reqs, ocf_req_end_t callback) + struct ocf_request *req, int dir, uint64_t offset, + uint64_t size, unsigned int reqs, ocf_req_end_t callback) { struct ocf_counters_block *cache_stats; uint64_t flags = req->io ? req->io->flags : 0; uint32_t class = req->io ? req->io->io_class : 0; uint64_t addr, bytes, total_bytes = 0; struct ocf_io *io; - uint32_t i; int err; + uint32_t i; + uint32_t entry = ocf_bytes_2_lines(cache, req->byte_position + offset) - + ocf_bytes_2_lines(cache, req->byte_position); + struct ocf_map_info *map_info = &req->map[entry]; + + ENV_BUG_ON(req->byte_length < offset + size); + ENV_BUG_ON(entry + reqs > req->core_line_count); cache_stats = &req->core->counters->cache_blocks; @@ -250,14 +256,14 @@ void ocf_submit_cache_reqs(struct ocf_cache *cache, map_info[0].coll_idx); addr *= ocf_line_size(cache); addr += cache->device->metadata_offset; - addr += (req->byte_position % ocf_line_size(cache)); - bytes = req->byte_length; + addr += ((req->byte_position + offset) % ocf_line_size(cache)); + bytes = size; ocf_io_configure(io, addr, bytes, dir, class, flags); ocf_io_set_queue(io, req->io_queue); ocf_io_set_cmpl(io, req, callback, ocf_submit_volume_req_cmpl); - err = ocf_io_set_data(io, req->data, 0); + err = ocf_io_set_data(io, req->data, offset); if (err) { ocf_io_put(io); callback(req, err); @@ -265,7 +271,7 @@ void ocf_submit_cache_reqs(struct ocf_cache *cache, } ocf_volume_submit_io(io); - total_bytes = req->byte_length; + total_bytes = bytes; goto update_stats; } @@ -288,24 +294,27 @@ void ocf_submit_cache_reqs(struct ocf_cache *cache, bytes = ocf_line_size(cache); if (i == 0) { - uint64_t seek = (req->byte_position % + uint64_t seek = ((req->byte_position + offset) % ocf_line_size(cache)); addr += seek; bytes -= seek; } else if (i == (reqs - 1)) { uint64_t skip = (ocf_line_size(cache) - - ((req->byte_position + req->byte_length) % + ((req->byte_position + offset + size) % ocf_line_size(cache))) % ocf_line_size(cache); bytes -= skip; } + bytes = OCF_MIN(bytes, size - total_bytes); + ENV_BUG_ON(bytes == 0); + ocf_io_configure(io, addr, bytes, dir, class, flags); ocf_io_set_queue(io, req->io_queue); ocf_io_set_cmpl(io, req, callback, ocf_submit_volume_req_cmpl); - err = ocf_io_set_data(io, req->data, total_bytes); + err = ocf_io_set_data(io, req->data, offset + total_bytes); if (err) { ocf_io_put(io); /* Finish all IOs which left with ERROR */ @@ -317,6 +326,8 @@ void ocf_submit_cache_reqs(struct ocf_cache *cache, total_bytes += bytes; } + ENV_BUG_ON(total_bytes != size); + update_stats: if (dir == OCF_WRITE) env_atomic64_add(total_bytes, &cache_stats->write_bytes); diff --git a/src/utils/utils_io.h b/src/utils/utils_io.h index 2a31f16..b1a5d47 100644 --- a/src/utils/utils_io.h +++ b/src/utils/utils_io.h @@ -60,10 +60,9 @@ void ocf_submit_cache_page(ocf_cache_t cache, uint64_t addr, int dir, void ocf_submit_volume_req(ocf_volume_t volume, struct ocf_request *req, ocf_req_end_t callback); - void ocf_submit_cache_reqs(struct ocf_cache *cache, - struct ocf_map_info *map_info, struct ocf_request *req, int dir, - unsigned int reqs, ocf_req_end_t callback); + struct ocf_request *req, int dir, uint64_t offset, + uint64_t size, unsigned int reqs, ocf_req_end_t callback); static inline struct ocf_io *ocf_new_cache_io(struct ocf_cache *cache) {