Write-only cache mode

Write-only cache mode is similar to writeback, however read
operations do not promote data to cache. Reads are mostly serviced
by the core device, only dirty sectors are fetched from the cache.

Signed-off-by: Adam Rutkowski <adam.j.rutkowski@intel.com>

inc/ocf_def.h

@@ -162,6 +162,9 @@ typedef enum {
 	ocf_cache_mode_wi,
 		/*!< Write invalidate cache mode */
 
+	ocf_cache_mode_wo,
+		/*!< Write-only cache mode */
+
 	ocf_cache_mode_max,
 		/*!< Stopper of cache mode enumerator */
 

src/engine/cache_engine.c

@@ -15,6 +15,7 @@
 #include "engine_wi.h"
 #include "engine_wa.h"
 #include "engine_wb.h"
+#include "engine_wo.h"
 #include "engine_fast.h"
 #include "engine_discard.h"
 #include "engine_d2c.h"
@@ -32,6 +33,7 @@ enum ocf_io_if_type {
 	OCF_IO_WA_IF,
 	OCF_IO_WI_IF,
 	OCF_IO_PT_IF,
+	OCF_IO_WO_IF,
 	OCF_IO_MAX_IF,
 
 	/* Private OCF interfaces */
@@ -68,6 +70,11 @@ static const struct ocf_io_if IO_IFS[OCF_IO_PRIV_MAX_IF] = {
 		.write = ocf_write_wi,
 		.name = "Pass Through",
 	},
+	[OCF_IO_WO_IF] = {
+		.read = ocf_read_wo,
+		.write = ocf_write_wb,
+		.name = "Write Only",
+	},
 	[OCF_IO_FAST_IF] = {
 		.read = ocf_read_fast,
 		.write = ocf_write_fast,
@@ -95,6 +102,7 @@ static const struct ocf_io_if *cache_mode_io_if_map[ocf_req_cache_mode_max] = {
 	[ocf_req_cache_mode_wb] = &IO_IFS[OCF_IO_WB_IF],
 	[ocf_req_cache_mode_wa] = &IO_IFS[OCF_IO_WA_IF],
 	[ocf_req_cache_mode_wi] = &IO_IFS[OCF_IO_WI_IF],
+	[ocf_req_cache_mode_wo] = &IO_IFS[OCF_IO_WO_IF],
 	[ocf_req_cache_mode_pt] = &IO_IFS[OCF_IO_PT_IF],
 	[ocf_req_cache_mode_fast] = &IO_IFS[OCF_IO_FAST_IF],
 	[ocf_req_cache_mode_d2c] = &IO_IFS[OCF_IO_D2C_IF],

src/engine/cache_engine.h

@@ -20,6 +20,7 @@ typedef enum {
 	ocf_req_cache_mode_wa = ocf_cache_mode_wa,
 	ocf_req_cache_mode_pt = ocf_cache_mode_pt,
 	ocf_req_cache_mode_wi = ocf_cache_mode_wi,
+	ocf_req_cache_mode_wo = ocf_cache_mode_wo,
 
 	/* internal modes */
 	ocf_req_cache_mode_fast,

src/engine/engine_bf.c

@@ -84,8 +84,8 @@ static int _ocf_backfill_do(struct ocf_request *req)
 
 	req->data = req->cp_data;
 
-	ocf_submit_cache_reqs(req->cache, req->map, req, OCF_WRITE, reqs_to_issue,
+	ocf_submit_cache_reqs(req->cache, req, OCF_WRITE, 0, req->byte_length,
-			      _ocf_backfill_complete);
+				reqs_to_issue, _ocf_backfill_complete);
 
 	return 0;
 }

src/engine/engine_common.h

@@ -7,6 +7,7 @@
 #define ENGINE_COMMON_H_
 
 #include "../ocf_request.h"
+#include "../utils/utils_cache_line.h"
 
 /**
  * @file engine_common.h
@@ -109,6 +110,37 @@ static inline uint32_t ocf_engine_io_count(struct ocf_request *req)
 	return req->info.seq_req ? 1 : req->core_line_count;
 }
 
+static inline
+bool ocf_engine_map_all_sec_dirty(struct ocf_request *req, uint32_t line)
+{
+	uint8_t start = ocf_map_line_start_sector(req, line);
+	uint8_t end = ocf_map_line_end_sector(req, line);
+
+	if (req->map[line].status != LOOKUP_HIT)
+		return false;
+
+	return metadata_test_dirty_all_sec(req->cache, req->map[line].coll_idx,
+		start, end);
+}
+
+static inline
+bool ocf_engine_map_all_sec_clean(struct ocf_request *req, uint32_t line)
+{
+	uint8_t start = ocf_map_line_start_sector(req, line);
+	uint8_t end = ocf_map_line_end_sector(req, line);
+
+	if (req->map[line].status != LOOKUP_HIT)
+		return false;
+
+	if (!metadata_test_valid_sec(req->cache, req->map[line].coll_idx,
+			start, end)) {
+		return false;
+	}
+
+	return !metadata_test_dirty_sec(req->cache, req->map[line].coll_idx,
+			start, end);
+}
+
 /**
  * @brief Clean request (flush dirty data to the core device)
  *

src/engine/engine_fast.c

@@ -87,7 +87,7 @@ static int _ocf_read_fast_do(struct ocf_request *req)
 	/* Submit IO */
 	OCF_DEBUG_RQ(req, "Submit");
 	env_atomic_set(&req->req_remaining, ocf_engine_io_count(req));
-	ocf_submit_cache_reqs(req->cache, req->map, req, OCF_READ,
+	ocf_submit_cache_reqs(req->cache, req, OCF_READ, 0, req->byte_length,
 		ocf_engine_io_count(req), _ocf_read_fast_complete);
 
 

src/engine/engine_ops.c

@@ -51,7 +51,7 @@ int ocf_engine_ops(struct ocf_request *req)
 	ocf_submit_volume_req(&req->core->volume, req,
 			_ocf_engine_ops_complete);
 
-	ocf_submit_cache_reqs(cache, req->map, req, req->rw,
+	ocf_submit_cache_reqs(cache, req, req->rw, 0, req->byte_length,
 			1, _ocf_engine_ops_complete);
 
 	/* Put OCF request - decrease reference counter */

src/engine/engine_rd.c

@@ -105,7 +105,7 @@ static inline void _ocf_read_generic_submit_hit(struct ocf_request *req)
 {
 	env_atomic_set(&req->req_remaining, ocf_engine_io_count(req));
 
-	ocf_submit_cache_reqs(req->cache, req->map, req, OCF_READ,
+	ocf_submit_cache_reqs(req->cache, req, OCF_READ, 0, req->byte_length,
 		ocf_engine_io_count(req), _ocf_read_generic_hit_complete);
 }
 
@@ -135,7 +135,7 @@ err_alloc:
 	_ocf_read_generic_miss_complete(req, -OCF_ERR_NO_MEM);
 }
 
-static int _ocf_read_generic_do(struct ocf_request *req)
+int ocf_read_generic_do(struct ocf_request *req)
 {
 	struct ocf_cache *cache = req->cache;
 
@@ -195,7 +195,7 @@ static int _ocf_read_generic_do(struct ocf_request *req)
 	else
 		_ocf_read_generic_submit_miss(req);
 
-	/* Updata statistics */
+	/* Update statistics */
 	ocf_engine_update_request_stats(req);
 	ocf_engine_update_block_stats(req);
 
@@ -206,8 +206,8 @@ static int _ocf_read_generic_do(struct ocf_request *req)
 }
 
 static const struct ocf_io_if _io_if_read_generic_resume = {
-	.read = _ocf_read_generic_do,
+	.read = ocf_read_generic_do,
-	.write = _ocf_read_generic_do,
+	.write = ocf_read_generic_do,
 	.resume = ocf_engine_on_resume,
 };
 
@@ -294,7 +294,7 @@ int ocf_read_generic(struct ocf_request *req)
 				OCF_DEBUG_RQ(req, "NO LOCK");
 			} else {
 				/* Lock was acquired can perform IO */
-				_ocf_read_generic_do(req);
+				ocf_read_generic_do(req);
 			}
 		} else {
 			OCF_DEBUG_RQ(req, "LOCK ERROR %d", lock);

src/engine/engine_rd.h

@@ -8,4 +8,6 @@
 
 int ocf_read_generic(struct ocf_request *req);
 
+int ocf_read_generic_do(struct ocf_request *req);
+
 #endif /* ENGINE_RD_H_ */

src/engine/engine_wb.c

@@ -141,7 +141,7 @@ static inline void _ocf_write_wb_submit(struct ocf_request *req)
 	OCF_DEBUG_RQ(req, "Submit Data");
 
 	/* Data IO */
-	ocf_submit_cache_reqs(cache, req->map, req, OCF_WRITE,
+	ocf_submit_cache_reqs(cache, req, OCF_WRITE, 0, req->byte_length,
 			ocf_engine_io_count(req), _ocf_write_wb_complete);
 }
 

src/engine/engine_wo.c

@@ -0,0 +1,258 @@
+/*
+ * Copyright(c) 2019 Intel Corporation
+ * SPDX-License-Identifier: BSD-3-Clause-Clear
+ */
+
+#include "ocf/ocf.h"
+#include "../ocf_cache_priv.h"
+#include "cache_engine.h"
+#include "engine_common.h"
+#include "engine_rd.h"
+#include "engine_pt.h"
+#include "../metadata/metadata.h"
+#include "../utils/utils_io.h"
+#include "../utils/utils_cache_line.h"
+#include "../utils/utils_part.h"
+#include "../concurrency/ocf_concurrency.h"
+
+#define OCF_ENGINE_DEBUG_IO_NAME "wo"
+#include "engine_debug.h"
+
+static void ocf_read_wo_cache_complete(struct ocf_request *req, int error)
+{
+	if (error) {
+		env_atomic_inc(&req->core->counters->cache_errors.read);
+		req->error |= error;
+	}
+
+	if (env_atomic_dec_return(&req->req_remaining))
+		return;
+
+	OCF_DEBUG_RQ(req, "Completion");
+
+	if (req->error)
+		ocf_engine_error(req, true, "Failed to read data from cache");
+
+	ocf_req_unlock_rd(req);
+
+	/* Complete request */
+	req->complete(req, req->error);
+
+	/* Release OCF request */
+	ocf_req_put(req);
+}
+
+static void ocf_read_wo_cache_io(struct ocf_request *req, uint64_t offset,
+		uint64_t size)
+{
+	OCF_DEBUG_RQ(req, "Submit cache");
+	env_atomic_inc(&req->req_remaining);
+	ocf_submit_cache_reqs(req->cache, req, OCF_READ, offset, size, 1,
+			ocf_read_wo_cache_complete);
+}
+
+static int ocf_read_wo_cache_do(struct ocf_request *req)
+{
+	ocf_cache_t cache = req->cache;
+	uint32_t s, e, i;
+	uint64_t line;
+	struct ocf_map_info *entry;
+	bool dirty = false;
+	bool io = false;
+	uint64_t phys_prev, phys_curr = 0;
+	uint64_t io_start = 0;
+	uint64_t offset = 0;
+	uint64_t increment;
+
+	env_atomic_set(&req->req_remaining, 1);
+
+	for (line = 0; line < req->core_line_count; ++line) {
+		entry = &req->map[line];
+		s = ocf_map_line_start_sector(req, line);
+		e = ocf_map_line_end_sector(req, line);
+
+		/* if cacheline mapping is not sequential, send cache IO to
+		 * previous cacheline(s) */
+		phys_prev = phys_curr;
+		if (entry->status != LOOKUP_MISS)
+			phys_curr = ocf_metadata_map_lg2phy(cache,
+					entry->coll_idx);
+		if (io && phys_prev + 1 != phys_curr) {
+			ocf_read_wo_cache_io(req, io_start, offset - io_start);
+			io = false;
+		}
+
+		/* try to seek directly to the last sector */
+		if (entry->status == LOOKUP_MISS ||
+				ocf_engine_map_all_sec_clean(req, line)) {
+			/* all sectors invalid or clean */
+			i = e + 1;
+			increment = SECTORS_TO_BYTES(e - s + 1);
+			dirty = false;
+		}
+		else if (ocf_engine_map_all_sec_dirty(req, line)) {
+			/* all sectors dirty */
+			i = e + 1;
+			increment = SECTORS_TO_BYTES(e - s + 1);
+			dirty = true;
+		} else {
+			/* need to iterate through CL sector by sector */
+			i = s;
+		}
+
+		do {
+			if (i <= e) {
+				 dirty = metadata_test_dirty_one(cache,
+						entry->coll_idx, i);
+				 increment = 0;
+				 do {
+					++i;
+					increment += SECTORS_TO_BYTES(1);
+				 } while (i <= e && metadata_test_dirty_one(
+						cache, entry->coll_idx, i)
+						== dirty);
+			}
+
+			if (io && !dirty) {
+				/* end of sequential dirty region */
+				ocf_read_wo_cache_io(req, io_start,
+						offset - io_start);
+				io = false;
+			}
+
+			if (!io && dirty) {
+				/* beginning of sequential dirty region */
+				io = true;
+				io_start = offset;
+			}
+
+			offset += increment;
+		} while (i <= e);
+	}
+
+	if (io)
+		ocf_read_wo_cache_io(req, io_start, offset - io_start);
+
+	ocf_read_wo_cache_complete(req, 0);
+
+	return 0;
+}
+
+static const struct ocf_io_if _io_if_wo_cache_read = {
+	.read = ocf_read_wo_cache_do,
+	.write = ocf_read_wo_cache_do,
+};
+
+static void _ocf_read_wo_core_complete(struct ocf_request *req, int error)
+{
+	if (error) {
+		req->error |= error;
+		req->info.core_error = 1;
+		env_atomic_inc(&req->core->counters->core_errors.read);
+	}
+
+	/* if all mapped cachelines are clean, the data we've read from core
+	 * is valid and we can complete the request */
+	if (!req->info.dirty_any || req->error) {
+		OCF_DEBUG_RQ(req, "Completion");
+		req->complete(req, req->error);
+		ocf_req_unlock_rd(req);
+		ocf_req_put(req);
+		return;
+	}
+
+	req->io_if = &_io_if_wo_cache_read;
+	ocf_engine_push_req_front(req, true);
+}
+
+int ocf_read_wo_do(struct ocf_request *req)
+{
+	ocf_cache_t cache = req->cache;
+
+	if (ocf_engine_is_hit(req)) {
+		/* read hit - just fetch the data from cache using standard read path
+		 */
+		ocf_read_generic_do(req);
+		return 0;
+	}
+
+	ocf_req_get(req);
+
+	if (req->info.re_part) {
+		OCF_DEBUG_RQ(req, "Re-Part");
+
+		OCF_METADATA_LOCK_WR();
+
+		/* Probably some cache lines are assigned into wrong
+		 * partition. Need to move it to new one
+		 */
+		ocf_part_move(req);
+
+		OCF_METADATA_UNLOCK_WR();
+	}
+
+	OCF_DEBUG_RQ(req, "Submit core");
+	ocf_submit_volume_req(&req->core->volume, req, _ocf_read_wo_core_complete);
+
+	ocf_engine_update_request_stats(req);
+	ocf_engine_update_block_stats(req);
+
+	ocf_req_put(req);
+
+	return 0;
+}
+
+static const struct ocf_io_if _io_if_wo_resume = {
+	.read = ocf_read_wo_do,
+	.write = ocf_read_wo_do,
+	.resume = ocf_engine_on_resume,
+};
+
+int ocf_read_wo(struct ocf_request *req)
+{
+	ocf_cache_t cache = req->cache;
+	int lock = OCF_LOCK_ACQUIRED;
+
+	OCF_DEBUG_TRACE(req->cache);
+
+	ocf_io_start(req->io);
+
+	/* Get OCF request - increase reference counter */
+	ocf_req_get(req);
+
+	/* Set resume call backs */
+	req->io_if = &_io_if_wo_resume;
+
+	OCF_METADATA_LOCK_RD(); /*- Metadata RD access -----------------------*/
+
+	/* Traverse request to check if there are mapped cache lines */
+	ocf_engine_traverse(req);
+
+	if (ocf_engine_mapped_count(req)) {
+		/* There are mapped cache lines,
+		 * lock request for READ access
+		 */
+		lock = ocf_req_trylock_rd(req);
+	}
+
+	OCF_METADATA_UNLOCK_RD(); /*- END Metadata RD access -----------------*/
+
+	if (lock >= 0) {
+		if (lock != OCF_LOCK_ACQUIRED) {
+			/* Lock was not acquired, need to wait for resume */
+			OCF_DEBUG_RQ(req, "NO LOCK");
+		} else {
+			/* Lock was acquired can perform IO */
+			ocf_read_wo_do(req);
+		}
+	} else {
+		OCF_DEBUG_RQ(req, "LOCK ERROR %d", lock);
+		req->complete(req, lock);
+		ocf_req_put(req);
+	}
+
+	/* Put OCF request - decrease reference counter */
+	ocf_req_put(req);
+
+	return 0;
+}

src/engine/engine_wo.h

@@ -0,0 +1,11 @@
+/*
+ * Copyright(c) 2019 Intel Corporation
+ * SPDX-License-Identifier: BSD-3-Clause-Clear
+ */
+
+#ifndef ENGINE_WO_H_
+#define ENGINE_WO_H_
+
+int ocf_read_wo(struct ocf_request *req);
+
+#endif /* ENGINE_WO_H_ */

src/engine/engine_wt.c

@@ -87,7 +87,7 @@ static inline void _ocf_write_wt_submit(struct ocf_request *req)
 	}
 
 	/* To cache */
-	ocf_submit_cache_reqs(cache, req->map, req, OCF_WRITE,
+	ocf_submit_cache_reqs(cache, req, OCF_WRITE, 0, req->byte_length,
 			ocf_engine_io_count(req), _ocf_write_wt_cache_complete);
 
 	/* To core */
@@ -145,7 +145,7 @@ static int _ocf_write_wt_do(struct ocf_request *req)
 	/* Submit IO */
 	_ocf_write_wt_submit(req);
 
-	/* Updata statistics */
+	/* Update statistics */
 	ocf_engine_update_request_stats(req);
 	ocf_engine_update_block_stats(req);
 

src/mngt/ocf_mngt_cache.c

@@ -1600,6 +1600,7 @@ static const char *_ocf_cache_mode_names[ocf_cache_mode_max] = {
 	[ocf_cache_mode_wa] = "wa",
 	[ocf_cache_mode_pt] = "pt",
 	[ocf_cache_mode_wi] = "wi",
+	[ocf_cache_mode_wo] = "wo",
 };
 
 static const char *_ocf_cache_mode_get_name(ocf_cache_mode_t cache_mode)

src/ocf_request.h

@@ -144,7 +144,7 @@ struct ocf_request {
 	/*!< Copy of request data */
 
 	uint64_t byte_position;
-	/*!< LBA byte position of request in code domain */
+	/*!< LBA byte position of request in core domain */
 
 	uint64_t core_line_first;
 	/*! First core line */

src/utils/utils_io.c

@@ -226,16 +226,22 @@ static void ocf_submit_volume_req_cmpl(struct ocf_io *io, int error)
 }
 
 void ocf_submit_cache_reqs(struct ocf_cache *cache,
-		struct ocf_map_info *map_info, struct ocf_request *req, int dir,
+		struct ocf_request *req, int dir, uint64_t offset,
-		unsigned int reqs, ocf_req_end_t callback)
+		uint64_t size, unsigned int reqs, ocf_req_end_t callback)
 {
 	struct ocf_counters_block *cache_stats;
 	uint64_t flags = req->io ? req->io->flags : 0;
 	uint32_t class = req->io ? req->io->io_class : 0;
 	uint64_t addr, bytes, total_bytes = 0;
 	struct ocf_io *io;
-	uint32_t i;
 	int err;
+	uint32_t i;
+	uint32_t entry = ocf_bytes_2_lines(cache, req->byte_position + offset) -
+			ocf_bytes_2_lines(cache, req->byte_position);
+	struct ocf_map_info *map_info = &req->map[entry];
+
+	ENV_BUG_ON(req->byte_length < offset + size);
+	ENV_BUG_ON(entry + reqs > req->core_line_count);
 
 	cache_stats = &req->core->counters->cache_blocks;
 
@@ -250,14 +256,14 @@ void ocf_submit_cache_reqs(struct ocf_cache *cache,
 					map_info[0].coll_idx);
 		addr *= ocf_line_size(cache);
 		addr += cache->device->metadata_offset;
-		addr += (req->byte_position % ocf_line_size(cache));
+		addr += ((req->byte_position + offset) % ocf_line_size(cache));
-		bytes = req->byte_length;
+		bytes = size;
 
 		ocf_io_configure(io, addr, bytes, dir, class, flags);
 		ocf_io_set_queue(io, req->io_queue);
 		ocf_io_set_cmpl(io, req, callback, ocf_submit_volume_req_cmpl);
 
-		err = ocf_io_set_data(io, req->data, 0);
+		err = ocf_io_set_data(io, req->data, offset);
 		if (err) {
 			ocf_io_put(io);
 			callback(req, err);
@@ -265,7 +271,7 @@ void ocf_submit_cache_reqs(struct ocf_cache *cache,
 		}
 
 		ocf_volume_submit_io(io);
-		total_bytes = req->byte_length;
+		total_bytes = bytes;
 
 		goto update_stats;
 	}
@@ -288,24 +294,27 @@ void ocf_submit_cache_reqs(struct ocf_cache *cache,
 		bytes = ocf_line_size(cache);
 
 		if (i == 0) {
-			uint64_t seek = (req->byte_position %
+			uint64_t seek = ((req->byte_position + offset) %
 					ocf_line_size(cache));
 
 			addr += seek;
 			bytes -= seek;
 		} else  if (i == (reqs - 1)) {
 			uint64_t skip = (ocf_line_size(cache) -
-				((req->byte_position + req->byte_length) %
+				((req->byte_position + offset + size) %
 				ocf_line_size(cache))) % ocf_line_size(cache);
 
 			bytes -= skip;
 		}
 
+		bytes = OCF_MIN(bytes, size - total_bytes);
+		ENV_BUG_ON(bytes == 0);
+
 		ocf_io_configure(io, addr, bytes, dir, class, flags);
 		ocf_io_set_queue(io, req->io_queue);
 		ocf_io_set_cmpl(io, req, callback, ocf_submit_volume_req_cmpl);
 
-		err = ocf_io_set_data(io, req->data, total_bytes);
+		err = ocf_io_set_data(io, req->data, offset + total_bytes);
 		if (err) {
 			ocf_io_put(io);
 			/* Finish all IOs which left with ERROR */
@@ -317,6 +326,8 @@ void ocf_submit_cache_reqs(struct ocf_cache *cache,
 		total_bytes += bytes;
 	}
 
+	ENV_BUG_ON(total_bytes != size);
+
 update_stats:
 	if (dir == OCF_WRITE)
 		env_atomic64_add(total_bytes, &cache_stats->write_bytes);

src/utils/utils_io.h

@@ -60,10 +60,9 @@ void ocf_submit_cache_page(ocf_cache_t cache, uint64_t addr, int dir,
 void ocf_submit_volume_req(ocf_volume_t volume, struct ocf_request *req,
 		ocf_req_end_t callback);
 
-
 void ocf_submit_cache_reqs(struct ocf_cache *cache,
-		struct ocf_map_info *map_info, struct ocf_request *req, int dir,
+		struct ocf_request *req, int dir, uint64_t offset,
-		unsigned int reqs, ocf_req_end_t callback);
+		uint64_t size, unsigned int reqs, ocf_req_end_t callback);
 
 static inline struct ocf_io *ocf_new_cache_io(struct ocf_cache *cache)
 {