From a7bdaa751d75f11ebe88da21d4471568ce8f42f7 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Tue, 26 Oct 2021 10:53:35 +0200 Subject: [PATCH 01/16] Add error messages on superblock mismatch Signed-off-by: Michal Mielewczyk --- src/mngt/ocf_mngt_cache.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 1059336..da13f00 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -2183,11 +2183,15 @@ static void _ocf_mngt_activate_check_superblock_complete(void *priv, int error) OCF_PL_FINISH_RET(context->pipeline, result); if (cache->conf_meta->metadata_layout != cache->metadata.layout) { + ocf_cache_log(cache, log_err, "Failed to activate standby instance: " + "invaild metadata layout\n"); OCF_PL_FINISH_RET(context->pipeline, -OCF_ERR_METADATA_LAYOUT_MISMATCH); } if (cache->conf_meta->line_size != cache->metadata.line_size) { + ocf_cache_log(cache, log_err, "Failed to activate standby instance: " + "invaild cache line size\n"); OCF_PL_FINISH_RET(context->pipeline, -OCF_ERR_CACHE_LINE_SIZE_MISMATCH); } From ccd0abfea54cbc95bc1155ce1e39380b1a299ecd Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Fri, 10 Sep 2021 11:08:04 +0200 Subject: [PATCH 02/16] Add cache line recovery utils to OCF internal API Signed-off-by: Michal Mielewczyk --- src/mngt/ocf_mngt_cache.c | 39 +----------------------------------- src/utils/utils_cache_line.c | 37 ++++++++++++++++++++++++++++++++++ src/utils/utils_cache_line.h | 4 ++++ 3 files changed, 42 insertions(+), 38 deletions(-) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index da13f00..50ec9af 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -481,43 +481,6 @@ err: OCF_PL_FINISH_RET(pipeline, -OCF_ERR_START_CACHE_FAIL); } -static void _recovery_rebuild_cline_metadata(ocf_cache_t cache, - ocf_core_id_t core_id, uint64_t core_line, - ocf_cache_line_t cache_line) -{ - ocf_core_t core = ocf_cache_get_core(cache, core_id); - ocf_part_id_t part_id; - ocf_cache_line_t hash_index; - struct ocf_part_runtime *part; - - part_id = PARTITION_DEFAULT; - part = cache->user_parts[part_id].part.runtime; - - ocf_metadata_set_partition_id(cache, cache_line, part_id); - env_atomic_inc(&part->curr_size); - - hash_index = ocf_metadata_hash_func(cache, core_line, core_id); - ocf_metadata_add_to_collision(cache, core_id, core_line, hash_index, - cache_line); - - ocf_lru_init_cline(cache, cache_line); - - ocf_lru_add(cache, cache_line); - - env_atomic_inc(&core->runtime_meta->cached_clines); - env_atomic_inc(&core->runtime_meta-> - part_counters[part_id].cached_clines); - - if (metadata_test_dirty(cache, cache_line)) { - env_atomic_inc(&core->runtime_meta->dirty_clines); - env_atomic_inc(&core->runtime_meta-> - part_counters[part_id].dirty_clines); - if (!env_atomic64_read(&core->runtime_meta->dirty_since)) - env_atomic64_cmpxchg(&core->runtime_meta->dirty_since, 0, - env_ticks_to_secs(env_get_tick_count())); - } -} - static void _recovery_reset_cline_metadata(struct ocf_cache *cache, ocf_cache_line_t cline) { @@ -546,7 +509,7 @@ static void _ocf_mngt_recovery_rebuild_metadata(ocf_cache_t cache) (!dirty_only || metadata_test_dirty(cache, cline))) { /* Rebuild metadata for mapped cache line */ - _recovery_rebuild_cline_metadata(cache, core_id, + ocf_cline_rebuild_metadata(cache, core_id, core_line, cline); if (dirty_only) metadata_clear_valid_if_clean(cache, cline); diff --git a/src/utils/utils_cache_line.c b/src/utils/utils_cache_line.c index 8dfbae2..d7df67b 100644 --- a/src/utils/utils_cache_line.c +++ b/src/utils/utils_cache_line.c @@ -167,3 +167,40 @@ void set_cache_line_dirty(struct ocf_cache *cache, uint8_t start_bit, ocf_cleaning_set_hot_cache_line(cache, line); } + +void ocf_cline_rebuild_metadata(ocf_cache_t cache, + ocf_core_id_t core_id, uint64_t core_line, + ocf_cache_line_t cache_line) +{ + ocf_core_t core = ocf_cache_get_core(cache, core_id); + ocf_part_id_t part_id; + ocf_cache_line_t hash_index; + struct ocf_part_runtime *part; + + part_id = PARTITION_DEFAULT; + part = cache->user_parts[part_id].part.runtime; + + ocf_metadata_set_partition_id(cache, cache_line, part_id); + env_atomic_inc(&part->curr_size); + + hash_index = ocf_metadata_hash_func(cache, core_line, core_id); + ocf_metadata_add_to_collision(cache, core_id, core_line, hash_index, + cache_line); + + ocf_lru_init_cline(cache, cache_line); + + ocf_lru_add(cache, cache_line); + + env_atomic_inc(&core->runtime_meta->cached_clines); + env_atomic_inc(&core->runtime_meta-> + part_counters[part_id].cached_clines); + + if (metadata_test_dirty(cache, cache_line)) { + env_atomic_inc(&core->runtime_meta->dirty_clines); + env_atomic_inc(&core->runtime_meta-> + part_counters[part_id].dirty_clines); + if (!env_atomic64_read(&core->runtime_meta->dirty_since)) + env_atomic64_cmpxchg(&core->runtime_meta->dirty_since, 0, + env_ticks_to_secs(env_get_tick_count())); + } +} diff --git a/src/utils/utils_cache_line.h b/src/utils/utils_cache_line.h index ccfafac..10e5078 100644 --- a/src/utils/utils_cache_line.h +++ b/src/utils/utils_cache_line.h @@ -123,6 +123,10 @@ void set_cache_line_clean(struct ocf_cache *cache, uint8_t start_bit, void set_cache_line_dirty(struct ocf_cache *cache, uint8_t start_bit, uint8_t end_bit, struct ocf_request *req, uint32_t map_idx); +void ocf_cline_rebuild_metadata(ocf_cache_t cache, + ocf_core_id_t core_id, uint64_t core_line, + ocf_cache_line_t cache_line); + /** * @brief Remove cache line from cleaning policy * From 5ca1404f06d857d11cd9987d6c87bdc9d44915de Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Thu, 14 Oct 2021 19:02:56 +0200 Subject: [PATCH 03/16] Fix spelling in the error message Signed-off-by: Michal Mielewczyk --- src/mngt/ocf_mngt_cache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 50ec9af..d2a909c 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -2186,7 +2186,7 @@ static void _ocf_mngt_activate_compare_superblock_end( OCF_PL_FINISH_RET(context->pipeline, result); if (diff) { - ocf_cache_log(cache, log_err, "Superblock missmatch!\n"); + ocf_cache_log(cache, log_err, "Superblock mismatch!\n"); OCF_PL_FINISH_RET(context->pipeline, -OCF_ERR_INVAL); } From 800190153b4349b517c4284bbdea0d9ae1936e4a Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Fri, 10 Sep 2021 11:11:24 +0200 Subject: [PATCH 04/16] Extend lru list API Signed-off-by: Michal Mielewczyk --- src/ocf_lru.c | 10 +++++----- src/ocf_lru.h | 4 ++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/ocf_lru.c b/src/ocf_lru.c index 171151f..86a5213 100644 --- a/src/ocf_lru.c +++ b/src/ocf_lru.c @@ -194,7 +194,7 @@ static void remove_lru_list_nobalance(ocf_cache_t cache, node->hot = false; } -static void remove_lru_list(ocf_cache_t cache, struct ocf_lru_list *list, +void ocf_lru_remove_locked(ocf_cache_t cache, struct ocf_lru_list *list, ocf_cache_line_t cline) { remove_lru_list_nobalance(cache, list, cline); @@ -221,7 +221,7 @@ void ocf_lru_init_cline(ocf_cache_t cache, ocf_cache_line_t cline) node->next = end_marker; } -static struct ocf_lru_list *ocf_lru_get_list(struct ocf_part *part, +struct ocf_lru_list *ocf_lru_get_list(struct ocf_part *part, uint32_t lru_idx, bool clean) { if (part->id == PARTITION_FREELIST) @@ -257,7 +257,7 @@ void ocf_lru_add(ocf_cache_t cache, ocf_cache_line_t cline) static inline void ocf_lru_move(ocf_cache_t cache, ocf_cache_line_t cline, struct ocf_lru_list *src_list, struct ocf_lru_list *dst_list) { - remove_lru_list(cache, src_list, cline); + ocf_lru_remove_locked(cache, src_list, cline); add_lru_head(cache, dst_list, cline); } @@ -846,7 +846,7 @@ void ocf_lru_clean_cline(ocf_cache_t cache, struct ocf_part *part, dirty_list = ocf_lru_get_list(part, lru_list, false); OCF_METADATA_LRU_WR_LOCK(cline); - remove_lru_list(cache, dirty_list, cline); + ocf_lru_remove_locked(cache, dirty_list, cline); add_lru_head(cache, clean_list, cline); OCF_METADATA_LRU_WR_UNLOCK(cline); } @@ -862,7 +862,7 @@ void ocf_lru_dirty_cline(ocf_cache_t cache, struct ocf_part *part, dirty_list = ocf_lru_get_list(part, lru_list, false); OCF_METADATA_LRU_WR_LOCK(cline); - remove_lru_list(cache, clean_list, cline); + ocf_lru_remove_locked(cache, clean_list, cline); add_lru_head(cache, dirty_list, cline); OCF_METADATA_LRU_WR_UNLOCK(cline); } diff --git a/src/ocf_lru.h b/src/ocf_lru.h index 4b7dabe..a8b0abd 100644 --- a/src/ocf_lru.h +++ b/src/ocf_lru.h @@ -32,5 +32,9 @@ void ocf_lru_repart(ocf_cache_t cache, ocf_cache_line_t cline, struct ocf_part *src_upart, struct ocf_part *dst_upart); uint32_t ocf_lru_num_free(ocf_cache_t cache); void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines); +struct ocf_lru_list *ocf_lru_get_list(struct ocf_part *part, + uint32_t lru_idx, bool clean); +void ocf_lru_remove_locked(ocf_cache_t cache, struct ocf_lru_list *list, + ocf_cache_line_t cline); #endif From bb0ff67fe9559a06b2ec09308e665dec2f72d3f6 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Mon, 4 Oct 2021 14:37:02 +0200 Subject: [PATCH 05/16] Metadata `clear_dirty_if_invalid()` utility Fix cacheline's metadata if it is dirty and invalid Signed-off-by: Michal Mielewczyk --- src/metadata/metadata.c | 25 +++++++++++++++++++++++++ src/metadata/metadata_bit.h | 18 ++++++++++++++++++ src/metadata/metadata_status.h | 9 +++++++++ 3 files changed, 52 insertions(+) diff --git a/src/metadata/metadata.c b/src/metadata/metadata.c index 08f9235..9660cb6 100644 --- a/src/metadata/metadata.c +++ b/src/metadata/metadata.c @@ -1566,6 +1566,31 @@ bool ocf_metadata_clear_valid_if_clean(struct ocf_cache *cache, } } +void ocf_metadata_clear_dirty_if_invalid(struct ocf_cache *cache, + ocf_cache_line_t line, uint8_t start, uint8_t stop) +{ + switch (cache->metadata.line_size) { + case ocf_cache_line_size_4: + return _ocf_metadata_clear_dirty_if_invalid_u8(cache, + line, start, stop); + case ocf_cache_line_size_8: + return _ocf_metadata_clear_dirty_if_invalid_u16(cache, + line, start, stop); + case ocf_cache_line_size_16: + return _ocf_metadata_clear_dirty_if_invalid_u32(cache, + line, start, stop); + case ocf_cache_line_size_32: + return _ocf_metadata_clear_dirty_if_invalid_u64(cache, + line, start, stop); + case ocf_cache_line_size_64: + return _ocf_metadata_clear_dirty_if_invalid_u128(cache, + line, start, stop); + case ocf_cache_line_size_none: + default: + ENV_BUG(); + } +} + int ocf_metadata_init(struct ocf_cache *cache, ocf_cache_line_size_t cache_line_size) { diff --git a/src/metadata/metadata_bit.h b/src/metadata/metadata_bit.h index a58dd95..6acaac9 100644 --- a/src/metadata/metadata_bit.h +++ b/src/metadata/metadata_bit.h @@ -245,6 +245,24 @@ static bool _ocf_metadata_clear_valid_if_clean_##type(struct ocf_cache *cache, \ return false; \ } \ } \ +\ +static void _ocf_metadata_clear_dirty_if_invalid_##type(struct ocf_cache *cache, \ + ocf_cache_line_t line, uint8_t start, uint8_t stop) \ +{ \ + type mask = _get_mask_##type(start, stop); \ +\ + struct ocf_metadata_ctrl *ctrl = \ + (struct ocf_metadata_ctrl *) cache->metadata.priv; \ +\ + struct ocf_metadata_raw *raw = \ + &ctrl->raw_desc[metadata_segment_collision]; \ +\ + struct ocf_metadata_map_##type *map = raw->mem_pool; \ +\ + _raw_bug_on(raw, line); \ +\ + map[line].dirty &= (mask & map[line].valid) | (~mask); \ +} \ #define ocf_metadata_bit_funcs(type) \ ocf_metadata_bit_struct(type); \ diff --git a/src/metadata/metadata_status.h b/src/metadata/metadata_status.h index cc888b6..21dc934 100644 --- a/src/metadata/metadata_status.h +++ b/src/metadata/metadata_status.h @@ -40,6 +40,8 @@ bool ocf_metadata_test_and_clear_valid(struct ocf_cache *cache, ocf_cache_line_t line, uint8_t start, uint8_t stop, bool all); bool ocf_metadata_clear_valid_if_clean(struct ocf_cache *cache, ocf_cache_line_t line, uint8_t start, uint8_t stop); +void ocf_metadata_clear_dirty_if_invalid(struct ocf_cache *cache, + ocf_cache_line_t line, uint8_t start, uint8_t stop); static inline void metadata_init_status_bits(struct ocf_cache *cache, ocf_cache_line_t line) @@ -249,6 +251,13 @@ static inline bool metadata_test_and_set_valid(struct ocf_cache *cache, ocf_line_end_sector(cache), true); } +static inline void metadata_clear_dirty_if_invalid(struct ocf_cache *cache, + ocf_cache_line_t line) +{ + ocf_metadata_clear_dirty_if_invalid(cache, line, 0, + ocf_line_end_sector(cache)); +} + /******************************************************************************* * Valid - Sector Implementation ******************************************************************************/ From 048bbedd71460fc9d0779777ae5624c624c0269f Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Mon, 4 Oct 2021 14:57:47 +0200 Subject: [PATCH 06/16] Fix metadata_clear_valid_if_clean() The function should return the cacheline's valid status Signed-off-by: Michal Mielewczyk --- src/metadata/metadata_status.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/metadata/metadata_status.h b/src/metadata/metadata_status.h index 21dc934..1d470e6 100644 --- a/src/metadata/metadata_status.h +++ b/src/metadata/metadata_status.h @@ -230,10 +230,10 @@ static inline void metadata_clear_valid(struct ocf_cache *cache, ocf_metadata_clear_valid(cache, line, 0, ocf_line_end_sector(cache)); } -static inline void metadata_clear_valid_if_clean(struct ocf_cache *cache, +static inline bool metadata_clear_valid_if_clean(struct ocf_cache *cache, ocf_cache_line_t line) { - ocf_metadata_clear_valid_if_clean(cache, line, 0, + return ocf_metadata_clear_valid_if_clean(cache, line, 0, ocf_line_end_sector(cache)); } From fc7c901c8b556e1988ee53101f38404baa72d4b8 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Mon, 4 Oct 2021 15:43:10 +0200 Subject: [PATCH 07/16] Skip collision init on cache start passive Recovery during passive start is based on the assuption that metadata collision section stored on disk might be partially valid. Reseting this data would make rebuilding metadata impossible. Signed-off-by: Michal Mielewczyk --- src/mngt/ocf_mngt_cache.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index d2a909c..5ba132d 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -321,10 +321,12 @@ static ocf_error_t init_attached_data_structures(ocf_cache_t cache) return 0; } -static void init_attached_data_structures_recovery(ocf_cache_t cache) +static void init_attached_data_structures_recovery(ocf_cache_t cache, + bool init_collision) { ocf_metadata_init_hash_table(cache); - ocf_metadata_init_collision(cache); + if (init_collision) + ocf_metadata_init_collision(cache); __init_parts_attached(cache); __reset_stats(cache); __init_metadata_version(cache); @@ -587,7 +589,7 @@ static void _ocf_mngt_load_init_instance_recovery( { ocf_cache_t cache = context->cache; - init_attached_data_structures_recovery(cache); + init_attached_data_structures_recovery(cache, true); ocf_cache_log(cache, log_info, "Initiating recovery sequence...\n"); @@ -2005,7 +2007,7 @@ static void _ocf_mngt_bind_init_attached_structures(ocf_pipeline_t pipeline, struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; - init_attached_data_structures_recovery(cache); + init_attached_data_structures_recovery(cache, false); ocf_pipeline_next(context->pipeline); } From 4deaa1e1338ddc912a4ef57b934e60326b162758 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Fri, 8 Oct 2021 09:19:49 +0200 Subject: [PATCH 08/16] Reset all the status bits during recovery Make sure all the invalid cachelines have reset status bits. This allows to recognize invalid cachelines easily during populate. Signed-off-by: Michal Mielewczyk --- src/mngt/ocf_mngt_cache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 5ba132d..1c42c1e 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -488,7 +488,7 @@ static void _recovery_reset_cline_metadata(struct ocf_cache *cache, { ocf_metadata_set_core_info(cache, cline, OCF_CORE_MAX, ULLONG_MAX); - metadata_clear_valid(cache, cline); + metadata_init_status_bits(cache, cline); ocf_cleaning_init_cache_block(cache, cline); } From 8f58add152803336721b4cb37eb188756d27e1f1 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Tue, 16 Nov 2021 10:31:27 +0100 Subject: [PATCH 09/16] Lru populate unsafe The unsafe mode is useful if the metadata of added cores is incomplete. Such scenario is possible when starting cache to standby mode from partially vaild metadata. Signed-off-by: Michal Mielewczyk --- src/mngt/ocf_mngt_cache.c | 15 +++++++++++---- src/ocf_lru.c | 12 ++++++++---- src/ocf_lru.h | 3 ++- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 1c42c1e..cfd410a 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -210,12 +210,19 @@ static void __init_parts_attached(ocf_cache_t cache) ocf_lru_init(cache, &cache->free); } -static void __populate_free(ocf_cache_t cache) +static void __populate_free_unsafe(ocf_cache_t cache) +{ + uint64_t free_clines = ocf_metadata_collision_table_entries(cache); + + ocf_lru_populate(cache, free_clines, false); +} + +static void __populate_free_safe(ocf_cache_t cache) { uint64_t free_clines = ocf_metadata_collision_table_entries(cache) - ocf_get_cache_occupancy(cache); - ocf_lru_populate(cache, free_clines); + ocf_lru_populate(cache, free_clines, true); } static ocf_error_t __init_cleaning_policy(ocf_cache_t cache) @@ -307,7 +314,7 @@ static ocf_error_t init_attached_data_structures(ocf_cache_t cache) ocf_metadata_init_hash_table(cache); ocf_metadata_init_collision(cache); __init_parts_attached(cache); - __populate_free(cache); + __populate_free_safe(cache); result = __init_cleaning_policy(cache); if (result) { @@ -536,7 +543,7 @@ static void _ocf_mngt_load_post_metadata_load(ocf_pipeline_t pipeline, if (context->metadata.shutdown_status != ocf_metadata_clean_shutdown) { _ocf_mngt_recovery_rebuild_metadata(cache); - __populate_free(cache); + __populate_free_safe(cache); } cleaning_policy = cache->conf_meta->cleaning_policy_type; diff --git a/src/ocf_lru.c b/src/ocf_lru.c index 86a5213..1613bba 100644 --- a/src/ocf_lru.c +++ b/src/ocf_lru.c @@ -891,7 +891,8 @@ static ocf_cache_line_t next_phys_invalid(ocf_cache_t cache, } /* put invalid cachelines on freelist partition lru list */ -void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines) +void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines, + bool safe) { ocf_cache_line_t phys, cline; ocf_cache_line_t collision_table_entries = @@ -905,7 +906,10 @@ void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines) for (i = 0; i < num_free_clines; i++) { /* find first invalid cacheline */ phys = next_phys_invalid(cache, phys); - ENV_BUG_ON(phys == collision_table_entries); + if (safe) + ENV_BUG_ON(phys == collision_table_entries); + else if (phys == collision_table_entries) + break; cline = ocf_metadata_map_phy2lg(cache, phys); ++phys; @@ -921,9 +925,9 @@ void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines) /* we should have reached the last invalid cache line */ phys = next_phys_invalid(cache, phys); - ENV_BUG_ON(phys != collision_table_entries); + ENV_BUG_ON(safe && phys != collision_table_entries); - env_atomic_set(&cache->free.runtime->curr_size, num_free_clines); + env_atomic_set(&cache->free.runtime->curr_size, i); } static bool _is_cache_line_acting(struct ocf_cache *cache, diff --git a/src/ocf_lru.h b/src/ocf_lru.h index a8b0abd..29dd14f 100644 --- a/src/ocf_lru.h +++ b/src/ocf_lru.h @@ -31,7 +31,8 @@ void ocf_lru_clean(ocf_cache_t cache, struct ocf_user_part *user_part, void ocf_lru_repart(ocf_cache_t cache, ocf_cache_line_t cline, struct ocf_part *src_upart, struct ocf_part *dst_upart); uint32_t ocf_lru_num_free(ocf_cache_t cache); -void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines); +void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines, + bool safe); struct ocf_lru_list *ocf_lru_get_list(struct ocf_part *part, uint32_t lru_idx, bool clean); void ocf_lru_remove_locked(ocf_cache_t cache, struct ocf_lru_list *list, From 11dacd6a84138f9a9497a29ba26b19bc8f5b5c86 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Tue, 16 Nov 2021 14:35:36 +0100 Subject: [PATCH 10/16] Set dirty shutdown status on standby init Since part of the recovery is done during `standby init`, the correct shutdown status has to be set Signed-off-by: Michal Mielewczyk --- src/mngt/ocf_mngt_cache.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index cfd410a..5b858ff 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -1997,6 +1997,22 @@ static void _ocf_mngt_load_unsafe_complete(void *priv, int error) ocf_pipeline_next(context->pipeline); } +static void _ocf_mngt_standby_init_properties(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) +{ + struct ocf_cache_attach_context *context = priv; + ocf_cache_t cache = context->cache; + + OCF_ASSERT_PLUGGED(cache); + + context->metadata.shutdown_status = ocf_metadata_dirty_shutdown; + context->metadata.dirty_flushed = DIRTY_FLUSHED; + context->metadata.line_size = context->cfg.cache_line_size ?: + cache->metadata.line_size; + + ocf_pipeline_next(pipeline); +} + static void _ocf_mngt_load_metadata_unsafe(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { @@ -2038,7 +2054,7 @@ struct ocf_pipeline_properties _ocf_mngt_cache_standby_pipeline_properties = { OCF_PL_STEP(_ocf_mngt_init_attached_nonpersistent), OCF_PL_STEP(_ocf_mngt_attach_cache_device), OCF_PL_STEP(_ocf_mngt_init_cache_front_volume), - OCF_PL_STEP(_ocf_mngt_init_properties), + OCF_PL_STEP(_ocf_mngt_standby_init_properties), OCF_PL_STEP(_ocf_mngt_attach_check_ram), OCF_PL_STEP(_ocf_mngt_test_volume), OCF_PL_STEP(_ocf_mngt_attach_prepare_metadata), From 6d4e6af5b61b445f2875c63e232a53ab1ee56d5d Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Tue, 16 Nov 2021 14:36:13 +0100 Subject: [PATCH 11/16] Recovery on passive start Adjust recovery procedure to allow rebuilding metadata from partialy valid metadata Signed-off-by: Michal Mielewczyk --- src/mngt/ocf_mngt_cache.c | 44 ++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 5b858ff..d21f6fe 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -500,34 +500,35 @@ static void _recovery_reset_cline_metadata(struct ocf_cache *cache, ocf_cleaning_init_cache_block(cache, cline); } -static void _ocf_mngt_recovery_rebuild_metadata(ocf_cache_t cache) +static void _ocf_mngt_rebuild_metadata(ocf_cache_t cache, bool initialized) { ocf_cache_line_t cline; ocf_core_id_t core_id; uint64_t core_line; unsigned char step = 0; - bool dirty_only = !ocf_volume_is_atomic(ocf_cache_get_volume(cache)); const uint64_t collision_table_entries = ocf_metadata_collision_table_entries(cache); ocf_metadata_start_exclusive_access(&cache->metadata.lock); for (cline = 0; cline < collision_table_entries; cline++) { - ocf_metadata_get_core_info(cache, cline, &core_id, &core_line); - if (core_id != OCF_CORE_MAX && - (!dirty_only || metadata_test_dirty(cache, - cline))) { - /* Rebuild metadata for mapped cache line */ - ocf_cline_rebuild_metadata(cache, core_id, - core_line, cline); - if (dirty_only) - metadata_clear_valid_if_clean(cache, cline); - } else { - /* Reset metadata for not mapped or clean cache line */ - _recovery_reset_cline_metadata(cache, cline); - } + bool any_valid = true; OCF_COND_RESCHED(step, 128); + ocf_metadata_get_core_info(cache, cline, &core_id, &core_line); + + if (!initialized) + metadata_clear_dirty_if_invalid(cache, cline); + + any_valid = metadata_clear_valid_if_clean(cache, cline); + if (!any_valid || core_id >= OCF_CORE_MAX) { + /* Reset metadata for not mapped or clean cache line */ + _recovery_reset_cline_metadata(cache, cline); + continue; + } + + /* Rebuild metadata for mapped cache line */ + ocf_cline_rebuild_metadata(cache, core_id, core_line, cline); } ocf_metadata_end_exclusive_access(&cache->metadata.lock); @@ -2035,6 +2036,18 @@ static void _ocf_mngt_bind_init_attached_structures(ocf_pipeline_t pipeline, ocf_pipeline_next(context->pipeline); } +static void _ocf_mngt_bind_recovery_unsafe(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) +{ + struct ocf_cache_attach_context *context = priv; + ocf_cache_t cache = context->cache; + + _ocf_mngt_bind_rebuild_metadata(cache); + __populate_free_unsafe(cache); + + ocf_pipeline_next(pipeline); +} + static void _ocf_mngt_bind_post_init(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { @@ -2060,6 +2073,7 @@ struct ocf_pipeline_properties _ocf_mngt_cache_standby_pipeline_properties = { OCF_PL_STEP(_ocf_mngt_attach_prepare_metadata), OCF_PL_STEP(_ocf_mngt_load_metadata_unsafe), OCF_PL_STEP(_ocf_mngt_bind_init_attached_structures), + OCF_PL_STEP(_ocf_mngt_bind_recovery_unsafe), OCF_PL_STEP(_ocf_mngt_bind_post_init), OCF_PL_STEP_TERMINATOR(), }, From 390e80794d73dbae2ebd15d59f2bf04b34a33526 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Tue, 5 Oct 2021 09:10:05 +0200 Subject: [PATCH 12/16] Refactor cleaning policy initialization Extract cleaning policy initialization to a separate function Signed-off-by: Michal Mielewczyk --- src/mngt/ocf_mngt_cache.c | 66 ++++++++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 12 deletions(-) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index d21f6fe..3581101 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -534,12 +534,60 @@ static void _ocf_mngt_rebuild_metadata(ocf_cache_t cache, bool initialized) ocf_metadata_end_exclusive_access(&cache->metadata.lock); } +static void _ocf_mngt_recovery_rebuild_metadata(ocf_cache_t cache) +{ + _ocf_mngt_rebuild_metadata(cache, false); +} + +static void _ocf_mngt_bind_rebuild_metadata(ocf_cache_t cache) +{ + ocf_cleaning_t clean_policy = cache->conf_meta->cleaning_policy_type; + cache->conf_meta->cleaning_policy_type = ocf_cleaning_nop; + + _ocf_mngt_rebuild_metadata(cache, true); + + cache->conf_meta->cleaning_policy_type = clean_policy; +} + +static inline ocf_error_t _ocf_init_cleaning_policy(ocf_cache_t cache, + ocf_cleaning_t cleaning_policy, + enum ocf_metadata_shutdown_status shutdown_status) +{ + ocf_error_t result; + + if (shutdown_status == ocf_metadata_clean_shutdown) + result = ocf_cleaning_initialize(cache, cleaning_policy, 0); + else + result = ocf_cleaning_initialize(cache, cleaning_policy, 1); + + if (result) + ocf_cache_log(cache, log_err, "Cannot initialize cleaning policy\n"); + + return result; +} + +static void _ocf_mngt_activate_init_cleaning(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) +{ + struct ocf_cache_attach_context *context = priv; + ocf_cache_t cache = context->cache; + ocf_error_t result; + + result = _ocf_init_cleaning_policy(cache, + cache->conf_meta->cleaning_policy_type, + context->metadata.shutdown_status); + + if (result) + OCF_PL_FINISH_RET(pipeline, result); + + ocf_pipeline_next(pipeline); +} + static void _ocf_mngt_load_post_metadata_load(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; - ocf_cleaning_t cleaning_policy; ocf_error_t result; if (context->metadata.shutdown_status != ocf_metadata_clean_shutdown) { @@ -547,18 +595,12 @@ static void _ocf_mngt_load_post_metadata_load(ocf_pipeline_t pipeline, __populate_free_safe(cache); } - cleaning_policy = cache->conf_meta->cleaning_policy_type; + result = _ocf_init_cleaning_policy(cache, + cache->conf_meta->cleaning_policy_type, + context->metadata.shutdown_status); - if (context->metadata.shutdown_status == ocf_metadata_clean_shutdown) - result = ocf_cleaning_initialize(cache, cleaning_policy, 0); - else - result = ocf_cleaning_initialize(cache, cleaning_policy, 1); - - if (result) { - ocf_cache_log(cache, log_err, - "Cannot initialize cleaning policy\n"); + if (result) OCF_PL_FINISH_RET(pipeline, result); - } ocf_pipeline_next(pipeline); } @@ -2339,7 +2381,7 @@ struct ocf_pipeline_properties _ocf_mngt_cache_activate_pipeline_properties = { OCF_PL_STEP(_ocf_mngt_init_cleaner), OCF_PL_STEP(_ocf_mngt_init_promotion), OCF_PL_STEP(_ocf_mngt_load_add_cores), - OCF_PL_STEP(_ocf_mngt_load_post_metadata_load), + OCF_PL_STEP(_ocf_mngt_activate_init_cleaning), OCF_PL_STEP(_ocf_mngt_attach_shutdown_status), OCF_PL_STEP(_ocf_mngt_attach_post_init), OCF_PL_STEP_TERMINATOR(), From 0e529479d6f6ad057ad4046f6a58b83ff3470716 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Thu, 7 Oct 2021 17:14:22 +0200 Subject: [PATCH 13/16] Init cleaner during passive start Initializing cleaning policy is very time consuming. To reduce the time required for activating cache instance the initialization sholud be done during passitve start Signed-off-by: Michal Mielewczyk --- src/mngt/ocf_mngt_cache.c | 50 ++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 3581101..f3de681 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -566,23 +566,6 @@ static inline ocf_error_t _ocf_init_cleaning_policy(ocf_cache_t cache, return result; } -static void _ocf_mngt_activate_init_cleaning(ocf_pipeline_t pipeline, - void *priv, ocf_pipeline_arg_t arg) -{ - struct ocf_cache_attach_context *context = priv; - ocf_cache_t cache = context->cache; - ocf_error_t result; - - result = _ocf_init_cleaning_policy(cache, - cache->conf_meta->cleaning_policy_type, - context->metadata.shutdown_status); - - if (result) - OCF_PL_FINISH_RET(pipeline, result); - - ocf_pipeline_next(pipeline); -} - static void _ocf_mngt_load_post_metadata_load(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { @@ -2090,6 +2073,23 @@ static void _ocf_mngt_bind_recovery_unsafe(ocf_pipeline_t pipeline, ocf_pipeline_next(pipeline); } +static void _ocf_mngt_bind_init_cleaning(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) +{ + struct ocf_cache_attach_context *context = priv; + ocf_cache_t cache = context->cache; + ocf_error_t result; + + result = _ocf_init_cleaning_policy(cache, + cache->conf_meta->cleaning_policy_type, + context->metadata.shutdown_status); + + if (result) + OCF_PL_FINISH_RET(pipeline, result); + + ocf_pipeline_next(pipeline); +} + static void _ocf_mngt_bind_post_init(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { @@ -2116,6 +2116,8 @@ struct ocf_pipeline_properties _ocf_mngt_cache_standby_pipeline_properties = { OCF_PL_STEP(_ocf_mngt_load_metadata_unsafe), OCF_PL_STEP(_ocf_mngt_bind_init_attached_structures), OCF_PL_STEP(_ocf_mngt_bind_recovery_unsafe), + OCF_PL_STEP(_ocf_mngt_init_cleaner), + OCF_PL_STEP(_ocf_mngt_bind_init_cleaning), OCF_PL_STEP(_ocf_mngt_bind_post_init), OCF_PL_STEP_TERMINATOR(), }, @@ -2378,10 +2380,8 @@ struct ocf_pipeline_properties _ocf_mngt_cache_activate_pipeline_properties = { OCF_PL_STEP(_ocf_mngt_activate_compare_superblock), OCF_PL_STEP(_ocf_mngt_activate_init_properties), OCF_PL_STEP(_ocf_mngt_test_volume), - OCF_PL_STEP(_ocf_mngt_init_cleaner), OCF_PL_STEP(_ocf_mngt_init_promotion), OCF_PL_STEP(_ocf_mngt_load_add_cores), - OCF_PL_STEP(_ocf_mngt_activate_init_cleaning), OCF_PL_STEP(_ocf_mngt_attach_shutdown_status), OCF_PL_STEP(_ocf_mngt_attach_post_init), OCF_PL_STEP_TERMINATOR(), @@ -2454,12 +2454,24 @@ static void _ocf_mngt_cache_load(ocf_cache_t cache, OCF_PL_NEXT_RET(pipeline); } +static void ocf_mngt_stop_standby_stop_cleaner(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) +{ + struct ocf_mngt_cache_stop_context *context = priv; + ocf_cache_t cache = context->cache; + + ocf_stop_cleaner(cache); + + ocf_pipeline_next(pipeline); +} + struct ocf_pipeline_properties ocf_mngt_cache_stop_standby_pipeline_properties = { .priv_size = sizeof(struct ocf_mngt_cache_stop_context), .finish = ocf_mngt_cache_stop_finish, .steps = { OCF_PL_STEP(ocf_mngt_cache_stop_wait_metadata_io), + OCF_PL_STEP(ocf_mngt_stop_standby_stop_cleaner), OCF_PL_STEP(ocf_mngt_cache_close_cache_volume), OCF_PL_STEP(ocf_mngt_cache_deinit_metadata), OCF_PL_STEP(ocf_mngt_cache_deinit_cache_volume), From 52824adaaf57904d0f2dfcc88a65bd34f0b8e192 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Thu, 14 Oct 2021 18:41:30 +0200 Subject: [PATCH 14/16] Additional cleaning policy info outside of the SB Starting cache in a standby mode requires access to a valid cleaning policy type. If the policy is stored only in the superblock, it may be overridden by one of the metadata passive updates. To prevent losing the information it should be stored in cache's runtime metadata. Signed-off-by: Michal Mielewczyk --- src/cleaning/cleaning.h | 1 + src/cleaning/cleaning_ops.h | 16 ++++++++-------- src/mngt/ocf_mngt_cache.c | 29 +++++++++++++++-------------- src/mngt/ocf_mngt_common.c | 7 +++++++ src/mngt/ocf_mngt_common.h | 3 +++ src/mngt/ocf_mngt_flush.c | 6 +++--- src/ocf_cache.c | 2 +- src/ocf_io_class.c | 2 +- 8 files changed, 39 insertions(+), 27 deletions(-) diff --git a/src/cleaning/cleaning.h b/src/cleaning/cleaning.h index 5eee0b8..c48d50e 100644 --- a/src/cleaning/cleaning.h +++ b/src/cleaning/cleaning.h @@ -41,6 +41,7 @@ struct cleaning_policy_meta { struct ocf_cleaner { struct ocf_refcnt refcnt __attribute__((aligned(64))); + ocf_cleaning_t policy; void *cleaning_policy_context; ocf_queue_t io_queue; ocf_cleaner_end_t end; diff --git a/src/cleaning/cleaning_ops.h b/src/cleaning/cleaning_ops.h index 3f72f02..a0f6177 100644 --- a/src/cleaning/cleaning_ops.h +++ b/src/cleaning/cleaning_ops.h @@ -91,7 +91,7 @@ static inline void ocf_cleaning_deinitialize(ocf_cache_t cache) { ocf_cleaning_t policy; - policy = cache->conf_meta->cleaning_policy_type; + policy = cache->cleaner.policy; ENV_BUG_ON(policy >= ocf_cleaning_max); @@ -110,7 +110,7 @@ static inline int ocf_cleaning_add_core(ocf_cache_t cache, if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt))) return -OCF_ERR_NO_LOCK; - policy = cache->conf_meta->cleaning_policy_type; + policy = cache->cleaner.policy; ENV_BUG_ON(policy >= ocf_cleaning_max); @@ -133,7 +133,7 @@ static inline void ocf_cleaning_remove_core(ocf_cache_t cache, if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt))) return; - policy = cache->conf_meta->cleaning_policy_type; + policy = cache->cleaner.policy; ENV_BUG_ON(policy >= ocf_cleaning_max); @@ -154,7 +154,7 @@ static inline void ocf_cleaning_init_cache_block(ocf_cache_t cache, if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt))) return; - policy = cache->conf_meta->cleaning_policy_type; + policy = cache->cleaner.policy; ENV_BUG_ON(policy >= ocf_cleaning_max); if (unlikely(!cleaning_policy_ops[policy].init_cache_block)) @@ -174,7 +174,7 @@ static inline void ocf_cleaning_purge_cache_block(ocf_cache_t cache, if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt))) return; - policy = cache->conf_meta->cleaning_policy_type; + policy = cache->cleaner.policy; ENV_BUG_ON(policy >= ocf_cleaning_max); if (unlikely(!cleaning_policy_ops[policy].purge_cache_block)) @@ -194,7 +194,7 @@ static inline void ocf_cleaning_purge_range(ocf_cache_t cache, if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt))) return; - policy = cache->conf_meta->cleaning_policy_type; + policy = cache->cleaner.policy; ENV_BUG_ON(policy >= ocf_cleaning_max); if (unlikely(!cleaning_policy_ops[policy].purge_range)) @@ -215,7 +215,7 @@ static inline void ocf_cleaning_set_hot_cache_line(ocf_cache_t cache, if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt))) return; - policy = cache->conf_meta->cleaning_policy_type; + policy = cache->cleaner.policy; ENV_BUG_ON(policy >= ocf_cleaning_max); if (unlikely(!cleaning_policy_ops[policy].set_hot_cache_line)) @@ -259,7 +259,7 @@ static inline void ocf_cleaning_perform_cleaning(ocf_cache_t cache, if (unlikely(!ocf_refcnt_inc(&cache->cleaner.refcnt))) return; - policy = cache->conf_meta->cleaning_policy_type; + policy = cache->cleaner.policy; ENV_BUG_ON(policy >= ocf_cleaning_max); if (unlikely(!cleaning_policy_ops[policy].perform_cleaning)) diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index f3de681..8581e67 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -227,7 +227,6 @@ static void __populate_free_safe(ocf_cache_t cache) static ocf_error_t __init_cleaning_policy(ocf_cache_t cache) { - ocf_cleaning_t cleaning_policy = ocf_cleaning_default; int i; OCF_ASSERT_PLUGGED(cache); @@ -237,9 +236,7 @@ static ocf_error_t __init_cleaning_policy(ocf_cache_t cache) for (i = 0; i < ocf_cleaning_max; i++) ocf_cleaning_setup(cache, i); - cache->conf_meta->cleaning_policy_type = ocf_cleaning_default; - - return ocf_cleaning_initialize(cache, cleaning_policy, 1); + return ocf_cleaning_initialize(cache, cache->cleaner.policy, 1); } static void __deinit_cleaning_policy(ocf_cache_t cache) @@ -541,12 +538,7 @@ static void _ocf_mngt_recovery_rebuild_metadata(ocf_cache_t cache) static void _ocf_mngt_bind_rebuild_metadata(ocf_cache_t cache) { - ocf_cleaning_t clean_policy = cache->conf_meta->cleaning_policy_type; - cache->conf_meta->cleaning_policy_type = ocf_cleaning_nop; - _ocf_mngt_rebuild_metadata(cache, true); - - cache->conf_meta->cleaning_policy_type = clean_policy; } static inline ocf_error_t _ocf_init_cleaning_policy(ocf_cache_t cache, @@ -578,8 +570,7 @@ static void _ocf_mngt_load_post_metadata_load(ocf_pipeline_t pipeline, __populate_free_safe(cache); } - result = _ocf_init_cleaning_policy(cache, - cache->conf_meta->cleaning_policy_type, + result = _ocf_init_cleaning_policy(cache, cache->cleaner.policy, context->metadata.shutdown_status); if (result) @@ -1222,6 +1213,7 @@ static void _ocf_mngt_cache_init(ocf_cache_t cache, cache->conf_meta->cache_mode = params->metadata.cache_mode; cache->conf_meta->metadata_layout = params->metadata.layout; cache->conf_meta->promotion_policy_type = params->metadata.promotion_policy; + __set_cleaning_policy(cache, ocf_cleaning_default); INIT_LIST_HEAD(&cache->io_queues); @@ -1410,6 +1402,7 @@ static void _ocf_mngt_load_superblock_complete(void *priv, int error) { struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; + ocf_cleaning_t loaded_clean_policy = cache->conf_meta->cleaning_policy_type; if (cache->conf_meta->cachelines != ocf_metadata_get_cachelines_count(cache)) { @@ -1419,6 +1412,15 @@ static void _ocf_mngt_load_superblock_complete(void *priv, int error) -OCF_ERR_START_CACHE_FAIL); } + if (loaded_clean_policy >= ocf_cleaning_max) { + ocf_cache_log(cache, log_err, + "ERROR: Invalid cleaning policy!\n"); + OCF_PL_FINISH_RET(context->pipeline, + -OCF_ERR_START_CACHE_FAIL); + } + + __set_cleaning_policy(cache, loaded_clean_policy); + if (error) { ocf_cache_log(cache, log_err, "ERROR: Cannot load cache state\n"); @@ -2080,8 +2082,7 @@ static void _ocf_mngt_bind_init_cleaning(ocf_pipeline_t pipeline, ocf_cache_t cache = context->cache; ocf_error_t result; - result = _ocf_init_cleaning_policy(cache, - cache->conf_meta->cleaning_policy_type, + result = _ocf_init_cleaning_policy(cache, cache->cleaner.policy, context->metadata.shutdown_status); if (result) @@ -2821,7 +2822,7 @@ static int _ocf_mngt_cache_load_core_log(ocf_core_t core, void *cntx) static void _ocf_mngt_cache_load_log(ocf_cache_t cache) { ocf_cache_mode_t cache_mode = ocf_cache_get_mode(cache); - ocf_cleaning_t cleaning_type = cache->conf_meta->cleaning_policy_type; + ocf_cleaning_t cleaning_type = cache->cleaner.policy; ocf_promotion_t promotion_type = cache->conf_meta->promotion_policy_type; ocf_cache_log(cache, log_info, "Successfully loaded\n"); diff --git a/src/mngt/ocf_mngt_common.c b/src/mngt/ocf_mngt_common.c index 6d39a3b..3668e65 100644 --- a/src/mngt/ocf_mngt_common.c +++ b/src/mngt/ocf_mngt_common.c @@ -151,6 +151,13 @@ void ocf_mngt_cache_put(ocf_cache_t cache) } } +void __set_cleaning_policy(ocf_cache_t cache, + ocf_cleaning_t new_cleaning_policy) +{ + cache->conf_meta->cleaning_policy_type = new_cleaning_policy; + cache->cleaner.policy = new_cleaning_policy; +} + int ocf_mngt_cache_get_by_name(ocf_ctx_t ctx, const char *name, size_t name_len, ocf_cache_t *cache) { diff --git a/src/mngt/ocf_mngt_common.h b/src/mngt/ocf_mngt_common.h index 5e8b301..751a2e0 100644 --- a/src/mngt/ocf_mngt_common.h +++ b/src/mngt/ocf_mngt_common.h @@ -30,4 +30,7 @@ void ocf_mngt_cache_lock_deinit(ocf_cache_t cache); bool ocf_mngt_cache_is_locked(ocf_cache_t cache); +void __set_cleaning_policy(ocf_cache_t cache, + ocf_cleaning_t new_cleaning_policy); + #endif /* __OCF_MNGT_COMMON_H__ */ diff --git a/src/mngt/ocf_mngt_flush.c b/src/mngt/ocf_mngt_flush.c index 4e5ec6a..440db41 100644 --- a/src/mngt/ocf_mngt_flush.c +++ b/src/mngt/ocf_mngt_flush.c @@ -968,7 +968,7 @@ static void _ocf_mngt_init_clean_policy(ocf_pipeline_t pipeline, void *priv, ocf_cleaning_get_name(new_policy)); } - cache->conf_meta->cleaning_policy_type = new_policy; + __set_cleaning_policy(cache, new_policy); ocf_refcnt_unfreeze(&cache->cleaner.refcnt); ocf_metadata_end_exclusive_access(&cache->metadata.lock); @@ -1014,7 +1014,7 @@ void ocf_mngt_cache_cleaning_set_policy(ocf_cache_t cache, if (ocf_cache_is_standby(cache)) OCF_CMPL_RET(priv, -OCF_ERR_CACHE_STANDBY); - old_policy = cache->conf_meta->cleaning_policy_type; + old_policy = cache->cleaner.policy; if (new_policy == old_policy) { ocf_cache_log(cache, log_info, "Cleaning policy %s is already " @@ -1047,7 +1047,7 @@ int ocf_mngt_cache_cleaning_get_policy(ocf_cache_t cache, ocf_cleaning_t *type) if (ocf_cache_is_standby(cache)) return -OCF_ERR_CACHE_STANDBY; - *type = cache->conf_meta->cleaning_policy_type; + *type = cache->cleaner.policy; return 0; } diff --git a/src/ocf_cache.c b/src/ocf_cache.c index f0fe2bb..2522f61 100644 --- a/src/ocf_cache.c +++ b/src/ocf_cache.c @@ -196,7 +196,7 @@ int ocf_cache_get_info(ocf_cache_t cache, struct ocf_cache_info *info) info->fallback_pt.error_counter = env_atomic_read(&cache->fallback_pt_error_counter); - info->cleaning_policy = cache->conf_meta->cleaning_policy_type; + info->cleaning_policy = cache->cleaner.policy; info->promotion_policy = cache->conf_meta->promotion_policy_type; info->cache_line_size = ocf_line_size(cache); diff --git a/src/ocf_io_class.c b/src/ocf_io_class.c index 3a3554a..b856e90 100644 --- a/src/ocf_io_class.c +++ b/src/ocf_io_class.c @@ -42,7 +42,7 @@ int ocf_cache_io_class_get_info(ocf_cache_t cache, uint32_t io_class, info->min_size = cache->user_parts[part_id].config->min_size; info->max_size = cache->user_parts[part_id].config->max_size; - info->cleaning_policy_type = cache->conf_meta->cleaning_policy_type; + info->cleaning_policy_type = cache->cleaner.policy; info->cache_mode = cache->user_parts[part_id].config->cache_mode; From a6989d1881f4e2a46cf1ef0e2f56fd310ce7804c Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Wed, 3 Nov 2021 15:19:21 +0100 Subject: [PATCH 15/16] Pio concurrency --- src/concurrency/ocf_pio_concurrency.c | 231 ++++++++++++++++++++++++++ src/concurrency/ocf_pio_concurrency.h | 20 +++ 2 files changed, 251 insertions(+) create mode 100644 src/concurrency/ocf_pio_concurrency.c create mode 100644 src/concurrency/ocf_pio_concurrency.h diff --git a/src/concurrency/ocf_pio_concurrency.c b/src/concurrency/ocf_pio_concurrency.c new file mode 100644 index 0000000..854572f --- /dev/null +++ b/src/concurrency/ocf_pio_concurrency.c @@ -0,0 +1,231 @@ +/* + * Copyright(c) 2021 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "ocf_concurrency.h" +#include "../metadata/metadata_internal.h" +#include "../metadata/metadata_io.h" +#include "../ocf_priv.h" +#include "../ocf_request.h" +#include "../utils/utils_alock.h" +#include "../utils/utils_cache_line.h" + +struct pio_ctx { + uint32_t segments_number; + struct { + enum ocf_metadata_segment_id id; + uint64_t first_entry; + uint64_t begin; + uint64_t end; + } segments[5]; +}; + +#define OUT_OF_RANGE -1 + +#define get_pio_ctx(__alock) (void*)__alock + ocf_alock_obj_size(); + +static inline bool page_belongs_to_section(struct pio_ctx *pio_ctx, + enum ocf_metadata_segment_id segment_id, uint32_t page) +{ + return page >= pio_ctx->segments[segment_id].begin && + page < pio_ctx->segments[segment_id].end; +} + +static inline ocf_cache_line_t page_to_entry(struct pio_ctx *pio_ctx, + enum ocf_metadata_segment_id segment_id, uint32_t page) +{ + uint32_t id_within_section; + + id_within_section = page - pio_ctx->segments[segment_id].begin; + + return pio_ctx->segments[segment_id].first_entry + id_within_section; +} + +static ocf_cache_line_t ocf_pio_lock_get_entry(struct ocf_alock *alock, + struct ocf_request *req, uint32_t id) +{ + uint32_t page; + enum ocf_metadata_segment_id segment_id; + struct pio_ctx *pio_ctx = get_pio_ctx(alock); + page = req->core_line_first + id; + + for (segment_id = 0; segment_id < pio_ctx->segments_number; segment_id++) { + if (page_belongs_to_section(pio_ctx, segment_id, page)) + return page_to_entry(pio_ctx, segment_id, page); + } + + return OUT_OF_RANGE; +} + +static int ocf_pio_lock_fast(struct ocf_alock *alock, + struct ocf_request *req, int rw) +{ + ocf_cache_line_t entry; + int ret = OCF_LOCK_ACQUIRED; + int32_t i; + ENV_BUG_ON(rw != OCF_WRITE); + + for (i = 0; i < req->core_line_count; i++) { + entry = ocf_pio_lock_get_entry(alock, req, i); + if (entry == OUT_OF_RANGE) + continue; + + ENV_BUG_ON(ocf_alock_is_index_locked(alock, req, i)); + + if (ocf_alock_trylock_entry_wr(alock, entry)) { + ocf_alock_mark_index_locked(alock, req, i, true); + } else { + ret = OCF_LOCK_NOT_ACQUIRED; + break; + } + } + + if (ret != OCF_LOCK_NOT_ACQUIRED) + return ret; + + /* Request is not locked, discard acquired locks */ + for (; i >= 0; i--) { + entry = ocf_pio_lock_get_entry(alock, req, i); + if (entry == OUT_OF_RANGE) + continue; + + if (ocf_alock_is_index_locked(alock, req, i)) { + ocf_alock_unlock_one_wr(alock, entry); + ocf_alock_mark_index_locked(alock, req, i, false); + } + } + + return ret; +} + +static int ocf_pio_lock_slow(struct ocf_alock *alock, + struct ocf_request *req, int rw, ocf_req_async_lock_cb cmpl) +{ + int32_t i; + ocf_cache_line_t entry; + int ret = OCF_LOCK_ACQUIRED; + ENV_BUG_ON(rw != OCF_WRITE); + + for (i = 0; i < req->core_line_count; i++) { + entry = ocf_pio_lock_get_entry(alock, req, i); + if (entry == OUT_OF_RANGE) + continue; + + ENV_BUG_ON(ocf_alock_is_index_locked(alock, req, i)); + + if (!ocf_alock_lock_one_wr(alock, entry, cmpl, req, i)) { + ENV_BUG(); + /* lock not acquired and not added to wait list */ + ret = -OCF_ERR_NO_MEM; + goto err; + } + } + + return ret; + +err: + for (; i >= 0; i--) { + entry = ocf_pio_lock_get_entry(alock, req, i); + if (entry == OUT_OF_RANGE) + continue; + + ocf_alock_waitlist_remove_entry(alock, req, i, entry, OCF_WRITE); + } + + return ret; +} + +static struct ocf_alock_lock_cbs ocf_pio_conc_cbs = { + .lock_entries_fast = ocf_pio_lock_fast, + .lock_entries_slow = ocf_pio_lock_slow +}; + +int ocf_pio_async_lock(struct ocf_alock *alock, struct ocf_request *req, + ocf_req_async_lock_cb cmpl) +{ + return ocf_alock_lock_wr(alock, req, cmpl); +} + +void ocf_pio_async_unlock(struct ocf_alock *alock, struct ocf_request *req) +{ + ocf_cache_line_t entry; + int i; + + for (i = 0; i < req->core_line_count; i++) { + if (!ocf_alock_is_index_locked(alock, req, i)) + continue; + + entry = ocf_pio_lock_get_entry(alock, req, i); + if (entry == OUT_OF_RANGE) + continue; + + ocf_alock_unlock_one_wr(alock, entry); + ocf_alock_mark_index_locked(alock, req, i, false); + } + + req->alock_status = 0; +} + +#define ALLOCATOR_NAME_FMT "ocf_%s_pio_conc" +#define ALLOCATOR_NAME_MAX (sizeof(ALLOCATOR_NAME_FMT) + OCF_CACHE_NAME_SIZE) + +int ocf_pio_concurrency_init(struct ocf_alock **self, ocf_cache_t cache) +{ + struct ocf_metadata_ctrl *ctrl = cache->metadata.priv; + struct ocf_alock *alock; + struct pio_ctx *pio_ctx; + size_t base_size = ocf_alock_obj_size(); + char name[ALLOCATOR_NAME_MAX]; + int ret; + uint32_t pages_to_alloc = 0; + enum ocf_metadata_segment_id update_segments[] = { + metadata_segment_sb_config, + metadata_segment_part_config, + metadata_segment_core_config, + metadata_segment_core_uuid, + metadata_segment_collision, + }; + int i; + + ENV_BUILD_BUG_ON( + ARRAY_SIZE(update_segments) > ARRAY_SIZE(pio_ctx->segments)); + + ret = snprintf(name, sizeof(name), ALLOCATOR_NAME_FMT, + ocf_cache_get_name(cache)); + if (ret < 0) + return ret; + if (ret >= ALLOCATOR_NAME_MAX) + return -OCF_ERR_NO_MEM; + + alock = env_vzalloc(base_size + sizeof(struct pio_ctx)); + if (!alock) + return -OCF_ERR_NO_MEM; + + pio_ctx = get_pio_ctx(alock); + pio_ctx->segments_number = ARRAY_SIZE(update_segments); + + for (i = 0; i < pio_ctx->segments_number; i++) { + struct ocf_metadata_raw *raw = &(ctrl->raw_desc[update_segments[i]]); + pio_ctx->segments[i].first_entry = pages_to_alloc; + pio_ctx->segments[i].id = update_segments[i]; + pio_ctx->segments[i].begin = raw->ssd_pages_offset; + pio_ctx->segments[i].end = raw->ssd_pages_offset + raw->ssd_pages; + + pages_to_alloc += raw->ssd_pages; + } + + ret = ocf_alock_init_inplace(alock, pages_to_alloc, name, &ocf_pio_conc_cbs, cache); + if (ret) { + env_vfree(alock); + return ret; + } + + *self = alock; + return 0; +} + +void ocf_pio_concurrency_deinit(struct ocf_alock **self) +{ + ocf_alock_deinit(self); +} diff --git a/src/concurrency/ocf_pio_concurrency.h b/src/concurrency/ocf_pio_concurrency.h new file mode 100644 index 0000000..7318bdb --- /dev/null +++ b/src/concurrency/ocf_pio_concurrency.h @@ -0,0 +1,20 @@ +/* + * Copyright(c) 2021-2021 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef __OCF_PIO_CONCURRENCY_H_ +#define __OCF_PIO_CONCURRENCY_H_ + +#include "../utils/utils_alock.h" + +int ocf_pio_async_lock(struct ocf_alock *alock, struct ocf_request *req, + ocf_req_async_lock_cb cmpl); + +void ocf_pio_async_unlock(struct ocf_alock *alock, struct ocf_request *req); + +int ocf_pio_concurrency_init(struct ocf_alock **self, ocf_cache_t cache); + +void ocf_pio_concurrency_deinit(struct ocf_alock **self); + +#endif From 4ab22ee2dcc66df682950ebc34d4341cf8be6bfb Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Tue, 5 Oct 2021 15:52:55 +0200 Subject: [PATCH 16/16] Maintain runtime struct during failover standby To allow the fastest switching from the passive-standby to active mode, the runtime metadata must be kept 100% synced with the metadata on the drive and in the RAM thus recovery is required after each collision section update. To avoid long-lasting recovering of all the cachelines each time the collision section is being updated, the passive update procedure recovers only those which have its MD entries on the updated pages. Signed-off-by: Michal Mielewczyk --- src/metadata/metadata.c | 48 --- src/metadata/metadata.h | 3 +- src/metadata/metadata_passive_update.c | 388 +++++++++++++++++++++++++ src/metadata/metadata_passive_update.h | 16 + src/metadata/metadata_raw.c | 2 +- src/mngt/ocf_mngt_cache.c | 38 ++- src/mngt/ocf_mngt_core.c | 1 + src/ocf_cache.c | 50 +++- src/ocf_cache_priv.h | 5 + 9 files changed, 486 insertions(+), 65 deletions(-) create mode 100644 src/metadata/metadata_passive_update.c create mode 100644 src/metadata/metadata_passive_update.h diff --git a/src/metadata/metadata.c b/src/metadata/metadata.c index 9660cb6..5037021 100644 --- a/src/metadata/metadata.c +++ b/src/metadata/metadata.c @@ -2044,51 +2044,3 @@ void ocf_metadata_probe_cores(ocf_ctx_t ctx, ocf_volume_t volume, ocf_metadata_query_cores(ctx, volume, uuids, uuids_count, ocf_metadata_probe_cores_end, context); } - -int ocf_metadata_passive_update(ocf_cache_t cache, struct ocf_io *io) -{ - struct ocf_metadata_ctrl *ctrl = cache->metadata.priv; - ctx_data_t *data = ocf_io_get_data(io); - uint64_t io_start_page = BYTES_TO_PAGES(io->addr); - uint64_t io_end_page = io_start_page + BYTES_TO_PAGES(io->bytes); - enum ocf_metadata_segment_id update_segments[] = { - metadata_segment_sb_config, - metadata_segment_part_config, - metadata_segment_core_config, - metadata_segment_core_uuid, - metadata_segment_collision, - }; - int i; - - if (io->dir == OCF_READ) - return 0; - - if (io->addr % PAGE_SIZE || io->bytes % PAGE_SIZE) { - ocf_cache_log(cache, log_crit, - "Metadata update not aligned to page size!\n"); - return -OCF_ERR_INVAL; - } - - if (io_start_page >= ctrl->count_pages) - return 0; - - for (i = 0; i < ARRAY_SIZE(update_segments); i++) { - enum ocf_metadata_segment_id seg = update_segments[i]; - struct ocf_metadata_raw *raw = &(ctrl->raw_desc[seg]); - uint64_t raw_start_page = raw->ssd_pages_offset; - uint64_t raw_end_page = raw_start_page + raw->ssd_pages; - uint64_t overlap_start = OCF_MAX(io_start_page, raw_start_page); - uint64_t overlap_end = OCF_MIN(io_end_page, raw_end_page); - uint64_t overlap_start_data = overlap_start - io_start_page; - - if (overlap_start < overlap_end) { - ctx_data_seek(cache->owner, data, ctx_data_seek_begin, - PAGES_TO_BYTES(overlap_start_data)); - ocf_metadata_raw_update(cache, raw, data, - overlap_start - raw_start_page, - overlap_end - overlap_start); - } - } - - return 0; -} diff --git a/src/metadata/metadata.h b/src/metadata/metadata.h index f6db6ea..29e3968 100644 --- a/src/metadata/metadata.h +++ b/src/metadata/metadata.h @@ -18,6 +18,7 @@ #include "metadata_collision.h" #include "metadata_core.h" #include "metadata_misc.h" +#include "metadata_passive_update.h" #define INVALID 0 #define VALID 1 @@ -226,6 +227,4 @@ static inline ocf_cache_line_t ocf_metadata_collision_table_entries( return cache->device->collision_table_entries; } -int ocf_metadata_passive_update(ocf_cache_t cache, struct ocf_io *io); - #endif /* METADATA_H_ */ diff --git a/src/metadata/metadata_passive_update.c b/src/metadata/metadata_passive_update.c new file mode 100644 index 0000000..ed6551f --- /dev/null +++ b/src/metadata/metadata_passive_update.c @@ -0,0 +1,388 @@ +/* + * Copyright(c) 2012-2021 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "ocf/ocf.h" + +#include "metadata.h" +#include "metadata_passive_update.h" +#include "metadata_collision.h" +#include "metadata_segment_id.h" +#include "metadata_internal.h" +#include "metadata_io.h" +#include "metadata_raw.h" +#include "metadata_segment.h" +#include "../concurrency/ocf_concurrency.h" +#include "../ocf_def_priv.h" +#include "../ocf_priv.h" +#include "../utils/utils_cache_line.h" +#include "../utils/utils_io.h" +#include "../utils/utils_pipeline.h" +#include "../concurrency/ocf_pio_concurrency.h" +#include "../engine/engine_common.h" + +#define MAX_PASSIVE_IO_SIZE (32*MiB) + + +static inline void _reset_cline(ocf_cache_t cache, ocf_cache_line_t cline) +{ + /* The cacheline used to be dirty but it is not anymore so it needs to be + moved to a clean lru list */ + ocf_lru_clean_cline(cache, &cache->user_parts[PARTITION_DEFAULT].part, + cline); + + ocf_lru_rm_cline(cache, cline); + ocf_metadata_set_partition_id(cache, cline, PARTITION_FREELIST); +} + +static inline void remove_from_freelist(ocf_cache_t cache, + ocf_cache_line_t cline) +{ + ocf_part_id_t lru_list; + struct ocf_lru_list *list; + + lru_list = (cline % OCF_NUM_LRU_LISTS); + list = ocf_lru_get_list(&cache->free, lru_list, true); + + OCF_METADATA_LRU_WR_LOCK(cline); + ocf_lru_remove_locked(cache, list, cline); + OCF_METADATA_LRU_WR_UNLOCK(cline); +} + +static inline void remove_from_default(ocf_cache_t cache, + ocf_cache_line_t cline) +{ + ocf_part_id_t part_id = PARTITION_DEFAULT; + ocf_part_id_t lru_list; + struct ocf_lru_list *list; + + lru_list = (cline % OCF_NUM_LRU_LISTS); + list = ocf_lru_get_list(&cache->user_parts[part_id].part, lru_list, false); + + OCF_METADATA_LRU_WR_LOCK(cline); + ocf_lru_remove_locked(cache, list, cline); + OCF_METADATA_LRU_WR_UNLOCK(cline); + + env_atomic_dec(&cache->user_parts[part_id].part.runtime->curr_size); + +} + +static void handle_previously_invalid(ocf_cache_t cache, + ocf_cache_line_t cline) +{ + ocf_core_id_t core_id; + uint64_t core_line; + uint32_t lock_idx = ocf_metadata_concurrency_next_idx(cache->mngt_queue); + + /* Pio lock provides exclusive access to the collision page thus either + mapping or status bits can't be changed by a concurrent thread */ + + ocf_metadata_get_core_info(cache, cline, &core_id, &core_line); + + if (metadata_test_dirty(cache, cline) && core_id < OCF_CORE_MAX) { + ENV_BUG_ON(!metadata_test_valid_any(cache, cline)); + /* Moving cline from the freelist to the default partitioin */ + remove_from_freelist(cache, cline); + + ocf_hb_cline_prot_lock_wr(&cache->metadata.lock, lock_idx, core_id, + core_line); + OCF_METADATA_LRU_WR_LOCK(cline); + ocf_cline_rebuild_metadata(cache, core_id, core_line, cline); + OCF_METADATA_LRU_WR_UNLOCK(cline); + ocf_hb_cline_prot_unlock_wr(&cache->metadata.lock, lock_idx, core_id, + core_line); + ocf_cleaning_init_cache_block(cache, cline); + ocf_cleaning_set_hot_cache_line(cache, cline); + + } else { + /* Cline stays on the freelist*/ + + /* To prevent random values in the metadata fill it with the defaults */ + metadata_init_status_bits(cache, cline); + ocf_metadata_set_core_info(cache, cline, OCF_CORE_MAX, ULLONG_MAX); + } +} + +static void handle_previously_valid(ocf_cache_t cache, + ocf_cache_line_t cline) +{ + ocf_core_id_t core_id; + uint64_t core_line; + uint32_t lock_idx = ocf_metadata_concurrency_next_idx(cache->mngt_queue); + + /* Pio lock provides exclusive access to the collision page thus either + mapping or status bits can't be changed by a concurrent thread */ + + ocf_metadata_get_core_info(cache, cline, &core_id, &core_line); + + if (metadata_test_dirty(cache, cline) && core_id < OCF_CORE_MAX) { + /* Cline stays on the default partition*/ + ENV_BUG_ON(!metadata_test_valid_any(cache, cline)); + + remove_from_default(cache, cline); + + ocf_hb_cline_prot_lock_wr(&cache->metadata.lock, lock_idx, core_id, + core_line); + OCF_METADATA_LRU_WR_LOCK(cline); + ocf_cline_rebuild_metadata(cache, core_id, core_line, cline); + OCF_METADATA_LRU_WR_UNLOCK(cline); + ocf_hb_cline_prot_unlock_wr(&cache->metadata.lock, lock_idx, core_id, + core_line); + ocf_cleaning_set_hot_cache_line(cache, cline); + + } else { + /* Moving cline from the default partition to the freelist */ + ocf_cleaning_purge_cache_block(cache, cline); + _reset_cline(cache, cline); + } +} + +static inline void update_list_segment(ocf_cache_t cache, + ocf_cache_line_t start, ocf_cache_line_t count) +{ + ocf_cache_line_t cline, end; + + for (cline = start, end = start + count; cline < end; cline++) { + ocf_part_id_t part_id; + + metadata_clear_dirty_if_invalid(cache, cline); + metadata_clear_valid_if_clean(cache, cline); + + part_id = ocf_metadata_get_partition_id(cache, cline); + switch (part_id) { + case PARTITION_FREELIST: + handle_previously_invalid(cache, cline); + break; + case PARTITION_DEFAULT: + handle_previously_valid(cache, cline); + break; + default: + ocf_cache_log(cache, log_crit, "Passive update: invalid " + "part id for cacheline %u: %hu\n", cline, part_id); + ENV_BUG(); + break; + } + } +} + +static void _dec_core_stats(ocf_core_t core) +{ + ocf_part_id_t part = PARTITION_DEFAULT; + + env_atomic *core_occupancy_counter = &core->runtime_meta->cached_clines; + env_atomic *part_occupancy_counter = + &core->runtime_meta->part_counters[part].cached_clines; + + env_atomic *core_dirty_counter = &core->runtime_meta->dirty_clines; + env_atomic *part_dirty_counter = + &core->runtime_meta->part_counters[part].dirty_clines; + + ENV_BUG_ON(env_atomic_dec_return(core_occupancy_counter) < 0); + ENV_BUG_ON(env_atomic_dec_return(part_occupancy_counter) < 0); + + ENV_BUG_ON(env_atomic_dec_return(core_dirty_counter) < 0); + ENV_BUG_ON(env_atomic_dec_return(part_dirty_counter) < 0); +} + +static void cleanup_old_mapping(ocf_cache_t cache, ocf_cache_line_t start, + ocf_cache_line_t count) +{ + ocf_cache_line_t cline, end; + uint32_t lock_idx = ocf_metadata_concurrency_next_idx(cache->mngt_queue); + + for (cline = start, end = start + count; cline < end; cline++) { + ocf_core_id_t core_id; + uint64_t core_line; + ocf_core_t core; + + ocf_metadata_get_core_info(cache, cline, &core_id, &core_line); + + ENV_BUG_ON(core_id > OCF_CORE_ID_INVALID); + + core = ocf_cache_get_core(cache, core_id); + if (!core) + continue; + + _dec_core_stats(core); + + ocf_hb_cline_prot_lock_wr(&cache->metadata.lock, lock_idx, core_id, + core_line); + ocf_metadata_remove_from_collision(cache, cline, PARTITION_DEFAULT); + ocf_hb_cline_prot_unlock_wr(&cache->metadata.lock, lock_idx, core_id, + core_line); + } +} + +static int passive_io_resume(struct ocf_request *req) +{ + ocf_cache_t cache = req->cache; + struct ocf_metadata_ctrl *ctrl = cache->metadata.priv; + struct ocf_io *io = (struct ocf_io*) req->data; + ctx_data_t *data = ocf_io_get_data(io); + uint64_t io_start_page = BYTES_TO_PAGES(io->addr); + uint64_t io_pages_count = BYTES_TO_PAGES(io->bytes); + uint64_t io_end_page = io_start_page + io_pages_count - 1; + ocf_end_io_t io_cmpl = req->master_io_req; + ocf_cache_line_t cache_etries = ocf_metadata_collision_table_entries(cache); + enum ocf_metadata_segment_id update_segments[] = { + metadata_segment_sb_config, + metadata_segment_part_config, + metadata_segment_core_config, + metadata_segment_core_uuid, + metadata_segment_collision, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(update_segments); i++) { + ocf_cache_line_t cache_line_count, cache_line_range_start; + enum ocf_metadata_segment_id seg = update_segments[i]; + struct ocf_metadata_raw *raw = &(ctrl->raw_desc[seg]); + uint64_t raw_start_page = raw->ssd_pages_offset; + uint64_t raw_end_page = raw_start_page + raw->ssd_pages - 1; + uint64_t overlap_start = OCF_MAX(io_start_page, raw_start_page); + uint64_t overlap_end = OCF_MIN(io_end_page, raw_end_page); + uint64_t overlap_start_data = overlap_start - io_start_page; + uint64_t overlap_page; + uint64_t overlap_count; + + if (overlap_start > overlap_end) + continue; + + overlap_page = overlap_start - raw_start_page; + overlap_count = overlap_end - overlap_start + 1; + + if (seg == metadata_segment_collision) { + /* The range of cachelines with potentially updated collision + section */ + cache_line_range_start = overlap_page * raw->entries_in_page; + cache_line_count = raw->entries_in_page * overlap_count; + + /* The last page of collision section may contain fewer entries than + entries_in_page */ + cache_line_count = OCF_MIN(cache_etries - cache_line_range_start, + cache_line_count); + + /* The collision is not updated yet but the range of affected + cachelines is already known. Remove the old mapping related info + from the metadata */ + cleanup_old_mapping(cache, cache_line_range_start, + cache_line_count); + } + + ctx_data_seek(cache->owner, data, ctx_data_seek_begin, + PAGES_TO_BYTES(overlap_start_data)); + ocf_metadata_raw_update(cache, raw, data, overlap_page, overlap_count); + + if (seg != metadata_segment_collision) + continue; + + update_list_segment(cache, cache_line_range_start, + cache_line_count); + } + + ocf_pio_async_unlock(req->cache->standby.concurrency, req); + io_cmpl(io, 0); + env_allocator_del(cache->standby.allocator, req); + return 0; +} + +static struct ocf_io_if passive_io_restart_if = { + .read = passive_io_resume, + .write = passive_io_resume, +}; + +static void passive_io_page_lock_acquired(struct ocf_request *req) +{ + ocf_engine_push_req_front(req, true); +} + +int ocf_metadata_passive_update(ocf_cache_t cache, struct ocf_io *io, + ocf_end_io_t io_cmpl) +{ + struct ocf_metadata_ctrl *ctrl = cache->metadata.priv; + struct ocf_request *req; + uint64_t io_start_page = BYTES_TO_PAGES(io->addr); + uint64_t io_end_page = io_start_page + BYTES_TO_PAGES(io->bytes); + int lock = 0; + + if (io->dir == OCF_READ) { + io_cmpl(io, 0); + return 0; + } + + if (io_start_page >= ctrl->count_pages) { + io_cmpl(io, 0); + return 0; + } + + if (io->addr % PAGE_SIZE || io->bytes % PAGE_SIZE) { + ocf_cache_log(cache, log_warn, + "Metadata update not aligned to page size!\n"); + io_cmpl(io, -OCF_ERR_INVAL); + return -OCF_ERR_INVAL; + } + + if (io->bytes > MAX_PASSIVE_IO_SIZE) { + //FIXME handle greater IOs + ocf_cache_log(cache, log_warn, + "IO size exceedes max supported size!\n"); + io_cmpl(io, -OCF_ERR_INVAL); + return -OCF_ERR_INVAL; + } + + req = (struct ocf_request*)env_allocator_new(cache->standby.allocator); + if (!req) { + io_cmpl(io, -OCF_ERR_NO_MEM); + return -OCF_ERR_NO_MEM; + } + + req->io_queue = io->io_queue;; + req->info.internal = true; + req->io_if = &passive_io_restart_if; + req->rw = OCF_WRITE; + req->data = io; + req->master_io_req = io_cmpl; + req->cache = cache; + env_atomic_set(&req->lock_remaining, 0); + + req->core_line_first = io_start_page; + req->core_line_count = io_end_page - io_start_page; + req->alock_status = (uint8_t*)&req->map; + + lock = ocf_pio_async_lock(req->cache->standby.concurrency, + req, passive_io_page_lock_acquired); + if (lock < 0) { + env_allocator_del(cache->standby.allocator, req); + io_cmpl(io, lock); + return lock; + } + + if (lock == OCF_LOCK_ACQUIRED) + passive_io_resume(req); + + return 0; +} + +int ocf_metadata_passive_io_ctx_init(ocf_cache_t cache) +{ + char *name = "ocf_cache_pio"; + size_t element_size, header_size, size; + + header_size = sizeof(struct ocf_request); + /* Only one bit per page is required. Since `alock_status` has `uint8_t*` + type, one entry can carry status for 8 pages. */ + element_size = OCF_DIV_ROUND_UP(BYTES_TO_PAGES(MAX_PASSIVE_IO_SIZE), 8); + size = header_size + element_size; + + cache->standby.allocator = env_allocator_create(size, name, true); + if (cache->standby.allocator == NULL) + return -1; + + return 0; +} + +void ocf_metadata_passive_io_ctx_deinit(ocf_cache_t cache) +{ + env_allocator_destroy(cache->standby.allocator); +} diff --git a/src/metadata/metadata_passive_update.h b/src/metadata/metadata_passive_update.h new file mode 100644 index 0000000..18657a9 --- /dev/null +++ b/src/metadata/metadata_passive_update.h @@ -0,0 +1,16 @@ +/* + * Copyright(c) 2012-2021 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef __OCF_METADATA_PASSIVE_IO_H__ +#define __OCF_METADATA_PASSIVE_IO_H__ + +int ocf_metadata_passive_update(ocf_cache_t cache, struct ocf_io *io, + ocf_end_io_t io_cmpl); + +int ocf_metadata_passive_io_ctx_init(ocf_cache_t cache); + +void ocf_metadata_passive_io_ctx_deinit(ocf_cache_t cache); + +#endif diff --git a/src/metadata/metadata_raw.c b/src/metadata/metadata_raw.c index 5933101..99a532b 100644 --- a/src/metadata/metadata_raw.c +++ b/src/metadata/metadata_raw.c @@ -191,7 +191,6 @@ static void *_raw_ram_access(ocf_cache_t cache, static int _raw_ram_drain_page(ocf_cache_t cache, struct ocf_metadata_raw *raw, ctx_data_t *data, uint32_t page) { - uint32_t size = raw->entry_size * raw->entries_in_page; ocf_cache_line_t line; @@ -222,6 +221,7 @@ static int _raw_ram_update(ocf_cache_t cache, _raw_ram_drain_page(cache, raw, data, page + i); return 0; + } struct _raw_ram_load_all_context { diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 8581e67..88c742f 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -25,6 +25,7 @@ #include "../ocf_ctx_priv.h" #include "../cleaning/cleaning.h" #include "../promotion/ops.h" +#include "../concurrency/ocf_pio_concurrency.h" #define OCF_ASSERT_PLUGGED(cache) ENV_BUG_ON(!(cache)->device) @@ -144,6 +145,10 @@ struct ocf_cache_attach_context { */ bool concurrency_inited : 1; + + bool pio_mpool : 1; + + bool pio_concurrency : 1; } flags; struct { @@ -433,8 +438,8 @@ static void _ocf_mngt_load_add_cores(ocf_pipeline_t pipeline, } else { core->opened = true; } - } + } env_bit_set(core_id, cache->conf_meta->valid_core_bitmap); core->added = true; cache->conf_meta->core_count++; @@ -1646,6 +1651,12 @@ static void _ocf_mngt_attach_handle_error( cache->device = NULL; } + if (context->flags.pio_concurrency) + ocf_pio_concurrency_deinit(&cache->standby.concurrency); + + if (context->flags.pio_mpool) + ocf_metadata_passive_io_ctx_deinit(cache); + ocf_pipeline_destroy(cache->stop_pipeline); } @@ -2052,15 +2063,34 @@ static void _ocf_mngt_load_metadata_unsafe(ocf_pipeline_t pipeline, _ocf_mngt_load_unsafe_complete, context); } +static void _ocf_mngt_bind_preapre_mempool(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) +{ + struct ocf_cache_attach_context *context = priv; + ocf_cache_t cache = context->cache; + int result; + + result = ocf_metadata_passive_io_ctx_init(cache); + if(!result) + context->flags.pio_mpool = true; + + OCF_PL_NEXT_ON_SUCCESS_RET(context->pipeline, result); +} + static void _ocf_mngt_bind_init_attached_structures(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; + int result; init_attached_data_structures_recovery(cache, false); - ocf_pipeline_next(context->pipeline); + result = ocf_pio_concurrency_init(&cache->standby.concurrency, cache); + if (!result) + context->flags.pio_concurrency = true; + + OCF_PL_NEXT_ON_SUCCESS_RET(context->pipeline, result); } static void _ocf_mngt_bind_recovery_unsafe(ocf_pipeline_t pipeline, @@ -2115,6 +2145,7 @@ struct ocf_pipeline_properties _ocf_mngt_cache_standby_pipeline_properties = { OCF_PL_STEP(_ocf_mngt_test_volume), OCF_PL_STEP(_ocf_mngt_attach_prepare_metadata), OCF_PL_STEP(_ocf_mngt_load_metadata_unsafe), + OCF_PL_STEP(_ocf_mngt_bind_preapre_mempool), OCF_PL_STEP(_ocf_mngt_bind_init_attached_structures), OCF_PL_STEP(_ocf_mngt_bind_recovery_unsafe), OCF_PL_STEP(_ocf_mngt_init_cleaner), @@ -2936,6 +2967,9 @@ static void _ocf_mngt_cache_activate_complete(ocf_cache_t cache, void *priv1, _ocf_mngt_cache_set_active(cache); ocf_cache_log(cache, log_info, "Successfully activated\n"); + ocf_pio_concurrency_deinit(&cache->standby.concurrency); + ocf_metadata_passive_io_ctx_deinit(cache); + OCF_CMPL_RET(cache, priv2, 0); } diff --git a/src/mngt/ocf_mngt_core.c b/src/mngt/ocf_mngt_core.c index b535068..a261069 100644 --- a/src/mngt/ocf_mngt_core.c +++ b/src/mngt/ocf_mngt_core.c @@ -508,6 +508,7 @@ static void ocf_mngt_cache_add_core_finish(ocf_pipeline_t pipeline, ocf_core_t core = context->core; if (error) { + _ocf_mngt_cache_add_core_handle_error(context); if (error == -OCF_ERR_CORE_NOT_AVAIL) { diff --git a/src/ocf_cache.c b/src/ocf_cache.c index 2522f61..1a5d203 100644 --- a/src/ocf_cache.c +++ b/src/ocf_cache.c @@ -5,6 +5,7 @@ #include "ocf/ocf.h" #include "metadata/metadata.h" +#include "metadata/metadata.h" #include "engine/cache_engine.h" #include "utils/utils_cache_line.h" #include "ocf_request.h" @@ -260,6 +261,7 @@ struct ocf_cache_volume_io_priv { struct ocf_io *io; struct ctx_data_t *data; env_atomic remaining; + env_atomic error; }; struct ocf_cache_volume { @@ -273,7 +275,8 @@ static inline ocf_cache_t ocf_volume_to_cache(ocf_volume_t volume) return cache_volume->cache; } -static void ocf_cache_volume_io_complete(struct ocf_io *vol_io, int error) +static void ocf_cache_volume_io_complete_generic(struct ocf_io *vol_io, + int error) { struct ocf_cache_volume_io_priv *priv; struct ocf_io *io = vol_io->priv1; @@ -289,6 +292,33 @@ static void ocf_cache_volume_io_complete(struct ocf_io *vol_io, int error) ocf_refcnt_dec(&cache->refcnt.metadata); } +static void ocf_cache_io_complete(struct ocf_io *io, int error) +{ + struct ocf_cache_volume_io_priv *priv; + ocf_cache_t cache; + + cache = ocf_volume_to_cache(ocf_io_get_volume(io)); + + priv = ocf_io_get_priv(io); + + env_atomic_cmpxchg(&priv->error, 0, error); + + if (env_atomic_dec_return(&priv->remaining)) + return; + + ocf_refcnt_dec(&cache->refcnt.metadata); + ocf_io_end(io, env_atomic_read(&priv->error)); +} + +static void ocf_cache_volume_io_complete(struct ocf_io *vol_io, int error) +{ + struct ocf_io *io = vol_io->priv1; + + ocf_io_put(vol_io); + + ocf_cache_io_complete(io, error); +} + static int ocf_cache_volume_prepare_vol_io(struct ocf_io *io, struct ocf_io **vol_io) { @@ -313,14 +343,11 @@ static int ocf_cache_volume_prepare_vol_io(struct ocf_io *io, return result; } - ocf_io_set_cmpl(tmp_io, io, NULL, ocf_cache_volume_io_complete); - *vol_io = tmp_io; return 0; } - static void ocf_cache_volume_submit_io(struct ocf_io *io) { struct ocf_cache_volume_io_priv *priv; @@ -336,7 +363,8 @@ static void ocf_cache_volume_submit_io(struct ocf_io *io) return; } - env_atomic_set(&priv->remaining, 2); + env_atomic_set(&priv->remaining, 3); + env_atomic_set(&priv->error, 0); result = ocf_cache_volume_prepare_vol_io(io, &vol_io); if (result) { @@ -344,20 +372,16 @@ static void ocf_cache_volume_submit_io(struct ocf_io *io) return; } + ocf_io_set_cmpl(vol_io, io, NULL, ocf_cache_volume_io_complete); ocf_volume_submit_io(vol_io); - result = ocf_metadata_passive_update(cache, io); + result = ocf_metadata_passive_update(cache, io, ocf_cache_io_complete); if (result) { ocf_cache_log(cache, log_crit, "Metadata update error (error=%d)!\n", result); } - if (env_atomic_dec_return(&priv->remaining)) - return; - - ocf_io_put(vol_io); - ocf_io_end(io, 0); - ocf_refcnt_dec(&cache->refcnt.metadata); + ocf_cache_io_complete(io, 0); } @@ -383,6 +407,7 @@ static void ocf_cache_volume_submit_flush(struct ocf_io *io) ocf_io_end(io, result); return; } + ocf_io_set_cmpl(vol_io, io, NULL, ocf_cache_volume_io_complete_generic); ocf_volume_submit_flush(vol_io); } @@ -410,6 +435,7 @@ static void ocf_cache_volume_submit_discard(struct ocf_io *io) ocf_io_end(io, result); return; } + ocf_io_set_cmpl(vol_io, io, NULL, ocf_cache_volume_io_complete_generic); ocf_volume_submit_discard(vol_io); } diff --git a/src/ocf_cache_priv.h b/src/ocf_cache_priv.h index 6771f0e..fdb6e42 100644 --- a/src/ocf_cache_priv.h +++ b/src/ocf_cache_priv.h @@ -85,6 +85,11 @@ struct ocf_cache { struct ocf_refcnt metadata __attribute__((aligned(64))); } refcnt; + struct { + env_allocator *allocator; + struct ocf_alock *concurrency; + } standby; + struct ocf_core core[OCF_CORE_MAX]; ocf_pipeline_t stop_pipeline;