diff --git a/env/posix/ocf_env.h b/env/posix/ocf_env.h index 4fa163a..e1ab63e 100644 --- a/env/posix/ocf_env.h +++ b/env/posix/ocf_env.h @@ -306,7 +306,7 @@ static inline int env_rwsem_init(env_rwsem *s) static inline void env_rwsem_up_read(env_rwsem *s) { - pthread_rwlock_unlock(&s->lock); + ENV_BUG_ON(pthread_rwlock_unlock(&s->lock)); } static inline void env_rwsem_down_read(env_rwsem *s) diff --git a/src/cleaning/acp.c b/src/cleaning/acp.c index 07f6a76..841bf47 100644 --- a/src/cleaning/acp.c +++ b/src/cleaning/acp.c @@ -235,32 +235,6 @@ void cleaning_policy_acp_deinitialize(struct ocf_cache *cache) cache->cleaner.cleaning_policy_context = NULL; } -static void _acp_rebuild(struct ocf_cache *cache) -{ - ocf_cache_line_t cline; - ocf_core_id_t cline_core_id; - uint32_t step = 0; - - for (cline = 0; cline < cache->device->collision_table_entries; cline++) { - ocf_metadata_get_core_and_part_id(cache, cline, &cline_core_id, - NULL); - - OCF_COND_RESCHED_DEFAULT(step); - - if (cline_core_id == OCF_CORE_MAX) - continue; - - cleaning_policy_acp_init_cache_block(cache, cline); - - if (!metadata_test_dirty(cache, cline)) - continue; - - cleaning_policy_acp_set_hot_cache_line(cache, cline); - } - - ocf_cache_log(cache, log_info, "Finished rebuilding ACP metadata\n"); -} - void cleaning_policy_acp_setup(struct ocf_cache *cache) { struct acp_cleaning_policy_config *config; @@ -271,7 +245,7 @@ void cleaning_policy_acp_setup(struct ocf_cache *cache) config->flush_max_buffers = OCF_ACP_DEFAULT_FLUSH_MAX_BUFFERS; } -int cleaning_policy_acp_init_common(ocf_cache_t cache) +int cleaning_policy_acp_initialize(ocf_cache_t cache, int kick_cleaner) { struct acp_context *acp; int err, i; @@ -317,26 +291,15 @@ int cleaning_policy_acp_init_common(ocf_cache_t cache) } } - return 0; -} - -int cleaning_policy_acp_initialize(ocf_cache_t cache, int init_metadata) -{ - int result; - - result = cleaning_policy_acp_init_common(cache); - if (result) - return result; - - _acp_rebuild(cache); - ocf_kick_cleaner(cache); + if (kick_cleaner) + ocf_kick_cleaner(cache); return 0; } -#define OCF_ACP_RECOVERY_SHARDS_CNT 32 +#define OCF_ACP_POPULATE_SHARDS_CNT 32 -struct ocf_acp_recovery_context { +struct ocf_acp_populate_context { ocf_cache_t cache; struct { @@ -344,16 +307,16 @@ struct ocf_acp_recovery_context { struct { struct list_head chunk_list; } bucket[ACP_MAX_BUCKETS]; - } shard[OCF_ACP_RECOVERY_SHARDS_CNT]; + } shard[OCF_ACP_POPULATE_SHARDS_CNT]; - ocf_cleaning_recovery_end_t cmpl; + ocf_cleaning_populate_end_t cmpl; void *priv; }; -static int ocf_acp_recovery_handle(ocf_parallelize_t parallelize, +static int ocf_acp_populate_handle(ocf_parallelize_t parallelize, void *priv, unsigned shard_id, unsigned shards_cnt) { - struct ocf_acp_recovery_context *context = priv; + struct ocf_acp_populate_context *context = priv; ocf_cache_t cache = context->cache; ocf_cache_line_t entries = cache->device->collision_table_entries; ocf_cache_line_t cline, portion; @@ -390,7 +353,7 @@ static int ocf_acp_recovery_handle(ocf_parallelize_t parallelize, return 0; } -static void ocf_acp_recovery_chunk(struct ocf_acp_recovery_context *context, +static void ocf_acp_populate_chunk(struct ocf_acp_populate_context *context, struct acp_chunk_info *chunk) { ocf_cache_t cache = context->cache; @@ -400,7 +363,7 @@ static void ocf_acp_recovery_chunk(struct ocf_acp_recovery_context *context, uint8_t bucket_id; chunk->num_dirty = 0; - for (shard_id = 0; shard_id < OCF_ACP_RECOVERY_SHARDS_CNT; shard_id++) { + for (shard_id = 0; shard_id < OCF_ACP_POPULATE_SHARDS_CNT; shard_id++) { chunk->num_dirty += context->shard[shard_id] .chunk[chunk->core_id][chunk->chunk_id]; } @@ -417,10 +380,10 @@ static void ocf_acp_recovery_chunk(struct ocf_acp_recovery_context *context, list_move_tail(&chunk->list, &bucket->chunk_list); } -static void ocf_acp_recovery_finish(ocf_parallelize_t parallelize, +static void ocf_acp_populate_finish(ocf_parallelize_t parallelize, void *priv, int error) { - struct ocf_acp_recovery_context *context = priv; + struct ocf_acp_populate_context *context = priv; ocf_cache_t cache = context->cache; struct acp_context *acp = _acp_get_ctx_from_cache(cache); ocf_core_id_t core_id; @@ -435,7 +398,7 @@ static void ocf_acp_recovery_finish(ocf_parallelize_t parallelize, num_chunks = OCF_DIV_ROUND_UP(core_size, ACP_CHUNK_SIZE); for (chunk_id = 0; chunk_id < num_chunks; chunk_id++) { - ocf_acp_recovery_chunk(context, + ocf_acp_populate_chunk(context, &acp->chunk_info[core_id][chunk_id]); OCF_COND_RESCHED_DEFAULT(step); } @@ -455,14 +418,14 @@ static void ocf_acp_recovery_finish(ocf_parallelize_t parallelize, ocf_parallelize_destroy(parallelize); } -void cleaning_policy_acp_recovery(ocf_cache_t cache, - ocf_cleaning_recovery_end_t cmpl, void *priv) +void cleaning_policy_acp_populate(ocf_cache_t cache, + ocf_cleaning_populate_end_t cmpl, void *priv) { - struct ocf_acp_recovery_context *context; + struct ocf_acp_populate_context *context; ocf_parallelize_t parallelize; ocf_core_id_t core_id; ocf_core_t core; - unsigned shards_cnt = OCF_ACP_RECOVERY_SHARDS_CNT; + unsigned shards_cnt = OCF_ACP_POPULATE_SHARDS_CNT; unsigned shard_id; uint64_t core_size; uint64_t num_chunks; @@ -470,8 +433,8 @@ void cleaning_policy_acp_recovery(ocf_cache_t cache, int result; result = ocf_parallelize_create(¶llelize, cache, - OCF_ACP_RECOVERY_SHARDS_CNT, sizeof(*context), - ocf_acp_recovery_handle, ocf_acp_recovery_finish); + OCF_ACP_POPULATE_SHARDS_CNT, sizeof(*context), + ocf_acp_populate_handle, ocf_acp_populate_finish); if (result) { cmpl(priv, result); return; @@ -498,10 +461,6 @@ void cleaning_policy_acp_recovery(ocf_cache_t cache, } } - result = cleaning_policy_acp_init_common(cache); - if (result) - goto err; - ocf_parallelize_run(parallelize); return; diff --git a/src/cleaning/acp.h b/src/cleaning/acp.h index 52e9fa1..d260501 100644 --- a/src/cleaning/acp.h +++ b/src/cleaning/acp.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ #ifndef __LAYER_CLEANING_POLICY_AGGRESSIVE_H__ @@ -10,10 +10,10 @@ void cleaning_policy_acp_setup(ocf_cache_t cache); -int cleaning_policy_acp_initialize(ocf_cache_t cache, int init_metadata); +int cleaning_policy_acp_initialize(ocf_cache_t cache, int kick_cleaner); -void cleaning_policy_acp_recovery(ocf_cache_t cache, - ocf_cleaning_recovery_end_t cmpl, void *priv); +void cleaning_policy_acp_populate(ocf_cache_t cache, + ocf_cleaning_populate_end_t cmpl, void *priv); void cleaning_policy_acp_deinitialize(ocf_cache_t cache); diff --git a/src/cleaning/alru.c b/src/cleaning/alru.c index a2ec098..d3065c5 100644 --- a/src/cleaning/alru.c +++ b/src/cleaning/alru.c @@ -349,42 +349,6 @@ void cleaning_policy_alru_set_hot_cache_line(struct ocf_cache *cache, env_spinlock_unlock(&ctx->list_lock[part_id]); } -static void _alru_rebuild(struct ocf_cache *cache) -{ - struct ocf_user_part *user_part; - struct alru_cleaning_policy *part_alru; - ocf_part_id_t part_id; - ocf_core_id_t core_id; - ocf_cache_line_t cline; - uint32_t step = 0; - - for_each_user_part(cache, user_part, part_id) { - /* ALRU initialization */ - part_alru = &user_part->clean_pol->policy.alru; - env_atomic_set(&part_alru->size, 0); - part_alru->lru_head = cache->device->collision_table_entries; - part_alru->lru_tail = cache->device->collision_table_entries; - cache->device->runtime_meta->cleaning_thread_access = 0; - } - - for (cline = 0; cline < cache->device->collision_table_entries; cline++) { - ocf_metadata_get_core_and_part_id(cache, cline, &core_id, - NULL); - - OCF_COND_RESCHED_DEFAULT(step); - - if (core_id == OCF_CORE_MAX) - continue; - - cleaning_policy_alru_init_cache_block(cache, cline); - - if (!metadata_test_dirty(cache, cline)) - continue; - - cleaning_policy_alru_set_hot_cache_line(cache, cline); - } -} - void cleaning_policy_alru_setup(struct ocf_cache *cache) { struct alru_cleaning_policy_config *config; @@ -397,7 +361,7 @@ void cleaning_policy_alru_setup(struct ocf_cache *cache) config->activity_threshold = OCF_ALRU_DEFAULT_ACTIVITY_THRESHOLD; } -int cleaning_policy_alru_init_common(ocf_cache_t cache) +int cleaning_policy_alru_initialize(ocf_cache_t cache, int kick_cleaner) { struct alru_context *ctx; int error = 0; @@ -426,41 +390,28 @@ int cleaning_policy_alru_init_common(ocf_cache_t cache) cache->cleaner.cleaning_policy_context = ctx; - return 0; -} - -int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata) -{ - int result; - - result = cleaning_policy_alru_init_common(cache); - if (result) - return result; - - if (init_metadata) - _alru_rebuild(cache); - - ocf_kick_cleaner(cache); + if (kick_cleaner) + ocf_kick_cleaner(cache); return 0; } -#define OCF_ALRU_RECOVERY_SHARDS_CNT 32 +#define OCF_ALRU_POPULATE_SHARDS_CNT 32 -struct ocf_alru_recovery_context { +struct ocf_alru_populate_context { ocf_cache_t cache; struct { struct { ocf_cache_line_t head; ocf_cache_line_t tail; } part[OCF_USER_IO_CLASS_MAX]; - } shard[OCF_ALRU_RECOVERY_SHARDS_CNT] __attribute__((aligned(64))); + } shard[OCF_ALRU_POPULATE_SHARDS_CNT] __attribute__((aligned(64))); - ocf_cleaning_recovery_end_t cmpl; + ocf_cleaning_populate_end_t cmpl; void *priv; }; -static void add_alru_head_recovery(struct ocf_alru_recovery_context *context, +static void add_alru_head_populate(struct ocf_alru_populate_context *context, unsigned shard_id, ocf_core_id_t part_id, ocf_cache_line_t cline) { @@ -499,10 +450,10 @@ static void add_alru_head_recovery(struct ocf_alru_recovery_context *context, } } -static int ocf_alru_recovery_handle(ocf_parallelize_t parallelize, +static int ocf_alru_populate_handle(ocf_parallelize_t parallelize, void *priv, unsigned shard_id, unsigned shards_cnt) { - struct ocf_alru_recovery_context *context = priv; + struct ocf_alru_populate_context *context = priv; ocf_cache_t cache = context->cache; ocf_cache_line_t entries = cache->device->collision_table_entries; ocf_cache_line_t terminator = entries; @@ -537,7 +488,7 @@ static int ocf_alru_recovery_handle(ocf_parallelize_t parallelize, if (!metadata_test_dirty(cache, cline)) { cleaning_policy_alru_init_cache_block(cache, cline); } else { - add_alru_head_recovery(context, shard_id, + add_alru_head_populate(context, shard_id, part_id, cline); ++part_size[part_id]; } @@ -551,10 +502,10 @@ static int ocf_alru_recovery_handle(ocf_parallelize_t parallelize, return 0; } -static void ocf_alru_recovery_finish(ocf_parallelize_t parallelize, +static void ocf_alru_populate_finish(ocf_parallelize_t parallelize, void *priv, int error) { - struct ocf_alru_recovery_context *context = priv; + struct ocf_alru_populate_context *context = priv; ocf_cache_t cache = context->cache; ocf_part_id_t part_id; ocf_cache_line_t head, tail; @@ -564,7 +515,7 @@ static void ocf_alru_recovery_finish(ocf_parallelize_t parallelize, goto end; for (part_id = 0; part_id < OCF_USER_IO_CLASS_MAX; part_id++) { - for (shard = 0; shard < OCF_ALRU_RECOVERY_SHARDS_CNT; shard++) { + for (shard = 0; shard < OCF_ALRU_POPULATE_SHARDS_CNT; shard++) { head = context->shard[shard].part[part_id].head; tail = context->shard[shard].part[part_id].tail; @@ -580,10 +531,10 @@ end: ocf_parallelize_destroy(parallelize); } -void cleaning_policy_alru_recovery(ocf_cache_t cache, - ocf_cleaning_recovery_end_t cmpl, void *priv) +void cleaning_policy_alru_populate(ocf_cache_t cache, + ocf_cleaning_populate_end_t cmpl, void *priv) { - struct ocf_alru_recovery_context *context; + struct ocf_alru_populate_context *context; ocf_parallelize_t parallelize; struct alru_cleaning_policy *part_alru; struct ocf_user_part *user_part; @@ -591,21 +542,13 @@ void cleaning_policy_alru_recovery(ocf_cache_t cache, int result; result = ocf_parallelize_create(¶llelize, cache, - OCF_ALRU_RECOVERY_SHARDS_CNT, sizeof(*context), - ocf_alru_recovery_handle, ocf_alru_recovery_finish); + OCF_ALRU_POPULATE_SHARDS_CNT, sizeof(*context), + ocf_alru_populate_handle, ocf_alru_populate_finish); if (result) { cmpl(priv, result); return; } - - result = cleaning_policy_alru_init_common(cache); - if (result) { - ocf_parallelize_destroy(parallelize); - cmpl(priv, result); - return; - } - for_each_user_part(cache, user_part, part_id) { /* ALRU initialization */ part_alru = &user_part->clean_pol->policy.alru; diff --git a/src/cleaning/alru.h b/src/cleaning/alru.h index b72fa3a..8d8c6b5 100644 --- a/src/cleaning/alru.h +++ b/src/cleaning/alru.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ #ifndef __LAYER_CLEANING_POLICY_ALRU_H__ @@ -10,9 +10,9 @@ #include "alru_structs.h" void cleaning_policy_alru_setup(ocf_cache_t cache); -int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata); -void cleaning_policy_alru_recovery(ocf_cache_t cache, - ocf_cleaning_recovery_end_t cmpl, void *priv); +int cleaning_policy_alru_initialize(ocf_cache_t cache, int kick_cleaner); +void cleaning_policy_alru_populate(ocf_cache_t cache, + ocf_cleaning_populate_end_t cmpl, void *priv); void cleaning_policy_alru_deinitialize(ocf_cache_t cache); void cleaning_policy_alru_init_cache_block(ocf_cache_t cache, uint32_t cache_line); diff --git a/src/cleaning/cleaning.h b/src/cleaning/cleaning.h index f85c002..99aa1f1 100644 --- a/src/cleaning/cleaning.h +++ b/src/cleaning/cleaning.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ @@ -54,6 +54,6 @@ void ocf_kick_cleaner(ocf_cache_t cache); void ocf_stop_cleaner(ocf_cache_t cache); -typedef void (*ocf_cleaning_recovery_end_t)(void *priv, int error); +typedef void (*ocf_cleaning_populate_end_t)(void *priv, int error); #endif diff --git a/src/cleaning/cleaning_ops.h b/src/cleaning/cleaning_ops.h index 134d324..60493a8 100644 --- a/src/cleaning/cleaning_ops.h +++ b/src/cleaning/cleaning_ops.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ @@ -14,8 +14,8 @@ struct cleaning_policy_ops { void (*setup)(ocf_cache_t cache); int (*initialize)(ocf_cache_t cache, int init_metadata); - void (*recovery)(ocf_cache_t cache, - ocf_cleaning_recovery_end_t cmpl, void *priv); + void (*populate)(ocf_cache_t cache, + ocf_cleaning_populate_end_t cmpl, void *priv); void (*deinitialize)(ocf_cache_t cache); int (*add_core)(ocf_cache_t cache, ocf_core_id_t core_id); void (*remove_core)(ocf_cache_t cache, ocf_core_id_t core_id); @@ -45,7 +45,7 @@ static struct cleaning_policy_ops cleaning_policy_ops[ocf_cleaning_max] = { .purge_range = cleaning_policy_alru_purge_range, .set_hot_cache_line = cleaning_policy_alru_set_hot_cache_line, .initialize = cleaning_policy_alru_initialize, - .recovery = cleaning_policy_alru_recovery, + .populate = cleaning_policy_alru_populate, .deinitialize = cleaning_policy_alru_deinitialize, .set_cleaning_param = cleaning_policy_alru_set_cleaning_param, .get_cleaning_param = cleaning_policy_alru_get_cleaning_param, @@ -59,7 +59,7 @@ static struct cleaning_policy_ops cleaning_policy_ops[ocf_cleaning_max] = { .purge_range = cleaning_policy_acp_purge_range, .set_hot_cache_line = cleaning_policy_acp_set_hot_cache_line, .initialize = cleaning_policy_acp_initialize, - .recovery = cleaning_policy_acp_recovery, + .populate = cleaning_policy_acp_populate, .deinitialize = cleaning_policy_acp_deinitialize, .set_cleaning_param = cleaning_policy_acp_set_cleaning_param, .get_cleaning_param = cleaning_policy_acp_get_cleaning_param, @@ -81,28 +81,28 @@ static inline void ocf_cleaning_setup(ocf_cache_t cache, ocf_cleaning_t policy) } static inline int ocf_cleaning_initialize(ocf_cache_t cache, - ocf_cleaning_t policy, int init_metadata) + ocf_cleaning_t policy, int kick_cleaner) { ENV_BUG_ON(policy >= ocf_cleaning_max); if (unlikely(!cleaning_policy_ops[policy].initialize)) return 0; - return cleaning_policy_ops[policy].initialize(cache, init_metadata); + return cleaning_policy_ops[policy].initialize(cache, kick_cleaner); } -static inline void ocf_cleaning_recovery(ocf_cache_t cache, +static inline void ocf_cleaning_populate(ocf_cache_t cache, ocf_cleaning_t policy, - ocf_cleaning_recovery_end_t cmpl, void *priv) + ocf_cleaning_populate_end_t cmpl, void *priv) { ENV_BUG_ON(policy >= ocf_cleaning_max); - if (unlikely(!cleaning_policy_ops[policy].recovery)) { + if (unlikely(!cleaning_policy_ops[policy].populate)) { cmpl(priv, 0); return; } - cleaning_policy_ops[policy].recovery(cache, cmpl, priv); + cleaning_policy_ops[policy].populate(cache, cmpl, priv); } static inline void ocf_cleaning_deinitialize(ocf_cache_t cache) diff --git a/src/metadata/metadata.c b/src/metadata/metadata.c index d5bff02..96a9aeb 100644 --- a/src/metadata/metadata.c +++ b/src/metadata/metadata.c @@ -18,6 +18,7 @@ #include "../utils/utils_cache_line.h" #include "../utils/utils_io.h" #include "../utils/utils_pipeline.h" +#include "../utils/utils_parallelize.h" #define OCF_METADATA_DEBUG 0 @@ -831,35 +832,118 @@ static inline void _ocf_init_collision_entry(struct ocf_cache *cache, /* * Initialize collision table */ -void ocf_metadata_init_collision(struct ocf_cache *cache) -{ - unsigned int i; - unsigned int step = 0; - for (i = 0; i < cache->device->collision_table_entries; i++) { - _ocf_init_collision_entry(cache, i); +static int ocf_metadata_init_collision_handle(ocf_parallelize_t parallelize, + void *priv, unsigned shard_id, unsigned shards_cnt) +{ + struct ocf_init_metadata_context *context = priv; + ocf_cache_t cache = context->cache; + ocf_cache_line_t collision_table_entries = cache->device->collision_table_entries; + uint32_t entry, portion, begin, end, step=0; + + portion = OCF_DIV_ROUND_UP((uint64_t)collision_table_entries, shards_cnt); + begin = portion*shard_id; + end = OCF_MIN(portion*(shard_id + 1), collision_table_entries); + + for (entry = begin; entry < end; entry++) { OCF_COND_RESCHED_DEFAULT(step); + + if (entry >= collision_table_entries) + break; + + _ocf_init_collision_entry(cache, entry); } + + return 0; +} + +static void ocf_metadata_init_finish(ocf_parallelize_t parallelize, + void *priv, int error) +{ + struct ocf_init_metadata_context *context = priv; + + ocf_pipeline_next(context->pipeline); + + ocf_parallelize_destroy(parallelize); +} + +void ocf_metadata_init_collision(ocf_pipeline_t pipeline, void *priv, + ocf_pipeline_arg_t arg) +{ + struct ocf_init_metadata_context *context = priv; + struct ocf_init_metadata_context *parallel_context; + ocf_cache_t cache = context->cache; + ocf_parallelize_t parallelize; + int result; + + if (context->skip_collision) + OCF_PL_NEXT_RET(pipeline); + + result = ocf_parallelize_create(¶llelize, cache, + ocf_cache_get_queue_count(cache), sizeof(*context), + ocf_metadata_init_collision_handle, + ocf_metadata_init_finish); + if (result) + OCF_PL_FINISH_RET(pipeline, result); + + parallel_context = ocf_parallelize_get_priv(parallelize); + + parallel_context->pipeline = pipeline; + parallel_context->cache = cache; + + ocf_parallelize_run(parallelize); } /* * Initialize hash table */ -void ocf_metadata_init_hash_table(struct ocf_cache *cache) +static int ocf_metadata_init_hash_table_handle(ocf_parallelize_t parallelize, + void *priv, unsigned shard_id, unsigned shards_cnt) { - unsigned int i; - unsigned int hash_table_entries = cache->device->hash_table_entries; + struct ocf_init_metadata_context *context = priv; + ocf_cache_t cache = context->cache; + uint32_t hash_table_entries = cache->device->hash_table_entries; ocf_cache_line_t invalid_idx = cache->device->collision_table_entries; + uint32_t entry, portion, begin, end, step=0; - /* Init hash table */ - for (i = 0; i < hash_table_entries; i++) { - /* hash_table contains indexes from collision_table - * thus it shall be initialized in improper values - * from collision_table - **/ - ocf_metadata_set_hash(cache, i, invalid_idx); + portion = OCF_DIV_ROUND_UP((uint64_t)hash_table_entries, shards_cnt); + begin = portion*shard_id; + end = OCF_MIN(portion*(shard_id + 1), hash_table_entries); + + for (entry = begin; entry < end; entry++) { + OCF_COND_RESCHED_DEFAULT(step); + + if (entry >= hash_table_entries) + break; + + ocf_metadata_set_hash(cache, entry, invalid_idx); } + return 0; +} + +void ocf_metadata_init_hash_table(ocf_pipeline_t pipeline, void *priv, + ocf_pipeline_arg_t arg) +{ + struct ocf_init_metadata_context *context = priv; + struct ocf_init_metadata_context *parallel_context; + ocf_cache_t cache = context->cache; + ocf_parallelize_t parallelize; + int result; + + result = ocf_parallelize_create(¶llelize, cache, + ocf_cache_get_queue_count(cache), sizeof(*context), + ocf_metadata_init_hash_table_handle, + ocf_metadata_init_finish); + if (result) + OCF_PL_FINISH_RET(pipeline, result); + + parallel_context = ocf_parallelize_get_priv(parallelize); + + parallel_context->pipeline = pipeline; + parallel_context->cache = cache; + + ocf_parallelize_run(parallelize); } /* @@ -994,7 +1078,7 @@ static bool ocf_check_if_cleaner_enabled(ocf_pipeline_t pipeline, void* priv, ocf_pipeline_arg_t arg) { struct ocf_metadata_context *context = priv; - + return !context->cache->conf_meta->cleaner_disabled; } @@ -1002,7 +1086,7 @@ struct ocf_pipeline_properties ocf_metadata_flush_all_pipeline_props = { .priv_size = sizeof(struct ocf_metadata_context), .finish = ocf_metadata_flush_all_finish, .steps = { - + OCF_PL_STEP_COND_ARG_INT(ocf_check_if_cleaner_enabled, ocf_metadata_flush_segment, metadata_segment_cleaning), @@ -1156,7 +1240,7 @@ struct ocf_pipeline_properties ocf_metadata_load_all_pipeline_props = { metadata_segment_cleaning), OCF_PL_STEP_FOREACH(ocf_metadata_load_segment, ocf_metadata_load_all_args), - + OCF_PL_STEP_COND_ARG_INT(ocf_check_if_cleaner_enabled, ocf_metadata_check_crc, metadata_segment_cleaning), diff --git a/src/metadata/metadata.h b/src/metadata/metadata.h index f26115d..6546702 100644 --- a/src/metadata/metadata.h +++ b/src/metadata/metadata.h @@ -25,6 +25,16 @@ #define CLEAN 2 #define DIRTY 3 +typedef void (*ocf_mngt_init_metadata_end_t)(void *priv, int error); + +struct ocf_init_metadata_context { + ocf_mngt_init_metadata_end_t cmpl; + ocf_pipeline_t pipeline; + ocf_cache_t cache; + uint8_t skip_collision; + void *priv; +}; + /** * @brief Initialize metadata * @@ -60,14 +70,16 @@ void ocf_metadata_init_freelist_partition(struct ocf_cache *cache); * * @param cache - Cache instance */ -void ocf_metadata_init_hash_table(struct ocf_cache *cache); +void ocf_metadata_init_hash_table(ocf_pipeline_t pipeline, void *priv, + ocf_pipeline_arg_t arg); /** * @brief Initialize collision table * * @param cache - Cache instance */ -void ocf_metadata_init_collision(struct ocf_cache *cache); +void ocf_metadata_init_collision(ocf_pipeline_t pipeline, void *priv, + ocf_pipeline_arg_t arg); /** * @brief De-Initialize metadata diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 7cc1aa8..b249fdb 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -130,6 +130,9 @@ struct ocf_cache_attach_context { bool promotion_initialized : 1; /*!< Promotion policy has been started */ + bool cleaning_initialized : 1; + /*!< Cleaning policy has been initialized */ + bool cores_opened : 1; /*!< underlying cores are opened (happens only during * load or recovery @@ -200,14 +203,19 @@ static void __init_partitions(ocf_cache_t cache) } } -static void __init_parts_attached(ocf_cache_t cache) +static void _init_parts_attached(ocf_pipeline_t pipeline, void *priv, + ocf_pipeline_arg_t arg) { + struct ocf_init_metadata_context *context = priv; + ocf_cache_t cache = context->cache; ocf_part_id_t part_id; for (part_id = 0; part_id < OCF_USER_IO_CLASS_MAX; part_id++) ocf_lru_init(cache, &cache->user_parts[part_id].part); ocf_lru_init(cache, &cache->free); + + ocf_pipeline_next(pipeline); } static ocf_error_t __init_cleaning_policy(ocf_cache_t cache) @@ -221,7 +229,7 @@ static ocf_error_t __init_cleaning_policy(ocf_cache_t cache) for (i = 0; i < ocf_cleaning_max; i++) ocf_cleaning_setup(cache, i); - return ocf_cleaning_initialize(cache, cache->cleaner.policy, 1); + return ocf_cleaning_initialize(cache, cache->cleaner.policy, false); } static void __deinit_cleaning_policy(ocf_cache_t cache) @@ -267,8 +275,11 @@ static void __init_metadata_version(ocf_cache_t cache) cache->conf_meta->metadata_version = METADATA_VERSION(); } -static void __reset_stats(ocf_cache_t cache) +static void _reset_stats(ocf_pipeline_t pipeline, void *priv, + ocf_pipeline_arg_t arg) { + struct ocf_init_metadata_context *context = priv; + ocf_cache_t cache = context->cache; ocf_core_t core; ocf_core_id_t core_id; ocf_part_id_t i; @@ -285,17 +296,65 @@ static void __reset_stats(ocf_cache_t cache) part_counters[i].dirty_clines, 0); } } + + ocf_pipeline_next(pipeline); } -static void init_attached_data_structures_recovery(ocf_cache_t cache, - bool init_collision) +static void _init_metadata_version(ocf_pipeline_t pipeline, void *priv, + ocf_pipeline_arg_t arg) { - ocf_metadata_init_hash_table(cache); - if (init_collision) - ocf_metadata_init_collision(cache); - __init_parts_attached(cache); - __reset_stats(cache); + struct ocf_init_metadata_context *context = priv; + ocf_cache_t cache = context->cache; + __init_metadata_version(cache); + + ocf_pipeline_next(pipeline); +} + +static void _ocf_mngt_init_metadata_finish(ocf_pipeline_t pipeline, + void *priv, int error) +{ + struct ocf_init_metadata_context *context = priv; + + context->cmpl(context->priv, error); + + ocf_pipeline_destroy(pipeline); +} + +struct ocf_pipeline_properties ocf_init_attached_recovery_props = { + .priv_size = sizeof(struct ocf_init_metadata_context), + .finish = _ocf_mngt_init_metadata_finish, + .steps = { + OCF_PL_STEP(ocf_metadata_init_hash_table), + OCF_PL_STEP(ocf_metadata_init_collision), + OCF_PL_STEP(_init_parts_attached), + OCF_PL_STEP(_reset_stats), + OCF_PL_STEP(_init_metadata_version), + OCF_PL_STEP_TERMINATOR(), + }, +}; + +static void init_attached_data_structures_recovery(ocf_cache_t cache, + ocf_mngt_init_metadata_end_t cmpl, void *priv, bool skip_collision) +{ + struct ocf_init_metadata_context *context; + ocf_pipeline_t pipeline; + int result; + + result = ocf_pipeline_create(&pipeline, cache, + &ocf_init_attached_recovery_props); + if (result) + OCF_CMPL_RET(priv, result); + + context = ocf_pipeline_get_priv(pipeline); + + context->cmpl = cmpl; + context->priv = priv; + context->pipeline = pipeline; + context->cache = cache; + context->skip_collision = skip_collision; + + OCF_PL_NEXT_RET(pipeline); } /**************************************************************** @@ -650,7 +709,7 @@ static void _ocf_mngt_load_rebuild_metadata(ocf_pipeline_t pipeline, ocf_pipeline_next(pipeline); } -static void _ocf_mngt_cleaning_recovery_complete(void *priv, int error) +static void _ocf_mngt_cleaning_populate_complete(void *priv, int error) { struct ocf_cache_attach_context *context = priv; @@ -665,13 +724,46 @@ static void _ocf_mngt_load_init_cleaning(ocf_pipeline_t pipeline, ocf_error_t result; if (context->metadata.shutdown_status == ocf_metadata_clean_shutdown) { - result = ocf_cleaning_initialize(cache, - cache->cleaner.policy, 0); + /* Cleaning policy structures have been loaded so no need to populate + them for the second time */ + result = ocf_cleaning_initialize(cache, cache->cleaner.policy, true); OCF_PL_NEXT_ON_SUCCESS_RET(pipeline, result); + + } else { + result = ocf_cleaning_initialize(cache, cache->cleaner.policy, false); + if (result) + OCF_PL_FINISH_RET(pipeline, result); + + ocf_cleaning_populate(cache, cache->cleaner.policy, + _ocf_mngt_cleaning_populate_complete, context); + } +} + +static void _ocf_mngt_init_metadata_complete(void *priv, int error) +{ + struct ocf_cache_attach_context *context = priv; + ocf_cache_t cache = context->cache; + + if (error) { + ocf_cache_log(cache, log_err, + "ERROR: Cannot initialize cache metadata\n"); + OCF_PL_FINISH_RET(context->pipeline, -OCF_ERR_NO_MEM); } - ocf_cleaning_recovery(cache, cache->cleaner.policy, - _ocf_mngt_cleaning_recovery_complete, context); + ocf_pipeline_next(context->pipeline); +} + +static void _ocf_mngt_load_init_structures(ocf_pipeline_t pipeline, + void *priv, ocf_pipeline_arg_t arg) +{ + struct ocf_cache_attach_context *context = priv; + ocf_cache_t cache = context->cache; + + if (context->metadata.shutdown_status == ocf_metadata_clean_shutdown) + OCF_PL_NEXT_RET(pipeline); + + init_attached_data_structures_recovery(cache, + _ocf_mngt_init_metadata_complete, context, false); } void _ocf_mngt_load_metadata_complete(void *priv, int error) @@ -725,8 +817,6 @@ static void _ocf_mngt_load_metadata(ocf_pipeline_t pipeline, } else { ocf_cache_log(cache, log_warn, "ERROR: Cache device did not shut down properly!\n"); - init_attached_data_structures_recovery(cache, true); - ocf_cache_log(cache, log_info, "Initiating recovery sequence...\n"); _ocf_mngt_load_metadata_recovery(pipeline, priv, arg); } @@ -1177,6 +1267,39 @@ static void _ocf_mngt_attach_prepare_metadata(ocf_pipeline_t pipeline, ocf_pipeline_next(pipeline); } +struct ocf_pipeline_properties ocf_init_metadata_pipeline_props = { + .priv_size = sizeof(struct ocf_init_metadata_context), + .finish = _ocf_mngt_init_metadata_finish, + .steps = { + OCF_PL_STEP(ocf_metadata_init_hash_table), + OCF_PL_STEP(ocf_metadata_init_collision), + OCF_PL_STEP(_init_parts_attached), + OCF_PL_STEP_TERMINATOR(), + }, +}; + +static void _ocf_mngt_init_metadata(ocf_cache_t cache, + ocf_mngt_init_metadata_end_t cmpl, void *priv) +{ + struct ocf_init_metadata_context *context; + ocf_pipeline_t pipeline; + int result; + + result = ocf_pipeline_create(&pipeline, cache, + &ocf_init_metadata_pipeline_props); + if (result) + OCF_CMPL_RET(priv, result); + + context = ocf_pipeline_get_priv(pipeline); + + context->cmpl = cmpl; + context->priv = priv; + context->pipeline = pipeline; + context->cache = cache; + + OCF_PL_NEXT_RET(pipeline); +} + /** * @brief initializing cache anew (not loading or recovering) */ @@ -1186,11 +1309,8 @@ static void _ocf_mngt_attach_init_metadata(ocf_pipeline_t pipeline, struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; - ocf_metadata_init_hash_table(cache); - ocf_metadata_init_collision(cache); - __init_parts_attached(cache); - - ocf_pipeline_next(pipeline); + _ocf_mngt_init_metadata(cache, _ocf_mngt_init_metadata_complete, + context); } static void _ocf_mngt_attach_populate_free_complete(void *priv, int error) @@ -1210,6 +1330,24 @@ static void _ocf_mngt_attach_populate_free(ocf_pipeline_t pipeline, context); } +static void _ocf_mngt_cleaning_populate_init_complete(void *priv, int error) +{ + struct ocf_cache_attach_context *context = priv; + ocf_cache_t cache = context->cache; + + if (error) + OCF_PL_FINISH_RET(context->pipeline, error); + + /* In initial cache state there is no dirty data, so all dirty data is + considered to be flushed + */ + cache->conf_meta->dirty_flushed = true; + + context->flags.cleaning_initialized = true; + + OCF_PL_NEXT_RET(context->pipeline); +} + static void _ocf_mngt_attach_init_services(ocf_pipeline_t pipeline, void *priv, ocf_pipeline_arg_t arg) { @@ -1217,6 +1355,8 @@ static void _ocf_mngt_attach_init_services(ocf_pipeline_t pipeline, ocf_cache_t cache = context->cache; ocf_error_t result; + __setup_promotion_policy(cache); + if (context->metadata.cleaner_disabled) __set_cleaning_policy(cache, ocf_cleaning_nop); @@ -1227,14 +1367,8 @@ static void _ocf_mngt_attach_init_services(ocf_pipeline_t pipeline, OCF_PL_FINISH_RET(pipeline, result); } - __setup_promotion_policy(cache); - - /* In initial cache state there is no dirty data, so all dirty data is - considered to be flushed - */ - cache->conf_meta->dirty_flushed = true; - - ocf_pipeline_next(pipeline); + ocf_cleaning_populate(cache, cache->cleaner.policy, + _ocf_mngt_cleaning_populate_init_complete, context); } uint64_t _ocf_mngt_calculate_ram_needed(ocf_cache_line_size_t line_size, @@ -1728,6 +1862,9 @@ static void _ocf_mngt_attach_handle_error( if (context->flags.promotion_initialized) __deinit_promotion_policy(cache); + if (context->flags.cleaning_initialized) + __deinit_cleaning_policy(cache); + if (context->flags.cores_opened) _ocf_mngt_deinit_added_cores(context); @@ -1819,6 +1956,7 @@ struct ocf_pipeline_properties _ocf_mngt_cache_load_pipeline_properties = { OCF_PL_STEP(_ocf_mngt_init_cleaner), OCF_PL_STEP(_ocf_mngt_init_promotion), OCF_PL_STEP(_ocf_mngt_load_add_cores), + OCF_PL_STEP(_ocf_mngt_load_init_structures), OCF_PL_STEP(_ocf_mngt_load_metadata), OCF_PL_STEP(_ocf_mngt_load_rebuild_metadata), OCF_PL_STEP(_ocf_mngt_load_init_cleaning), @@ -2159,9 +2297,8 @@ static void _ocf_mngt_standby_init_structures_attach(ocf_pipeline_t pipeline, struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; - init_attached_data_structures_recovery(cache, true); - - ocf_pipeline_next(pipeline); + init_attached_data_structures_recovery(cache, + _ocf_mngt_init_metadata_complete, context, false); } static void _ocf_mngt_standby_init_structures_load(ocf_pipeline_t pipeline, @@ -2170,9 +2307,8 @@ static void _ocf_mngt_standby_init_structures_load(ocf_pipeline_t pipeline, struct ocf_cache_attach_context *context = priv; ocf_cache_t cache = context->cache; - init_attached_data_structures_recovery(cache, false); - - ocf_pipeline_next(pipeline); + init_attached_data_structures_recovery(cache, + _ocf_mngt_init_metadata_complete, context, true); } static void _ocf_mngt_standby_init_pio_concurrency(ocf_pipeline_t pipeline, diff --git a/src/mngt/ocf_mngt_flush.c b/src/mngt/ocf_mngt_flush.c index 5702ffc..aa894bb 100644 --- a/src/mngt/ocf_mngt_flush.c +++ b/src/mngt/ocf_mngt_flush.c @@ -961,18 +961,16 @@ static void _ocf_mngt_deinit_clean_policy(ocf_pipeline_t pipeline, void *priv, _ocf_mngt_cleaning_deinit_complete, context); } -static void _ocf_mngt_init_clean_policy(ocf_pipeline_t pipeline, void *priv, - ocf_pipeline_arg_t arg) +static void _ocf_mngt_cleaning_init_complete(void *priv, int error) { - int result; struct ocf_mngt_cache_set_cleaning_context *context = priv; + ocf_pipeline_t pipeline = context->pipeline; ocf_cache_t cache = context->cache; ocf_cleaning_t old_policy = context->old_policy; ocf_cleaning_t new_policy = context->new_policy; ocf_cleaning_t emergency_policy = ocf_cleaning_nop; - result = ocf_cleaning_initialize(cache, new_policy, 1); - if (result) { + if (error) { ocf_cache_log(cache, log_info, "Failed to initialize %s cleaning " "policy. Setting %s instead\n", ocf_cleaning_get_name(new_policy), @@ -986,16 +984,35 @@ static void _ocf_mngt_init_clean_policy(ocf_pipeline_t pipeline, void *priv, __set_cleaning_policy(cache, new_policy); - ocf_refcnt_unfreeze(&cache->cleaner.refcnt); - ocf_metadata_end_exclusive_access(&cache->metadata.lock); + OCF_PL_NEXT_ON_SUCCESS_RET(pipeline, error); - OCF_PL_NEXT_ON_SUCCESS_RET(pipeline, result); +} + +static void _ocf_mngt_init_clean_policy(ocf_pipeline_t pipeline, void *priv, + ocf_pipeline_arg_t arg) +{ + int result; + struct ocf_mngt_cache_set_cleaning_context *context = priv; + ocf_cache_t cache = context->cache; + ocf_cleaning_t new_policy = context->new_policy; + + result = ocf_cleaning_initialize(cache, new_policy, false); + if (result) { + _ocf_mngt_cleaning_init_complete(context, result); + } else { + ocf_cleaning_populate(cache, new_policy, + _ocf_mngt_cleaning_init_complete, context); + } } static void _ocf_mngt_set_cleaning_finish(ocf_pipeline_t pipeline, void *priv, int error) { struct ocf_mngt_cache_set_cleaning_context *context = priv; + ocf_cache_t cache = context->cache; + + ocf_refcnt_unfreeze(&cache->cleaner.refcnt); + ocf_metadata_end_exclusive_access(&cache->metadata.lock); context->cmpl(context->priv, error); diff --git a/src/ocf_cache.c b/src/ocf_cache.c index be11244..f0665ac 100644 --- a/src/ocf_cache.c +++ b/src/ocf_cache.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ @@ -25,6 +25,17 @@ ocf_volume_t ocf_cache_get_front_volume(ocf_cache_t cache) return cache->device ? &cache->device->front_volume : NULL; } +uint32_t ocf_cache_get_queue_count(ocf_cache_t cache) +{ + ocf_queue_t queue = NULL; + uint32_t cnt = 0; + + list_for_each_entry(queue, &cache->io_queues, list) + cnt++; + + return cnt; +} + int ocf_cache_set_name(ocf_cache_t cache, const char *src, size_t src_size) { int result; diff --git a/src/ocf_cache_priv.h b/src/ocf_cache_priv.h index 041bc6f..807483a 100644 --- a/src/ocf_cache_priv.h +++ b/src/ocf_cache_priv.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ @@ -19,6 +19,7 @@ #include "ocf_stats_priv.h" #include "cleaning/cleaning.h" #include "ocf_logger_priv.h" +#include "ocf_queue_priv.h" #include "promotion/promotion.h" #define DIRTY_FLUSHED 1 @@ -175,6 +176,8 @@ static inline uint64_t ocf_get_cache_occupancy(ocf_cache_t cache) return result; } +uint32_t ocf_cache_get_queue_count(ocf_cache_t cache); + int ocf_cache_set_name(ocf_cache_t cache, const char *src, size_t src_size); int ocf_cache_volume_type_init(ocf_ctx_t ctx); diff --git a/src/utils/utils_parallelize.c b/src/utils/utils_parallelize.c index 8265aa5..fc0ce6e 100644 --- a/src/utils/utils_parallelize.c +++ b/src/utils/utils_parallelize.c @@ -57,8 +57,7 @@ int ocf_parallelize_create(ocf_parallelize_t *parallelize, unsigned queue_count = 0; int result, i; - list_for_each_entry(queue, &cache->io_queues, list) - queue_count++; + queue_count = ocf_cache_get_queue_count(cache); if (shards_cnt == 0) shards_cnt = queue_count;