diff --git a/env/posix/ocf_env.h b/env/posix/ocf_env.h index ad4e7c6..7e81924 100644 --- a/env/posix/ocf_env.h +++ b/env/posix/ocf_env.h @@ -653,5 +653,4 @@ unsigned env_get_execution_context(void); void env_put_execution_context(unsigned ctx); unsigned env_get_execution_context_count(void); - #endif /* __OCF_ENV_H__ */ diff --git a/src/engine/cache_engine.c b/src/engine/cache_engine.c index 41eff2e..64799c3 100644 --- a/src/engine/cache_engine.c +++ b/src/engine/cache_engine.c @@ -169,7 +169,8 @@ static inline bool ocf_seq_cutoff_is_on(ocf_cache_t cache) if (!ocf_cache_is_device_attached(cache)) return false; - return (cache->device->freelist_part->curr_size <= SEQ_CUTOFF_FULL_MARGIN); + return (ocf_freelist_num_free(cache->freelist) <= + SEQ_CUTOFF_FULL_MARGIN); } bool ocf_seq_cutoff_check(ocf_core_t core, uint32_t dir, uint64_t addr, diff --git a/src/engine/engine_common.c b/src/engine/engine_common.c index 53a5fcc..b01148d 100644 --- a/src/engine/engine_common.c +++ b/src/engine/engine_common.c @@ -7,6 +7,7 @@ #include "../ocf_priv.h" #include "../ocf_cache_priv.h" #include "../ocf_queue_priv.h" +#include "../ocf_freelist.h" #include "engine_common.h" #define OCF_ENGINE_DEBUG_IO_NAME "common" #include "engine_debug.h" @@ -250,19 +251,11 @@ static void ocf_engine_map_cache_line(struct ocf_request *req, ocf_part_id_t part_id = req->part_id; ocf_cleaning_t clean_policy_type; - if (cache->device->freelist_part->curr_size == 0) { + if (!ocf_freelist_get_cache_line(cache->freelist, cache_line)) { req->info.mapping_error = 1; return; } - *cache_line = cache->device->freelist_part->head; - - /* add_to_collision_list changes .next_col and other fields for entry - * so updated last_cache_line_give must be updated before calling it. - */ - - ocf_metadata_remove_from_free_list(cache, *cache_line); - ocf_metadata_add_to_partition(cache, part_id, *cache_line); /* Add the block to the corresponding collision list */ diff --git a/src/eviction/eviction.c b/src/eviction/eviction.c index 0bacee8..2d76db4 100644 --- a/src/eviction/eviction.c +++ b/src/eviction/eviction.c @@ -107,11 +107,13 @@ int space_managment_evict_do(struct ocf_cache *cache, struct ocf_request *req, uint32_t evict_cline_no) { uint32_t evicted; + uint32_t free; - if (evict_cline_no <= cache->device->freelist_part->curr_size) + free = ocf_freelist_num_free(cache->freelist); + if (evict_cline_no <= free) return LOOKUP_MAPPED; - evict_cline_no -= cache->device->freelist_part->curr_size; + evict_cline_no -= free; evicted = ocf_evict_do(cache, req->io_queue, evict_cline_no, req->part_id); diff --git a/src/eviction/eviction.h b/src/eviction/eviction.h index 7ebafea..6444521 100644 --- a/src/eviction/eviction.h +++ b/src/eviction/eviction.h @@ -32,8 +32,7 @@ union eviction_policy_meta { * set core_id to -2 to purge the whole cache partition */ struct eviction_policy_ops { - void (*init_cline)(ocf_cache_t cache, - ocf_cache_line_t cline); + void (*init_cline)(ocf_cache_t cache, ocf_cache_line_t cline); void (*rm_cline)(ocf_cache_t cache, ocf_cache_line_t cline); bool (*can_evict)(ocf_cache_t cache); diff --git a/src/eviction/lru.h b/src/eviction/lru.h index 79488a4..fa996f7 100644 --- a/src/eviction/lru.h +++ b/src/eviction/lru.h @@ -8,8 +8,7 @@ #include "eviction.h" #include "lru_structs.h" -void evp_lru_init_cline(struct ocf_cache *cache, - ocf_cache_line_t cline); +void evp_lru_init_cline(struct ocf_cache *cache, ocf_cache_line_t cline); void evp_lru_rm_cline(struct ocf_cache *cache, ocf_cache_line_t cline); bool evp_lru_can_evict(struct ocf_cache *cache); uint32_t evp_lru_req_clines(struct ocf_cache *cache, ocf_queue_t io_queue, diff --git a/src/metadata/metadata.c b/src/metadata/metadata.c index 35b79cf..1b31132 100644 --- a/src/metadata/metadata.c +++ b/src/metadata/metadata.c @@ -53,18 +53,18 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache, uint64_t device_siz cache_line_size, layout); } -void ocf_metadata_init_freelist_partition(struct ocf_cache *cache) -{ - OCF_DEBUG_TRACE(cache); - cache->metadata.iface.init_freelist(cache); -} - void ocf_metadata_init_hash_table(struct ocf_cache *cache) { OCF_DEBUG_TRACE(cache); cache->metadata.iface.init_hash_table(cache); } +void ocf_metadata_init_collision(struct ocf_cache *cache) +{ + OCF_DEBUG_TRACE(cache); + cache->metadata.iface.init_collision(cache); +} + void ocf_metadata_deinit(struct ocf_cache *cache) { OCF_DEBUG_TRACE(cache); diff --git a/src/metadata/metadata.h b/src/metadata/metadata.h index 7893fed..5c75b89 100644 --- a/src/metadata/metadata.h +++ b/src/metadata/metadata.h @@ -60,6 +60,13 @@ void ocf_metadata_init_freelist_partition(struct ocf_cache *cache); */ void ocf_metadata_init_hash_table(struct ocf_cache *cache); +/** + * @brief Initialize collision table + * + * @param cache - Cache instance + */ +void ocf_metadata_init_collision(struct ocf_cache *cache); + /** * @brief De-Initialize metadata * @@ -207,4 +214,10 @@ typedef void (*ocf_metadata_load_properties_end_t)(void *priv, int error, void ocf_metadata_load_properties(ocf_volume_t volume, ocf_metadata_load_properties_end_t cmpl, void *priv); +static inline ocf_cache_line_t ocf_metadata_collision_table_entries( + struct ocf_cache *cache) +{ + return cache->device->collision_table_entries; +} + #endif /* METADATA_H_ */ diff --git a/src/metadata/metadata_hash.c b/src/metadata/metadata_hash.c index 0fb669c..1c0c35d 100644 --- a/src/metadata/metadata_hash.c +++ b/src/metadata/metadata_hash.c @@ -13,6 +13,7 @@ #include "../utils/utils_pipeline.h" #include "../ocf_def_priv.h" #include "../ocf_priv.h" +#include "../ocf_freelist.h" #define OCF_METADATA_HASH_DEBUG 0 @@ -1025,14 +1026,10 @@ finalize: } static inline void _ocf_init_collision_entry(struct ocf_cache *cache, - ocf_cache_line_t idx, ocf_cache_line_t next, - ocf_cache_line_t prev) + ocf_cache_line_t idx) { ocf_cache_line_t invalid_idx = cache->device->collision_table_entries; - ocf_part_id_t invalid_part_id = PARTITION_INVALID; - ocf_metadata_set_partition_info(cache, idx, - invalid_part_id, next, prev); ocf_metadata_set_collision_info(cache, idx, invalid_idx, invalid_idx); ocf_metadata_set_core_info(cache, idx, OCF_CORE_MAX, ULONG_MAX); @@ -1040,36 +1037,17 @@ static inline void _ocf_init_collision_entry(struct ocf_cache *cache, } /* - * Modified initialization of freelist partition + * Initialize collision table */ -static void ocf_metadata_hash_init_freelist(struct ocf_cache *cache) +static void ocf_metadata_hash_init_collision(struct ocf_cache *cache) { - uint32_t step = 0; unsigned int i; - ocf_cache_line_t prev, next; - ocf_cache_line_t idx; - ocf_cache_line_t collision_table_entries = - cache->device->collision_table_entries; - prev = collision_table_entries; - idx = 0; - for (i = 0; i < cache->device->collision_table_entries - 1; i++) { - next = ocf_metadata_map_phy2lg(cache, i + 1); - _ocf_init_collision_entry(cache, idx, next, prev); - prev = idx; - idx = next; - OCF_COND_RESCHED_DEFAULT(step); + for (i = 0; i < cache->device->collision_table_entries; i++) { + _ocf_init_collision_entry(cache, i); } - _ocf_init_collision_entry(cache, idx, collision_table_entries, prev); - - /* Initialize freelist partition */ - cache->device->freelist_part->head = 0; - cache->device->freelist_part->tail = idx; - cache->device->freelist_part->curr_size = cache->device-> - collision_table_entries; } - /* * Initialize hash table */ @@ -1850,8 +1828,6 @@ static void _recovery_rebuild_cline_metadata(ocf_cache_t cache, part_id = PARTITION_DEFAULT; - ocf_metadata_remove_from_free_list(cache, cache_line); - ocf_metadata_add_to_partition(cache, part_id, cache_line); hash_index = ocf_metadata_hash_func(cache, core_line, core_id); @@ -1917,10 +1893,12 @@ static void _recovery_rebuild_metadata(ocf_pipeline_t pipeline, ocf_core_id_t core_id; uint64_t core_line; unsigned char step = 0; + const uint64_t collision_table_entries = + ocf_metadata_collision_table_entries(cache); OCF_METADATA_LOCK_WR(); - for (cline = 0; cline < cache->device->collision_table_entries; cline++) { + for (cline = 0; cline < collision_table_entries; cline++) { ocf_metadata_get_core_info(cache, cline, &core_id, &core_line); if (core_id != OCF_CORE_MAX && (!dirty_only || metadata_test_dirty(cache, @@ -2648,7 +2626,7 @@ static const struct ocf_metadata_iface metadata_hash_iface = { .init_variable_size = ocf_metadata_hash_init_variable_size, .deinit_variable_size = ocf_metadata_hash_deinit_variable_size, .init_hash_table = ocf_metadata_hash_init_hash_table, - .init_freelist = ocf_metadata_hash_init_freelist, + .init_collision = ocf_metadata_hash_init_collision, .layout_iface = NULL, .pages = ocf_metadata_hash_pages, diff --git a/src/metadata/metadata_misc.c b/src/metadata/metadata_misc.c index 9a0187c..ff17e77 100644 --- a/src/metadata/metadata_misc.c +++ b/src/metadata/metadata_misc.c @@ -5,6 +5,7 @@ #include "ocf/ocf.h" #include "metadata.h" +#include "../ocf_freelist.h" #include "../utils/utils_cache_line.h" static bool _is_cache_line_acting(struct ocf_cache *cache, @@ -100,7 +101,7 @@ void ocf_metadata_sparse_cache_line(struct ocf_cache *cache, ocf_metadata_remove_from_partition(cache, partition_id, cache_line); - ocf_metadata_add_to_free_list(cache, cache_line); + ocf_freelist_put_cache_line(cache->freelist, cache_line); } static void _ocf_metadata_sparse_cache_line(struct ocf_cache *cache, diff --git a/src/metadata/metadata_partition.c b/src/metadata/metadata_partition.c index de02dcc..b802101 100644 --- a/src/metadata/metadata_partition.c +++ b/src/metadata/metadata_partition.c @@ -16,99 +16,6 @@ static void update_partition_head(struct ocf_cache *cache, part->runtime->head = line; } -void ocf_metadata_remove_from_free_list(struct ocf_cache *cache, - ocf_cache_line_t cline) -{ - struct ocf_part *free_list = cache->device->freelist_part; - int is_head, is_tail; - ocf_part_id_t invalid_part_id = PARTITION_INVALID; - ocf_cache_line_t prev, next; - ocf_cache_line_t line_entries = cache->device->collision_table_entries; - - ENV_BUG_ON(cline >= line_entries); - - /* Get Partition info */ - ocf_metadata_get_partition_info(cache, cline, NULL, &next, &prev); - - /* Find out if this node is Partition _head_ */ - is_head = (prev == line_entries); - is_tail = (next == line_entries); - - /* Case 1: If we are head and there is only one node. So unlink node - * and set that there is no node left in the list. - */ - if (is_head && (free_list->curr_size == 1)) { - ocf_metadata_set_partition_info(cache, cline, invalid_part_id, - line_entries, line_entries); - free_list->head = line_entries; - free_list->tail = line_entries; - } else if (is_head) { - /* Case 2: else if this collision_index is partition list head, - * but many nodes, update head and return - */ - ENV_BUG_ON(next >= line_entries); - - free_list->head = next; - ocf_metadata_set_partition_prev(cache, next, line_entries); - ocf_metadata_set_partition_next(cache, cline, line_entries); - } else if (is_tail) { - /* Case 3: else if this cline is partition list tail */ - ENV_BUG_ON(prev >= line_entries); - - free_list->tail = prev; - ocf_metadata_set_partition_prev(cache, cline, line_entries); - ocf_metadata_set_partition_next(cache, prev, line_entries); - } else { - /* Case 4: else this collision_index is a middle node. - * There is no change to the head and the tail pointers. - */ - - ENV_BUG_ON(next >= line_entries || prev >= line_entries); - - /* Update prev and next nodes */ - ocf_metadata_set_partition_prev(cache, next, prev); - ocf_metadata_set_partition_next(cache, prev, next); - - /* Update the given node */ - ocf_metadata_set_partition_info(cache, cline, invalid_part_id, - line_entries, line_entries); - } - - free_list->curr_size--; -} - -void ocf_metadata_add_to_free_list(struct ocf_cache *cache, - ocf_cache_line_t line) -{ - struct ocf_part *free_list = cache->device->freelist_part; - ocf_cache_line_t tail; - ocf_cache_line_t line_entries = cache->device->collision_table_entries; - ocf_part_id_t invalid_part_id = PARTITION_INVALID; - - ENV_BUG_ON(line >= line_entries); - - if (free_list->curr_size == 0) { - free_list->head = line; - free_list->tail = line; - - ocf_metadata_set_partition_info(cache, line, invalid_part_id, - line_entries, line_entries); - } else { - tail = free_list->tail; - - ENV_BUG_ON(tail >= line_entries); - - ocf_metadata_set_partition_info(cache, line, invalid_part_id, - line_entries, tail); - ocf_metadata_set_partition_next(cache, tail, line); - - free_list->tail = line; - } - - free_list->curr_size++; -} - - /* Adds the given collision_index to the _head_ of the Partition list */ void ocf_metadata_add_to_partition(struct ocf_cache *cache, ocf_part_id_t part_id, ocf_cache_line_t line) diff --git a/src/metadata/metadata_partition.h b/src/metadata/metadata_partition.h index 8aee83f..00605e0 100644 --- a/src/metadata/metadata_partition.h +++ b/src/metadata/metadata_partition.h @@ -75,12 +75,6 @@ static inline void ocf_metadata_set_partition_info( next_line, prev_line); } -void ocf_metadata_add_to_free_list(struct ocf_cache *cache, - ocf_cache_line_t cline); - -void ocf_metadata_remove_from_free_list(struct ocf_cache *cache, - ocf_cache_line_t cline); - void ocf_metadata_add_to_partition(struct ocf_cache *cache, ocf_part_id_t part_id, ocf_cache_line_t line); diff --git a/src/metadata/metadata_partition_structs.h b/src/metadata/metadata_partition_structs.h index c8ed593..58d7e56 100644 --- a/src/metadata/metadata_partition_structs.h +++ b/src/metadata/metadata_partition_structs.h @@ -10,12 +10,6 @@ #include "../cleaning/cleaning.h" #include "../eviction/eviction.h" -struct ocf_part { - ocf_cache_line_t head; - ocf_cache_line_t tail; - uint32_t curr_size; -}; - struct ocf_user_part_config { char name[OCF_IO_CLASS_NAME_MAX]; uint32_t min_size; diff --git a/src/metadata/metadata_structs.h b/src/metadata/metadata_structs.h index 278e628..e9b3ac5 100644 --- a/src/metadata/metadata_structs.h +++ b/src/metadata/metadata_structs.h @@ -124,13 +124,6 @@ struct ocf_metadata_iface { struct ocf_volume_uuid *uuid, uint32_t count, ocf_metadata_query_cores_end_t cmpl, void *priv); - /** - * @brief Initialize freelist partition - * - * @param cache - Cache instance - */ - - void (*init_freelist)(struct ocf_cache *cache); /** * @brief Metadata cache line location on pages interface @@ -144,6 +137,13 @@ struct ocf_metadata_iface { */ void (*init_hash_table)(struct ocf_cache *cache); + /** + * @brief Initialize collision table + * + * @param cache - Cache instance + */ + void (*init_collision)(struct ocf_cache *cache); + /** * @brief De-Initialize metadata * diff --git a/src/metadata/metadata_superblock.h b/src/metadata/metadata_superblock.h index 0baa76f..c08e90d 100644 --- a/src/metadata/metadata_superblock.h +++ b/src/metadata/metadata_superblock.h @@ -59,8 +59,6 @@ struct ocf_superblock_config { * @brief OCF cache metadata runtime superblock */ struct ocf_superblock_runtime { - struct ocf_part freelist_part; - uint32_t cleaning_thread_access; }; diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 52e8e62..2481623 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -21,6 +21,7 @@ #include "../concurrency/ocf_concurrency.h" #include "../eviction/ops.h" #include "../ocf_ctx_priv.h" +#include "../ocf_freelist.h" #include "../cleaning/cleaning.h" #define OCF_ASSERT_PLUGGED(cache) ENV_BUG_ON(!(cache)->device) @@ -117,6 +118,8 @@ struct ocf_cache_attach_context { * load or recovery */ + bool freelist_inited : 1; + bool concurrency_inited : 1; } flags; @@ -156,18 +159,6 @@ struct ocf_cache_attach_context { ocf_pipeline_t pipeline; }; -static void __init_hash_table(ocf_cache_t cache) -{ - /* Initialize hash table*/ - ocf_metadata_init_hash_table(cache); -} - -static void __init_freelist(ocf_cache_t cache) -{ - /* Initialize free list partition*/ - ocf_metadata_init_freelist_partition(cache); -} - static void __init_partitions(ocf_cache_t cache) { ocf_part_id_t i_part; @@ -204,6 +195,14 @@ static void __init_partitions_attached(ocf_cache_t cache) } } +static void __init_freelist(ocf_cache_t cache) +{ + uint64_t free_clines = ocf_metadata_collision_table_entries(cache) - + ocf_get_cache_occupancy(cache); + + ocf_freelist_populate(cache->freelist, free_clines); +} + static ocf_error_t __init_cleaning_policy(ocf_cache_t cache) { ocf_cleaning_t cleaning_policy = ocf_cleaning_default; @@ -295,9 +294,10 @@ static ocf_error_t init_attached_data_structures(ocf_cache_t cache, /* Lock to ensure consistency */ OCF_METADATA_LOCK_WR(); - __init_hash_table(cache); - __init_freelist(cache); + ocf_metadata_init_hash_table(cache); + ocf_metadata_init_collision(cache); __init_partitions_attached(cache); + __init_freelist(cache); result = __init_cleaning_policy(cache); if (result) { @@ -325,8 +325,8 @@ static ocf_error_t init_attached_data_structures(ocf_cache_t cache, static void init_attached_data_structures_recovery(ocf_cache_t cache) { OCF_METADATA_LOCK_WR(); - __init_hash_table(cache); - __init_freelist(cache); + ocf_metadata_init_hash_table(cache); + ocf_metadata_init_collision(cache); __init_partitions_attached(cache); __reset_stats(cache); __init_metadata_version(cache); @@ -469,6 +469,8 @@ void _ocf_mngt_init_instance_load_complete(void *priv, int error) OCF_PL_FINISH_RET(context->pipeline, -OCF_ERR_START_CACHE_FAIL); } + __init_freelist(cache); + result = __init_promotion_policy(cache); if (result) { ocf_cache_log(cache, log_err, @@ -985,8 +987,10 @@ static void _ocf_mngt_attach_prepare_metadata(ocf_pipeline_t pipeline, OCF_PL_FINISH_RET(context->pipeline, -OCF_ERR_START_CACHE_FAIL); } - - cache->device->freelist_part = &cache->device->runtime_meta->freelist_part; + cache->freelist = ocf_freelist_init(cache); + if (!cache->freelist) + OCF_PL_FINISH_RET(context->pipeline, -OCF_ERR_START_CACHE_FAIL); + context->flags.freelist_inited = true; ret = ocf_concurrency_init(cache); if (ret) @@ -1143,6 +1147,9 @@ static void _ocf_mngt_attach_handle_error( if (context->flags.concurrency_inited) ocf_concurrency_deinit(cache); + if (context->flags.freelist_inited) + ocf_freelist_deinit(cache->freelist); + if (context->flags.volume_inited) ocf_volume_deinit(&cache->device->volume); @@ -1725,6 +1732,7 @@ static void _ocf_mngt_cache_unplug_complete(void *priv, int error) ocf_metadata_deinit_variable_size(cache); ocf_concurrency_deinit(cache); + ocf_freelist_deinit(cache->freelist); ocf_volume_deinit(&cache->device->volume); diff --git a/src/ocf_cache_priv.h b/src/ocf_cache_priv.h index a32b64e..7f54881 100644 --- a/src/ocf_cache_priv.h +++ b/src/ocf_cache_priv.h @@ -21,6 +21,7 @@ #include "ocf_logger_priv.h" #include "ocf/ocf_trace.h" #include "promotion/promotion.h" +#include "ocf_freelist.h" #define DIRTY_FLUSHED 1 #define DIRTY_NOT_FLUSHED 0 @@ -82,8 +83,6 @@ struct ocf_cache_device { uint64_t metadata_offset; - struct ocf_part *freelist_part; - struct { struct ocf_cache_line_concurrency *cache_line; } concurrency; @@ -110,6 +109,8 @@ struct ocf_cache { struct ocf_metadata metadata; + ocf_freelist_t freelist; + ocf_eviction_t eviction_policy_init; struct { @@ -199,4 +200,16 @@ static inline ocf_core_t ocf_cache_get_core(ocf_cache_t cache, #define ocf_cache_log_rl(cache) \ ocf_log_rl(ocf_cache_get_ctx(cache)) +static inline uint64_t ocf_get_cache_occupancy(ocf_cache_t cache) +{ + uint64_t result = 0; + ocf_core_t core; + ocf_core_id_t core_id; + + for_each_core(cache, core, core_id) + result += env_atomic_read(&core->runtime_meta->cached_clines); + + return result; +} + #endif /* __OCF_CACHE_PRIV_H__ */ diff --git a/src/ocf_freelist.c b/src/ocf_freelist.c new file mode 100644 index 0000000..fdff6bb --- /dev/null +++ b/src/ocf_freelist.c @@ -0,0 +1,420 @@ +/* + * Copyright(c) 2019-2019 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause-Clear + */ + +#include "ocf/ocf.h" +#include "metadata/metadata.h" + +struct ocf_part { + ocf_cache_line_t head; + ocf_cache_line_t tail; + env_atomic64 curr_size; +}; + +struct ocf_freelist { + /* parent cache */ + struct ocf_cache *cache; + + /* partition list array */ + struct ocf_part *part; + + /* freelist lock array */ + env_spinlock *lock; + + /* number of free lists */ + uint32_t count; + + /* next slowpath victim idx */ + env_atomic slowpath_victim_idx; + + /* total number of free lines */ + env_atomic64 total_free; +}; + +static void ocf_freelist_lock(ocf_freelist_t freelist, uint32_t ctx) +{ + env_spinlock_lock(&freelist->lock[ctx]); +} + +static int ocf_freelist_trylock(ocf_freelist_t freelist, uint32_t ctx) +{ + return env_spinlock_trylock(&freelist->lock[ctx]); +} + +static void ocf_freelist_unlock(ocf_freelist_t freelist, uint32_t ctx) +{ + env_spinlock_unlock(&freelist->lock[ctx]); +} + +/* Sets the given collision_index as the new _head_ of the Partition list. */ +static void _ocf_freelist_remove_cache_line(ocf_freelist_t freelist, + uint32_t ctx, ocf_cache_line_t cline) +{ + struct ocf_cache *cache = freelist->cache; + struct ocf_part *freelist_part = &freelist->part[ctx]; + int is_head, is_tail; + ocf_part_id_t invalid_part_id = PARTITION_INVALID; + ocf_cache_line_t prev, next; + ocf_cache_line_t line_entries = ocf_metadata_collision_table_entries( + freelist->cache); + uint32_t free; + + ENV_BUG_ON(cline >= line_entries); + + /* Get Partition info */ + ocf_metadata_get_partition_info(cache, cline, NULL, &next, &prev); + + /* Find out if this node is Partition _head_ */ + is_head = (prev == line_entries); + is_tail = (next == line_entries); + + free = env_atomic64_read(&freelist_part->curr_size); + + /* Case 1: If we are head and there is only one node. So unlink node + * and set that there is no node left in the list. + */ + if (is_head && free == 1) { + ocf_metadata_set_partition_info(cache, cline, invalid_part_id, + line_entries, line_entries); + freelist_part->head = line_entries; + freelist_part->tail = line_entries; + } else if (is_head) { + /* Case 2: else if this collision_index is partition list head, + * but many nodes, update head and return + */ + ENV_BUG_ON(next >= line_entries); + + freelist_part->head = next; + ocf_metadata_set_partition_prev(cache, next, line_entries); + ocf_metadata_set_partition_next(cache, cline, line_entries); + } else if (is_tail) { + /* Case 3: else if this cline is partition list tail */ + ENV_BUG_ON(prev >= line_entries); + + freelist_part->tail = prev; + ocf_metadata_set_partition_prev(cache, cline, line_entries); + ocf_metadata_set_partition_next(cache, prev, line_entries); + } else { + /* Case 4: else this collision_index is a middle node. + * There is no change to the head and the tail pointers. + */ + + ENV_BUG_ON(next >= line_entries || prev >= line_entries); + + /* Update prev and next nodes */ + ocf_metadata_set_partition_prev(cache, next, prev); + ocf_metadata_set_partition_next(cache, prev, next); + + /* Update the given node */ + ocf_metadata_set_partition_info(cache, cline, invalid_part_id, + line_entries, line_entries); + } + + env_atomic64_dec(&freelist_part->curr_size); + env_atomic64_dec(&freelist->total_free); +} + +static ocf_cache_line_t next_phys_invalid(ocf_cache_t cache, + ocf_cache_line_t phys) +{ + ocf_cache_line_t lg; + ocf_cache_line_t collision_table_entries = + ocf_metadata_collision_table_entries(cache); + + if (phys == collision_table_entries) + return collision_table_entries; + + lg = ocf_metadata_map_phy2lg(cache, phys); + while (metadata_test_valid_any(cache, lg)) { + ++phys; + + if (phys == collision_table_entries) + break; + + lg = ocf_metadata_map_phy2lg(cache, phys); + } + + return phys; +} + +/* Assign unused cachelines to freelist */ +void ocf_freelist_populate(ocf_freelist_t freelist, + ocf_cache_line_t num_free_clines) +{ + unsigned step = 0; + ocf_cache_t cache = freelist->cache; + unsigned num_freelists = freelist->count; + ocf_cache_line_t prev, next, idx; + ocf_cache_line_t phys; + ocf_cache_line_t collision_table_entries = + ocf_metadata_collision_table_entries(cache); + unsigned freelist_idx; + uint64_t freelist_size; + + phys = 0; + for (freelist_idx = 0; freelist_idx < num_freelists; freelist_idx++) + { + /* calculate current freelist size */ + freelist_size = num_free_clines / num_freelists; + if (freelist_idx < (num_free_clines % num_freelists)) + ++freelist_size; + + env_atomic64_set(&freelist->part[freelist_idx].curr_size, + freelist_size); + + if (!freelist_size) { + /* init empty freelist and move to next one */ + freelist->part[freelist_idx].head = + collision_table_entries; + freelist->part[freelist_idx].tail = + collision_table_entries; + continue; + } + + /* find first invalid cacheline */ + phys = next_phys_invalid(cache, phys); + ENV_BUG_ON(phys == collision_table_entries); + idx = ocf_metadata_map_phy2lg(cache, phys); + ++phys; + + /* store freelist head */ + freelist->part[freelist_idx].head = idx; + + /* link freelist elements using partition list */ + prev = collision_table_entries; + while (--freelist_size) { + phys = next_phys_invalid(cache, phys); + ENV_BUG_ON(phys == collision_table_entries); + next = ocf_metadata_map_phy2lg(cache, phys); + ++phys; + + ocf_metadata_set_partition_info(cache, idx, + PARTITION_INVALID, next, prev); + + prev = idx; + idx = next; + + OCF_COND_RESCHED_DEFAULT(step); + } + + /* terminate partition list */ + ocf_metadata_set_partition_info(cache, idx, PARTITION_INVALID, + collision_table_entries, prev); + + /* store freelist tail */ + freelist->part[freelist_idx].tail = idx; + } + + /* we should have reached the last invalid cache line */ + phys = next_phys_invalid(cache, phys); + ENV_BUG_ON(phys != collision_table_entries); + + env_atomic64_set(&freelist->total_free, num_free_clines); +} + +static void ocf_freelist_add_cache_line(ocf_freelist_t freelist, + uint32_t ctx, ocf_cache_line_t line) +{ + struct ocf_cache *cache = freelist->cache; + struct ocf_part *freelist_part = &freelist->part[ctx]; + ocf_cache_line_t tail; + ocf_cache_line_t line_entries = ocf_metadata_collision_table_entries( + freelist->cache); + ocf_part_id_t invalid_part_id = PARTITION_INVALID; + + ENV_BUG_ON(line >= line_entries); + + if (env_atomic64_read(&freelist_part->curr_size) == 0) { + freelist_part->head = line; + freelist_part->tail = line; + + ocf_metadata_set_partition_info(cache, line, invalid_part_id, + line_entries, line_entries); + } else { + tail = freelist_part->tail; + + ENV_BUG_ON(tail >= line_entries); + + ocf_metadata_set_partition_info(cache, line, invalid_part_id, + line_entries, tail); + ocf_metadata_set_partition_next(cache, tail, line); + + freelist_part->tail = line; + } + + env_atomic64_inc(&freelist_part->curr_size); + env_atomic64_inc(&freelist->total_free); +} + +typedef enum { + OCF_FREELIST_ERR_NOLOCK = 1, + OCF_FREELIST_ERR_LIST_EMPTY, +} ocf_freelist_get_err_t; + +static ocf_freelist_get_err_t ocf_freelist_get_cache_line_ctx( + ocf_freelist_t freelist, uint32_t ctx, bool can_wait, + ocf_cache_line_t *cline) +{ + if (env_atomic64_read(&freelist->part[ctx].curr_size) == 0) + return -OCF_FREELIST_ERR_LIST_EMPTY; + + if (!can_wait && ocf_freelist_trylock(freelist, ctx)) + return -OCF_FREELIST_ERR_NOLOCK; + + if (can_wait) + ocf_freelist_lock(freelist, ctx); + + if (env_atomic64_read(&freelist->part[ctx].curr_size) == 0) { + ocf_freelist_unlock(freelist, ctx); + return -OCF_FREELIST_ERR_LIST_EMPTY; + } + + *cline = freelist->part[ctx].head; + _ocf_freelist_remove_cache_line(freelist, ctx, *cline); + + ocf_freelist_unlock(freelist, ctx); + + return 0; +} + +static int get_next_victim_freelist(ocf_freelist_t freelist) +{ + int ctx, next; + + do { + ctx = env_atomic_read(&freelist->slowpath_victim_idx); + next = (ctx + 1) % freelist->count; + } while (ctx != env_atomic_cmpxchg(&freelist->slowpath_victim_idx, ctx, + next)); + + return ctx; +} + +static bool ocf_freelist_get_cache_line_slow(ocf_freelist_t freelist, + ocf_cache_line_t *cline) +{ + int i, ctx; + int err; + bool lock_err; + + /* try slowpath without waiting on lock */ + lock_err = false; + for (i = 0; i < freelist->count; i++) { + ctx = get_next_victim_freelist(freelist); + err = ocf_freelist_get_cache_line_ctx(freelist, ctx, false, + cline); + if (!err) + return true; + if (err == -OCF_FREELIST_ERR_NOLOCK) + lock_err = true; + } + + if (!lock_err) { + /* Slowpath failed due to empty freelists - no point in + * iterating through contexts to attempt slowpath with full + * lock */ + return false; + } + + /* slow path with waiting on lock */ + for (i = 0; i < freelist->count; i++) { + ctx = get_next_victim_freelist(freelist); + if (!ocf_freelist_get_cache_line_ctx(freelist, ctx, true, + cline)) { + return true; + } + } + + return false; +} + +static bool ocf_freelist_get_cache_line_fast(ocf_freelist_t freelist, + ocf_cache_line_t *cline) +{ + bool ret; + uint32_t ctx = env_get_execution_context(); + + ret = !ocf_freelist_get_cache_line_ctx(freelist, ctx, false, cline); + + env_put_execution_context(ctx); + + return ret; +} + +bool ocf_freelist_get_cache_line(ocf_freelist_t freelist, + ocf_cache_line_t *cline) +{ + if (env_atomic64_read(&freelist->total_free) == 0) + return false; + + if (!ocf_freelist_get_cache_line_fast(freelist, cline)) + return ocf_freelist_get_cache_line_slow(freelist, cline); + + return true; +} + +void ocf_freelist_put_cache_line(ocf_freelist_t freelist, + ocf_cache_line_t cline) +{ + uint32_t ctx = env_get_execution_context(); + + ocf_freelist_lock(freelist, ctx); + ocf_freelist_add_cache_line(freelist, ctx, cline); + ocf_freelist_unlock(freelist, ctx); + env_put_execution_context(ctx); +} + +ocf_freelist_t ocf_freelist_init(struct ocf_cache *cache) +{ + uint32_t num; + int i; + ocf_freelist_t freelist; + ocf_cache_line_t line_entries = ocf_metadata_collision_table_entries( + cache); + + freelist = env_vzalloc(sizeof(*freelist)); + if (!freelist) + return NULL; + + num = env_get_execution_context_count(); + + freelist->cache = cache; + freelist->count = num; + env_atomic64_set(&freelist->total_free, 0); + freelist->lock = env_vzalloc(sizeof(freelist->lock[0]) * num); + freelist->part = env_vzalloc(sizeof(freelist->part[0]) * num); + + if (!freelist->lock || !freelist->part) { + env_vfree(freelist->lock); + env_vfree(freelist->part); + env_vfree(freelist); + return NULL; + } + + for (i = 0; i < num; i++) { + env_spinlock_init(&freelist->lock[i]); + freelist->part[i].head = line_entries; + freelist->part[i].tail = line_entries; + env_atomic64_set(&freelist->part[i].curr_size, 0); + } + + return freelist; +} + +void ocf_freelist_deinit(ocf_freelist_t freelist) +{ + int i; + + for (i = 0; i < freelist->count; i++) + env_spinlock_destroy(&freelist->lock[i]); + env_vfree(freelist->lock); + env_vfree(freelist->part); + env_vfree(freelist); +} + +ocf_cache_line_t ocf_freelist_num_free(ocf_freelist_t freelist) +{ + return env_atomic64_read(&freelist->total_free); +} + diff --git a/src/ocf_freelist.h b/src/ocf_freelist.h new file mode 100644 index 0000000..3b67b2a --- /dev/null +++ b/src/ocf_freelist.h @@ -0,0 +1,34 @@ +/* + * Copyright(c) 2019-2019 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause-Clear + */ + +#ifndef __OCF_FREELIST_H__ +#define __OCF_FREELIST_H__ + +#include "ocf_cache_priv.h" + +struct ocf_freelist; + +typedef struct ocf_freelist *ocf_freelist_t; + +/* Init / deinit freelist runtime structures */ +ocf_freelist_t ocf_freelist_init(struct ocf_cache *cache); +void ocf_freelist_deinit(ocf_freelist_t freelist); + +/* Assign unused cachelines to freelist */ +void ocf_freelist_populate(ocf_freelist_t freelist, + ocf_cache_line_t num_free_clines); + +/* Get cacheline from freelist */ +bool ocf_freelist_get_cache_line(ocf_freelist_t freelist, + ocf_cache_line_t *cline); + +/* Put cacheline back to freelist */ +void ocf_freelist_put_cache_line(ocf_freelist_t freelist, + ocf_cache_line_t cline); + +/* Return total number of free cachelines */ +ocf_cache_line_t ocf_freelist_num_free(ocf_freelist_t freelist); + +#endif /* __OCF_FREELIST_H__ */ diff --git a/src/ocf_stats_builder.c b/src/ocf_stats_builder.c index f1d6948..91847b9 100644 --- a/src/ocf_stats_builder.c +++ b/src/ocf_stats_builder.c @@ -44,18 +44,6 @@ static uint64_t _bytes4k(uint64_t bytes) return (bytes + 4095UL) >> 12; } -static uint64_t _get_cache_occupancy(ocf_cache_t cache) -{ - uint64_t result = 0; - ocf_core_t core; - ocf_core_id_t core_id; - - for_each_core(cache, core, core_id) - result += env_atomic_read(&core->runtime_meta->cached_clines); - - return result; -} - static void _set(struct ocf_stat *stat, uint64_t value, uint64_t denominator) { stat->value = value; @@ -169,7 +157,7 @@ int ocf_stats_collect_core(ocf_core_t core, cache = ocf_core_get_cache(core); cache_line_size = ocf_cache_get_line_size(cache); cache_size = cache->conf_meta->cachelines; - cache_occupancy = _get_cache_occupancy(cache); + cache_occupancy = ocf_get_cache_occupancy(cache); _ocf_stats_zero(usage); _ocf_stats_zero(req); diff --git a/src/promotion/nhit/nhit.c b/src/promotion/nhit/nhit.c index 7648768..5c9b340 100644 --- a/src/promotion/nhit/nhit.c +++ b/src/promotion/nhit/nhit.c @@ -181,7 +181,7 @@ bool nhit_req_should_promote(ocf_promotion_policy_t policy, uint64_t core_line; uint64_t occupied_cachelines = ocf_metadata_get_cachelines_count(policy->owner) - - policy->owner->device->freelist_part->curr_size; + ocf_freelist_num_free(policy->owner->freelist); if (occupied_cachelines > env_atomic64_read(&ctx->trigger_threshold)) return true;