diff --git a/env/posix/ocf_env.c b/env/posix/ocf_env.c index d4c0e00..aae1960 100644 --- a/env/posix/ocf_env.c +++ b/env/posix/ocf_env.c @@ -130,8 +130,64 @@ void env_stack_trace(void) } /* *** CRC *** */ - uint32_t env_crc32(uint32_t crc, uint8_t const *data, size_t len) { return crc32(crc, data, len); } + +/* *** execution contexts *** */ +pthread_mutex_t *exec_context_mutex; + +static void __attribute__((constructor)) init_execution_context(void) +{ + unsigned count = env_get_execution_context_count(); + unsigned i; + + ENV_BUG_ON(count == 0); + exec_context_mutex = malloc(count * sizeof(exec_context_mutex[0])); + ENV_BUG_ON(exec_context_mutex == NULL); + for (i = 0; i < count; i++) + ENV_BUG_ON(pthread_mutex_init(&exec_context_mutex[i], NULL)); +} + +static void __attribute__((destructor)) deinit_execution_context(void) +{ + unsigned count = env_get_execution_context_count(); + unsigned i; + + ENV_BUG_ON(count == 0); + ENV_BUG_ON(exec_context_mutex == NULL); + + for (i = 0; i < count; i++) + ENV_BUG_ON(pthread_mutex_destroy(&exec_context_mutex[i])); + free(exec_context_mutex); +} + +/* get_execuction_context must assure that after the call finishes, the caller + * will not get preempted from current execution context. For userspace env + * we simulate this behavior by acquiring per execution context mutex. As a + * result the caller might actually get preempted, but no other thread will + * execute in this context by the time the caller puts current execution ctx. */ +unsigned env_get_execution_context(void) +{ + unsigned cpu; + + cpu = sched_getcpu(); + cpu = (cpu == -1) ? 0 : cpu; + + ENV_BUG_ON(pthread_mutex_lock(&exec_context_mutex[cpu])); + + return cpu; +} + +void env_put_execution_context(unsigned ctx) +{ + pthread_mutex_unlock(&exec_context_mutex[ctx]); +} + +unsigned env_get_execution_context_count(void) +{ + int num = sysconf(_SC_NPROCESSORS_ONLN); + + return (num == -1) ? 0 : num; +} diff --git a/env/posix/ocf_env.h b/env/posix/ocf_env.h index 8baeeeb..7e81924 100644 --- a/env/posix/ocf_env.h +++ b/env/posix/ocf_env.h @@ -455,6 +455,11 @@ static inline void env_spinlock_init(env_spinlock *l) ENV_BUG_ON(pthread_spin_init(&l->lock, 0)); } +static inline int env_spinlock_trylock(env_spinlock *l) +{ + return pthread_spin_trylock(&l->lock) ? -OCF_ERR_NO_LOCK : 0; +} + static inline void env_spinlock_lock(env_spinlock *l) { ENV_BUG_ON(pthread_spin_lock(&l->lock)); @@ -644,4 +649,8 @@ uint32_t env_crc32(uint32_t crc, uint8_t const *data, size_t len); #define ENV_PRIu64 "lu" +unsigned env_get_execution_context(void); +void env_put_execution_context(unsigned ctx); +unsigned env_get_execution_context_count(void); + #endif /* __OCF_ENV_H__ */ diff --git a/src/engine/cache_engine.c b/src/engine/cache_engine.c index 41eff2e..64799c3 100644 --- a/src/engine/cache_engine.c +++ b/src/engine/cache_engine.c @@ -169,7 +169,8 @@ static inline bool ocf_seq_cutoff_is_on(ocf_cache_t cache) if (!ocf_cache_is_device_attached(cache)) return false; - return (cache->device->freelist_part->curr_size <= SEQ_CUTOFF_FULL_MARGIN); + return (ocf_freelist_num_free(cache->freelist) <= + SEQ_CUTOFF_FULL_MARGIN); } bool ocf_seq_cutoff_check(ocf_core_t core, uint32_t dir, uint64_t addr, diff --git a/src/engine/engine_common.c b/src/engine/engine_common.c index 53a5fcc..b01148d 100644 --- a/src/engine/engine_common.c +++ b/src/engine/engine_common.c @@ -7,6 +7,7 @@ #include "../ocf_priv.h" #include "../ocf_cache_priv.h" #include "../ocf_queue_priv.h" +#include "../ocf_freelist.h" #include "engine_common.h" #define OCF_ENGINE_DEBUG_IO_NAME "common" #include "engine_debug.h" @@ -250,19 +251,11 @@ static void ocf_engine_map_cache_line(struct ocf_request *req, ocf_part_id_t part_id = req->part_id; ocf_cleaning_t clean_policy_type; - if (cache->device->freelist_part->curr_size == 0) { + if (!ocf_freelist_get_cache_line(cache->freelist, cache_line)) { req->info.mapping_error = 1; return; } - *cache_line = cache->device->freelist_part->head; - - /* add_to_collision_list changes .next_col and other fields for entry - * so updated last_cache_line_give must be updated before calling it. - */ - - ocf_metadata_remove_from_free_list(cache, *cache_line); - ocf_metadata_add_to_partition(cache, part_id, *cache_line); /* Add the block to the corresponding collision list */ diff --git a/src/eviction/eviction.c b/src/eviction/eviction.c index 0bacee8..2d76db4 100644 --- a/src/eviction/eviction.c +++ b/src/eviction/eviction.c @@ -107,11 +107,13 @@ int space_managment_evict_do(struct ocf_cache *cache, struct ocf_request *req, uint32_t evict_cline_no) { uint32_t evicted; + uint32_t free; - if (evict_cline_no <= cache->device->freelist_part->curr_size) + free = ocf_freelist_num_free(cache->freelist); + if (evict_cline_no <= free) return LOOKUP_MAPPED; - evict_cline_no -= cache->device->freelist_part->curr_size; + evict_cline_no -= free; evicted = ocf_evict_do(cache, req->io_queue, evict_cline_no, req->part_id); diff --git a/src/eviction/eviction.h b/src/eviction/eviction.h index 7ebafea..6444521 100644 --- a/src/eviction/eviction.h +++ b/src/eviction/eviction.h @@ -32,8 +32,7 @@ union eviction_policy_meta { * set core_id to -2 to purge the whole cache partition */ struct eviction_policy_ops { - void (*init_cline)(ocf_cache_t cache, - ocf_cache_line_t cline); + void (*init_cline)(ocf_cache_t cache, ocf_cache_line_t cline); void (*rm_cline)(ocf_cache_t cache, ocf_cache_line_t cline); bool (*can_evict)(ocf_cache_t cache); diff --git a/src/eviction/lru.h b/src/eviction/lru.h index 79488a4..fa996f7 100644 --- a/src/eviction/lru.h +++ b/src/eviction/lru.h @@ -8,8 +8,7 @@ #include "eviction.h" #include "lru_structs.h" -void evp_lru_init_cline(struct ocf_cache *cache, - ocf_cache_line_t cline); +void evp_lru_init_cline(struct ocf_cache *cache, ocf_cache_line_t cline); void evp_lru_rm_cline(struct ocf_cache *cache, ocf_cache_line_t cline); bool evp_lru_can_evict(struct ocf_cache *cache); uint32_t evp_lru_req_clines(struct ocf_cache *cache, ocf_queue_t io_queue, diff --git a/src/metadata/metadata.c b/src/metadata/metadata.c index 35b79cf..1b31132 100644 --- a/src/metadata/metadata.c +++ b/src/metadata/metadata.c @@ -53,18 +53,18 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache, uint64_t device_siz cache_line_size, layout); } -void ocf_metadata_init_freelist_partition(struct ocf_cache *cache) -{ - OCF_DEBUG_TRACE(cache); - cache->metadata.iface.init_freelist(cache); -} - void ocf_metadata_init_hash_table(struct ocf_cache *cache) { OCF_DEBUG_TRACE(cache); cache->metadata.iface.init_hash_table(cache); } +void ocf_metadata_init_collision(struct ocf_cache *cache) +{ + OCF_DEBUG_TRACE(cache); + cache->metadata.iface.init_collision(cache); +} + void ocf_metadata_deinit(struct ocf_cache *cache) { OCF_DEBUG_TRACE(cache); diff --git a/src/metadata/metadata.h b/src/metadata/metadata.h index 7893fed..5c75b89 100644 --- a/src/metadata/metadata.h +++ b/src/metadata/metadata.h @@ -60,6 +60,13 @@ void ocf_metadata_init_freelist_partition(struct ocf_cache *cache); */ void ocf_metadata_init_hash_table(struct ocf_cache *cache); +/** + * @brief Initialize collision table + * + * @param cache - Cache instance + */ +void ocf_metadata_init_collision(struct ocf_cache *cache); + /** * @brief De-Initialize metadata * @@ -207,4 +214,10 @@ typedef void (*ocf_metadata_load_properties_end_t)(void *priv, int error, void ocf_metadata_load_properties(ocf_volume_t volume, ocf_metadata_load_properties_end_t cmpl, void *priv); +static inline ocf_cache_line_t ocf_metadata_collision_table_entries( + struct ocf_cache *cache) +{ + return cache->device->collision_table_entries; +} + #endif /* METADATA_H_ */ diff --git a/src/metadata/metadata_hash.c b/src/metadata/metadata_hash.c index 0fb669c..1c0c35d 100644 --- a/src/metadata/metadata_hash.c +++ b/src/metadata/metadata_hash.c @@ -13,6 +13,7 @@ #include "../utils/utils_pipeline.h" #include "../ocf_def_priv.h" #include "../ocf_priv.h" +#include "../ocf_freelist.h" #define OCF_METADATA_HASH_DEBUG 0 @@ -1025,14 +1026,10 @@ finalize: } static inline void _ocf_init_collision_entry(struct ocf_cache *cache, - ocf_cache_line_t idx, ocf_cache_line_t next, - ocf_cache_line_t prev) + ocf_cache_line_t idx) { ocf_cache_line_t invalid_idx = cache->device->collision_table_entries; - ocf_part_id_t invalid_part_id = PARTITION_INVALID; - ocf_metadata_set_partition_info(cache, idx, - invalid_part_id, next, prev); ocf_metadata_set_collision_info(cache, idx, invalid_idx, invalid_idx); ocf_metadata_set_core_info(cache, idx, OCF_CORE_MAX, ULONG_MAX); @@ -1040,36 +1037,17 @@ static inline void _ocf_init_collision_entry(struct ocf_cache *cache, } /* - * Modified initialization of freelist partition + * Initialize collision table */ -static void ocf_metadata_hash_init_freelist(struct ocf_cache *cache) +static void ocf_metadata_hash_init_collision(struct ocf_cache *cache) { - uint32_t step = 0; unsigned int i; - ocf_cache_line_t prev, next; - ocf_cache_line_t idx; - ocf_cache_line_t collision_table_entries = - cache->device->collision_table_entries; - prev = collision_table_entries; - idx = 0; - for (i = 0; i < cache->device->collision_table_entries - 1; i++) { - next = ocf_metadata_map_phy2lg(cache, i + 1); - _ocf_init_collision_entry(cache, idx, next, prev); - prev = idx; - idx = next; - OCF_COND_RESCHED_DEFAULT(step); + for (i = 0; i < cache->device->collision_table_entries; i++) { + _ocf_init_collision_entry(cache, i); } - _ocf_init_collision_entry(cache, idx, collision_table_entries, prev); - - /* Initialize freelist partition */ - cache->device->freelist_part->head = 0; - cache->device->freelist_part->tail = idx; - cache->device->freelist_part->curr_size = cache->device-> - collision_table_entries; } - /* * Initialize hash table */ @@ -1850,8 +1828,6 @@ static void _recovery_rebuild_cline_metadata(ocf_cache_t cache, part_id = PARTITION_DEFAULT; - ocf_metadata_remove_from_free_list(cache, cache_line); - ocf_metadata_add_to_partition(cache, part_id, cache_line); hash_index = ocf_metadata_hash_func(cache, core_line, core_id); @@ -1917,10 +1893,12 @@ static void _recovery_rebuild_metadata(ocf_pipeline_t pipeline, ocf_core_id_t core_id; uint64_t core_line; unsigned char step = 0; + const uint64_t collision_table_entries = + ocf_metadata_collision_table_entries(cache); OCF_METADATA_LOCK_WR(); - for (cline = 0; cline < cache->device->collision_table_entries; cline++) { + for (cline = 0; cline < collision_table_entries; cline++) { ocf_metadata_get_core_info(cache, cline, &core_id, &core_line); if (core_id != OCF_CORE_MAX && (!dirty_only || metadata_test_dirty(cache, @@ -2648,7 +2626,7 @@ static const struct ocf_metadata_iface metadata_hash_iface = { .init_variable_size = ocf_metadata_hash_init_variable_size, .deinit_variable_size = ocf_metadata_hash_deinit_variable_size, .init_hash_table = ocf_metadata_hash_init_hash_table, - .init_freelist = ocf_metadata_hash_init_freelist, + .init_collision = ocf_metadata_hash_init_collision, .layout_iface = NULL, .pages = ocf_metadata_hash_pages, diff --git a/src/metadata/metadata_misc.c b/src/metadata/metadata_misc.c index 9a0187c..ff17e77 100644 --- a/src/metadata/metadata_misc.c +++ b/src/metadata/metadata_misc.c @@ -5,6 +5,7 @@ #include "ocf/ocf.h" #include "metadata.h" +#include "../ocf_freelist.h" #include "../utils/utils_cache_line.h" static bool _is_cache_line_acting(struct ocf_cache *cache, @@ -100,7 +101,7 @@ void ocf_metadata_sparse_cache_line(struct ocf_cache *cache, ocf_metadata_remove_from_partition(cache, partition_id, cache_line); - ocf_metadata_add_to_free_list(cache, cache_line); + ocf_freelist_put_cache_line(cache->freelist, cache_line); } static void _ocf_metadata_sparse_cache_line(struct ocf_cache *cache, diff --git a/src/metadata/metadata_partition.c b/src/metadata/metadata_partition.c index de02dcc..b802101 100644 --- a/src/metadata/metadata_partition.c +++ b/src/metadata/metadata_partition.c @@ -16,99 +16,6 @@ static void update_partition_head(struct ocf_cache *cache, part->runtime->head = line; } -void ocf_metadata_remove_from_free_list(struct ocf_cache *cache, - ocf_cache_line_t cline) -{ - struct ocf_part *free_list = cache->device->freelist_part; - int is_head, is_tail; - ocf_part_id_t invalid_part_id = PARTITION_INVALID; - ocf_cache_line_t prev, next; - ocf_cache_line_t line_entries = cache->device->collision_table_entries; - - ENV_BUG_ON(cline >= line_entries); - - /* Get Partition info */ - ocf_metadata_get_partition_info(cache, cline, NULL, &next, &prev); - - /* Find out if this node is Partition _head_ */ - is_head = (prev == line_entries); - is_tail = (next == line_entries); - - /* Case 1: If we are head and there is only one node. So unlink node - * and set that there is no node left in the list. - */ - if (is_head && (free_list->curr_size == 1)) { - ocf_metadata_set_partition_info(cache, cline, invalid_part_id, - line_entries, line_entries); - free_list->head = line_entries; - free_list->tail = line_entries; - } else if (is_head) { - /* Case 2: else if this collision_index is partition list head, - * but many nodes, update head and return - */ - ENV_BUG_ON(next >= line_entries); - - free_list->head = next; - ocf_metadata_set_partition_prev(cache, next, line_entries); - ocf_metadata_set_partition_next(cache, cline, line_entries); - } else if (is_tail) { - /* Case 3: else if this cline is partition list tail */ - ENV_BUG_ON(prev >= line_entries); - - free_list->tail = prev; - ocf_metadata_set_partition_prev(cache, cline, line_entries); - ocf_metadata_set_partition_next(cache, prev, line_entries); - } else { - /* Case 4: else this collision_index is a middle node. - * There is no change to the head and the tail pointers. - */ - - ENV_BUG_ON(next >= line_entries || prev >= line_entries); - - /* Update prev and next nodes */ - ocf_metadata_set_partition_prev(cache, next, prev); - ocf_metadata_set_partition_next(cache, prev, next); - - /* Update the given node */ - ocf_metadata_set_partition_info(cache, cline, invalid_part_id, - line_entries, line_entries); - } - - free_list->curr_size--; -} - -void ocf_metadata_add_to_free_list(struct ocf_cache *cache, - ocf_cache_line_t line) -{ - struct ocf_part *free_list = cache->device->freelist_part; - ocf_cache_line_t tail; - ocf_cache_line_t line_entries = cache->device->collision_table_entries; - ocf_part_id_t invalid_part_id = PARTITION_INVALID; - - ENV_BUG_ON(line >= line_entries); - - if (free_list->curr_size == 0) { - free_list->head = line; - free_list->tail = line; - - ocf_metadata_set_partition_info(cache, line, invalid_part_id, - line_entries, line_entries); - } else { - tail = free_list->tail; - - ENV_BUG_ON(tail >= line_entries); - - ocf_metadata_set_partition_info(cache, line, invalid_part_id, - line_entries, tail); - ocf_metadata_set_partition_next(cache, tail, line); - - free_list->tail = line; - } - - free_list->curr_size++; -} - - /* Adds the given collision_index to the _head_ of the Partition list */ void ocf_metadata_add_to_partition(struct ocf_cache *cache, ocf_part_id_t part_id, ocf_cache_line_t line) diff --git a/src/metadata/metadata_partition.h b/src/metadata/metadata_partition.h index 8aee83f..00605e0 100644 --- a/src/metadata/metadata_partition.h +++ b/src/metadata/metadata_partition.h @@ -75,12 +75,6 @@ static inline void ocf_metadata_set_partition_info( next_line, prev_line); } -void ocf_metadata_add_to_free_list(struct ocf_cache *cache, - ocf_cache_line_t cline); - -void ocf_metadata_remove_from_free_list(struct ocf_cache *cache, - ocf_cache_line_t cline); - void ocf_metadata_add_to_partition(struct ocf_cache *cache, ocf_part_id_t part_id, ocf_cache_line_t line); diff --git a/src/metadata/metadata_partition_structs.h b/src/metadata/metadata_partition_structs.h index c8ed593..58d7e56 100644 --- a/src/metadata/metadata_partition_structs.h +++ b/src/metadata/metadata_partition_structs.h @@ -10,12 +10,6 @@ #include "../cleaning/cleaning.h" #include "../eviction/eviction.h" -struct ocf_part { - ocf_cache_line_t head; - ocf_cache_line_t tail; - uint32_t curr_size; -}; - struct ocf_user_part_config { char name[OCF_IO_CLASS_NAME_MAX]; uint32_t min_size; diff --git a/src/metadata/metadata_structs.h b/src/metadata/metadata_structs.h index 278e628..e9b3ac5 100644 --- a/src/metadata/metadata_structs.h +++ b/src/metadata/metadata_structs.h @@ -124,13 +124,6 @@ struct ocf_metadata_iface { struct ocf_volume_uuid *uuid, uint32_t count, ocf_metadata_query_cores_end_t cmpl, void *priv); - /** - * @brief Initialize freelist partition - * - * @param cache - Cache instance - */ - - void (*init_freelist)(struct ocf_cache *cache); /** * @brief Metadata cache line location on pages interface @@ -144,6 +137,13 @@ struct ocf_metadata_iface { */ void (*init_hash_table)(struct ocf_cache *cache); + /** + * @brief Initialize collision table + * + * @param cache - Cache instance + */ + void (*init_collision)(struct ocf_cache *cache); + /** * @brief De-Initialize metadata * diff --git a/src/metadata/metadata_superblock.h b/src/metadata/metadata_superblock.h index 0baa76f..c08e90d 100644 --- a/src/metadata/metadata_superblock.h +++ b/src/metadata/metadata_superblock.h @@ -59,8 +59,6 @@ struct ocf_superblock_config { * @brief OCF cache metadata runtime superblock */ struct ocf_superblock_runtime { - struct ocf_part freelist_part; - uint32_t cleaning_thread_access; }; diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 52e8e62..2481623 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -21,6 +21,7 @@ #include "../concurrency/ocf_concurrency.h" #include "../eviction/ops.h" #include "../ocf_ctx_priv.h" +#include "../ocf_freelist.h" #include "../cleaning/cleaning.h" #define OCF_ASSERT_PLUGGED(cache) ENV_BUG_ON(!(cache)->device) @@ -117,6 +118,8 @@ struct ocf_cache_attach_context { * load or recovery */ + bool freelist_inited : 1; + bool concurrency_inited : 1; } flags; @@ -156,18 +159,6 @@ struct ocf_cache_attach_context { ocf_pipeline_t pipeline; }; -static void __init_hash_table(ocf_cache_t cache) -{ - /* Initialize hash table*/ - ocf_metadata_init_hash_table(cache); -} - -static void __init_freelist(ocf_cache_t cache) -{ - /* Initialize free list partition*/ - ocf_metadata_init_freelist_partition(cache); -} - static void __init_partitions(ocf_cache_t cache) { ocf_part_id_t i_part; @@ -204,6 +195,14 @@ static void __init_partitions_attached(ocf_cache_t cache) } } +static void __init_freelist(ocf_cache_t cache) +{ + uint64_t free_clines = ocf_metadata_collision_table_entries(cache) - + ocf_get_cache_occupancy(cache); + + ocf_freelist_populate(cache->freelist, free_clines); +} + static ocf_error_t __init_cleaning_policy(ocf_cache_t cache) { ocf_cleaning_t cleaning_policy = ocf_cleaning_default; @@ -295,9 +294,10 @@ static ocf_error_t init_attached_data_structures(ocf_cache_t cache, /* Lock to ensure consistency */ OCF_METADATA_LOCK_WR(); - __init_hash_table(cache); - __init_freelist(cache); + ocf_metadata_init_hash_table(cache); + ocf_metadata_init_collision(cache); __init_partitions_attached(cache); + __init_freelist(cache); result = __init_cleaning_policy(cache); if (result) { @@ -325,8 +325,8 @@ static ocf_error_t init_attached_data_structures(ocf_cache_t cache, static void init_attached_data_structures_recovery(ocf_cache_t cache) { OCF_METADATA_LOCK_WR(); - __init_hash_table(cache); - __init_freelist(cache); + ocf_metadata_init_hash_table(cache); + ocf_metadata_init_collision(cache); __init_partitions_attached(cache); __reset_stats(cache); __init_metadata_version(cache); @@ -469,6 +469,8 @@ void _ocf_mngt_init_instance_load_complete(void *priv, int error) OCF_PL_FINISH_RET(context->pipeline, -OCF_ERR_START_CACHE_FAIL); } + __init_freelist(cache); + result = __init_promotion_policy(cache); if (result) { ocf_cache_log(cache, log_err, @@ -985,8 +987,10 @@ static void _ocf_mngt_attach_prepare_metadata(ocf_pipeline_t pipeline, OCF_PL_FINISH_RET(context->pipeline, -OCF_ERR_START_CACHE_FAIL); } - - cache->device->freelist_part = &cache->device->runtime_meta->freelist_part; + cache->freelist = ocf_freelist_init(cache); + if (!cache->freelist) + OCF_PL_FINISH_RET(context->pipeline, -OCF_ERR_START_CACHE_FAIL); + context->flags.freelist_inited = true; ret = ocf_concurrency_init(cache); if (ret) @@ -1143,6 +1147,9 @@ static void _ocf_mngt_attach_handle_error( if (context->flags.concurrency_inited) ocf_concurrency_deinit(cache); + if (context->flags.freelist_inited) + ocf_freelist_deinit(cache->freelist); + if (context->flags.volume_inited) ocf_volume_deinit(&cache->device->volume); @@ -1725,6 +1732,7 @@ static void _ocf_mngt_cache_unplug_complete(void *priv, int error) ocf_metadata_deinit_variable_size(cache); ocf_concurrency_deinit(cache); + ocf_freelist_deinit(cache->freelist); ocf_volume_deinit(&cache->device->volume); diff --git a/src/ocf_cache_priv.h b/src/ocf_cache_priv.h index a32b64e..7f54881 100644 --- a/src/ocf_cache_priv.h +++ b/src/ocf_cache_priv.h @@ -21,6 +21,7 @@ #include "ocf_logger_priv.h" #include "ocf/ocf_trace.h" #include "promotion/promotion.h" +#include "ocf_freelist.h" #define DIRTY_FLUSHED 1 #define DIRTY_NOT_FLUSHED 0 @@ -82,8 +83,6 @@ struct ocf_cache_device { uint64_t metadata_offset; - struct ocf_part *freelist_part; - struct { struct ocf_cache_line_concurrency *cache_line; } concurrency; @@ -110,6 +109,8 @@ struct ocf_cache { struct ocf_metadata metadata; + ocf_freelist_t freelist; + ocf_eviction_t eviction_policy_init; struct { @@ -199,4 +200,16 @@ static inline ocf_core_t ocf_cache_get_core(ocf_cache_t cache, #define ocf_cache_log_rl(cache) \ ocf_log_rl(ocf_cache_get_ctx(cache)) +static inline uint64_t ocf_get_cache_occupancy(ocf_cache_t cache) +{ + uint64_t result = 0; + ocf_core_t core; + ocf_core_id_t core_id; + + for_each_core(cache, core, core_id) + result += env_atomic_read(&core->runtime_meta->cached_clines); + + return result; +} + #endif /* __OCF_CACHE_PRIV_H__ */ diff --git a/src/ocf_freelist.c b/src/ocf_freelist.c new file mode 100644 index 0000000..fdff6bb --- /dev/null +++ b/src/ocf_freelist.c @@ -0,0 +1,420 @@ +/* + * Copyright(c) 2019-2019 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause-Clear + */ + +#include "ocf/ocf.h" +#include "metadata/metadata.h" + +struct ocf_part { + ocf_cache_line_t head; + ocf_cache_line_t tail; + env_atomic64 curr_size; +}; + +struct ocf_freelist { + /* parent cache */ + struct ocf_cache *cache; + + /* partition list array */ + struct ocf_part *part; + + /* freelist lock array */ + env_spinlock *lock; + + /* number of free lists */ + uint32_t count; + + /* next slowpath victim idx */ + env_atomic slowpath_victim_idx; + + /* total number of free lines */ + env_atomic64 total_free; +}; + +static void ocf_freelist_lock(ocf_freelist_t freelist, uint32_t ctx) +{ + env_spinlock_lock(&freelist->lock[ctx]); +} + +static int ocf_freelist_trylock(ocf_freelist_t freelist, uint32_t ctx) +{ + return env_spinlock_trylock(&freelist->lock[ctx]); +} + +static void ocf_freelist_unlock(ocf_freelist_t freelist, uint32_t ctx) +{ + env_spinlock_unlock(&freelist->lock[ctx]); +} + +/* Sets the given collision_index as the new _head_ of the Partition list. */ +static void _ocf_freelist_remove_cache_line(ocf_freelist_t freelist, + uint32_t ctx, ocf_cache_line_t cline) +{ + struct ocf_cache *cache = freelist->cache; + struct ocf_part *freelist_part = &freelist->part[ctx]; + int is_head, is_tail; + ocf_part_id_t invalid_part_id = PARTITION_INVALID; + ocf_cache_line_t prev, next; + ocf_cache_line_t line_entries = ocf_metadata_collision_table_entries( + freelist->cache); + uint32_t free; + + ENV_BUG_ON(cline >= line_entries); + + /* Get Partition info */ + ocf_metadata_get_partition_info(cache, cline, NULL, &next, &prev); + + /* Find out if this node is Partition _head_ */ + is_head = (prev == line_entries); + is_tail = (next == line_entries); + + free = env_atomic64_read(&freelist_part->curr_size); + + /* Case 1: If we are head and there is only one node. So unlink node + * and set that there is no node left in the list. + */ + if (is_head && free == 1) { + ocf_metadata_set_partition_info(cache, cline, invalid_part_id, + line_entries, line_entries); + freelist_part->head = line_entries; + freelist_part->tail = line_entries; + } else if (is_head) { + /* Case 2: else if this collision_index is partition list head, + * but many nodes, update head and return + */ + ENV_BUG_ON(next >= line_entries); + + freelist_part->head = next; + ocf_metadata_set_partition_prev(cache, next, line_entries); + ocf_metadata_set_partition_next(cache, cline, line_entries); + } else if (is_tail) { + /* Case 3: else if this cline is partition list tail */ + ENV_BUG_ON(prev >= line_entries); + + freelist_part->tail = prev; + ocf_metadata_set_partition_prev(cache, cline, line_entries); + ocf_metadata_set_partition_next(cache, prev, line_entries); + } else { + /* Case 4: else this collision_index is a middle node. + * There is no change to the head and the tail pointers. + */ + + ENV_BUG_ON(next >= line_entries || prev >= line_entries); + + /* Update prev and next nodes */ + ocf_metadata_set_partition_prev(cache, next, prev); + ocf_metadata_set_partition_next(cache, prev, next); + + /* Update the given node */ + ocf_metadata_set_partition_info(cache, cline, invalid_part_id, + line_entries, line_entries); + } + + env_atomic64_dec(&freelist_part->curr_size); + env_atomic64_dec(&freelist->total_free); +} + +static ocf_cache_line_t next_phys_invalid(ocf_cache_t cache, + ocf_cache_line_t phys) +{ + ocf_cache_line_t lg; + ocf_cache_line_t collision_table_entries = + ocf_metadata_collision_table_entries(cache); + + if (phys == collision_table_entries) + return collision_table_entries; + + lg = ocf_metadata_map_phy2lg(cache, phys); + while (metadata_test_valid_any(cache, lg)) { + ++phys; + + if (phys == collision_table_entries) + break; + + lg = ocf_metadata_map_phy2lg(cache, phys); + } + + return phys; +} + +/* Assign unused cachelines to freelist */ +void ocf_freelist_populate(ocf_freelist_t freelist, + ocf_cache_line_t num_free_clines) +{ + unsigned step = 0; + ocf_cache_t cache = freelist->cache; + unsigned num_freelists = freelist->count; + ocf_cache_line_t prev, next, idx; + ocf_cache_line_t phys; + ocf_cache_line_t collision_table_entries = + ocf_metadata_collision_table_entries(cache); + unsigned freelist_idx; + uint64_t freelist_size; + + phys = 0; + for (freelist_idx = 0; freelist_idx < num_freelists; freelist_idx++) + { + /* calculate current freelist size */ + freelist_size = num_free_clines / num_freelists; + if (freelist_idx < (num_free_clines % num_freelists)) + ++freelist_size; + + env_atomic64_set(&freelist->part[freelist_idx].curr_size, + freelist_size); + + if (!freelist_size) { + /* init empty freelist and move to next one */ + freelist->part[freelist_idx].head = + collision_table_entries; + freelist->part[freelist_idx].tail = + collision_table_entries; + continue; + } + + /* find first invalid cacheline */ + phys = next_phys_invalid(cache, phys); + ENV_BUG_ON(phys == collision_table_entries); + idx = ocf_metadata_map_phy2lg(cache, phys); + ++phys; + + /* store freelist head */ + freelist->part[freelist_idx].head = idx; + + /* link freelist elements using partition list */ + prev = collision_table_entries; + while (--freelist_size) { + phys = next_phys_invalid(cache, phys); + ENV_BUG_ON(phys == collision_table_entries); + next = ocf_metadata_map_phy2lg(cache, phys); + ++phys; + + ocf_metadata_set_partition_info(cache, idx, + PARTITION_INVALID, next, prev); + + prev = idx; + idx = next; + + OCF_COND_RESCHED_DEFAULT(step); + } + + /* terminate partition list */ + ocf_metadata_set_partition_info(cache, idx, PARTITION_INVALID, + collision_table_entries, prev); + + /* store freelist tail */ + freelist->part[freelist_idx].tail = idx; + } + + /* we should have reached the last invalid cache line */ + phys = next_phys_invalid(cache, phys); + ENV_BUG_ON(phys != collision_table_entries); + + env_atomic64_set(&freelist->total_free, num_free_clines); +} + +static void ocf_freelist_add_cache_line(ocf_freelist_t freelist, + uint32_t ctx, ocf_cache_line_t line) +{ + struct ocf_cache *cache = freelist->cache; + struct ocf_part *freelist_part = &freelist->part[ctx]; + ocf_cache_line_t tail; + ocf_cache_line_t line_entries = ocf_metadata_collision_table_entries( + freelist->cache); + ocf_part_id_t invalid_part_id = PARTITION_INVALID; + + ENV_BUG_ON(line >= line_entries); + + if (env_atomic64_read(&freelist_part->curr_size) == 0) { + freelist_part->head = line; + freelist_part->tail = line; + + ocf_metadata_set_partition_info(cache, line, invalid_part_id, + line_entries, line_entries); + } else { + tail = freelist_part->tail; + + ENV_BUG_ON(tail >= line_entries); + + ocf_metadata_set_partition_info(cache, line, invalid_part_id, + line_entries, tail); + ocf_metadata_set_partition_next(cache, tail, line); + + freelist_part->tail = line; + } + + env_atomic64_inc(&freelist_part->curr_size); + env_atomic64_inc(&freelist->total_free); +} + +typedef enum { + OCF_FREELIST_ERR_NOLOCK = 1, + OCF_FREELIST_ERR_LIST_EMPTY, +} ocf_freelist_get_err_t; + +static ocf_freelist_get_err_t ocf_freelist_get_cache_line_ctx( + ocf_freelist_t freelist, uint32_t ctx, bool can_wait, + ocf_cache_line_t *cline) +{ + if (env_atomic64_read(&freelist->part[ctx].curr_size) == 0) + return -OCF_FREELIST_ERR_LIST_EMPTY; + + if (!can_wait && ocf_freelist_trylock(freelist, ctx)) + return -OCF_FREELIST_ERR_NOLOCK; + + if (can_wait) + ocf_freelist_lock(freelist, ctx); + + if (env_atomic64_read(&freelist->part[ctx].curr_size) == 0) { + ocf_freelist_unlock(freelist, ctx); + return -OCF_FREELIST_ERR_LIST_EMPTY; + } + + *cline = freelist->part[ctx].head; + _ocf_freelist_remove_cache_line(freelist, ctx, *cline); + + ocf_freelist_unlock(freelist, ctx); + + return 0; +} + +static int get_next_victim_freelist(ocf_freelist_t freelist) +{ + int ctx, next; + + do { + ctx = env_atomic_read(&freelist->slowpath_victim_idx); + next = (ctx + 1) % freelist->count; + } while (ctx != env_atomic_cmpxchg(&freelist->slowpath_victim_idx, ctx, + next)); + + return ctx; +} + +static bool ocf_freelist_get_cache_line_slow(ocf_freelist_t freelist, + ocf_cache_line_t *cline) +{ + int i, ctx; + int err; + bool lock_err; + + /* try slowpath without waiting on lock */ + lock_err = false; + for (i = 0; i < freelist->count; i++) { + ctx = get_next_victim_freelist(freelist); + err = ocf_freelist_get_cache_line_ctx(freelist, ctx, false, + cline); + if (!err) + return true; + if (err == -OCF_FREELIST_ERR_NOLOCK) + lock_err = true; + } + + if (!lock_err) { + /* Slowpath failed due to empty freelists - no point in + * iterating through contexts to attempt slowpath with full + * lock */ + return false; + } + + /* slow path with waiting on lock */ + for (i = 0; i < freelist->count; i++) { + ctx = get_next_victim_freelist(freelist); + if (!ocf_freelist_get_cache_line_ctx(freelist, ctx, true, + cline)) { + return true; + } + } + + return false; +} + +static bool ocf_freelist_get_cache_line_fast(ocf_freelist_t freelist, + ocf_cache_line_t *cline) +{ + bool ret; + uint32_t ctx = env_get_execution_context(); + + ret = !ocf_freelist_get_cache_line_ctx(freelist, ctx, false, cline); + + env_put_execution_context(ctx); + + return ret; +} + +bool ocf_freelist_get_cache_line(ocf_freelist_t freelist, + ocf_cache_line_t *cline) +{ + if (env_atomic64_read(&freelist->total_free) == 0) + return false; + + if (!ocf_freelist_get_cache_line_fast(freelist, cline)) + return ocf_freelist_get_cache_line_slow(freelist, cline); + + return true; +} + +void ocf_freelist_put_cache_line(ocf_freelist_t freelist, + ocf_cache_line_t cline) +{ + uint32_t ctx = env_get_execution_context(); + + ocf_freelist_lock(freelist, ctx); + ocf_freelist_add_cache_line(freelist, ctx, cline); + ocf_freelist_unlock(freelist, ctx); + env_put_execution_context(ctx); +} + +ocf_freelist_t ocf_freelist_init(struct ocf_cache *cache) +{ + uint32_t num; + int i; + ocf_freelist_t freelist; + ocf_cache_line_t line_entries = ocf_metadata_collision_table_entries( + cache); + + freelist = env_vzalloc(sizeof(*freelist)); + if (!freelist) + return NULL; + + num = env_get_execution_context_count(); + + freelist->cache = cache; + freelist->count = num; + env_atomic64_set(&freelist->total_free, 0); + freelist->lock = env_vzalloc(sizeof(freelist->lock[0]) * num); + freelist->part = env_vzalloc(sizeof(freelist->part[0]) * num); + + if (!freelist->lock || !freelist->part) { + env_vfree(freelist->lock); + env_vfree(freelist->part); + env_vfree(freelist); + return NULL; + } + + for (i = 0; i < num; i++) { + env_spinlock_init(&freelist->lock[i]); + freelist->part[i].head = line_entries; + freelist->part[i].tail = line_entries; + env_atomic64_set(&freelist->part[i].curr_size, 0); + } + + return freelist; +} + +void ocf_freelist_deinit(ocf_freelist_t freelist) +{ + int i; + + for (i = 0; i < freelist->count; i++) + env_spinlock_destroy(&freelist->lock[i]); + env_vfree(freelist->lock); + env_vfree(freelist->part); + env_vfree(freelist); +} + +ocf_cache_line_t ocf_freelist_num_free(ocf_freelist_t freelist) +{ + return env_atomic64_read(&freelist->total_free); +} + diff --git a/src/ocf_freelist.h b/src/ocf_freelist.h new file mode 100644 index 0000000..3b67b2a --- /dev/null +++ b/src/ocf_freelist.h @@ -0,0 +1,34 @@ +/* + * Copyright(c) 2019-2019 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause-Clear + */ + +#ifndef __OCF_FREELIST_H__ +#define __OCF_FREELIST_H__ + +#include "ocf_cache_priv.h" + +struct ocf_freelist; + +typedef struct ocf_freelist *ocf_freelist_t; + +/* Init / deinit freelist runtime structures */ +ocf_freelist_t ocf_freelist_init(struct ocf_cache *cache); +void ocf_freelist_deinit(ocf_freelist_t freelist); + +/* Assign unused cachelines to freelist */ +void ocf_freelist_populate(ocf_freelist_t freelist, + ocf_cache_line_t num_free_clines); + +/* Get cacheline from freelist */ +bool ocf_freelist_get_cache_line(ocf_freelist_t freelist, + ocf_cache_line_t *cline); + +/* Put cacheline back to freelist */ +void ocf_freelist_put_cache_line(ocf_freelist_t freelist, + ocf_cache_line_t cline); + +/* Return total number of free cachelines */ +ocf_cache_line_t ocf_freelist_num_free(ocf_freelist_t freelist); + +#endif /* __OCF_FREELIST_H__ */ diff --git a/src/ocf_stats_builder.c b/src/ocf_stats_builder.c index f1d6948..91847b9 100644 --- a/src/ocf_stats_builder.c +++ b/src/ocf_stats_builder.c @@ -44,18 +44,6 @@ static uint64_t _bytes4k(uint64_t bytes) return (bytes + 4095UL) >> 12; } -static uint64_t _get_cache_occupancy(ocf_cache_t cache) -{ - uint64_t result = 0; - ocf_core_t core; - ocf_core_id_t core_id; - - for_each_core(cache, core, core_id) - result += env_atomic_read(&core->runtime_meta->cached_clines); - - return result; -} - static void _set(struct ocf_stat *stat, uint64_t value, uint64_t denominator) { stat->value = value; @@ -169,7 +157,7 @@ int ocf_stats_collect_core(ocf_core_t core, cache = ocf_core_get_cache(core); cache_line_size = ocf_cache_get_line_size(cache); cache_size = cache->conf_meta->cachelines; - cache_occupancy = _get_cache_occupancy(cache); + cache_occupancy = ocf_get_cache_occupancy(cache); _ocf_stats_zero(usage); _ocf_stats_zero(req); diff --git a/src/promotion/nhit/nhit.c b/src/promotion/nhit/nhit.c index 7648768..5c9b340 100644 --- a/src/promotion/nhit/nhit.c +++ b/src/promotion/nhit/nhit.c @@ -181,7 +181,7 @@ bool nhit_req_should_promote(ocf_promotion_policy_t policy, uint64_t core_line; uint64_t occupied_cachelines = ocf_metadata_get_cachelines_count(policy->owner) - - policy->owner->device->freelist_part->curr_size; + ocf_freelist_num_free(policy->owner->freelist); if (occupied_cachelines > env_atomic64_read(&ctx->trigger_threshold)) return true; diff --git a/tests/unit/framework/add_new_test_file.py b/tests/unit/framework/add_new_test_file.py index 92e8713..45ffb33 100755 --- a/tests/unit/framework/add_new_test_file.py +++ b/tests/unit/framework/add_new_test_file.py @@ -104,7 +104,7 @@ class TestGenerator(object): def get_empty_test_function(self): ret = "static void " + self.get_tested_function_name() + "_test01(void **state)\n" ret += "{\n" - ret += "\tprint_test_description(\"Put test description here\");\n" + ret += "\tprint_test_description(\"Put test description here\\n\");\n" ret += "\tassert_int_equal(1,1);\n" ret += "}\n\n" @@ -116,7 +116,7 @@ class TestGenerator(object): ret += "\tconst struct CMUnitTest tests[] = {\n" ret += "\t\tcmocka_unit_test(" + self.get_tested_function_name() + "_test01)\n" ret += "\t};\n\n" - ret += "\tprint_message(\"Unit test of " + self.get_tested_file_path() + "\");\n\n" + ret += "\tprint_message(\"Unit test for " + self.get_tested_function_name() + "\\n\");\n\n" ret += "\treturn cmocka_run_group_tests(tests, NULL, NULL);\n" ret += "}" diff --git a/tests/unit/framework/tests_config.py b/tests/unit/framework/tests_config.py index 59d299c..654b3cb 100644 --- a/tests/unit/framework/tests_config.py +++ b/tests/unit/framework/tests_config.py @@ -14,7 +14,7 @@ MAIN_DIRECTORY_OF_UNIT_TESTS = "../tests/" # Paths to all directories, in which tests are stored. All paths should be relative to # MAIN_DIRECTORY_OF_UNIT_TESTS DIRECTORIES_WITH_TESTS_LIST = ["cleaning/", "metadata/", "mngt/", "concurrency/", "engine/", - "eviction/", "utils/", "promotion/"] + "eviction/", "utils/", "promotion/", "ocf_freelist.c/"] # Paths to all directories containing files with sources. All paths should be relative to # MAIN_DIRECTORY_OF_TESTED_PROJECT diff --git a/tests/unit/tests/ocf_env/ocf_env.c b/tests/unit/tests/ocf_env/ocf_env.c index d6cbd96..7b0c417 100644 --- a/tests/unit/tests/ocf_env/ocf_env.c +++ b/tests/unit/tests/ocf_env/ocf_env.c @@ -150,28 +150,25 @@ void env_completion_complete(env_completion *completion) int env_mutex_init(env_mutex *mutex) { - function_called(); - check_expected_ptr(mutex); - return mock(); + return 0; +} + +int env_mutex_destroy(env_mutex *mutex) +{ + return 0; } void env_mutex_lock(env_mutex *mutex) { - function_called(); - check_expected_ptr(mutex); } int env_mutex_lock_interruptible(env_mutex *mutex) { - function_called(); - check_expected_ptr(mutex); - return mock(); + return 0; } void env_mutex_unlock(env_mutex *mutex) { - function_called(); - check_expected_ptr(mutex); } int env_rmutex_init(env_rmutex *rmutex) @@ -361,22 +358,27 @@ long env_atomic64_cmpxchg(env_atomic64 *a, long old, long new) return oldval; } -void env_spinlock_init(env_spinlock *l) +int env_spinlock_init(env_spinlock *l) { - function_called(); - check_expected_ptr(l); + return 0; +} + +int env_spinlock_destroy(env_spinlock *l) +{ + return 0; } void env_spinlock_lock(env_spinlock *l) { - function_called(); - check_expected_ptr(l); +} + +int env_spinlock_trylock(env_spinlock *l) +{ + return 0; } void env_spinlock_unlock(env_spinlock *l) { - function_called(); - check_expected_ptr(l); } void env_rwlock_init(env_rwlock *l) @@ -535,3 +537,7 @@ uint32_t env_crc32(uint32_t crc, uint8_t const *data, size_t len) check_expected_ptr(data); return mock(); } + +void env_cond_resched(void) +{ +} diff --git a/tests/unit/tests/ocf_env/ocf_env.h b/tests/unit/tests/ocf_env/ocf_env.h index ebec304..6fdc2e1 100644 --- a/tests/unit/tests/ocf_env/ocf_env.h +++ b/tests/unit/tests/ocf_env/ocf_env.h @@ -71,7 +71,13 @@ typedef uint64_t sector_t; abort(); \ }) -#define ENV_BUG_ON(cond) bug_on((int)cond); +#define ENV_BUG_ON(cond) ({ \ + int eval = cond; \ + if (eval) { \ + print_message("%s:%u BUG: %s\n", __FILE__, __LINE__, #cond); \ + bug_on(eval); \ + } \ + }) #define container_of(ptr, type, member) ({ \ const typeof( ((type *)0)->member ) *__mptr = (ptr); \ @@ -128,6 +134,8 @@ typedef struct { int env_mutex_init(env_mutex *mutex); +int env_mutex_destroy(env_mutex *mutex); + void env_mutex_lock(env_mutex *mutex); int env_mutex_lock_interruptible(env_mutex *mutex); @@ -231,10 +239,14 @@ void env_completion_complete(env_completion *completion); typedef struct { } env_spinlock; -void env_spinlock_init(env_spinlock *l); +int env_spinlock_init(env_spinlock *l); + +int env_spinlock_destroy(env_spinlock *l); void env_spinlock_lock(env_spinlock *l); +int env_spinlock_trylock(env_spinlock *l); + void env_spinlock_unlock(env_spinlock *l); #define env_spinlock_lock_irqsave(l, flags) \ @@ -327,4 +339,6 @@ void env_msleep(uint64_t n); uint32_t env_crc32(uint32_t crc, uint8_t const *data, size_t len); +void env_cond_resched(void); + #endif /* __OCF_ENV_H__ */ diff --git a/tests/unit/tests/ocf_freelist.c/ocf_freelist_get_put.c b/tests/unit/tests/ocf_freelist.c/ocf_freelist_get_put.c new file mode 100644 index 0000000..7ee9181 --- /dev/null +++ b/tests/unit/tests/ocf_freelist.c/ocf_freelist_get_put.c @@ -0,0 +1,382 @@ +/* + * src/ocf_freelist.c + * ocf_freelist_get_cache_line + * + * ocf_freelist_init + * ocf_freelist_deinit + * ocf_freelist_populate + * next_phys_invalid + * ocf_freelist_lock + * ocf_freelist_trylock + * ocf_freelist_unlock + * _ocf_freelist_remove_cache_line + * ocf_freelist_get_cache_line_fast + * ocf_freelist_get_cache_line_slow + * ocf_freelist_add_cache_line + * ocf_freelist_get_cache_line_ctx + * get_next_victim_freelist + * ocf_freelist_put_cache_line + * + */ + +#undef static + +#undef inline + + +#include +#include +#include +#include +#include "print_desc.h" + +#include "ocf/ocf.h" +#include "metadata/metadata.h" + +#include "ocf_freelist.c/ocf_freelist_get_put_generated_warps.c" + +ocf_cache_line_t __wrap_ocf_metadata_collision_table_entries(ocf_cache_t cache) +{ + return mock(); +} + +unsigned __wrap_env_get_execution_context_count(void) +{ + return mock(); +} + +unsigned __wrap_env_get_execution_context(void) +{ + return mock(); +} + +void __wrap_env_put_execution_context(unsigned ctx) +{ +} + +/* simulate no striping */ +ocf_cache_line_t __wrap_ocf_metadata_map_phy2lg(ocf_cache_t cache, ocf_cache_line_t phy) +{ + return phy; +} + +bool __wrap_metadata_test_valid_any(ocf_cache_t cache, ocf_cache_line_t cline) +{ + return mock(); +} + +/* metadata partition info interface mock: */ + +#define max_clines 100 + +struct { + ocf_cache_line_t prev; + ocf_cache_line_t next; +} partition_list[max_clines]; + + +void __wrap_ocf_metadata_set_partition_info(struct ocf_cache *cache, + ocf_cache_line_t line, ocf_part_id_t part_id, + ocf_cache_line_t next_line, ocf_cache_line_t prev_line) +{ + assert_int_equal(part_id, PARTITION_INVALID); + partition_list[line].prev = prev_line; + partition_list[line].next = next_line; +} + +void __wrap_ocf_metadata_get_partition_info(struct ocf_cache *cache, + ocf_cache_line_t line, ocf_part_id_t *part_id, + ocf_cache_line_t *next_line, ocf_cache_line_t *prev_line) +{ + if (part_id) + *part_id = PARTITION_INVALID; + if (prev_line) + *prev_line = partition_list[line].prev; + if (next_line) + *next_line = partition_list[line].next; +} + +void __wrap_ocf_metadata_set_partition_prev(struct ocf_cache *cache, + ocf_cache_line_t line, ocf_cache_line_t prev_line) +{ + partition_list[line].prev = prev_line; +} + +void __wrap_ocf_metadata_set_partition_next(struct ocf_cache *cache, + ocf_cache_line_t line, ocf_cache_line_t next_line) +{ + partition_list[line].next = next_line; +} + +static void ocf_freelist_get_cache_line_get_fast(void **state) +{ + unsigned num_cls = 8; + unsigned num_ctxts = 3; + ocf_freelist_t freelist; + unsigned ctx_iter, cl_iter; + ocf_cache_line_t line; + + print_test_description("Verify get free cache line get fast path"); + + will_return_maybe(__wrap_ocf_metadata_collision_table_entries, num_cls); + will_return_maybe(__wrap_env_get_execution_context_count, num_ctxts); + will_return_maybe(__wrap_metadata_test_valid_any, false); + + freelist = ocf_freelist_init(NULL); + + ocf_freelist_populate(freelist, num_cls); + + /* now there are following cachelines on per-context lists: + * ctx 0: 0, 1, 2 + * ctx 1: 3, 4, 5 + * ctx 2: 6, 7 + */ + + /* get cline from context 1 */ + will_return(__wrap_env_get_execution_context, 1); + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 3); + + /* ctx 0: 0, 1, 2 + * ctx 1: _, 4, 5 + * ctx 2: 6, 7 */ + + /* get cline from context 2 */ + will_return(__wrap_env_get_execution_context, 2); + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 6); + + /* ctx 0: 0, 1, 2 + * ctx 1: _, 4, 5 + * ctx 2: _, 7 */ + + /* get cline from context 1 */ + will_return(__wrap_env_get_execution_context, 1); + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 4); + + /* ctx 0: 0, 1, 2 + * ctx 1: _, _, 5 + * ctx 2: _, 7 */ + + /* get cline from context 0 */ + will_return(__wrap_env_get_execution_context, 0); + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 0); + + /* ctx 0: _, 1, 2 + * ctx 1: _, _, 5 + * ctx 2: _, 7 */ + + /* get cline from context 0 */ + will_return(__wrap_env_get_execution_context, 0); + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 1); + + /* ctx 0: _, _, 2 + * ctx 1: _, _, 5 + * ctx 2: _, 7 */ + + /* get cline from context 0 */ + will_return(__wrap_env_get_execution_context, 0); + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 2); + + /* ctx 0: _, _, _, + * ctx 1: _, _, 5 + * ctx 2: _, 7 */ + + /* get cline from context 2 */ + will_return(__wrap_env_get_execution_context, 2); + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 7); + + /* ctx 0: _, _, _, + * ctx 1: _, _, _5 + * ctx 2: _, _ */ + + /* get cline from context 1 */ + will_return(__wrap_env_get_execution_context, 1); + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 5); + + /* ctx 0: _, _, _, + * ctx 1: _, _, _ + * ctx 2: _, _ */ + + ocf_freelist_deinit(freelist); +} + +static void ocf_freelist_get_cache_line_get_slow(void **state) +{ + unsigned num_cls = 8; + unsigned num_ctxts = 3; + ocf_freelist_t freelist; + unsigned ctx_iter, cl_iter; + ocf_cache_line_t line; + + print_test_description("Verify get free cache line get slow path"); + + will_return_maybe(__wrap_ocf_metadata_collision_table_entries, num_cls); + will_return_maybe(__wrap_env_get_execution_context_count, num_ctxts); + will_return_maybe(__wrap_metadata_test_valid_any, false); + + /* always return exec ctx 0 */ + will_return_maybe(__wrap_env_get_execution_context, 0); + + freelist = ocf_freelist_init(NULL); + + ocf_freelist_populate(freelist, num_cls); + + /* now there are following cachelines on per-context lists: + * ctx 0: 0, 1, 2 + * ctx 1: 3, 4, 5 + * ctx 2: 6, 7 + */ + + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 0); + + /* ctx 0: _, 1, 2 + * ctx 1: 3, 4, 5 + * ctx 2: 6, 7 */ + + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 1); + + /* ctx 0: _, _, 2 + * ctx 1: 3, 4, 5 + * ctx 2: 6, 7 */ + + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 2); + + /* ctx 0: _, _, _ + * ctx 1: 3, 4, 5 + * ctx 2: 6, 7 */ + + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 3); + + /* ctx 0: _, _, _ + * ctx 1: _, 4, 5 + * ctx 2: 6, 7 */ + + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 6); + + /* ctx 0: _, _, _ + * ctx 1: _, 4, 5 + * ctx 2: _, 7 */ + + + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 4); + + /* ctx 0: _, _, _ + * ctx 1: _, _, 5 + * ctx 2: _, 7 */ + + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 7); + + /* ctx 0: _, _, _ + * ctx 1: _, _, 5 + * ctx 2: _, _ */ + + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 5); + + /* ctx 0: _, _, _, + * ctx 1: _, _, _ + * ctx 2: _, _ */ + + ocf_freelist_deinit(freelist); +} + +static void ocf_freelist_get_cache_line_put(void **state) +{ + unsigned num_cls = 8; + unsigned num_ctxts = 3; + ocf_freelist_t freelist; + unsigned ctx_iter, cl_iter; + ocf_cache_line_t line; + + print_test_description("Verify freelist cacheline put"); + + will_return_maybe(__wrap_ocf_metadata_collision_table_entries, num_cls); + will_return_maybe(__wrap_env_get_execution_context_count, num_ctxts); + will_return_maybe(__wrap_metadata_test_valid_any, false); + + freelist = ocf_freelist_init(NULL); + + ocf_freelist_populate(freelist, num_cls); + + /* get some clines from the freelists */ + will_return(__wrap_env_get_execution_context, 0); + ocf_freelist_get_cache_line(freelist, &line); + will_return(__wrap_env_get_execution_context, 0); + ocf_freelist_get_cache_line(freelist, &line); + will_return(__wrap_env_get_execution_context, 0); + ocf_freelist_get_cache_line(freelist, &line); + will_return(__wrap_env_get_execution_context, 0); + ocf_freelist_get_cache_line(freelist, &line); + will_return(__wrap_env_get_execution_context, 0); + ocf_freelist_get_cache_line(freelist, &line); + + /* ctx 0: + * ctx 1: 4, 5 + * ctx 2: 7 */ + + will_return(__wrap_env_get_execution_context, 1); + ocf_freelist_put_cache_line(freelist, 0); + + will_return(__wrap_env_get_execution_context, 1); + ocf_freelist_put_cache_line(freelist, 2); + + will_return(__wrap_env_get_execution_context, 2); + ocf_freelist_put_cache_line(freelist, 3); + + /* ctx 0: + * ctx 1: 4, 5, 0, 2 + * ctx 2: 7, 3*/ + + will_return(__wrap_env_get_execution_context, 1); + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 4); + + will_return(__wrap_env_get_execution_context, 1); + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 5); + + will_return(__wrap_env_get_execution_context, 1); + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 0); + + will_return(__wrap_env_get_execution_context, 1); + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 2); + + will_return(__wrap_env_get_execution_context, 2); + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 7); + + will_return(__wrap_env_get_execution_context, 2); + assert(ocf_freelist_get_cache_line(freelist, &line)); + assert_int_equal(line, 3); + + ocf_freelist_deinit(freelist); +} + +int main(void) +{ + const struct CMUnitTest tests[] = { + cmocka_unit_test(ocf_freelist_get_cache_line_get_fast), + cmocka_unit_test(ocf_freelist_get_cache_line_get_slow), + cmocka_unit_test(ocf_freelist_get_cache_line_put) + }; + + print_message("Unit test for ocf_freelist_get_cache_line\n"); + + return cmocka_run_group_tests(tests, NULL, NULL); +} diff --git a/tests/unit/tests/ocf_freelist.c/ocf_freelist_init.c b/tests/unit/tests/ocf_freelist.c/ocf_freelist_init.c new file mode 100644 index 0000000..c81cf83 --- /dev/null +++ b/tests/unit/tests/ocf_freelist.c/ocf_freelist_init.c @@ -0,0 +1,68 @@ +/* + * src/ocf_freelist.c + * ocf_freelist_populate + * + * ocf_freelist_init + * ocf_freelist_deinit + * + */ + +#undef static + +#undef inline + + +#include +#include +#include +#include +#include "print_desc.h" + +#include "ocf/ocf.h" +#include "metadata/metadata.h" + +#include "ocf_freelist.c/ocf_freelist_init_generated_warps.c" + +ocf_cache_line_t __wrap_ocf_metadata_collision_table_entries(ocf_cache_t cache) +{ + function_called(); + return mock(); +} + +ocf_cache_line_t __wrap_env_get_execution_context_count(ocf_cache_t cache) +{ + function_called(); + return mock(); +} + +static void ocf_freelist_init_test01(void **state) +{ + unsigned num_cls = 9; + unsigned num_ctxts = 3; + ocf_freelist_t freelist; + ocf_cache_t cache = 0x1234; + + print_test_description("Freelist initialization test"); + + expect_function_call(__wrap_ocf_metadata_collision_table_entries); + will_return(__wrap_ocf_metadata_collision_table_entries, num_cls); + + expect_function_call(__wrap_env_get_execution_context_count); + will_return(__wrap_env_get_execution_context_count, num_ctxts); + + freelist = ocf_freelist_init(cache); + assert(freelist != NULL); + + ocf_freelist_deinit(freelist); +} + +int main(void) +{ + const struct CMUnitTest tests[] = { + cmocka_unit_test(ocf_freelist_init_test01) + }; + + print_message("Unit test of ocf_freelist_init\n"); + + return cmocka_run_group_tests(tests, NULL, NULL); +} diff --git a/tests/unit/tests/ocf_freelist.c/ocf_freelist_locks.c b/tests/unit/tests/ocf_freelist.c/ocf_freelist_locks.c new file mode 100644 index 0000000..0a05094 --- /dev/null +++ b/tests/unit/tests/ocf_freelist.c/ocf_freelist_locks.c @@ -0,0 +1,213 @@ +/* + * src/ocf_freelist.c + * ocf_freelist_get_cache_line + * + * ocf_freelist_init + * ocf_freelist_deinit + * ocf_freelist_populate + * next_phys_invalid + * ocf_freelist_unlock + * _ocf_freelist_remove_cache_line + * ocf_freelist_get_cache_line_fast + * ocf_freelist_get_cache_line_slow + * ocf_freelist_add_cache_line + * ocf_freelist_get_cache_line_ctx + * get_next_victim_freelist + * ocf_freelist_put_cache_line + * + */ + +#undef static + +#undef inline + + +#include +#include +#include +#include +#include "print_desc.h" + +#include "ocf/ocf.h" +#include "metadata/metadata.h" + +#include "ocf_freelist.c/ocf_freelist_get_put_generated_warps.c" + +ocf_cache_line_t __wrap_ocf_metadata_collision_table_entries(ocf_cache_t cache) +{ + return mock(); +} + +unsigned __wrap_env_get_execution_context_count(void) +{ + return mock(); +} + +unsigned __wrap_env_get_execution_context(void) +{ + return mock(); +} + +void __wrap_env_put_execution_context(unsigned ctx) +{ +} + +/* simulate no striping */ +ocf_cache_line_t __wrap_ocf_metadata_map_phy2lg(ocf_cache_t cache, ocf_cache_line_t phy) +{ + return phy; +} + +bool __wrap_metadata_test_valid_any(ocf_cache_t cache, ocf_cache_line_t cline) +{ + return mock(); +} + +void __wrap_ocf_freelist_lock(ocf_freelist_t freelist, uint32_t ctx) +{ + function_called(); + check_expected(ctx); +} + +int __wrap_ocf_freelist_trylock(ocf_freelist_t freelist, uint32_t ctx) +{ + function_called(); + check_expected(ctx); + return mock(); +} + +/* metadata partition info interface mock: */ + +#define max_clines 100 + +struct { + ocf_cache_line_t prev; + ocf_cache_line_t next; +} partition_list[max_clines]; + + +void __wrap_ocf_metadata_set_partition_info(struct ocf_cache *cache, + ocf_cache_line_t line, ocf_part_id_t part_id, + ocf_cache_line_t next_line, ocf_cache_line_t prev_line) +{ + assert_int_equal(part_id, PARTITION_INVALID); + partition_list[line].prev = prev_line; + partition_list[line].next = next_line; +} + +void __wrap_ocf_metadata_get_partition_info(struct ocf_cache *cache, + ocf_cache_line_t line, ocf_part_id_t *part_id, + ocf_cache_line_t *next_line, ocf_cache_line_t *prev_line) +{ + if (part_id) + *part_id = PARTITION_INVALID; + if (prev_line) + *prev_line = partition_list[line].prev; + if (next_line) + *next_line = partition_list[line].next; +} + +void __wrap_ocf_metadata_set_partition_prev(struct ocf_cache *cache, + ocf_cache_line_t line, ocf_cache_line_t prev_line) +{ + partition_list[line].prev = prev_line; +} + +void __wrap_ocf_metadata_set_partition_next(struct ocf_cache *cache, + ocf_cache_line_t line, ocf_cache_line_t next_line) +{ + partition_list[line].next = next_line; +} + +static void ocf_freelist_get_put_locks(void **state) +{ + unsigned num_cls = 4; + unsigned num_ctxts = 3; + ocf_freelist_t freelist; + unsigned ctx_iter, cl_iter; + ocf_cache_line_t line; + + print_test_description("Verify lock/trylock sequence in get free cacheline"); + + will_return_maybe(__wrap_ocf_metadata_collision_table_entries, num_cls); + will_return_maybe(__wrap_env_get_execution_context_count, num_ctxts); + will_return_maybe(__wrap_metadata_test_valid_any, false); + + /* simulate context 1 for the entire test duration */ + will_return_maybe(__wrap_env_get_execution_context, 1); + + freelist = ocf_freelist_init(NULL); + + ocf_freelist_populate(freelist, num_cls); + + /****************************************************************/ + /* verify fast path locking - scucessfull trylock */ + + /* ctx 0: 0, 3 + * ctx 1: 1 + * ctx 2: 2 + * slowpath next victim: 0 + */ + + expect_value(__wrap_ocf_freelist_trylock, ctx, 1); + expect_function_call(__wrap_ocf_freelist_trylock); + will_return(__wrap_ocf_freelist_trylock, 0); + ocf_freelist_get_cache_line(freelist, &line); + + /****************************************************************/ + /* verify fast path locking - scucessfull trylock in slowpath */ + + /* ctx 0: 0, 3 + * ctx 1: + * ctx 2: 2 + * slowpath next victim: 0 */ + + /* we expect trylock for context 0, since context 1 has empty list */ + expect_value(__wrap_ocf_freelist_trylock, ctx, 0); + expect_function_call(__wrap_ocf_freelist_trylock); + will_return(__wrap_ocf_freelist_trylock, 0); + ocf_freelist_get_cache_line(freelist, &line); + + /****************************************************************/ + /* verify fast path locking - trylock failure in slowpath */ + + /* ctx 0: 3 + * ctx 1: + * ctx 2: 2 + * slowpath next victim: 1 */ + + /* fastpath will fail immediately - context 1 list is empty */ + /* next slowpath victim context (1) is empty - will move to ctx 2 */ + /* so now we expect trylock for context no 2 - injecting error here*/ + expect_value(__wrap_ocf_freelist_trylock, ctx, 2); + expect_function_call(__wrap_ocf_freelist_trylock); + will_return(__wrap_ocf_freelist_trylock, 1); + + /* slowpath will attempt to trylock next non-empty context - 0 + * - injecting error here as well */ + expect_value(__wrap_ocf_freelist_trylock, ctx, 0); + expect_function_call(__wrap_ocf_freelist_trylock); + will_return(__wrap_ocf_freelist_trylock, 1); + + /* slowpath trylock loop failed - expecting full lock */ + expect_value(__wrap_ocf_freelist_lock, ctx, 2); + expect_function_call(__wrap_ocf_freelist_lock); + + /* execute freelist_get_cache_line */ + ocf_freelist_get_cache_line(freelist, &line); + + /****************************************************************/ + + ocf_freelist_deinit(freelist); +} + +int main(void) +{ + const struct CMUnitTest tests[] = { + cmocka_unit_test(ocf_freelist_get_put_locks) + }; + + print_message("Unit test for ocf_freelist_get_cache_line locking\n"); + + return cmocka_run_group_tests(tests, NULL, NULL); +} diff --git a/tests/unit/tests/ocf_freelist.c/ocf_freelist_populate.c b/tests/unit/tests/ocf_freelist.c/ocf_freelist_populate.c new file mode 100644 index 0000000..419f240 --- /dev/null +++ b/tests/unit/tests/ocf_freelist.c/ocf_freelist_populate.c @@ -0,0 +1,138 @@ +/* + * src/ocf_freelist.c + * ocf_freelist_populate + * + * ocf_freelist_init + * ocf_freelist_deinit + * ocf_freelist_populate + * next_phys_invalid + * + */ + +#undef static + +#undef inline + + +#include +#include +#include +#include +#include "print_desc.h" + +#include "ocf/ocf.h" +#include "metadata/metadata.h" + +#include "ocf_freelist.c/ocf_freelist_populate_generated_warps.c" + +ocf_cache_line_t __wrap_ocf_metadata_collision_table_entries(ocf_cache_t cache) +{ + return mock(); +} + +ocf_cache_line_t __wrap_env_get_execution_context_count(ocf_cache_t cache) +{ + return mock(); +} + +/* simulate no striping */ +ocf_cache_line_t __wrap_ocf_metadata_map_phy2lg(ocf_cache_t cache, ocf_cache_line_t phy) +{ + return phy; +} + +bool __wrap_metadata_test_valid_any(ocf_cache_t cache, ocf_cache_line_t cline) +{ + return mock(); +} + +void __wrap_ocf_metadata_set_partition_info(struct ocf_cache *cache, + ocf_cache_line_t line, ocf_part_id_t part_id, + ocf_cache_line_t next_line, ocf_cache_line_t prev_line) +{ + print_message("%s %u %u %u\n", __func__, prev_line, line, next_line); + check_expected(line); + check_expected(part_id); + check_expected(next_line); + check_expected(prev_line); +} + +#define expect_set_info(curr, part, next, prev) \ + expect_value(__wrap_ocf_metadata_set_partition_info, line, curr); \ + expect_value(__wrap_ocf_metadata_set_partition_info, part_id, part); \ + expect_value(__wrap_ocf_metadata_set_partition_info, next_line, next); \ + expect_value(__wrap_ocf_metadata_set_partition_info, prev_line, prev); + +static void ocf_freelist_populate_test01(void **state) +{ + unsigned num_cls = 8; + unsigned num_ctxts = 3; + ocf_freelist_t freelist; + unsigned ctx_iter, cl_iter; + + print_test_description("Verify proper set_partition_info order and arguments - empty cache"); + + will_return_maybe(__wrap_ocf_metadata_collision_table_entries, num_cls); + will_return_maybe(__wrap_env_get_execution_context_count, num_ctxts); + will_return_maybe(__wrap_metadata_test_valid_any, false); + + freelist = ocf_freelist_init(NULL); + + expect_set_info(0, PARTITION_INVALID, 1 , num_cls); + expect_set_info(1, PARTITION_INVALID, 2 , 0); + expect_set_info(2, PARTITION_INVALID, num_cls, 1); + expect_set_info(3, PARTITION_INVALID, 4 , num_cls); + expect_set_info(4, PARTITION_INVALID, 5 , 3); + expect_set_info(5, PARTITION_INVALID, num_cls, 4); + expect_set_info(6, PARTITION_INVALID, 7 , num_cls); + expect_set_info(7, PARTITION_INVALID, num_cls, 6); + + ocf_freelist_populate(freelist, num_cls); + + ocf_freelist_deinit(freelist); +} + +static void ocf_freelist_populate_test02(void **state) +{ + unsigned num_cls = 8; + unsigned num_ctxts = 3; + ocf_freelist_t freelist; + unsigned ctx_iter, cl_iter; + + print_test_description("Verify proper set_partition_info order and arguments - some valid clines"); + + will_return_maybe(__wrap_ocf_metadata_collision_table_entries, num_cls); + will_return_maybe(__wrap_env_get_execution_context_count, num_ctxts); + + freelist = ocf_freelist_init(NULL); + + /* simulate only cachelines 2, 3, 4, 7 invalid */ + will_return(__wrap_metadata_test_valid_any, true); + will_return(__wrap_metadata_test_valid_any, true); + will_return(__wrap_metadata_test_valid_any, false); + will_return(__wrap_metadata_test_valid_any, false); + will_return(__wrap_metadata_test_valid_any, false); + will_return(__wrap_metadata_test_valid_any, true); + will_return(__wrap_metadata_test_valid_any, true); + will_return(__wrap_metadata_test_valid_any, false); + + expect_set_info(2, PARTITION_INVALID, 3 , num_cls); + expect_set_info(3, PARTITION_INVALID, num_cls, 2); + expect_set_info(4, PARTITION_INVALID, num_cls, num_cls); + expect_set_info(7, PARTITION_INVALID, num_cls, num_cls); + + ocf_freelist_populate(freelist, 4); + + ocf_freelist_deinit(freelist); +} +int main(void) +{ + const struct CMUnitTest tests[] = { + cmocka_unit_test(ocf_freelist_populate_test01), + cmocka_unit_test(ocf_freelist_populate_test02) + }; + + print_message("Unit test of src/ocf_freelist.c\n"); + + return cmocka_run_group_tests(tests, NULL, NULL); +}