From 5684b53d9b1874f71c709ddee8fb0cfc96eea799 Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Mon, 23 Sep 2019 11:08:27 -0400 Subject: [PATCH] Adding collision table locks Signed-off-by: Adam Rutkowski --- src/concurrency/ocf_cache_line_concurrency.c | 6 +- src/concurrency/ocf_metadata_concurrency.c | 74 +++++++++++++++-- src/concurrency/ocf_metadata_concurrency.h | 11 ++- src/metadata/metadata_collision.h | 12 +++ src/metadata/metadata_hash.c | 86 +++++++++++++++----- src/metadata/metadata_raw.c | 14 ++++ src/metadata/metadata_raw.h | 14 ++++ src/metadata/metadata_raw_dynamic.c | 10 +++ src/metadata/metadata_raw_dynamic.h | 5 ++ src/metadata/metadata_structs.h | 8 ++ src/mngt/ocf_mngt_cache.c | 3 +- 11 files changed, 210 insertions(+), 33 deletions(-) diff --git a/src/concurrency/ocf_cache_line_concurrency.c b/src/concurrency/ocf_cache_line_concurrency.c index 5b50ee6..c0cb54c 100644 --- a/src/concurrency/ocf_cache_line_concurrency.c +++ b/src/concurrency/ocf_cache_line_concurrency.c @@ -71,6 +71,8 @@ int ocf_cache_line_concurrency_init(struct ocf_cache *cache) int error = 0; struct ocf_cache_line_concurrency *c; char name[ALLOCATOR_NAME_MAX]; + ocf_cache_line_t line_entries = ocf_metadata_collision_table_entries( + cache); ENV_BUG_ON(cache->device->concurrency.cache_line); @@ -85,8 +87,8 @@ int ocf_cache_line_concurrency_init(struct ocf_cache *cache) cache->device->concurrency.cache_line = c; OCF_REALLOC_INIT(&c->access, &c->access_limit); - OCF_REALLOC_CP(&c->access, sizeof(c->access[0]), - cache->device->collision_table_entries, &c->access_limit); + OCF_REALLOC_CP(&c->access, sizeof(c->access[0]), line_entries, + &c->access_limit); if (!c->access) { error = __LINE__; diff --git a/src/concurrency/ocf_metadata_concurrency.c b/src/concurrency/ocf_metadata_concurrency.c index 2d0a551..058271b 100644 --- a/src/concurrency/ocf_metadata_concurrency.c +++ b/src/concurrency/ocf_metadata_concurrency.c @@ -22,19 +22,44 @@ void ocf_metadata_concurrency_deinit(struct ocf_metadata_lock *metadata_lock) int ocf_metadata_concurrency_attached_init( struct ocf_metadata_lock *metadata_lock, ocf_cache_t cache, - uint64_t hash_table_entries) + uint64_t hash_table_entries, uint32_t colision_table_pages) { uint64_t i; + int err = 0; - metadata_lock->cache = cache; - metadata_lock->num_hash_entries = hash_table_entries; metadata_lock->hash = env_vzalloc(sizeof(env_rwsem) * hash_table_entries); - if (!metadata_lock->hash) + metadata_lock->collision_pages = env_vzalloc(sizeof(env_rwsem) * + colision_table_pages); + if (!metadata_lock->hash || + !metadata_lock->collision_pages) { + env_vfree(metadata_lock->hash); + env_vfree(metadata_lock->collision_pages); + metadata_lock->hash = NULL; + metadata_lock->collision_pages = NULL; return -OCF_ERR_NO_MEM; + } for (i = 0; i < hash_table_entries; i++) env_rwsem_init(&metadata_lock->hash[i]); + for (i = 0; i < colision_table_pages; i++) { + err = env_rwsem_init(&metadata_lock->collision_pages[i]); + if (err) + break; + } + + if (err) { + while (i--) + env_rwsem_destroy(&metadata_lock->collision_pages[i]); + env_vfree(metadata_lock->collision_pages); + metadata_lock->collision_pages = NULL; + ocf_metadata_concurrency_attached_deinit(metadata_lock); + return err; + } + + metadata_lock->cache = cache; + metadata_lock->num_hash_entries = hash_table_entries; + metadata_lock->num_collision_pages = colision_table_pages; return 0; } @@ -44,10 +69,21 @@ void ocf_metadata_concurrency_attached_deinit( { uint64_t i; - for (i = 0; i < metadata_lock->num_hash_entries; i++) - env_rwsem_destroy(&metadata_lock->hash[i]); + if (metadata_lock->hash) { + for (i = 0; i < metadata_lock->num_hash_entries; i++) + env_rwsem_destroy(&metadata_lock->hash[i]); + env_vfree(metadata_lock->hash); + metadata_lock->hash = NULL; + metadata_lock->num_hash_entries = 0; + } - env_vfree(metadata_lock->hash); + if (metadata_lock->collision_pages) { + for (i = 0; i < metadata_lock->num_collision_pages; i++) + env_rwsem_destroy(&metadata_lock->collision_pages[i]); + env_vfree(metadata_lock->collision_pages); + metadata_lock->collision_pages = NULL; + metadata_lock->num_collision_pages = 0; + } } void ocf_metadata_start_exclusive_access( @@ -266,3 +302,27 @@ void ocf_req_hash_unlock_wr(struct ocf_request *req) } ocf_metadata_end_shared_access(&req->cache->metadata.lock); } + +void ocf_collision_start_shared_access(struct ocf_metadata_lock *metadata_lock, + uint32_t page) +{ + env_rwsem_down_read(&metadata_lock->collision_pages[page]); +} + +void ocf_collision_end_shared_access(struct ocf_metadata_lock *metadata_lock, + uint32_t page) +{ + env_rwsem_up_read(&metadata_lock->collision_pages[page]); +} + +void ocf_collision_start_exclusive_access(struct ocf_metadata_lock *metadata_lock, + uint32_t page) +{ + env_rwsem_down_write(&metadata_lock->collision_pages[page]); +} + +void ocf_collision_end_exclusive_access(struct ocf_metadata_lock *metadata_lock, + uint32_t page) +{ + env_rwsem_up_write(&metadata_lock->collision_pages[page]); +} diff --git a/src/concurrency/ocf_metadata_concurrency.h b/src/concurrency/ocf_metadata_concurrency.h index 0875237..ffef123 100644 --- a/src/concurrency/ocf_metadata_concurrency.h +++ b/src/concurrency/ocf_metadata_concurrency.h @@ -16,7 +16,7 @@ void ocf_metadata_concurrency_deinit(struct ocf_metadata_lock *metadata_lock); int ocf_metadata_concurrency_attached_init( struct ocf_metadata_lock *metadata_lock, ocf_cache_t cache, - uint64_t hash_table_entries); + uint64_t hash_table_entries, uint32_t colision_table_pages); void ocf_metadata_concurrency_attached_deinit( struct ocf_metadata_lock *metadata_lock); @@ -111,4 +111,13 @@ void ocf_req_hash_lock_wr(struct ocf_request *req); void ocf_req_hash_unlock_wr(struct ocf_request *req); void ocf_req_hash_lock_upgrade(struct ocf_request *req); +/* collision table page lock interface */ +void ocf_collision_start_shared_access(struct ocf_metadata_lock *metadata_lock, + uint32_t page); +void ocf_collision_end_shared_access(struct ocf_metadata_lock *metadata_lock, + uint32_t page); +void ocf_collision_start_exclusive_access(struct ocf_metadata_lock *metadata_lock, + uint32_t page); +void ocf_collision_end_exclusive_access(struct ocf_metadata_lock *metadata_lock, + uint32_t page); #endif diff --git a/src/metadata/metadata_collision.h b/src/metadata/metadata_collision.h index 05e6664..33459b9 100644 --- a/src/metadata/metadata_collision.h +++ b/src/metadata/metadata_collision.h @@ -105,4 +105,16 @@ void ocf_metadata_add_to_collision(struct ocf_cache *cache, void ocf_metadata_remove_from_collision(struct ocf_cache *cache, ocf_cache_line_t line, ocf_part_id_t part_id); +static inline void ocf_metadata_start_collision_shared_access( + struct ocf_cache *cache, ocf_cache_line_t line) +{ + cache->metadata.iface.start_collision_shared_access(cache, line); +} + +static inline void ocf_metadata_end_collision_shared_access( + struct ocf_cache *cache, ocf_cache_line_t line) +{ + cache->metadata.iface.end_collision_shared_access(cache, line); +} + #endif /* METADATA_COLLISION_H_ */ diff --git a/src/metadata/metadata_hash.c b/src/metadata/metadata_hash.c index 8bb59f3..b1c1737 100644 --- a/src/metadata/metadata_hash.c +++ b/src/metadata/metadata_hash.c @@ -414,6 +414,8 @@ static void ocf_metadata_hash_deinit_variable_size(struct ocf_cache *cache) OCF_DEBUG_TRACE(cache); + ocf_metadata_concurrency_attached_deinit(&cache->metadata.lock); + /* * De initialize RAW types */ @@ -982,29 +984,30 @@ finalize: * Hash De-Init also contains RAW deinitialization */ ocf_metadata_hash_deinit_variable_size(cache); - } else { - cache->device->runtime_meta = METADATA_MEM_POOL(ctrl, - metadata_segment_sb_runtime); - - cache->device->collision_table_entries = ctrl->cachelines; - - cache->device->hash_table_entries = - ctrl->raw_desc[metadata_segment_hash].entries; - - cache->device->metadata_offset = ctrl->count_pages * PAGE_SIZE; - - cache->conf_meta->cachelines = ctrl->cachelines; - cache->conf_meta->line_size = cache_line_size; - - ocf_metadata_hash_raw_info(cache, ctrl); - - ocf_cache_log(cache, log_info, "Cache line size: %llu kiB\n", - settings->size / KiB); - - ocf_cache_log(cache, log_info, "Metadata capacity: %llu MiB\n", - (uint64_t)ocf_metadata_size_of(cache) / MiB); + return result; } + cache->device->runtime_meta = METADATA_MEM_POOL(ctrl, + metadata_segment_sb_runtime); + + cache->device->collision_table_entries = ctrl->cachelines; + + cache->device->hash_table_entries = + ctrl->raw_desc[metadata_segment_hash].entries; + + cache->device->metadata_offset = ctrl->count_pages * PAGE_SIZE; + + cache->conf_meta->cachelines = ctrl->cachelines; + cache->conf_meta->line_size = cache_line_size; + + ocf_metadata_hash_raw_info(cache, ctrl); + + ocf_cache_log(cache, log_info, "Cache line size: %llu kiB\n", + settings->size / KiB); + + ocf_cache_log(cache, log_info, "Metadata capacity: %llu MiB\n", + (uint64_t)ocf_metadata_size_of(cache) / MiB); + /* * Self test of metadata */ @@ -1029,7 +1032,18 @@ finalize: "OCF metadata self-test ERROR\n"); } - return result; + result = ocf_metadata_concurrency_attached_init(&cache->metadata.lock, + cache, ctrl->raw_desc[metadata_segment_hash].entries, + (uint32_t)ctrl->raw_desc[metadata_segment_collision]. + ssd_pages); + if (result) { + ocf_cache_log(cache, log_err, "Failed to initialize attached " + "metadata concurrency\n"); + ocf_metadata_hash_deinit_variable_size(cache); + return result; + } + + return 0; } static inline void _ocf_init_collision_entry(struct ocf_cache *cache, @@ -2533,6 +2547,30 @@ static void ocf_metadata_hash_get_collision_info( } } +void ocf_metadata_hash_start_collision_shared_access(struct ocf_cache *cache, + ocf_cache_line_t line) +{ + struct ocf_metadata_hash_ctrl *ctrl = + (struct ocf_metadata_hash_ctrl *) cache->metadata.iface_priv; + struct ocf_metadata_raw *raw = + &ctrl->raw_desc[metadata_segment_collision]; + uint32_t page = ocf_metadata_raw_page(raw, line); + + ocf_collision_start_shared_access(&cache->metadata.lock, page); +} + +void ocf_metadata_hash_end_collision_shared_access(struct ocf_cache *cache, + ocf_cache_line_t line) +{ + struct ocf_metadata_hash_ctrl *ctrl = + (struct ocf_metadata_hash_ctrl *) cache->metadata.iface_priv; + struct ocf_metadata_raw *raw = + &ctrl->raw_desc[metadata_segment_collision]; + uint32_t page = ocf_metadata_raw_page(raw, line); + + ocf_collision_start_shared_access(&cache->metadata.lock, page); +} + /******************************************************************************* * Partition ******************************************************************************/ @@ -2682,6 +2720,10 @@ static const struct ocf_metadata_iface metadata_hash_iface = { .set_collision_info = ocf_metadata_hash_set_collision_info, .set_collision_next = ocf_metadata_hash_set_collision_next, .set_collision_prev = ocf_metadata_hash_set_collision_prev, + .start_collision_shared_access = + ocf_metadata_hash_start_collision_shared_access, + .end_collision_shared_access = + ocf_metadata_hash_end_collision_shared_access, /* * Partition Info diff --git a/src/metadata/metadata_raw.c b/src/metadata/metadata_raw.c index 7c0e3ca..633e64b 100644 --- a/src/metadata/metadata_raw.c +++ b/src/metadata/metadata_raw.c @@ -149,6 +149,16 @@ static uint32_t _raw_ram_checksum(ocf_cache_t cache, return crc; } +/* + * RAM Implementation - Entry page number + */ +uint32_t _raw_ram_page(struct ocf_metadata_raw *raw, uint32_t entry) +{ + ENV_BUG_ON(entry >= raw->entries); + + return _RAW_RAM_PAGE(raw, entry); +} + /* * RAM Implementation - Get entry */ @@ -541,6 +551,7 @@ static const struct raw_iface IRAW[metadata_raw_type_max] = { .size_of = _raw_ram_size_of, .size_on_ssd = _raw_ram_size_on_ssd, .checksum = _raw_ram_checksum, + .page = _raw_ram_page, .get = _raw_ram_get, .set = _raw_ram_set, .access = _raw_ram_access, @@ -555,6 +566,7 @@ static const struct raw_iface IRAW[metadata_raw_type_max] = { .size_of = raw_dynamic_size_of, .size_on_ssd = raw_dynamic_size_on_ssd, .checksum = raw_dynamic_checksum, + .page = raw_dynamic_page, .get = raw_dynamic_get, .set = raw_dynamic_set, .access = raw_dynamic_access, @@ -569,6 +581,7 @@ static const struct raw_iface IRAW[metadata_raw_type_max] = { .size_of = _raw_ram_size_of, .size_on_ssd = raw_volatile_size_on_ssd, .checksum = raw_volatile_checksum, + .page = _raw_ram_page, .get = _raw_ram_get, .set = _raw_ram_set, .access = _raw_ram_access, @@ -583,6 +596,7 @@ static const struct raw_iface IRAW[metadata_raw_type_max] = { .size_of = _raw_ram_size_of, .size_on_ssd = _raw_ram_size_on_ssd, .checksum = _raw_ram_checksum, + .page = _raw_ram_page, .get = _raw_ram_get, .set = _raw_ram_set, .access = _raw_ram_access, diff --git a/src/metadata/metadata_raw.h b/src/metadata/metadata_raw.h index 2da5137..66b10ad 100644 --- a/src/metadata/metadata_raw.h +++ b/src/metadata/metadata_raw.h @@ -102,6 +102,7 @@ struct raw_iface { uint32_t (*checksum)(ocf_cache_t cache, struct ocf_metadata_raw *raw); + uint32_t (*page)(struct ocf_metadata_raw *raw, uint32_t entry); int (*get)(ocf_cache_t cache, struct ocf_metadata_raw *raw, uint32_t entry, void *data); @@ -183,6 +184,19 @@ static inline uint32_t ocf_metadata_raw_checksum(struct ocf_cache* cache, return raw->iface->checksum(cache, raw); } +/** + * @brief Calculate entry page index + * + * @param raw - RAW descriptor + * @param entry - Entry number + * @return Page index + */ +static inline uint32_t ocf_metadata_raw_page(struct ocf_metadata_raw* raw, + uint32_t entry) +{ + return raw->iface->page(raw, entry); +} + /** * @brief Get specified element of metadata * diff --git a/src/metadata/metadata_raw_dynamic.c b/src/metadata/metadata_raw_dynamic.c index e45ade1..9ef666b 100644 --- a/src/metadata/metadata_raw_dynamic.c +++ b/src/metadata/metadata_raw_dynamic.c @@ -219,6 +219,16 @@ uint32_t raw_dynamic_checksum(ocf_cache_t cache, return crc; } +/* + * RAM DYNAMIC Implementation - Entry page number + */ +uint32_t raw_dynamic_page(struct ocf_metadata_raw *raw, uint32_t entry) +{ + ENV_BUG_ON(entry >= raw->entries); + + return _RAW_DYNAMIC_PAGE(raw, entry); +} + /* * RAM DYNAMIC Implementation - Get */ diff --git a/src/metadata/metadata_raw_dynamic.h b/src/metadata/metadata_raw_dynamic.h index 94c0657..b8f541b 100644 --- a/src/metadata/metadata_raw_dynamic.h +++ b/src/metadata/metadata_raw_dynamic.h @@ -40,6 +40,11 @@ uint32_t raw_dynamic_size_on_ssd(struct ocf_metadata_raw *raw); uint32_t raw_dynamic_checksum(ocf_cache_t cache, struct ocf_metadata_raw *raw); +/* + * RAM DYNAMIC Implementation - Entry page number + */ +uint32_t raw_dynamic_page(struct ocf_metadata_raw *raw, uint32_t entry); + /* * RAW DYNAMIC - Get specified entry */ diff --git a/src/metadata/metadata_structs.h b/src/metadata/metadata_structs.h index 22b3703..9d6e681 100644 --- a/src/metadata/metadata_structs.h +++ b/src/metadata/metadata_structs.h @@ -360,6 +360,12 @@ struct ocf_metadata_iface { void (*set_collision_prev)(struct ocf_cache *cache, ocf_cache_line_t line, ocf_cache_line_t prev); + void (*start_collision_shared_access)(struct ocf_cache *cache, + ocf_cache_line_t line); + + void (*end_collision_shared_access)(struct ocf_cache *cache, + ocf_cache_line_t line); + void (*get_partition_info)(struct ocf_cache *cache, ocf_cache_line_t line, ocf_part_id_t *part_id, ocf_cache_line_t *next_line, @@ -434,7 +440,9 @@ struct ocf_metadata_lock env_rwlock status; /*!< Fast lock for status bits */ env_spinlock eviction; /*!< Fast lock for eviction policy */ env_rwsem *hash; /*!< Hash bucket locks */ + env_rwsem *collision_pages; /*!< Collision table page locks */ uint32_t num_hash_entries; /*!< Hash bucket count */ + uint32_t num_collision_pages; /*!< Collision table page count */ ocf_cache_t cache; /*!< Parent cache object */ }; diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 84e2084..9e508b6 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -980,7 +980,8 @@ static void _ocf_mngt_attach_prepare_metadata(ocf_pipeline_t pipeline, context->flags.attached_metadata_inited = true; if (ocf_metadata_concurrency_attached_init(&cache->metadata.lock, - cache, cache->device->hash_table_entries)) { + cache, cache->device->hash_table_entries, + ocf_metadata_get_num_collision_pages(cache))) { ocf_cache_log(cache, log_err, "Failed to initialize attached " "metadata concurrency\n"); OCF_PL_FINISH_RET(context->pipeline, -OCF_ERR_START_CACHE_FAIL);