From d91012f4b47581e18cd4b0fd3661234254c86ad6 Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Thu, 1 Aug 2019 16:57:14 -0400 Subject: [PATCH] Introduce hash bucket locks There is one RW lock per hash bucket. Write lock is required to map cacheline, read lock is sufficient for traversing. Hash bucket locks are always acquired under global metadata read lock. This assures mutual exclusion with eviction and management paths, where global metadata write lock is held. Signed-off-by: Adam Rutkowski --- src/concurrency/ocf_metadata_concurrency.c | 216 ++++++++++++++++++++- src/concurrency/ocf_metadata_concurrency.h | 128 ++++++------ src/metadata/metadata.c | 4 +- src/metadata/metadata_structs.h | 15 +- src/mngt/ocf_mngt_cache.c | 4 +- src/ocf_request.c | 11 ++ src/ocf_request.h | 7 + 7 files changed, 298 insertions(+), 87 deletions(-) diff --git a/src/concurrency/ocf_metadata_concurrency.c b/src/concurrency/ocf_metadata_concurrency.c index 5304780..7952ac2 100644 --- a/src/concurrency/ocf_metadata_concurrency.c +++ b/src/concurrency/ocf_metadata_concurrency.c @@ -5,21 +5,219 @@ #include "ocf_metadata_concurrency.h" -void ocf_metadata_concurrency_init(struct ocf_cache *cache) +void ocf_metadata_concurrency_init(struct ocf_metadata_lock *metadata_lock) { - env_spinlock_init(&cache->metadata.lock.eviction); - env_rwlock_init(&cache->metadata.lock.status); - env_rwsem_init(&cache->metadata.lock.collision); + env_spinlock_init(&metadata_lock->eviction); + env_rwlock_init(&metadata_lock->status); + env_rwsem_init(&metadata_lock->global); } -void ocf_metadata_concurrency_deinit(struct ocf_cache *cache) +void ocf_metadata_concurrency_deinit(struct ocf_metadata_lock *metadata_lock) { - env_spinlock_destroy(&cache->metadata.lock.eviction); - env_rwlock_destroy(&cache->metadata.lock.status); - env_rwsem_destroy(&cache->metadata.lock.collision); + env_spinlock_destroy(&metadata_lock->eviction); + env_rwlock_destroy(&metadata_lock->status); + env_rwsem_destroy(&metadata_lock->global); } -int ocf_metadata_concurrency_attached_init(struct ocf_cache *cache) +int ocf_metadata_concurrency_attached_init( + struct ocf_metadata_lock *metadata_lock, + uint64_t hash_table_entries) { + uint64_t i; + + metadata_lock->num_hash_entries = hash_table_entries; + metadata_lock->hash = env_vzalloc(sizeof(env_rwsem) * + hash_table_entries); + if (!metadata_lock->hash) + return -OCF_ERR_NO_MEM; + + for (i = 0; i < hash_table_entries; i++) + env_rwsem_init(&metadata_lock->hash[i]); + return 0; } + +void ocf_metadata_concurrency_attached_deinit( + struct ocf_metadata_lock *metadata_lock) +{ + uint64_t i; + + for (i = 0; i < metadata_lock->num_hash_entries; i++) + env_rwsem_destroy(&metadata_lock->hash[i]); + + env_vfree(metadata_lock->hash); +} + +void ocf_metadata_start_exclusive_access( + struct ocf_metadata_lock *metadata_lock) +{ + env_rwsem_down_write(&metadata_lock->global); +} + +int ocf_metadata_try_start_exclusive_access( + struct ocf_metadata_lock *metadata_lock) +{ + return env_rwsem_down_write_trylock(&metadata_lock->global); +} + +void ocf_metadata_end_exclusive_access( + struct ocf_metadata_lock *metadata_lock) +{ + env_rwsem_up_write(&metadata_lock->global); +} + +void ocf_metadata_start_shared_access( + struct ocf_metadata_lock *metadata_lock) +{ + env_rwsem_down_read(&metadata_lock->global); +} + +int ocf_metadata_try_start_shared_access( + struct ocf_metadata_lock *metadata_lock) +{ + return env_rwsem_down_read_trylock(&metadata_lock->global); +} + +void ocf_metadata_end_shared_access(struct ocf_metadata_lock *metadata_lock) +{ + env_rwsem_up_read(&metadata_lock->global); +} + +void ocf_metadata_hash_lock(struct ocf_metadata_lock *metadata_lock, + ocf_cache_line_t hash, int rw) +{ + ENV_BUG_ON(hash >= metadata_lock->num_hash_entries); + + if (rw == OCF_METADATA_WR) + env_rwsem_down_write(&metadata_lock->hash[hash]); + else if (rw == OCF_METADATA_RD) + env_rwsem_down_read(&metadata_lock->hash[hash]); + else + ENV_BUG(); +} + +void ocf_metadata_hash_unlock(struct ocf_metadata_lock *metadata_lock, + ocf_cache_line_t hash, int rw) +{ + ENV_BUG_ON(hash >= metadata_lock->num_hash_entries); + + if (rw == OCF_METADATA_WR) + env_rwsem_up_write(&metadata_lock->hash[hash]); + else if (rw == OCF_METADATA_RD) + env_rwsem_up_read(&metadata_lock->hash[hash]); + else + ENV_BUG(); +} + +int ocf_metadata_hash_try_lock(struct ocf_metadata_lock *metadata_lock, + ocf_cache_line_t hash, int rw) +{ + int result = -1; + + ENV_BUG_ON(hash >= metadata_lock->num_hash_entries); + + if (rw == OCF_METADATA_WR) { + result = env_rwsem_down_write_trylock( + &metadata_lock->hash[hash]); + } else if (rw == OCF_METADATA_RD) { + result = env_rwsem_down_read_trylock( + &metadata_lock->hash[hash]); + } else { + ENV_BUG(); + } + + if (!result) + return -1; + + return 0; +} + +#define _NUM_HASH_ENTRIES req->cache->metadata.lock.num_hash_entries + +/* + * Iterate over hash buckets for all core lines in the request in ascending hash + * bucket value order. Each hash bucket is visited only once. + * + * @i is used as iteration counter, starting from 0 + * @hash stores hash values for each iteration + * @start is internal helper variable. It set to the index of first occurence + * of hash with minimal value within the request. + * + * Example hash iteration order for _NUM_HASH_ENTRIES == 5: + * Request hashes Iteration order start + * [2, 3, 4] [2, 3, 4] 0 + * [2, 3, 4, 0] [0, 2, 3, 4] 3 + * [2, 3, 4, 0, 1, 2, 3, 4, 0, 1] [0, 1, 2, 3, 4] 3 + * [4, 0] [0, 4] 1 + * [0, 1, 2, 3, 4, 0, 1] [0, 1, 2, 3, 4] 0 + * + */ +#define for_each_req_hash_asc(req, i, hash, start) \ + for (i = 0, start = (req->map[0].hash + req->core_line_count <= \ + _NUM_HASH_ENTRIES) ? 0 : (_NUM_HASH_ENTRIES - req->map[0].hash)\ + % _NUM_HASH_ENTRIES, hash = req->map[start].hash; \ + i < OCF_MIN(req->core_line_count, _NUM_HASH_ENTRIES); \ + i++, hash = req->map[(start + i) % req->core_line_count].hash) + +void ocf_req_hash_lock_rd(struct ocf_request *req) +{ + unsigned i, start; + ocf_cache_line_t hash; + + ocf_metadata_start_shared_access(&req->cache->metadata.lock); + for_each_req_hash_asc(req, i, hash, start) { + ocf_metadata_hash_lock(&req->cache->metadata.lock, hash, + OCF_METADATA_RD); + } +} + +void ocf_req_hash_unlock_rd(struct ocf_request *req) +{ + unsigned i, start; + ocf_cache_line_t hash; + + for_each_req_hash_asc(req, i, hash, start) { + ocf_metadata_hash_unlock(&req->cache->metadata.lock, hash, + OCF_METADATA_RD); + } + ocf_metadata_end_shared_access(&req->cache->metadata.lock); +} + +void ocf_req_hash_lock_wr(struct ocf_request *req) +{ + unsigned i, start; + ocf_cache_line_t hash; + + ocf_metadata_start_shared_access(&req->cache->metadata.lock); + for_each_req_hash_asc(req, i, hash, start) { + ocf_metadata_hash_lock(&req->cache->metadata.lock, hash, + OCF_METADATA_WR); + } +} + +void ocf_req_hash_lock_upgrade(struct ocf_request *req) +{ + unsigned i, start; + ocf_cache_line_t hash; + + for_each_req_hash_asc(req, i, hash, start) { + ocf_metadata_hash_unlock(&req->cache->metadata.lock, hash, + OCF_METADATA_RD); + } + for_each_req_hash_asc(req, i, hash, start) { + ocf_metadata_hash_lock(&req->cache->metadata.lock, hash, + OCF_METADATA_WR); + } +} + +void ocf_req_hash_unlock_wr(struct ocf_request *req) +{ + unsigned i, start; + ocf_cache_line_t hash; + + for_each_req_hash_asc(req, i, hash, start) { + ocf_metadata_hash_unlock(&req->cache->metadata.lock, hash, + OCF_METADATA_WR); + } + ocf_metadata_end_shared_access(&req->cache->metadata.lock); +} diff --git a/src/concurrency/ocf_metadata_concurrency.h b/src/concurrency/ocf_metadata_concurrency.h index 85b0c10..ccd996b 100644 --- a/src/concurrency/ocf_metadata_concurrency.h +++ b/src/concurrency/ocf_metadata_concurrency.h @@ -10,125 +10,113 @@ #define OCF_METADATA_RD 0 #define OCF_METADATA_WR 1 -void ocf_metadata_concurrency_init(struct ocf_cache *cache); +void ocf_metadata_concurrency_init(struct ocf_metadata_lock *metadata_lock); -void ocf_metadata_concurrency_deinit(struct ocf_cache *cache); +void ocf_metadata_concurrency_deinit(struct ocf_metadata_lock *metadata_lock); -int ocf_metadata_concurrency_attached_init(struct ocf_cache *cache); +int ocf_metadata_concurrency_attached_init( + struct ocf_metadata_lock *metadata_lock, + uint64_t hash_table_entries); -static inline void ocf_metadata_eviction_lock(struct ocf_cache *cache) +void ocf_metadata_concurrency_attached_deinit( + struct ocf_metadata_lock *metadata_lock); + +static inline void ocf_metadata_eviction_lock( + struct ocf_metadata_lock *metadata_lock) { - env_spinlock_lock(&cache->metadata.lock.eviction); + env_spinlock_lock(&metadata_lock->eviction); } -static inline void ocf_metadata_eviction_unlock(struct ocf_cache *cache) +static inline void ocf_metadata_eviction_unlock( + struct ocf_metadata_lock *metadata_lock) { - env_spinlock_unlock(&cache->metadata.lock.eviction); + env_spinlock_unlock(&metadata_lock->eviction); } #define OCF_METADATA_EVICTION_LOCK() \ - ocf_metadata_eviction_lock(cache) + ocf_metadata_eviction_lock(&cache->metadata.lock) #define OCF_METADATA_EVICTION_UNLOCK() \ - ocf_metadata_eviction_unlock(cache) + ocf_metadata_eviction_unlock(&cache->metadata.lock) -static inline void ocf_metadata_lock(struct ocf_cache *cache, int rw) -{ - if (rw == OCF_METADATA_WR) - env_rwsem_down_write(&cache->metadata.lock.collision); - else if (rw == OCF_METADATA_RD) - env_rwsem_down_read(&cache->metadata.lock.collision); - else - ENV_BUG(); -} +void ocf_metadata_start_exclusive_access( + struct ocf_metadata_lock *metadata_lock); +int ocf_metadata_try_start_exclusive_access( + struct ocf_metadata_lock *metadata_lock); -static inline void ocf_metadata_unlock(struct ocf_cache *cache, int rw) -{ - if (rw == OCF_METADATA_WR) - env_rwsem_up_write(&cache->metadata.lock.collision); - else if (rw == OCF_METADATA_RD) - env_rwsem_up_read(&cache->metadata.lock.collision); - else - ENV_BUG(); -} +void ocf_metadata_end_exclusive_access( + struct ocf_metadata_lock *metadata_lock); -static inline int ocf_metadata_try_lock(struct ocf_cache *cache, int rw) -{ - int result = 0; +int ocf_metadata_try_start_shared_access( + struct ocf_metadata_lock *metadata_lock); - if (rw == OCF_METADATA_WR) { - result = env_rwsem_down_write_trylock( - &cache->metadata.lock.collision); - } else if (rw == OCF_METADATA_RD) { - result = env_rwsem_down_read_trylock( - &cache->metadata.lock.collision); - } else { - ENV_BUG(); - } +void ocf_metadata_start_shared_access( + struct ocf_metadata_lock *metadata_lock); - if (result) - return -1; - - return 0; -} +void ocf_metadata_end_shared_access( + struct ocf_metadata_lock *metadata_lock); static inline void ocf_metadata_status_bits_lock( - struct ocf_cache *cache, int rw) + struct ocf_metadata_lock *metadata_lock, int rw) { if (rw == OCF_METADATA_WR) - env_rwlock_write_lock(&cache->metadata.lock.status); + env_rwlock_write_lock(&metadata_lock->status); else if (rw == OCF_METADATA_RD) - env_rwlock_read_lock(&cache->metadata.lock.status); + env_rwlock_read_lock(&metadata_lock->status); else ENV_BUG(); } static inline void ocf_metadata_status_bits_unlock( - struct ocf_cache *cache, int rw) + struct ocf_metadata_lock *metadata_lock, int rw) { if (rw == OCF_METADATA_WR) - env_rwlock_write_unlock(&cache->metadata.lock.status); + env_rwlock_write_unlock(&metadata_lock->status); else if (rw == OCF_METADATA_RD) - env_rwlock_read_unlock(&cache->metadata.lock.status); + env_rwlock_read_unlock(&metadata_lock->status); else ENV_BUG(); } -#define OCF_METADATA_LOCK_RD() \ - ocf_metadata_lock(cache, OCF_METADATA_RD) +#define OCF_METADATA_LOCK_RD() ocf_metadata_start_shared_access( \ + &cache->metadata.lock) -#define OCF_METADATA_UNLOCK_RD() \ - ocf_metadata_unlock(cache, OCF_METADATA_RD) +#define OCF_METADATA_UNLOCK_RD() ocf_metadata_end_shared_access( \ + &cache->metadata.lock) -#define OCF_METADATA_LOCK_RD_TRY() \ - ocf_metadata_try_lock(cache, OCF_METADATA_RD) +#define OCF_METADATA_LOCK_RD_TRY() ocf_metadata_try_start_shared_access( \ + &cache->metadata.lock) -#define OCF_METADATA_LOCK_WR() \ - ocf_metadata_lock(cache, OCF_METADATA_WR) +#define OCF_METADATA_LOCK_WR() ocf_metadata_start_exclusive_access( \ + &cache->metadata.lock) #define OCF_METADATA_LOCK_WR_TRY() \ - ocf_metadata_try_lock(cache, OCF_METADATA_WR) + ocf_metadata_try_start_exclusive_access(&cache->metadata.lock) -#define OCF_METADATA_UNLOCK_WR() \ - ocf_metadata_unlock(cache, OCF_METADATA_WR) +#define OCF_METADATA_UNLOCK_WR() ocf_metadata_end_exclusive_access( \ + &cache->metadata.lock) #define OCF_METADATA_BITS_LOCK_RD() \ - ocf_metadata_status_bits_lock(cache, OCF_METADATA_RD) + ocf_metadata_status_bits_lock(&cache->metadata.lock, \ + OCF_METADATA_RD) #define OCF_METADATA_BITS_UNLOCK_RD() \ - ocf_metadata_status_bits_unlock(cache, OCF_METADATA_RD) + ocf_metadata_status_bits_unlock(&cache->metadata.lock, \ + OCF_METADATA_RD) #define OCF_METADATA_BITS_LOCK_WR() \ - ocf_metadata_status_bits_lock(cache, OCF_METADATA_WR) + ocf_metadata_status_bits_lock(&cache->metadata.lock, \ + OCF_METADATA_WR) #define OCF_METADATA_BITS_UNLOCK_WR() \ - ocf_metadata_status_bits_unlock(cache, OCF_METADATA_WR) + ocf_metadata_status_bits_unlock(&cache->metadata.lock, \ + OCF_METADATA_WR) -#define OCF_METADATA_FLUSH_LOCK() \ - ocf_metadata_flush_lock(cache) - -#define OCF_METADATA_FLUSH_UNLOCK() \ - ocf_metadata_flush_unlock(cache) +void ocf_req_hash_lock_rd(struct ocf_request *req); +void ocf_req_hash_unlock_rd(struct ocf_request *req); +void ocf_req_hash_lock_wr(struct ocf_request *req); +void ocf_req_hash_unlock_wr(struct ocf_request *req); +void ocf_req_hash_lock_upgrade(struct ocf_request *req); #endif diff --git a/src/metadata/metadata.c b/src/metadata/metadata.c index 1b31132..5e5d02f 100644 --- a/src/metadata/metadata.c +++ b/src/metadata/metadata.c @@ -39,7 +39,7 @@ int ocf_metadata_init(struct ocf_cache *cache, return ret; } - ocf_metadata_concurrency_init(cache); + ocf_metadata_concurrency_init(&cache->metadata.lock); return 0; } @@ -73,7 +73,7 @@ void ocf_metadata_deinit(struct ocf_cache *cache) cache->metadata.iface.deinit(cache); } - ocf_metadata_concurrency_deinit(cache); + ocf_metadata_concurrency_deinit(&cache->metadata.lock); ocf_metadata_io_deinit(cache); } diff --git a/src/metadata/metadata_structs.h b/src/metadata/metadata_structs.h index e9b3ac5..ccea1a0 100644 --- a/src/metadata/metadata_structs.h +++ b/src/metadata/metadata_structs.h @@ -428,6 +428,15 @@ struct ocf_cache_line_settings { uint64_t sector_end; }; +struct ocf_metadata_lock +{ + env_rwsem global; /*!< global metadata lock (GML) */ + env_rwlock status; /*!< Fast lock for status bits */ + env_spinlock eviction; /*!< Fast lock for eviction policy */ + env_rwsem *hash; /*!< Hash bucket locks */ + uint32_t num_hash_entries; /*!< Hash bucket count */ +}; + /** * @brief Metadata control structure */ @@ -444,11 +453,7 @@ struct ocf_metadata { bool is_volatile; /*!< true if metadata used in volatile mode (RAM only) */ - struct { - env_rwsem collision; /*!< lock for collision table */ - env_rwlock status; /*!< Fast lock for status bits */ - env_spinlock eviction; /*!< Fast lock for eviction policy */ - } lock; + struct ocf_metadata_lock lock; }; #endif /* __METADATA_STRUCTS_H__ */ diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 4a496da..847efef 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -986,7 +986,8 @@ static void _ocf_mngt_attach_prepare_metadata(ocf_pipeline_t pipeline, context->flags.attached_metadata_inited = true; - if (ocf_metadata_concurrency_attached_init(cache)) { + if (ocf_metadata_concurrency_attached_init(&cache->metadata.lock, + cache->device->hash_table_entries)) { ocf_cache_log(cache, log_err, "Failed to initialize attached " "metadata concurrency\n"); OCF_PL_FINISH_RET(context->pipeline, -OCF_ERR_START_CACHE_FAIL); @@ -1734,6 +1735,7 @@ static void _ocf_mngt_cache_unplug_complete(void *priv, int error) ocf_volume_close(&cache->device->volume); + ocf_metadata_concurrency_attached_deinit(&cache->metadata.lock); ocf_metadata_deinit_variable_size(cache); ocf_concurrency_deinit(cache); ocf_freelist_deinit(cache->freelist); diff --git a/src/ocf_request.c b/src/ocf_request.c index 57cc838..8584b71 100644 --- a/src/ocf_request.c +++ b/src/ocf_request.c @@ -311,3 +311,14 @@ void ocf_req_clear_map(struct ocf_request *req) ENV_BUG_ON(env_memset(req->map, sizeof(req->map[0]) * req->core_line_count, 0)); } + +void ocf_req_hash(struct ocf_request *req) +{ + int i; + + for (i = 0; i < req->core_line_count; i++) { + req->map[i].hash = ocf_metadata_hash_func(req->cache, + req->core_line_first + i, + ocf_core_get_id(req->core)); + } +} diff --git a/src/ocf_request.h b/src/ocf_request.h index 1fa924b..fcb2a16 100644 --- a/src/ocf_request.h +++ b/src/ocf_request.h @@ -319,6 +319,13 @@ void ocf_req_clear_info(struct ocf_request *req); */ void ocf_req_clear_map(struct ocf_request *req); +/** + * @brief Calculate hashes for all core lines within the request + * + * @param req - OCF request + */ +void ocf_req_hash(struct ocf_request *req); + /** * @brief Clear OCF request *