github/ocf
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Split global metadata lock

Browse Source 
Divide single global lock instance into 4 to reduce contention
in multiple read-locks scenario.

Signed-off-by: Adam Rutkowski <adam.j.rutkowski@intel.com>
This commit is contained in:
Adam Rutkowski 2020-12-28 10:34:16 +01:00
parent 10c3c3de36
commit 05780c98ed
11 changed files with 148 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/cleaning/acp.c
View File

@ -13,6 +13,7 @@
#include "../cleaning/acp.h"
#include "../engine/engine_common.h"
#include "../concurrency/ocf_cache_line_concurrency.h"
#include "../concurrency/ocf_metadata_concurrency.h"
#include "cleaning_priv.h"
#define OCF_ACP_DEBUG 0
@ -385,8 +386,11 @@ static ocf_cache_line_t _acp_trylock_dirty(struct ocf_cache *cache,
{
struct ocf_map_info info;
bool locked = false;
unsigned lock_idx = ocf_metadata_concurrency_next_idx(
cache->cleaner.io_queue);
ocf_hb_cline_prot_lock_rd(&cache->metadata.lock, core_id, core_line);
ocf_hb_cline_prot_lock_rd(&cache->metadata.lock, lock_idx, core_id,
core_line);
ocf_engine_lookup_map_entry(cache, &info, core_id,
core_line);
@ -397,7 +401,8 @@ static ocf_cache_line_t _acp_trylock_dirty(struct ocf_cache *cache,
locked = true;
}
ocf_hb_cline_prot_unlock_rd(&cache->metadata.lock, core_id, core_line);
ocf_hb_cline_prot_unlock_rd(&cache->metadata.lock, lock_idx, core_id,
core_line);
return locked ? info.coll_idx : cache->device->collision_table_entries;
}

131
src/concurrency/ocf_metadata_concurrency.c
View File

@ -5,12 +5,14 @@
#include "ocf_metadata_concurrency.h"
#include "../metadata/metadata_misc.h"
#include "../ocf_queue_priv.h"
int ocf_metadata_concurrency_init(struct ocf_metadata_lock *metadata_lock)
{
int err = 0;
unsigned evp_iter;
unsigned part_iter;
unsigned global_iter;
for (evp_iter = 0; evp_iter < OCF_NUM_EVICTION_LISTS; evp_iter++) {
err = env_spinlock_init(&metadata_lock->eviction[evp_iter]);
@ -20,22 +22,29 @@ int ocf_metadata_concurrency_init(struct ocf_metadata_lock *metadata_lock)
env_rwlock_init(&metadata_lock->status);
err = env_rwsem_init(&metadata_lock->global);
for (global_iter = 0; global_iter < OCF_NUM_GLOBAL_META_LOCKS;
global_iter++) {
err = env_rwsem_init(&metadata_lock->global[global_iter]);
if (err)
goto rwsem_err;
goto global_err;
}
for (part_iter = 0; part_iter < OCF_IO_CLASS_MAX; part_iter++) {
err = env_spinlock_init(&metadata_lock->partition[part_iter]);
if (err)
goto spinlocks_err;
goto partition_err;
}
return err;
spinlocks_err:
partition_err:
while (part_iter--)
env_spinlock_destroy(&metadata_lock->partition[part_iter]);
rwsem_err:
global_err:
while (global_iter--)
env_rwsem_destroy(&metadata_lock->global[global_iter]);
env_rwlock_destroy(&metadata_lock->status);
eviction_err:
@ -55,8 +64,10 @@ void ocf_metadata_concurrency_deinit(struct ocf_metadata_lock *metadata_lock)
for (i = 0; i < OCF_NUM_EVICTION_LISTS; i++)
env_spinlock_destroy(&metadata_lock->eviction[i]);
for (i = 0; i < OCF_NUM_GLOBAL_META_LOCKS; i++)
env_rwsem_destroy(&metadata_lock->global[i]);
env_rwlock_destroy(&metadata_lock->status);
env_rwsem_destroy(&metadata_lock->global);
}
int ocf_metadata_concurrency_attached_init(
@ -140,34 +151,68 @@ void ocf_metadata_concurrency_attached_deinit(
void ocf_metadata_start_exclusive_access(
struct ocf_metadata_lock *metadata_lock)
{
env_rwsem_down_write(&metadata_lock->global);
unsigned i;
for (i = 0; i < OCF_NUM_GLOBAL_META_LOCKS; i++) {
env_rwsem_down_write(&metadata_lock->global[i]);
}
}
int ocf_metadata_try_start_exclusive_access(
struct ocf_metadata_lock *metadata_lock)
{
return env_rwsem_down_write_trylock(&metadata_lock->global);
unsigned i;
int error;
for (i = 0; i < OCF_NUM_GLOBAL_META_LOCKS; i++) {
error = env_rwsem_down_write_trylock(&metadata_lock->global[i]);
if (error)
break;
}
if (error) {
while (i--) {
env_rwsem_up_write(&metadata_lock->global[i]);
}
}
return error;
}
void ocf_metadata_end_exclusive_access(
struct ocf_metadata_lock *metadata_lock)
{
env_rwsem_up_write(&metadata_lock->global);
unsigned i;
for (i = OCF_NUM_GLOBAL_META_LOCKS; i > 0; i--)
env_rwsem_up_write(&metadata_lock->global[i - 1]);
}
void ocf_metadata_start_shared_access(struct ocf_metadata_lock *metadata_lock)
/* lock_idx determines which of underlying R/W locks is acquired for read. The goal
is to spread calls across all available underlying locks to reduce contention
on one single RW semaphor primitive. Technically any value is correct, but
picking wisely would allow for higher read througput:
* free running per-cpu counter sounds good,
* for rarely excercised code paths (e.g. management) any value would do.
*/
void ocf_metadata_start_shared_access(
struct ocf_metadata_lock *metadata_lock,
unsigned lock_idx)
{
env_rwsem_down_read(&metadata_lock->global);
env_rwsem_down_read(&metadata_lock->global[lock_idx]);
}
int ocf_metadata_try_start_shared_access(struct ocf_metadata_lock *metadata_lock)
int ocf_metadata_try_start_shared_access(
struct ocf_metadata_lock *metadata_lock,
unsigned lock_idx)
{
return env_rwsem_down_read_trylock(&metadata_lock->global);
return env_rwsem_down_read_trylock(&metadata_lock->global[lock_idx]);
}
void ocf_metadata_end_shared_access(struct ocf_metadata_lock *metadata_lock)
void ocf_metadata_end_shared_access(struct ocf_metadata_lock *metadata_lock,
unsigned lock_idx)
{
env_rwsem_up_read(&metadata_lock->global);
env_rwsem_up_read(&metadata_lock->global[lock_idx]);
}
/* NOTE: Calling 'naked' lock/unlock requires caller to hold global metadata
@ -267,74 +312,74 @@ void ocf_hb_cline_naked_unlock_wr(struct ocf_metadata_lock *metadata_lock,
/* common part of protected hash bucket lock routines */
static inline void ocf_hb_id_prot_lock_common(
struct ocf_metadata_lock *metadata_lock,
ocf_cache_line_t hash, int rw)
uint32_t lock_idx, ocf_cache_line_t hash, int rw)
{
ocf_metadata_start_shared_access(metadata_lock);
ocf_metadata_start_shared_access(metadata_lock, lock_idx);
ocf_hb_id_naked_lock(metadata_lock, hash, rw);
}
/* common part of protected hash bucket unlock routines */
static inline void ocf_hb_id_prot_unlock_common(
struct ocf_metadata_lock *metadata_lock,
ocf_cache_line_t hash, int rw)
uint32_t lock_idx, ocf_cache_line_t hash, int rw)
{
ocf_hb_id_naked_unlock(metadata_lock, hash, rw);
ocf_metadata_end_shared_access(metadata_lock);
ocf_metadata_end_shared_access(metadata_lock, lock_idx);
}
/* NOTE: caller can lock at most one hash bucket at a time using protected
variants of lock routines. */
void ocf_hb_cline_prot_lock_wr(struct ocf_metadata_lock *metadata_lock,
uint32_t core_id, uint64_t core_line)
uint32_t lock_idx, uint32_t core_id, uint64_t core_line)
{
ocf_cache_line_t hash = ocf_metadata_hash_func(metadata_lock->cache,
core_line, core_id);
ocf_hb_id_prot_lock_common(metadata_lock, hash,
OCF_METADATA_WR);
ocf_hb_id_prot_lock_common(metadata_lock, lock_idx,
hash, OCF_METADATA_WR);
}
void ocf_hb_cline_prot_unlock_wr(struct ocf_metadata_lock *metadata_lock,
uint32_t core_id, uint64_t core_line)
uint32_t lock_idx, uint32_t core_id, uint64_t core_line)
{
ocf_cache_line_t hash = ocf_metadata_hash_func(metadata_lock->cache,
core_line, core_id);
ocf_hb_id_prot_unlock_common(metadata_lock, hash,
OCF_METADATA_WR);
ocf_hb_id_prot_unlock_common(metadata_lock, lock_idx,
hash, OCF_METADATA_WR);
}
void ocf_hb_cline_prot_lock_rd(struct ocf_metadata_lock *metadata_lock,
uint32_t core_id, uint64_t core_line)
uint32_t lock_idx, uint32_t core_id, uint64_t core_line)
{
ocf_cache_line_t hash = ocf_metadata_hash_func(metadata_lock->cache,
core_line, core_id);
ocf_hb_id_prot_lock_common(metadata_lock, hash,
OCF_METADATA_RD);
ocf_hb_id_prot_lock_common(metadata_lock, lock_idx,
hash, OCF_METADATA_RD);
}
void ocf_hb_cline_prot_unlock_rd(struct ocf_metadata_lock *metadata_lock,
uint32_t core_id, uint64_t core_line)
uint32_t lock_idx, uint32_t core_id, uint64_t core_line)
{
ocf_cache_line_t hash = ocf_metadata_hash_func(metadata_lock->cache,
core_line, core_id);
ocf_hb_id_prot_unlock_common(metadata_lock, hash,
OCF_METADATA_RD);
ocf_hb_id_prot_unlock_common(metadata_lock, lock_idx,
hash, OCF_METADATA_RD);
}
void ocf_hb_hash_prot_lock_wr(struct ocf_metadata_lock *metadata_lock,
ocf_cache_line_t hash)
void ocf_hb_id_prot_lock_wr(struct ocf_metadata_lock *metadata_lock,
unsigned lock_idx, ocf_cache_line_t hash)
{
ocf_hb_id_prot_lock_common(metadata_lock, hash,
ocf_hb_id_prot_lock_common(metadata_lock, lock_idx, hash,
OCF_METADATA_WR);
}
void ocf_hb_hash_prot_unlock_wr(struct ocf_metadata_lock *metadata_lock,
ocf_cache_line_t hash)
void ocf_hb_id_prot_unlock_wr(struct ocf_metadata_lock *metadata_lock,
unsigned lock_idx, ocf_cache_line_t hash)
{
ocf_hb_id_prot_unlock_common(metadata_lock, hash,
ocf_hb_id_prot_unlock_common(metadata_lock, lock_idx, hash,
OCF_METADATA_WR);
}
@ -392,7 +437,8 @@ void ocf_hb_req_prot_lock_rd(struct ocf_request *req)
{
ocf_cache_line_t hash;
ocf_metadata_start_shared_access(&req->cache->metadata.lock);
ocf_metadata_start_shared_access(&req->cache->metadata.lock,
req->lock_idx);
for_each_req_hash_asc(req, hash) {
ocf_hb_id_naked_lock(&req->cache->metadata.lock, hash,
OCF_METADATA_RD);
@ -407,14 +453,16 @@ void ocf_hb_req_prot_unlock_rd(struct ocf_request *req)
ocf_hb_id_naked_unlock(&req->cache->metadata.lock, hash,
OCF_METADATA_RD);
}
ocf_metadata_end_shared_access(&req->cache->metadata.lock);
ocf_metadata_end_shared_access(&req->cache->metadata.lock,
req->lock_idx);
}
void ocf_hb_req_prot_lock_wr(struct ocf_request *req)
{
ocf_cache_line_t hash;
ocf_metadata_start_shared_access(&req->cache->metadata.lock);
ocf_metadata_start_shared_access(&req->cache->metadata.lock,
req->lock_idx);
for_each_req_hash_asc(req, hash) {
ocf_hb_id_naked_lock(&req->cache->metadata.lock, hash,
OCF_METADATA_WR);
@ -443,7 +491,8 @@ void ocf_hb_req_prot_unlock_wr(struct ocf_request *req)
ocf_hb_id_naked_unlock(&req->cache->metadata.lock, hash,
OCF_METADATA_WR);
}
ocf_metadata_end_shared_access(&req->cache->metadata.lock);
ocf_metadata_end_shared_access(&req->cache->metadata.lock,
req->lock_idx);
}
void ocf_collision_start_shared_access(struct ocf_metadata_lock *metadata_lock,

34
src/concurrency/ocf_metadata_concurrency.h
View File

@ -4,6 +4,7 @@
*/
#include "../ocf_cache_priv.h"
#include "../eviction/eviction.h"
#include "../ocf_queue_priv.h"
#ifndef __OCF_METADATA_CONCURRENCY_H__
#define __OCF_METADATA_CONCURRENCY_H__
@ -11,6 +12,11 @@
#define OCF_METADATA_RD 0
#define OCF_METADATA_WR 1
static inline unsigned ocf_metadata_concurrency_next_idx(ocf_queue_t q)
{
return q->lock_idx++ % OCF_NUM_GLOBAL_META_LOCKS;
}
int ocf_metadata_concurrency_init(struct ocf_metadata_lock *metadata_lock);
void ocf_metadata_concurrency_deinit(struct ocf_metadata_lock *metadata_lock);
@ -90,13 +96,16 @@ void ocf_metadata_end_exclusive_access(
struct ocf_metadata_lock *metadata_lock);
int ocf_metadata_try_start_shared_access(
struct ocf_metadata_lock *metadata_lock);
struct ocf_metadata_lock *metadata_lock,
unsigned lock_idx);
void ocf_metadata_start_shared_access(
struct ocf_metadata_lock *metadata_lock);
struct ocf_metadata_lock *metadata_lock,
unsigned lock_idx);
void ocf_metadata_end_shared_access(
struct ocf_metadata_lock *metadata_lock);
struct ocf_metadata_lock *metadata_lock,
unsigned lock_idx);
static inline void ocf_metadata_status_bits_lock(
struct ocf_metadata_lock *metadata_lock, int rw)
@ -137,14 +146,19 @@ static inline void ocf_metadata_status_bits_unlock(
OCF_METADATA_WR)
void ocf_hb_cline_prot_lock_rd(struct ocf_metadata_lock *metadata_lock,
uint32_t core_id, uint64_t core_line);
uint32_t lock_idx, uint32_t core_id, uint64_t core_line);
void ocf_hb_cline_prot_unlock_rd(struct ocf_metadata_lock *metadata_lock,
uint32_t core_id, uint64_t core_line);
uint32_t lock_idx, uint32_t core_id, uint64_t core_line);
void ocf_hb_cline_prot_lock_wr(struct ocf_metadata_lock *metadata_lock,
uint32_t lock_idx, uint32_t core_id, uint64_t core_line);
void ocf_hb_cline_prot_unlock_wr(struct ocf_metadata_lock *metadata_lock,
uint32_t lock_idx, uint32_t core_id, uint64_t core_line);
void ocf_hb_id_prot_lock_wr(struct ocf_metadata_lock *metadata_lock,
ocf_cache_line_t hash);
unsigned lock_idx, ocf_cache_line_t hash);
void ocf_hb_id_prot_unlock_wr(struct ocf_metadata_lock *metadata_lock,
ocf_cache_line_t hash);
unsigned lock_idx, ocf_cache_line_t hash);
/* caller must hold global metadata read lock */
bool ocf_hb_cline_naked_trylock_rd(struct ocf_metadata_lock *metadata_lock,
@ -152,12 +166,6 @@ bool ocf_hb_cline_naked_trylock_rd(struct ocf_metadata_lock *metadata_lock,
void ocf_hb_cline_naked_unlock_rd(struct ocf_metadata_lock *metadata_lock,
uint32_t core_id, uint64_t core_line);
void ocf_hb_cline_prot_lock_wr(struct ocf_metadata_lock *metadata_lock,
uint32_t core_id, uint64_t core_line);
void ocf_hb_cline_prot_unlock_wr(struct ocf_metadata_lock *metadata_lock,
uint32_t core_id, uint64_t core_line);
/* caller must hold global metadata read lock */
bool ocf_hb_cline_naked_trylock_wr(struct ocf_metadata_lock *metadata_lock,
uint32_t core_id, uint64_t core_line);
void ocf_hb_cline_naked_unlock_wr(struct ocf_metadata_lock *metadata_lock,

2
src/eviction/eviction.h
View File

@ -9,7 +9,6 @@
#include "ocf/ocf.h"
#include "lru.h"
#include "lru_structs.h"
#include "../ocf_request.h"
#define OCF_TO_EVICTION_MIN 128UL
#define OCF_PENDING_EVICTION_LIMIT 512UL
@ -17,6 +16,7 @@
#define OCF_NUM_EVICTION_LISTS 32
struct ocf_user_part;
struct ocf_request;
struct eviction_policy {
union {

6
src/metadata/metadata_io.c
View File

@ -215,9 +215,11 @@ static int metadata_io_restart_req(struct ocf_request *req)
/* Fill with the latest metadata. */
if (m_req->req.rw == OCF_WRITE) {
ocf_metadata_start_shared_access(&cache->metadata.lock);
ocf_metadata_start_shared_access(&cache->metadata.lock,
m_req->page % OCF_NUM_GLOBAL_META_LOCKS);
metadata_io_req_fill(m_req);
ocf_metadata_end_shared_access(&cache->metadata.lock);
ocf_metadata_end_shared_access(&cache->metadata.lock,
m_req->page % OCF_NUM_GLOBAL_META_LOCKS);
}
io = ocf_new_cache_io(cache, req->io_queue,

6
src/metadata/metadata_structs.h
View File

@ -43,9 +43,13 @@ struct ocf_cache_line_settings {
uint64_t sector_end;
};
#define OCF_METADATA_GLOBAL_LOCK_IDX_BITS 2
#define OCF_NUM_GLOBAL_META_LOCKS (1 << (OCF_METADATA_GLOBAL_LOCK_IDX_BITS))
struct ocf_metadata_lock
{
env_rwsem global; /*!< global metadata lock (GML) */
env_rwsem global[OCF_NUM_GLOBAL_META_LOCKS]; /*!< global metadata lock (GML) */
env_rwlock status; /*!< Fast lock for status bits */
env_spinlock eviction[OCF_NUM_EVICTION_LISTS]; /*!< Fast lock for eviction policy */
env_rwsem *hash; /*!< Hash bucket locks */

8
src/mngt/ocf_mngt_cache.c
View File

@ -2344,11 +2344,11 @@ ocf_promotion_t ocf_mngt_cache_promotion_get_policy(ocf_cache_t cache)
{
ocf_promotion_t result;
ocf_metadata_start_shared_access(&cache->metadata.lock);
ocf_metadata_start_shared_access(&cache->metadata.lock, 0);
result = cache->conf_meta->promotion_policy_type;
ocf_metadata_end_shared_access(&cache->metadata.lock);
ocf_metadata_end_shared_access(&cache->metadata.lock, 0);
return result;
}
@ -2358,11 +2358,11 @@ int ocf_mngt_cache_promotion_get_param(ocf_cache_t cache, ocf_promotion_t type,
{
int result;
ocf_metadata_start_shared_access(&cache->metadata.lock);
ocf_metadata_start_shared_access(&cache->metadata.lock, 0);
result = ocf_promotion_get_param(cache, type, param_id, param_value);
ocf_metadata_end_shared_access(&cache->metadata.lock);
ocf_metadata_end_shared_access(&cache->metadata.lock, 0);
return result;
}

7
src/mngt/ocf_mngt_common.c
View File

@ -59,10 +59,13 @@ void cache_mngt_core_deinit_attached_meta(ocf_core_t core)
ocf_core_id_t iter_core_id;
ocf_cache_line_t curr_cline, prev_cline;
uint32_t hash, num_hash = cache->device->hash_table_entries;
unsigned lock_idx;
for (hash = 0; hash < num_hash;) {
prev_cline = cache->device->collision_table_entries;
ocf_hb_id_prot_lock_wr(&cache->metadata.lock, hash);
lock_idx = ocf_metadata_concurrency_next_idx(cache->mngt_queue);
ocf_hb_id_prot_lock_wr(&cache->metadata.lock, lock_idx, hash);
curr_cline = ocf_metadata_get_hash(cache, hash);
while (curr_cline != cache->device->collision_table_entries) {
@ -91,7 +94,7 @@ void cache_mngt_core_deinit_attached_meta(ocf_core_t core)
else
curr_cline = ocf_metadata_get_hash(cache, hash);
}
ocf_hb_id_prot_unlock_wr(&cache->metadata.lock, hash);
ocf_hb_id_prot_unlock_wr(&cache->metadata.lock, lock_idx, hash);
/* Check whether all the cachelines from the hash bucket were sparsed */
if (curr_cline == cache->device->collision_table_entries)

3
src/ocf_queue_priv.h
View File

@ -18,6 +18,9 @@ struct ocf_queue {
struct list_head io_list;
env_spinlock io_list_lock;
/* per-queue free running global metadata lock index */
unsigned lock_idx;
/* Tracing reference counter */
env_atomic64 trace_ref_cntr;

4
src/ocf_request.c
View File

@ -6,7 +6,7 @@
#include "ocf/ocf.h"
#include "ocf_request.h"
#include "ocf_cache_priv.h"
#include "ocf_queue_priv.h"
#include "concurrency/ocf_metadata_concurrency.h"
#include "utils/utils_cache_line.h"
#define OCF_UTILS_RQ_DEBUG 0
@ -205,6 +205,8 @@ struct ocf_request *ocf_req_new(ocf_queue_t queue, ocf_core_t core,
req->discard.nr_sects = BYTES_TO_SECTORS(bytes);
req->discard.handled = 0;
req->lock_idx = ocf_metadata_concurrency_next_idx(queue);
return req;
}

4
src/ocf_request.h
View File

@ -9,6 +9,7 @@
#include "ocf_env.h"
#include "ocf_io_priv.h"
#include "engine/cache_engine.h"
#include "metadata/metadata_structs.h"
struct ocf_req_allocator;
@ -190,6 +191,9 @@ struct ocf_request {
uint8_t part_evict : 1;
/* !< Some cachelines from request's partition must be evicted */
uint8_t lock_idx : OCF_METADATA_GLOBAL_LOCK_IDX_BITS;
/* !< Selected global metadata read lock */
log_sid_t sid;
/*!< Tracing sequence ID */