diff --git a/src/cleaning/acp.c b/src/cleaning/acp.c index e8c17a4..6441749 100644 --- a/src/cleaning/acp.c +++ b/src/cleaning/acp.c @@ -396,9 +396,10 @@ static ocf_cache_line_t _acp_trylock_dirty(struct ocf_cache *cache, core_line); if (info.status == LOOKUP_HIT && - metadata_test_dirty(cache, info.coll_idx) && - ocf_cache_line_try_lock_rd(cache, info.coll_idx)) { - locked = true; + metadata_test_dirty(cache, info.coll_idx)) { + locked = ocf_cache_line_try_lock_rd( + cache->device->concurrency.cache_line, + info.coll_idx); } ocf_hb_cline_prot_unlock_rd(&cache->metadata.lock, lock_idx, core_id, @@ -472,7 +473,9 @@ static void _acp_flush_end(void *priv, int error) config = (void *)&cache->conf_meta->cleaning[ocf_cleaning_acp].data; for (i = 0; i < flush->size; i++) { - ocf_cache_line_unlock_rd(cache, flush->data[i].cache_line); + ocf_cache_line_unlock_rd( + cache->device->concurrency.cache_line, + flush->data[i].cache_line); ACP_DEBUG_END(acp, flush->data[i].cache_line); } diff --git a/src/cleaning/alru.c b/src/cleaning/alru.c index 97d3d48..dbea378 100644 --- a/src/cleaning/alru.c +++ b/src/cleaning/alru.c @@ -682,8 +682,11 @@ static bool block_is_busy(struct ocf_cache *cache, if (!cache->core[core_id].opened) return true; - if (ocf_cache_line_is_used(cache, cache_line)) + if (ocf_cache_line_is_used( + cache->device->concurrency.cache_line, + cache_line)) { return true; + } return false; } diff --git a/src/concurrency/ocf_cache_line_concurrency.c b/src/concurrency/ocf_cache_line_concurrency.c index e384370..62745fb 100644 --- a/src/concurrency/ocf_cache_line_concurrency.c +++ b/src/concurrency/ocf_cache_line_concurrency.c @@ -50,10 +50,12 @@ struct __waiters_list { }; struct ocf_cache_line_concurrency { - env_rwsem lock; + ocf_cache_t cache; + env_mutex lock; env_atomic *access; env_atomic waiting; size_t access_limit; + ocf_cache_line_t num_clines; env_allocator *allocator; struct __waiters_list waiters_lsts[_WAITERS_LIST_ENTRIES]; @@ -66,94 +68,103 @@ struct ocf_cache_line_concurrency { #define ALLOCATOR_NAME_FMT "ocf_%s_cache_concurrency" #define ALLOCATOR_NAME_MAX (sizeof(ALLOCATOR_NAME_FMT) + OCF_CACHE_NAME_SIZE) -int ocf_cache_line_concurrency_init(struct ocf_cache *cache) +int ocf_cache_line_concurrency_init(struct ocf_cache_line_concurrency **self, + unsigned num_clines, ocf_cache_t cache) { uint32_t i; int error = 0; struct ocf_cache_line_concurrency *c; char name[ALLOCATOR_NAME_MAX]; - ocf_cache_line_t line_entries = ocf_metadata_collision_table_entries( - cache); - - ENV_BUG_ON(cache->device->concurrency.cache_line); OCF_DEBUG_TRACE(cache); - c = env_vmalloc(sizeof(*c)); + c = env_vzalloc(sizeof(*c)); if (!c) { error = __LINE__; - goto ocf_cache_line_concurrency_init; + goto exit_err; } - error = env_rwsem_init(&c->lock); + c->cache = cache; + c->num_clines = num_clines; + + error = env_mutex_init(&c->lock); if (error) { - env_vfree(c); error = __LINE__; - goto ocf_cache_line_concurrency_init; + goto rwsem_err; } - cache->device->concurrency.cache_line = c; - OCF_REALLOC_INIT(&c->access, &c->access_limit); - OCF_REALLOC_CP(&c->access, sizeof(c->access[0]), line_entries, + OCF_REALLOC_CP(&c->access, sizeof(c->access[0]), num_clines, &c->access_limit); if (!c->access) { error = __LINE__; - goto ocf_cache_line_concurrency_init; + goto allocation_err; } if (snprintf(name, sizeof(name), ALLOCATOR_NAME_FMT, ocf_cache_get_name(cache)) < 0) { error = __LINE__; - goto ocf_cache_line_concurrency_init; + goto allocation_err; } c->allocator = env_allocator_create(sizeof(struct __waiter), name); if (!c->allocator) { error = __LINE__; - goto ocf_cache_line_concurrency_init; + goto allocation_err; } /* Init concurrency control table */ for (i = 0; i < _WAITERS_LIST_ENTRIES; i++) { INIT_LIST_HEAD(&c->waiters_lsts[i].head); error = env_spinlock_init(&c->waiters_lsts[i].lock); - if (error) + if (error) { + error = __LINE__; goto spinlock_err; + } } + *self = c; return 0; spinlock_err: while (i--) env_spinlock_destroy(&c->waiters_lsts[i].lock); -ocf_cache_line_concurrency_init: + +allocation_err: + if (c->allocator) + env_allocator_destroy(c->allocator); + + if (c->access) + OCF_REALLOC_DEINIT(&c->access, &c->access_limit); + +rwsem_err: + env_mutex_destroy(&c->lock); + +exit_err: ocf_cache_log(cache, log_err, "Cannot initialize cache concurrency, " "ERROR %d", error); + if (c) + env_vfree(c); - if (cache->device->concurrency.cache_line) - ocf_cache_line_concurrency_deinit(cache); - + *self = NULL; return -1; } /* * */ -void ocf_cache_line_concurrency_deinit(struct ocf_cache *cache) +void ocf_cache_line_concurrency_deinit(struct ocf_cache_line_concurrency **self) { + struct ocf_cache_line_concurrency *concurrency = *self; int i; - struct ocf_cache_line_concurrency *concurrency; - if (!cache->device->concurrency.cache_line) + if (!concurrency) return; - OCF_DEBUG_TRACE(cache); + OCF_DEBUG_TRACE(concurrency->cache); - concurrency = cache->device->concurrency.cache_line; - - env_rwsem_destroy(&concurrency->lock); + env_mutex_destroy(&concurrency->lock); for (i = 0; i < _WAITERS_LIST_ENTRIES; i++) env_spinlock_destroy(&concurrency->waiters_lsts[i].lock); @@ -166,10 +177,11 @@ void ocf_cache_line_concurrency_deinit(struct ocf_cache *cache) env_allocator_destroy(concurrency->allocator); env_vfree(concurrency); - cache->device->concurrency.cache_line = NULL; + + *self = NULL; } -size_t ocf_cache_line_concurrency_size_of(struct ocf_cache *cache) +size_t ocf_cache_line_concurrency_size_of(ocf_cache_t cache) { size_t size; @@ -365,12 +377,11 @@ static inline bool __try_lock_rd2rd(struct ocf_cache_line_concurrency *c, /* * */ -static void _req_on_lock(void *ctx, ocf_req_async_lock_cb cb, +static void _req_on_lock(struct ocf_cache_line_concurrency *c, + void *ctx, ocf_req_async_lock_cb cb, uint32_t ctx_id, ocf_cache_line_t line, int rw) { struct ocf_request *req = ctx; - struct ocf_cache_line_concurrency *c = req->cache->device->concurrency. - cache_line; if (rw == OCF_READ) req->map[ctx_id].rd_locked = true; @@ -396,55 +407,50 @@ static inline bool __lock_cache_line_wr(struct ocf_cache_line_concurrency *c, void *ctx, uint32_t ctx_id) { struct __waiter *waiter; - bool locked = false; bool waiting = false; unsigned long flags = 0; + ENV_BUG_ON(!cb); + if (__try_lock_wr(c, line)) { - /* No activity before look get */ - if (cb) - _req_on_lock(ctx, cb, ctx_id, line, OCF_WRITE); + /* lock was not owned by anyone */ + _req_on_lock(c, ctx, cb, ctx_id, line, OCF_WRITE); return true; } - waiter = NULL; - if (cb) { - /* Need to create waiter */ - waiter = env_allocator_new(c->allocator); - if (!waiter) - return false; - } + waiter = env_allocator_new(c->allocator); + if (!waiter) + return false; __lock_waiters_list(c, line, flags); /* At the moment list is protected, double check if the cache line is * unlocked */ - if (__try_lock_wr(c, line)) { - /* Look get */ - locked = true; - } else if (cb) { - /* Setup waiters filed */ - waiter->line = line; - waiter->ctx = ctx; - waiter->ctx_id = ctx_id; - waiter->cb = cb; - waiter->rw = OCF_WRITE; - INIT_LIST_HEAD(&waiter->item); + if (__try_lock_wr(c, line)) + goto unlock; - /* Add to waiters list */ - __add_waiter(c, line, waiter); - waiting = true; - } + /* Setup waiters filed */ + waiter->line = line; + waiter->ctx = ctx; + waiter->ctx_id = ctx_id; + waiter->cb = cb; + waiter->rw = OCF_WRITE; + INIT_LIST_HEAD(&waiter->item); + /* Add to waiters list */ + __add_waiter(c, line, waiter); + waiting = true; + +unlock: __unlock_waiters_list(c, line, flags); - if (locked && cb) - _req_on_lock(ctx, cb, ctx_id, line, OCF_WRITE); - if (!waiting && waiter) + if (!waiting) { + _req_on_lock(c, ctx, cb, ctx_id, line, OCF_WRITE); env_allocator_del(c->allocator, waiter); + } - return locked || waiting; + return true; } /* @@ -456,20 +462,21 @@ static inline bool __lock_cache_line_rd(struct ocf_cache_line_concurrency *c, void *ctx, uint32_t ctx_id) { struct __waiter *waiter; - bool locked = false; bool waiting = false; unsigned long flags = 0; - if (__try_lock_rd_idle(c, line)) { - /* No activity before look get, it is first reader */ - if (cb) - _req_on_lock(ctx, cb, ctx_id, line, OCF_READ); + ENV_BUG_ON(!cb); + + if( __try_lock_rd_idle(c, line)) { + /* lock was not owned by anyone */ + _req_on_lock(c, ctx, cb, ctx_id, line, OCF_READ); return true; } - waiter = NULL; + waiter = env_allocator_new(c->allocator); + if (!waiter) + return false; -repeat: /* Lock waiters list */ __lock_waiters_list(c, line, flags); @@ -479,23 +486,10 @@ repeat: /* Check if read lock can be obtained */ if (__try_lock_rd(c, line)) { /* Cache line locked */ - locked = true; goto unlock; } } - if (!cb) - goto unlock; - - if (!waiter) { - /* Need to create waiters and add it into list */ - __unlock_waiters_list(c, line, flags); - waiter = env_allocator_new(c->allocator); - if (!waiter) - goto end; - goto repeat; - } - /* Setup waiters field */ waiter->line = line; waiter->ctx = ctx; @@ -511,13 +505,12 @@ repeat: unlock: __unlock_waiters_list(c, line, flags); -end: - if (locked && cb) - _req_on_lock(ctx, cb, ctx_id, line, OCF_READ); - if (!waiting && waiter) + if (!waiting) { + _req_on_lock(c, ctx, cb, ctx_id, line, OCF_READ); env_allocator_del(c->allocator, waiter); + } - return locked || waiting; + return true; } static inline void __unlock_cache_line_rd_common(struct ocf_cache_line_concurrency *c, @@ -569,7 +562,7 @@ static inline void __unlock_cache_line_rd_common(struct ocf_cache_line_concurren exchanged = false; list_del(iter); - _req_on_lock(waiter->ctx, waiter->cb, waiter->ctx_id, + _req_on_lock(c, waiter->ctx, waiter->cb, waiter->ctx_id, line, waiter->rw); env_allocator_del(c->allocator, waiter); @@ -650,7 +643,7 @@ static inline void __unlock_cache_line_wr_common(struct ocf_cache_line_concurren exchanged = false; list_del(iter); - _req_on_lock(waiter->ctx, waiter->cb, waiter->ctx_id, line, + _req_on_lock(c, waiter->ctx, waiter->cb, waiter->ctx_id, line, waiter->rw); env_allocator_del(c->allocator, waiter); @@ -720,11 +713,10 @@ static inline void __remove_line_from_waiters_list(struct ocf_cache_line_concurr /* Try to read-lock request without adding waiters. Function should be called * under read lock, multiple threads may attempt to acquire the lock * concurrently. */ -static int _ocf_req_trylock_rd(struct ocf_request *req) +static int _ocf_req_trylock_rd(struct ocf_cache_line_concurrency *c, + struct ocf_request *req) { int32_t i; - struct ocf_cache_line_concurrency *c = req->cache->device->concurrency. - cache_line; ocf_cache_line_t line; int ret = OCF_LOCK_ACQUIRED; @@ -739,11 +731,11 @@ static int _ocf_req_trylock_rd(struct ocf_request *req) } line = req->map[i].coll_idx; - ENV_BUG_ON(line >= req->cache->device->collision_table_entries); + ENV_BUG_ON(line >= c->num_clines); ENV_BUG_ON(req->map[i].rd_locked); ENV_BUG_ON(req->map[i].wr_locked); - if (__lock_cache_line_rd(c, line, NULL, NULL, 0)) { + if( __try_lock_rd_idle(c, line)) { /* cache line locked */ req->map[i].rd_locked = true; } else { @@ -761,7 +753,7 @@ static int _ocf_req_trylock_rd(struct ocf_request *req) line = req->map[i].coll_idx; if (req->map[i].rd_locked) { - __unlock_rd(c, line); + __unlock_cache_line_rd(c, line); req->map[i].rd_locked = false; } } @@ -774,11 +766,10 @@ static int _ocf_req_trylock_rd(struct ocf_request *req) * Read-lock request cache lines. Must be called under cacheline concurrency * write lock. */ -static int _ocf_req_lock_rd(struct ocf_request *req, ocf_req_async_lock_cb cb) +static int _ocf_req_lock_rd(struct ocf_cache_line_concurrency *c, + struct ocf_request *req, ocf_req_async_lock_cb cb) { int32_t i; - struct ocf_cache_line_concurrency *c = req->cache->device->concurrency. - cache_line; ocf_cache_line_t line; int ret = OCF_LOCK_NOT_ACQUIRED; @@ -797,7 +788,7 @@ static int _ocf_req_lock_rd(struct ocf_request *req, ocf_req_async_lock_cb cb) } line = req->map[i].coll_idx; - ENV_BUG_ON(line >= req->cache->device->collision_table_entries); + ENV_BUG_ON(line >= c->num_clines); ENV_BUG_ON(req->map[i].rd_locked); ENV_BUG_ON(req->map[i].wr_locked); @@ -827,20 +818,17 @@ err: } -int ocf_req_async_lock_rd(struct ocf_request *req, ocf_req_async_lock_cb cb) +int ocf_req_async_lock_rd(struct ocf_cache_line_concurrency *c, + struct ocf_request *req, ocf_req_async_lock_cb cb) { - struct ocf_cache_line_concurrency *c = - req->cache->device->concurrency.cache_line; int lock; - env_rwsem_down_read(&c->lock); - lock = _ocf_req_trylock_rd(req); - env_rwsem_up_read(&c->lock); + lock = _ocf_req_trylock_rd(c, req); if (lock != OCF_LOCK_ACQUIRED) { - env_rwsem_down_write(&c->lock); - lock = _ocf_req_lock_rd(req, cb); - env_rwsem_up_write(&c->lock); + env_mutex_lock(&c->lock); + lock = _ocf_req_lock_rd(c, req, cb); + env_mutex_unlock(&c->lock); } return lock; @@ -849,11 +837,10 @@ int ocf_req_async_lock_rd(struct ocf_request *req, ocf_req_async_lock_cb cb) /* Try to write-lock request without adding waiters. Function should be called * under read lock, multiple threads may attempt to acquire the lock * concurrently. */ -static int _ocf_req_trylock_wr(struct ocf_request *req) +static int _ocf_req_trylock_wr(struct ocf_cache_line_concurrency *c, + struct ocf_request *req) { int32_t i; - struct ocf_cache_line_concurrency *c = req->cache->device->concurrency. - cache_line; ocf_cache_line_t line; int ret = OCF_LOCK_ACQUIRED; @@ -866,11 +853,11 @@ static int _ocf_req_trylock_wr(struct ocf_request *req) } line = req->map[i].coll_idx; - ENV_BUG_ON(line >= req->cache->device->collision_table_entries); + ENV_BUG_ON(line >= c->num_clines); ENV_BUG_ON(req->map[i].rd_locked); ENV_BUG_ON(req->map[i].wr_locked); - if (__lock_cache_line_wr(c, line, NULL, NULL, 0)) { + if (__try_lock_wr(c, line)) { /* cache line locked */ req->map[i].wr_locked = true; } else { @@ -888,7 +875,7 @@ static int _ocf_req_trylock_wr(struct ocf_request *req) line = req->map[i].coll_idx; if (req->map[i].wr_locked) { - __unlock_wr(c, line); + __unlock_cache_line_wr(c, line); req->map[i].wr_locked = false; } } @@ -901,11 +888,10 @@ static int _ocf_req_trylock_wr(struct ocf_request *req) * Write-lock request cache lines. Must be called under cacheline concurrency * write lock. */ -static int _ocf_req_lock_wr(struct ocf_request *req, ocf_req_async_lock_cb cb) +static int _ocf_req_lock_wr(struct ocf_cache_line_concurrency *c, + struct ocf_request *req, ocf_req_async_lock_cb cb) { int32_t i; - struct ocf_cache_line_concurrency *c = req->cache->device->concurrency. - cache_line; ocf_cache_line_t line; int ret = OCF_LOCK_NOT_ACQUIRED; @@ -925,7 +911,7 @@ static int _ocf_req_lock_wr(struct ocf_request *req, ocf_req_async_lock_cb cb) } line = req->map[i].coll_idx; - ENV_BUG_ON(line >= req->cache->device->collision_table_entries); + ENV_BUG_ON(line >= c->num_clines); ENV_BUG_ON(req->map[i].rd_locked); ENV_BUG_ON(req->map[i].wr_locked); @@ -954,20 +940,17 @@ err: return ret; } -int ocf_req_async_lock_wr(struct ocf_request *req, ocf_req_async_lock_cb cb) +int ocf_req_async_lock_wr(struct ocf_cache_line_concurrency *c, + struct ocf_request *req, ocf_req_async_lock_cb cb) { - struct ocf_cache_line_concurrency *c = - req->cache->device->concurrency.cache_line; int lock; - env_rwsem_down_read(&c->lock); - lock = _ocf_req_trylock_wr(req); - env_rwsem_up_read(&c->lock); + lock = _ocf_req_trylock_wr(c, req); if (lock != OCF_LOCK_ACQUIRED) { - env_rwsem_down_write(&c->lock); - lock = _ocf_req_lock_wr(req, cb); - env_rwsem_up_write(&c->lock); + env_mutex_lock(&c->lock); + lock = _ocf_req_lock_wr(c, req, cb); + env_mutex_unlock(&c->lock); } return lock; @@ -977,9 +960,8 @@ int ocf_req_async_lock_wr(struct ocf_request *req, ocf_req_async_lock_cb cb) /* * */ -void ocf_req_unlock_rd(struct ocf_request *req) +void ocf_req_unlock_rd(struct ocf_cache_line_concurrency *c, struct ocf_request *req) { - struct ocf_cache_line_concurrency *c = req->cache->device->concurrency.cache_line; int32_t i; ocf_cache_line_t line; @@ -995,7 +977,7 @@ void ocf_req_unlock_rd(struct ocf_request *req) line = req->map[i].coll_idx; ENV_BUG_ON(!req->map[i].rd_locked); - ENV_BUG_ON(line >= req->cache->device->collision_table_entries); + ENV_BUG_ON(line >= c->num_clines); __unlock_cache_line_rd(c, line); req->map[i].rd_locked = false; @@ -1005,9 +987,8 @@ void ocf_req_unlock_rd(struct ocf_request *req) /* * */ -void ocf_req_unlock_wr(struct ocf_request *req) +void ocf_req_unlock_wr(struct ocf_cache_line_concurrency *c, struct ocf_request *req) { - struct ocf_cache_line_concurrency *c = req->cache->device->concurrency.cache_line; int32_t i; ocf_cache_line_t line; @@ -1023,7 +1004,7 @@ void ocf_req_unlock_wr(struct ocf_request *req) line = req->map[i].coll_idx; ENV_BUG_ON(!req->map[i].wr_locked); - ENV_BUG_ON(line >= req->cache->device->collision_table_entries); + ENV_BUG_ON(line >= c->num_clines); __unlock_cache_line_wr(c, line); req->map[i].wr_locked = false; @@ -1033,9 +1014,8 @@ void ocf_req_unlock_wr(struct ocf_request *req) /* * */ -void ocf_req_unlock(struct ocf_request *req) +void ocf_req_unlock(struct ocf_cache_line_concurrency *c, struct ocf_request *req) { - struct ocf_cache_line_concurrency *c = req->cache->device->concurrency.cache_line; int32_t i; ocf_cache_line_t line; @@ -1049,7 +1029,7 @@ void ocf_req_unlock(struct ocf_request *req) } line = req->map[i].coll_idx; - ENV_BUG_ON(line >= req->cache->device->collision_table_entries); + ENV_BUG_ON(line >= c->num_clines); if (req->map[i].rd_locked && req->map[i].wr_locked) { ENV_BUG(); @@ -1068,11 +1048,9 @@ void ocf_req_unlock(struct ocf_request *req) /* * */ -void ocf_req_unlock_entry(struct ocf_cache *cache, +void ocf_req_unlock_entry(struct ocf_cache_line_concurrency *c, struct ocf_request *req, uint32_t entry) { - struct ocf_cache_line_concurrency *c = req->cache->device->concurrency.cache_line; - ENV_BUG_ON(req->map[entry].status == LOOKUP_MISS); if (req->map[entry].rd_locked && req->map[entry].wr_locked) { @@ -1091,17 +1069,15 @@ void ocf_req_unlock_entry(struct ocf_cache *cache, /* * */ -bool ocf_cache_line_is_used(struct ocf_cache *cache, +bool ocf_cache_line_is_used(struct ocf_cache_line_concurrency *c, ocf_cache_line_t line) { - struct ocf_cache_line_concurrency *c = cache->device->concurrency.cache_line; - - ENV_BUG_ON(line >= cache->device->collision_table_entries); + ENV_BUG_ON(line >= c->num_clines); if (env_atomic_read(&(c->access[line]))) return true; - if (ocf_cache_line_are_waiters(cache, line)) + if (ocf_cache_line_are_waiters(c, line)) return true; else return false; @@ -1110,14 +1086,13 @@ bool ocf_cache_line_is_used(struct ocf_cache *cache, /* * */ -bool ocf_cache_line_are_waiters(struct ocf_cache *cache, +bool ocf_cache_line_are_waiters(struct ocf_cache_line_concurrency *c, ocf_cache_line_t line) { - struct ocf_cache_line_concurrency *c = cache->device->concurrency.cache_line; bool are; unsigned long flags = 0; - ENV_BUG_ON(line >= cache->device->collision_table_entries); + ENV_BUG_ON(line >= c->num_clines); /* Lock waiters list */ __lock_waiters_list(c, line, flags); @@ -1132,42 +1107,35 @@ bool ocf_cache_line_are_waiters(struct ocf_cache *cache, /* * */ -uint32_t ocf_cache_line_concurrency_suspended_no(struct ocf_cache *cache) +uint32_t ocf_cache_line_concurrency_suspended_no(struct ocf_cache_line_concurrency *c) { - struct ocf_cache_line_concurrency *c = cache->device->concurrency.cache_line; - return env_atomic_read(&c->waiting); } -bool ocf_cache_line_try_lock_rd(struct ocf_cache *cache, ocf_cache_line_t line) +bool ocf_cache_line_try_lock_rd(struct ocf_cache_line_concurrency *c, + ocf_cache_line_t line) { - struct ocf_cache_line_concurrency *c = cache->device->concurrency.cache_line; - return __lock_cache_line_rd(c, line, NULL, NULL, 0); + return __try_lock_rd_idle(c, line); } /* * */ -void ocf_cache_line_unlock_rd(struct ocf_cache *cache, ocf_cache_line_t line) +void ocf_cache_line_unlock_rd(struct ocf_cache_line_concurrency *c, ocf_cache_line_t line) { - struct ocf_cache_line_concurrency *c = cache->device->concurrency.cache_line; - - OCF_DEBUG_RQ(cache, "Cache line = %u", line); - + OCF_DEBUG_RQ(c->cache, "Cache line = %u", line); __unlock_cache_line_rd(c, line); } -bool ocf_cache_line_try_lock_wr(struct ocf_cache *cache, ocf_cache_line_t line) +bool ocf_cache_line_try_lock_wr(struct ocf_cache_line_concurrency *c, + ocf_cache_line_t line) { - struct ocf_cache_line_concurrency *c = cache->device->concurrency.cache_line; - return __lock_cache_line_wr(c, line, NULL, NULL, 0); + return __try_lock_wr(c, line); } -void ocf_cache_line_unlock_wr(struct ocf_cache *cache, ocf_cache_line_t line) +void ocf_cache_line_unlock_wr(struct ocf_cache_line_concurrency *c, + ocf_cache_line_t line) { - struct ocf_cache_line_concurrency *c = cache->device->concurrency.cache_line; - - OCF_DEBUG_RQ(cache, "Cache line = %u", line); - + OCF_DEBUG_RQ(c->cache, "Cache line = %u", line); __unlock_cache_line_wr(c, line); } diff --git a/src/concurrency/ocf_cache_line_concurrency.h b/src/concurrency/ocf_cache_line_concurrency.h index 42a40cf..29895fe 100644 --- a/src/concurrency/ocf_cache_line_concurrency.h +++ b/src/concurrency/ocf_cache_line_concurrency.h @@ -19,27 +19,32 @@ struct ocf_cache_line_concurrency; /** * @brief Initialize OCF cache concurrency module * + * @param self - cacheline concurrency private data + * @param num_clines - cachelines count * @param cache - OCF cache instance + * @return 0 - Initialization successful, otherwise ERROR */ -int ocf_cache_line_concurrency_init(struct ocf_cache *cache); +int ocf_cache_line_concurrency_init(struct ocf_cache_line_concurrency **self, + unsigned num_clines, struct ocf_cache *cache); /** * @biref De-Initialize OCF cache concurrency module * - * @param cache - OCF cache instance + * @param self - cacheline concurrency private data */ -void ocf_cache_line_concurrency_deinit(struct ocf_cache *cache); +void ocf_cache_line_concurrency_deinit( + struct ocf_cache_line_concurrency **self); /** * @brief Get number of waiting (suspended) OCF requests in due to cache * overlapping * - * @param cache - OCF cache instance + * @param c - cacheline concurrency private data * * @return Number of suspended OCF requests */ -uint32_t ocf_cache_line_concurrency_suspended_no(struct ocf_cache *cache); +uint32_t ocf_cache_line_concurrency_suspended_no(struct ocf_cache_line_concurrency *c); /** * @brief Return memory footprint conusmed by cache concurrency module @@ -48,7 +53,7 @@ uint32_t ocf_cache_line_concurrency_suspended_no(struct ocf_cache *cache); * * @return Memory footprint of cache concurrency module */ -size_t ocf_cache_line_concurrency_size_of(struct ocf_cache *cache); +size_t ocf_cache_line_concurrency_size_of(ocf_cache_t cache); /* async request cacheline lock acquisition callback */ typedef void (*ocf_req_async_lock_cb)(struct ocf_request *req); @@ -56,6 +61,7 @@ typedef void (*ocf_req_async_lock_cb)(struct ocf_request *req); /** * @brief Lock OCF request for write access (Lock all cache lines in map info) * + * @param c - cacheline concurrency private data * @param req - OCF request * @param cb - async lock acquisition callback * @@ -65,11 +71,13 @@ typedef void (*ocf_req_async_lock_cb)(struct ocf_request *req); * @retval OCF_LOCK_NOT_ACQUIRED - OCF request lock not acquired, request was * added into waiting list. When lock will be acquired @cb cllback be called */ -int ocf_req_async_lock_wr(struct ocf_request *req, ocf_req_async_lock_cb cb); +int ocf_req_async_lock_wr(struct ocf_cache_line_concurrency *c, + struct ocf_request *req, ocf_req_async_lock_cb cb); /** * @brief Lock OCF request for read access (Lock all cache lines in map info) * + * @param c - cacheline concurrency private data * @param req - OCF request * @param cb - async lock acquisition callback * @@ -79,28 +87,35 @@ int ocf_req_async_lock_wr(struct ocf_request *req, ocf_req_async_lock_cb cb); * @retval OCF_LOCK_NOT_ACQUIRED - OCF request lock not acquired, request was * added into waiting list. When lock will be acquired @cb callback be called */ -int ocf_req_async_lock_rd(struct ocf_request *req, ocf_req_async_lock_cb cb); +int ocf_req_async_lock_rd(struct ocf_cache_line_concurrency *c, + struct ocf_request *req, ocf_req_async_lock_cb cb); /** * @brief Unlock OCF request from write access * + * @param c - cacheline concurrency private data * @param req - OCF request */ -void ocf_req_unlock_wr(struct ocf_request *req); +void ocf_req_unlock_wr(struct ocf_cache_line_concurrency *c, + struct ocf_request *req); /** * @brief Unlock OCF request from read access * + * @param c - cacheline concurrency private data * @param req - OCF request */ -void ocf_req_unlock_rd(struct ocf_request *req); +void ocf_req_unlock_rd(struct ocf_cache_line_concurrency *c, + struct ocf_request *req); /** * @brief Unlock OCF request from read or write access * + * @param c - cacheline concurrency private data * @param req - OCF request */ -void ocf_req_unlock(struct ocf_request *req); +void ocf_req_unlock(struct ocf_cache_line_concurrency *c, + struct ocf_request *req); /** * @Check if cache line is used. @@ -116,30 +131,30 @@ void ocf_req_unlock(struct ocf_request *req); * @retval true - cache line is used * @retval false - cache line is not used */ -bool ocf_cache_line_is_used(struct ocf_cache *cache, +bool ocf_cache_line_is_used(struct ocf_cache_line_concurrency *c, ocf_cache_line_t line); /** * @brief Check if for specified cache line there are waiters * on the waiting list * - * @param cache - OCF cache instance + * @param c - cacheline concurrency private data * @param line - Cache line to be checked for waiters * * @retval true - there are waiters * @retval false - No waiters */ -bool ocf_cache_line_are_waiters(struct ocf_cache *cache, +bool ocf_cache_line_are_waiters(struct ocf_cache_line_concurrency *c, ocf_cache_line_t line); /** * @brief un_lock request map info entry from from write or read access. * - * @param cache - OCF cache instance + * @param c - cacheline concurrency private data * @param req - OCF request * @param entry - request map entry number */ -void ocf_req_unlock_entry(struct ocf_cache *cache, +void ocf_req_unlock_entry(struct ocf_cache_line_concurrency *c, struct ocf_request *req, uint32_t entry); /** @@ -148,36 +163,40 @@ void ocf_req_unlock_entry(struct ocf_cache *cache, * @param cache - OCF cache instance * @param line - Cache line to be unlocked */ -void ocf_cache_line_unlock_rd(struct ocf_cache *cache, ocf_cache_line_t line); +void ocf_cache_line_unlock_rd(struct ocf_cache_line_concurrency *c, + ocf_cache_line_t line); /** * @brief Attempt to lock cache line for read * - * @param cache - OCF cache instance + * @param c - cacheline concurrency private data * @param line - Cache line to be checked for waiters * * @retval true - read lock successfully acquired * @retval false - failed to acquire read lock */ -bool ocf_cache_line_try_lock_rd(struct ocf_cache *cache, ocf_cache_line_t line); +bool ocf_cache_line_try_lock_rd(struct ocf_cache_line_concurrency *c, + ocf_cache_line_t line); /** * @brief Release cache line write lock * - * @param cache - OCF cache instance + * @param c - cacheline concurrency private data * @param line - Cache line to be unlocked */ -void ocf_cache_line_unlock_wr(struct ocf_cache *cache, ocf_cache_line_t line); +void ocf_cache_line_unlock_wr(struct ocf_cache_line_concurrency *c, + ocf_cache_line_t line); /** * @brief Attempt to lock cache line for write * - * @param cache - OCF cache instance + * @param c - cacheline concurrency private data * @param line - Cache line to be checked for waiters * * @retval true - write lock successfully acquired * @retval false - failed to acquire write lock */ -bool ocf_cache_line_try_lock_wr(struct ocf_cache *cache, ocf_cache_line_t line); +bool ocf_cache_line_try_lock_wr(struct ocf_cache_line_concurrency *c, + ocf_cache_line_t line); #endif /* OCF_CONCURRENCY_H_ */ diff --git a/src/concurrency/ocf_concurrency.c b/src/concurrency/ocf_concurrency.c index 24ebc8e..f469f7c 100644 --- a/src/concurrency/ocf_concurrency.c +++ b/src/concurrency/ocf_concurrency.c @@ -4,12 +4,16 @@ */ #include "ocf_concurrency.h" +#include "../metadata/metadata.h" int ocf_concurrency_init(struct ocf_cache *cache) { int result = 0; - result = ocf_cache_line_concurrency_init(cache); + result = ocf_cache_line_concurrency_init( + &cache->device->concurrency.cache_line, + ocf_metadata_collision_table_entries(cache), + cache); if (result) ocf_concurrency_deinit(cache); @@ -19,6 +23,7 @@ int ocf_concurrency_init(struct ocf_cache *cache) void ocf_concurrency_deinit(struct ocf_cache *cache) { - ocf_cache_line_concurrency_deinit(cache); + ocf_cache_line_concurrency_deinit( + &cache->device->concurrency.cache_line); } diff --git a/src/engine/engine_bf.c b/src/engine/engine_bf.c index afa59fa..79249ed 100644 --- a/src/engine/engine_bf.c +++ b/src/engine/engine_bf.c @@ -64,7 +64,7 @@ static void _ocf_backfill_complete(struct ocf_request *req, int error) ocf_core_stats_cache_error_update(req->core, OCF_WRITE); ocf_engine_invalidate(req); } else { - ocf_req_unlock(req); + ocf_req_unlock(cache->device->concurrency.cache_line, req); /* put the request at the last point of the completion path */ ocf_req_put(req); diff --git a/src/engine/engine_common.c b/src/engine/engine_common.c index db7569f..9401b6e 100644 --- a/src/engine/engine_common.c +++ b/src/engine/engine_common.c @@ -396,7 +396,8 @@ static void _ocf_engine_clean_end(void *private_data, int error) req->error |= error; /* End request and do not processing */ - ocf_req_unlock(req); + ocf_req_unlock(req->cache->device->concurrency.cache_line, + req); /* Complete request */ req->complete(req, error); @@ -414,12 +415,14 @@ static int lock_clines(struct ocf_request *req, const struct ocf_engine_callbacks *engine_cbs) { enum ocf_engine_lock_type lock_type = engine_cbs->get_lock_type(req); + struct ocf_cache_line_concurrency *c = + req->cache->device->concurrency.cache_line; switch (lock_type) { case ocf_engine_lock_write: - return ocf_req_async_lock_wr(req, engine_cbs->resume); + return ocf_req_async_lock_wr(c, req, engine_cbs->resume); case ocf_engine_lock_read: - return ocf_req_async_lock_rd(req, engine_cbs->resume); + return ocf_req_async_lock_rd(c, req, engine_cbs->resume); default: return OCF_LOCK_ACQUIRED; } @@ -703,7 +706,8 @@ static int _ocf_engine_refresh(struct ocf_request *req) req->complete(req, req->error); /* Release WRITE lock of request */ - ocf_req_unlock(req); + ocf_req_unlock(req->cache->device->concurrency.cache_line, + req); /* Release OCF request */ ocf_req_put(req); diff --git a/src/engine/engine_discard.c b/src/engine/engine_discard.c index a2d97c6..a5a8678 100644 --- a/src/engine/engine_discard.c +++ b/src/engine/engine_discard.c @@ -147,7 +147,7 @@ static void _ocf_discard_step_complete(struct ocf_request *req, int error) OCF_DEBUG_RQ(req, "Completion"); /* Release WRITE lock of request */ - ocf_req_unlock_wr(req); + ocf_req_unlock_wr(req->cache->device->concurrency.cache_line, req); if (req->error) { ocf_metadata_error(req->cache); @@ -235,7 +235,9 @@ static int _ocf_discard_step(struct ocf_request *req) if (ocf_engine_mapped_count(req)) { /* Some cache line are mapped, lock request for WRITE access */ - lock = ocf_req_async_lock_wr(req, _ocf_discard_on_resume); + lock = ocf_req_async_lock_wr( + cache->device->concurrency.cache_line, + req, _ocf_discard_on_resume); } else { lock = OCF_LOCK_ACQUIRED; } diff --git a/src/engine/engine_fast.c b/src/engine/engine_fast.c index abb9695..42e1047 100644 --- a/src/engine/engine_fast.c +++ b/src/engine/engine_fast.c @@ -46,7 +46,8 @@ static void _ocf_read_fast_complete(struct ocf_request *req, int error) ocf_core_stats_cache_error_update(req->core, OCF_READ); ocf_engine_push_req_front_pt(req); } else { - ocf_req_unlock(req); + ocf_req_unlock(req->cache->device->concurrency.cache_line, + req); /* Complete request */ req->complete(req, req->error); @@ -130,7 +131,9 @@ int ocf_read_fast(struct ocf_request *req) if (hit && part_has_space) { ocf_io_start(&req->ioi.io); - lock = ocf_req_async_lock_rd(req, ocf_engine_on_resume); + lock = ocf_req_async_lock_rd( + req->cache->device->concurrency.cache_line, + req, ocf_engine_on_resume); } ocf_hb_req_prot_unlock_rd(req); @@ -200,7 +203,9 @@ int ocf_write_fast(struct ocf_request *req) if (mapped && part_has_space) { ocf_io_start(&req->ioi.io); - lock = ocf_req_async_lock_wr(req, ocf_engine_on_resume); + lock = ocf_req_async_lock_wr( + req->cache->device->concurrency.cache_line, + req, ocf_engine_on_resume); } ocf_hb_req_prot_unlock_rd(req); diff --git a/src/engine/engine_inv.c b/src/engine/engine_inv.c index 0a15137..ee3af47 100644 --- a/src/engine/engine_inv.c +++ b/src/engine/engine_inv.c @@ -31,7 +31,7 @@ static void _ocf_invalidate_req(struct ocf_request *req, int error) if (req->error) ocf_engine_error(req, true, "Failed to flush metadata to cache"); - ocf_req_unlock(req); + ocf_req_unlock(req->cache->device->concurrency.cache_line, req); /* Put OCF request - decrease reference counter */ ocf_req_put(req); diff --git a/src/engine/engine_pt.c b/src/engine/engine_pt.c index 4ddb33b..3cb321d 100644 --- a/src/engine/engine_pt.c +++ b/src/engine/engine_pt.c @@ -34,7 +34,7 @@ static void _ocf_read_pt_complete(struct ocf_request *req, int error) /* Complete request */ req->complete(req, req->error); - ocf_req_unlock_rd(req); + ocf_req_unlock_rd(req->cache->device->concurrency.cache_line, req); /* Release OCF request */ ocf_req_put(req); @@ -127,7 +127,10 @@ int ocf_read_pt(struct ocf_request *req) /* There are mapped cache line, * lock request for READ access */ - lock = ocf_req_async_lock_rd(req, ocf_engine_on_resume); + lock = ocf_req_async_lock_rd( + req->cache->device->concurrency. + cache_line, + req, ocf_engine_on_resume); } else { /* No mapped cache lines, no need to get lock */ lock = OCF_LOCK_ACQUIRED; diff --git a/src/engine/engine_rd.c b/src/engine/engine_rd.c index b18369a..6a9f778 100644 --- a/src/engine/engine_rd.c +++ b/src/engine/engine_rd.c @@ -24,6 +24,9 @@ static void _ocf_read_generic_hit_complete(struct ocf_request *req, int error) { + struct ocf_cache_line_concurrency *c = + req->cache->device->concurrency.cache_line; + if (error) req->error |= error; @@ -41,8 +44,7 @@ static void _ocf_read_generic_hit_complete(struct ocf_request *req, int error) ocf_core_stats_cache_error_update(req->core, OCF_READ); ocf_engine_push_req_front_pt(req); } else { - - ocf_req_unlock(req); + ocf_req_unlock(c, req); /* Complete request */ req->complete(req, req->error); diff --git a/src/engine/engine_wb.c b/src/engine/engine_wb.c index 95bd8ee..70ab3cb 100644 --- a/src/engine/engine_wb.c +++ b/src/engine/engine_wb.c @@ -60,7 +60,7 @@ static void _ocf_write_wb_io_flush_metadata(struct ocf_request *req, int error) if (req->error) ocf_engine_error(req, true, "Failed to write data to cache"); - ocf_req_unlock_wr(req); + ocf_req_unlock_wr(req->cache->device->concurrency.cache_line, req); req->complete(req, req->error); diff --git a/src/engine/engine_wi.c b/src/engine/engine_wi.c index 52a9242..9b8d005 100644 --- a/src/engine/engine_wi.c +++ b/src/engine/engine_wi.c @@ -25,7 +25,7 @@ static const struct ocf_io_if _io_if_wi_update_metadata = { int _ocf_write_wi_next_pass(struct ocf_request *req) { - ocf_req_unlock_wr(req); + ocf_req_unlock_wr(req->cache->device->concurrency.cache_line, req); if (req->wi_second_pass) { req->complete(req, req->error); @@ -75,7 +75,7 @@ static void _ocf_write_wi_io_flush_metadata(struct ocf_request *req, int error) if (req->error) ocf_engine_error(req, true, "Failed to write data to cache"); - ocf_req_unlock_wr(req); + ocf_req_unlock_wr(req->cache->device->concurrency.cache_line, req); req->complete(req, req->error); @@ -128,7 +128,8 @@ static void _ocf_write_wi_core_complete(struct ocf_request *req, int error) OCF_DEBUG_RQ(req, "Completion"); if (req->error) { - ocf_req_unlock_wr(req); + ocf_req_unlock_wr(req->cache->device->concurrency.cache_line, + req); req->complete(req, req->error); @@ -198,7 +199,9 @@ int ocf_write_wi(struct ocf_request *req) if (ocf_engine_mapped_count(req)) { /* Some cache line are mapped, lock request for WRITE access */ - lock = ocf_req_async_lock_wr(req, _ocf_write_wi_on_resume); + lock = ocf_req_async_lock_wr( + req->cache->device->concurrency.cache_line, + req, _ocf_write_wi_on_resume); } else { lock = OCF_LOCK_ACQUIRED; } diff --git a/src/engine/engine_wo.c b/src/engine/engine_wo.c index 0b4a4bc..d210186 100644 --- a/src/engine/engine_wo.c +++ b/src/engine/engine_wo.c @@ -33,7 +33,7 @@ static void ocf_read_wo_cache_complete(struct ocf_request *req, int error) if (req->error) ocf_engine_error(req, true, "Failed to read data from cache"); - ocf_req_unlock_rd(req); + ocf_req_unlock_rd(req->cache->device->concurrency.cache_line, req); /* Complete request */ req->complete(req, req->error); @@ -169,7 +169,8 @@ static void _ocf_read_wo_core_complete(struct ocf_request *req, int error) if (!req->info.dirty_any || req->error) { OCF_DEBUG_RQ(req, "Completion"); req->complete(req, req->error); - ocf_req_unlock_rd(req); + ocf_req_unlock_rd(req->cache->device->concurrency.cache_line, + req); ocf_req_put(req); return; } @@ -236,7 +237,9 @@ int ocf_read_wo(struct ocf_request *req) /* There are mapped cache lines, * lock request for READ access */ - lock = ocf_req_async_lock_rd(req, ocf_engine_on_resume); + lock = ocf_req_async_lock_rd( + req->cache->device->concurrency.cache_line, + req, ocf_engine_on_resume); } ocf_hb_req_prot_unlock_rd(req); /*- END Metadata RD access -----------------*/ diff --git a/src/engine/engine_wt.c b/src/engine/engine_wt.c index d786a0f..a8eb087 100644 --- a/src/engine/engine_wt.c +++ b/src/engine/engine_wt.c @@ -34,7 +34,8 @@ static void _ocf_write_wt_req_complete(struct ocf_request *req) ocf_engine_invalidate(req); } else { /* Unlock reqest from WRITE access */ - ocf_req_unlock_wr(req); + ocf_req_unlock_wr(req->cache->device->concurrency.cache_line, + req); /* Complete request */ req->complete(req, req->info.core_error ? req->error : 0); diff --git a/src/engine/engine_zero.c b/src/engine/engine_zero.c index 2294064..4dd9c0a 100644 --- a/src/engine/engine_zero.c +++ b/src/engine/engine_zero.c @@ -31,7 +31,7 @@ static int ocf_zero_purge(struct ocf_request *req) ocf_hb_req_prot_unlock_wr(req); /*- END Metadata WR access ---------*/ } - ocf_req_unlock_wr(req); + ocf_req_unlock_wr(req->cache->device->concurrency.cache_line, req); req->complete(req, req->error); @@ -152,7 +152,9 @@ void ocf_engine_zero_line(struct ocf_request *req) req->io_if = &_io_if_ocf_zero_do; /* Some cache line are mapped, lock request for WRITE access */ - lock = ocf_req_async_lock_wr(req, ocf_engine_on_resume); + lock = ocf_req_async_lock_wr( + req->cache->device->concurrency.cache_line, + req, ocf_engine_on_resume); if (lock >= 0) { ENV_BUG_ON(lock != OCF_LOCK_ACQUIRED); diff --git a/src/eviction/lru.c b/src/eviction/lru.c index e63573c..c3bbc5a 100644 --- a/src/eviction/lru.c +++ b/src/eviction/lru.c @@ -357,7 +357,9 @@ static int evp_lru_clean_getter(ocf_cache_t cache, void *getter_context, break; /* Prevent evicting already locked items */ - if (ocf_cache_line_is_used(cache, cline)) { + if (ocf_cache_line_is_used( + cache->device->concurrency.cache_line, + cline)) { continue; } @@ -490,8 +492,11 @@ uint32_t evp_lru_req_clines(ocf_cache_t cache, ocf_queue_t io_queue, break; /* Prevent evicting already locked items */ - if (ocf_cache_line_is_used(cache, cline)) + if (ocf_cache_line_is_used( + cache->device->concurrency.cache_line, + cline)) { continue; + } ENV_BUG_ON(metadata_test_dirty(cache, cline)); diff --git a/src/metadata/metadata_misc.c b/src/metadata/metadata_misc.c index 62639d7..ff47f2e 100644 --- a/src/metadata/metadata_misc.c +++ b/src/metadata/metadata_misc.c @@ -51,6 +51,8 @@ int ocf_metadata_actor(struct ocf_cache *cache, ocf_cache_line_t i, next_i; uint64_t start_line, end_line; int ret = 0; + struct ocf_cache_line_concurrency *c = + cache->device->concurrency.cache_line; start_line = ocf_bytes_2_lines(cache, start_byte); end_line = ocf_bytes_2_lines(cache, end_byte); @@ -63,7 +65,7 @@ int ocf_metadata_actor(struct ocf_cache *cache, if (_is_cache_line_acting(cache, i, core_id, start_line, end_line)) { - if (ocf_cache_line_is_used(cache, i)) + if (ocf_cache_line_is_used(c, i)) ret = -OCF_ERR_AGAIN; else actor(cache, i); @@ -75,7 +77,7 @@ int ocf_metadata_actor(struct ocf_cache *cache, for (i = 0; i < cache->device->collision_table_entries; ++i) { if (_is_cache_line_acting(cache, i, core_id, start_line, end_line)) { - if (ocf_cache_line_is_used(cache, i)) + if (ocf_cache_line_is_used(c, i)) ret = -OCF_ERR_AGAIN; else actor(cache, i); diff --git a/src/mngt/ocf_mngt_common.c b/src/mngt/ocf_mngt_common.c index 8c63cdc..82a2873 100644 --- a/src/mngt/ocf_mngt_common.c +++ b/src/mngt/ocf_mngt_common.c @@ -80,14 +80,18 @@ void cache_mngt_core_deinit_attached_meta(ocf_core_t core) continue; } - if (!ocf_cache_line_try_lock_wr(cache, curr_cline)) + if (!ocf_cache_line_try_lock_wr( + cache->device->concurrency.cache_line, + curr_cline)) { break; + } if (metadata_test_dirty(cache, curr_cline)) ocf_purge_cleaning_policy(cache, curr_cline); ocf_metadata_sparse_cache_line(cache, curr_cline); - ocf_cache_line_unlock_wr(cache, curr_cline); + ocf_cache_line_unlock_wr(cache->device->concurrency.cache_line, + curr_cline); if (prev_cline != cache->device->collision_table_entries) curr_cline = ocf_metadata_get_collision_next(cache, prev_cline); diff --git a/src/utils/utils_cache_line.c b/src/utils/utils_cache_line.c index 02d2d36..0d3ab3d 100644 --- a/src/utils/utils_cache_line.c +++ b/src/utils/utils_cache_line.c @@ -43,7 +43,9 @@ static void __set_cache_line_invalid(struct ocf_cache *cache, uint8_t start_bit, * for this cache line which will use one, clear * only valid bits */ - if (!is_valid && !ocf_cache_line_are_waiters(cache, line)) { + if (!is_valid && !ocf_cache_line_are_waiters( + cache->device->concurrency.cache_line, + line)) { ocf_purge_eviction_policy(cache, line); ocf_metadata_remove_cache_line(cache, line); } diff --git a/src/utils/utils_cleaner.c b/src/utils/utils_cleaner.c index 2f536dc..5c07aa4 100644 --- a/src/utils/utils_cleaner.c +++ b/src/utils/utils_cleaner.c @@ -213,7 +213,9 @@ static int _ocf_cleaner_cache_line_lock(struct ocf_request *req) OCF_DEBUG_TRACE(req->cache); - return ocf_req_async_lock_rd(req, _ocf_cleaner_on_resume); + return ocf_req_async_lock_rd( + req->cache->device->concurrency.cache_line, + req, _ocf_cleaner_on_resume); } /* @@ -224,7 +226,8 @@ static void _ocf_cleaner_cache_line_unlock(struct ocf_request *req) { if (req->info.cleaner_cache_line_lock) { OCF_DEBUG_TRACE(req->cache); - ocf_req_unlock(req); + ocf_req_unlock(req->cache->device->concurrency.cache_line, + req); } } diff --git a/tests/unit/tests/concurrency/ocf_cache_line_concurrency.c/ocf_cache_line_concurrency.c b/tests/unit/tests/concurrency/ocf_cache_line_concurrency.c/ocf_cache_line_concurrency.c new file mode 100644 index 0000000..24c54d8 --- /dev/null +++ b/tests/unit/tests/concurrency/ocf_cache_line_concurrency.c/ocf_cache_line_concurrency.c @@ -0,0 +1,482 @@ +/* + * src/concurrency/ocf_cache_line_concurrency.c + * ocf_req_async_lock_rd + * + * ocf_cache_line_concurrency_init + * ocf_cache_line_concurrency_deinit + * ocf_req_async_lock_rd + * ocf_req_async_lock_wr + * ocf_req_unlock_wr + * ocf_req_unlock_rd + * ocf_req_unlock + * ocf_cache_line_unlock_rd + * ocf_cache_line_unlock_wr + * ocf_cache_line_try_lock_rd + * ocf_cache_line_try_lock_wr + * ocf_cache_line_is_used + * __are_waiters + * __add_waiter + * __try_lock_wr + * __try_lock_rd_idle + * __try_lock_rd + * __unlock_wr + * __unlock_rd + * __try_lock_wr2wr + * __try_lock_wr2rd + * __try_lock_rd2wr + * __try_lock_rd2rd + * __lock_cache_line_wr + * __lock_cache_line_rd + * __unlock_cache_line_rd_common + * __unlock_cache_line_rd + * __unlock_cache_line_wr_common + * __unlock_cache_line_wr + * __remove_line_from_waiters_list + * _ocf_req_needs_cl_lock + * _ocf_req_trylock_rd + * _ocf_req_lock_rd + * _ocf_req_lock_wr + * _ocf_req_trylock_wr + * _req_on_lock + * ocf_cache_line_are_waiters + * + */ + +#undef static + +#undef inline + +#define TEST_MAX_MAP_SIZE 32 + +#include +#include +#include +#include +#include +#include +#include "print_desc.h" + +#include "ocf_concurrency.h" +#include "../ocf_priv.h" +#include "../ocf_request.h" +#include "../utils/utils_cache_line.h" +#include "../utils/utils_realloc.h" + +#include "concurrency/ocf_cache_line_concurrency.c/ocf_cache_line_concurrency_generated_wraps.c" + +#define LOCK_WAIT_TIMEOUT 5 + +void __wrap___assert_fail (const char *__assertion, const char *__file, + unsigned int __line, const char *__function) +{ + print_message("assertion failure %s in %s:%u %s\n", + __assertion, __file, __line, __function); +} + +int __wrap_list_empty(struct list_head *l1) +{ + return l1->next == l1; +} + +void __wrap_list_del(struct list_head *it) +{ + it->next->prev = it->prev; + it->prev->next = it->next; +} + +void __wrap_list_add_tail(struct list_head *it, struct list_head *l1) +{ + it->prev = l1->prev; + it->next = l1; + + l1->prev->next = it; + l1->prev = it; +} + +void __wrap_ocf_realloc_cp(void** mem, size_t size, size_t count, size_t *limit) +{ + if (*mem) + free(*mem); + + if (count == 0) + return; + + *mem = malloc(size * count); + memset(*mem, 0, size * count); + + *limit = count; +} + +const char *__wrap_ocf_cache_get_name(ocf_cache_t cache) +{ + return "test"; +} + +int __wrap_snprintf (char *__restrict __s, size_t __maxlen, + const char *__restrict __format, ...) +{ + va_list args; + int ret; + + va_start(args, __format); + ret = vsnprintf(__s, __maxlen, __format, args); + va_end(args); + + return ret; +} + +static inline bool __wrap___lock_cache_line_wr(struct ocf_cache_line_concurrency *c, + const ocf_cache_line_t line, ocf_req_async_lock_cb cb, + void *ctx, uint32_t ctx_id) +{ + usleep(rand() % 100); + return __real___lock_cache_line_wr(c, line, cb, ctx, ctx_id); +} + +static inline bool __wrap___lock_cache_line_rd(struct ocf_cache_line_concurrency *c, + const ocf_cache_line_t line, ocf_req_async_lock_cb cb, + void *ctx, uint32_t ctx_id) +{ + usleep(rand() % 100); + return __real___lock_cache_line_rd(c, line, cb, ctx, ctx_id); +} + +int __wrap__ocf_req_lock_wr(struct ocf_request *req, ocf_req_async_lock_cb cb) +{ + usleep(rand() % 500); + return __real__ocf_req_lock_wr(req, cb); +} + +int __wrap__ocf_req_lock_rd(struct ocf_request *req, ocf_req_async_lock_cb cb) +{ + usleep(rand() % 500); + return __real__ocf_req_lock_wr(req, cb); +} + +unsigned long long progress; +pthread_cond_t prog_cond = PTHREAD_COND_INITIALIZER; +pthread_mutex_t prog_mutex = PTHREAD_MUTEX_INITIALIZER; + +struct test_req { + struct ocf_request r; + struct ocf_map_info map[TEST_MAX_MAP_SIZE]; + pthread_cond_t completion; + pthread_mutex_t completion_mutex; + bool finished; +}; + +static void req_async_lock_callback(struct ocf_request *req) +{ + struct test_req* treq = (struct test_req *)req; + + pthread_mutex_lock(&treq->completion_mutex); + treq->finished = true; + pthread_cond_signal(&treq->completion); + pthread_mutex_unlock(&treq->completion_mutex); +} + +bool req_lock_sync(struct ocf_cache_line_concurrency *c, struct ocf_request *req, + int (*pfn)(struct ocf_cache_line_concurrency *c, struct ocf_request *req, + void (*cmpl)(struct ocf_request *req)), + volatile int *finish) +{ + struct test_req* treq = (struct test_req *)req; + int result; + bool timeout = false; + struct timespec ts; + + treq->finished = false; + + result = pfn(c, req, req_async_lock_callback); + assert(result >= 0); + + if (result == OCF_LOCK_ACQUIRED) { + return true; + } + + pthread_mutex_lock(&treq->completion_mutex); + while (!treq->finished && !*finish) { + pthread_cond_wait(&treq->completion, + &treq->completion_mutex); + } + pthread_mutex_unlock(&treq->completion_mutex); + + return treq->finished; +} + +struct thread_ctx +{ + pthread_t t; + struct ocf_cache_line_concurrency *c; + unsigned num_iterations; + unsigned clines; + unsigned max_io_size; + bool timeout; + volatile int finished; + volatile int terminated; + struct test_req treq; +}; + +void shuffle(unsigned *array, unsigned size) +{ + int i, j; + unsigned tmp; + + for (i = size - 1; i >= 0; i--) + { + j = rand() % (i + 1); + + tmp = array[i]; + array[i] = array[j]; + array[j] = tmp; + } +} + +void thread(void *_ctx) +{ + struct thread_ctx *ctx = _ctx; + struct ocf_cache_line_concurrency *c = ctx->c; + struct ocf_request *req = &ctx->treq.r; + unsigned i; + unsigned cline; + unsigned *permutation; + bool rw; + bool single; + int (*lock_pfn)(struct ocf_cache_line_concurrency *c, struct ocf_request *req, + void (*cmpl)(struct ocf_request *req)); + unsigned max_io_size = min(min(TEST_MAX_MAP_SIZE, ctx->clines), ctx->max_io_size); + unsigned line; + bool locked; + + ctx->treq.r.map = &ctx->treq.map; + pthread_cond_init(&ctx->treq.completion, NULL); + pthread_mutex_init(&ctx->treq.completion_mutex, NULL); + + permutation = malloc(ctx->clines * sizeof(unsigned)); + + for (i = 0; i < ctx->clines; i++) + permutation[i] = i; + + i = ctx->num_iterations; + while (i-- && !ctx->terminated) + { + rw = rand() % 2; + single = (rand() % 4 == 0); + + if (!single) { + shuffle(permutation, ctx->clines); + req->core_line_count = (rand() % max_io_size) + 1; + + for (cline = 0; cline < req->core_line_count; cline++) { + req->map[cline].core_id = 0; + req->map[cline].core_line = 0; + req->map[cline].coll_idx = permutation[cline]; + req->map[cline].status = LOOKUP_HIT; + } + + lock_pfn = rw ? ocf_req_async_lock_wr : ocf_req_async_lock_rd; + + if (req_lock_sync(c, req, lock_pfn, &ctx->terminated)) { + usleep(rand() % 500); + if (rw) + ocf_req_unlock_wr(c, req); + else + ocf_req_unlock_rd(c, req); + usleep(rand() % 500); + } + } else { + line = rand() % ctx->clines; + if (rw) + locked = ocf_cache_line_try_lock_wr(c, line); + else + locked = ocf_cache_line_try_lock_rd(c, line); + + usleep(rand() % 500); + + if (locked) { + if (rw) + ocf_cache_line_unlock_wr(c, line); + else + ocf_cache_line_unlock_rd(c, line); + usleep(rand() % 500); + } + } + + pthread_mutex_lock(&prog_mutex); + progress++; + pthread_cond_signal(&prog_cond); + pthread_mutex_unlock(&prog_mutex); + } + + free(permutation); + ctx->finished = 1; +} + +int cmp_map(const void *p1, const void *p2) +{ + struct ocf_map_info * m1 = *( struct ocf_map_info **)p1; + struct ocf_map_info * m2 = *( struct ocf_map_info **)p2; + + if (m1->coll_idx > m2->coll_idx) + return 1; + if (m2->coll_idx > m1->coll_idx) + return -1; + + return 0; +} + +static void cctest(unsigned num_threads, unsigned num_iterations, unsigned clines, + unsigned max_io_size) +{ + struct ocf_cache_line_concurrency *c; + struct thread_ctx *threads; + unsigned i, j; + time_t t; + char desc[1024]; + unsigned randseed = (unsigned) time(&t); + unsigned last_progress = 0, curr_progress = 0; + struct timespec ts; + int result; + bool deadlocked = false; + unsigned timeout_s = max_io_size / 10 + 3; + + snprintf(desc, sizeof(desc), "cacheline concurrency deadlock detection " + "threads %u iterations %u cache size %u max io size %u randseed %u\n", + num_threads, num_iterations, clines, max_io_size, randseed); + + print_test_description(desc); + + progress = 0; + pthread_cond_init(&prog_cond, NULL); + pthread_mutex_init(&prog_mutex, NULL); + + srand(randseed); + + threads = malloc(num_threads * sizeof(threads[0])); + memset(threads, 0, num_threads * sizeof(threads[0])); + + assert_int_equal(0, ocf_cache_line_concurrency_init(&c, clines, NULL)); + + for (i = 0; i < num_threads; i++) + { + threads[i].timeout = false; + threads[i].finished = false; + threads[i].terminated = false; + threads[i].c = c; + threads[i].num_iterations = num_iterations; + threads[i].clines = clines; + threads[i].max_io_size = max_io_size; + pthread_create(&threads[i].t, NULL, thread, &threads[i]); + } + + do { + clock_gettime(CLOCK_REALTIME, &ts); + ts.tv_sec += timeout_s; + do { + last_progress = curr_progress; + pthread_mutex_lock(&prog_mutex); + result = pthread_cond_timedwait(&prog_cond, + &prog_mutex, &ts); + curr_progress = progress; + pthread_mutex_unlock(&prog_mutex); + } while (!result && curr_progress == last_progress); + } while (last_progress != curr_progress); + + for (i = 0; i < num_threads; i++) + { + if (!threads[i].finished) { + print_message("deadlocked\n"); + deadlocked = true; + break; + } + } + + if (!deadlocked) + goto join; + + /* print locks on which all stuck threads are hanging */ + for (i = 0; i < num_threads; i++) + { + if (!threads[i].finished) + { + unsigned num_clines = threads[i].treq.r.core_line_count; + struct ocf_map_info **clines = malloc(num_clines * + sizeof(*clines)); + for (j = 0; j < num_clines; j++) + { + clines[j] = &threads[i].treq.r.map[j]; + } + + qsort(clines, num_clines, sizeof(*clines), cmp_map); + + print_message("thread no %u\n", i); + for (j = 0; j < num_clines; j++) { + struct ocf_map_info *map = clines[j]; + const char *status = map->rd_locked ? "R" : + map->wr_locked ? "W" : "X"; + print_message("[%u] %u %s\n", j, map->coll_idx, status); + } + + free(clines); + } + } + + /* terminate all waiting threads */ + for (i = 0; i < num_threads; i++) { + threads[i].terminated = 1; + pthread_mutex_lock(&threads[i].treq.completion_mutex); + pthread_cond_signal(&threads[i].treq.completion); + pthread_mutex_unlock(&threads[i].treq.completion_mutex); + } + +join: + assert_int_equal((int)deadlocked, 0); + + for (i = 0; i < num_threads; i++) { + pthread_join(threads[i].t, NULL); + } + + ocf_cache_line_concurrency_deinit(&c); + + free(threads); +} + +static void ocf_req_async_lock_rd_test01(void **state) +{ + cctest(8, 10000, 16, 8); +} + +static void ocf_req_async_lock_rd_test02(void **state) +{ + cctest(64, 1000, 16, 8); +} + +static void ocf_req_async_lock_rd_test03(void **state) +{ + cctest(64, 1000, 128, 32); +} + +static void ocf_req_async_lock_rd_test04(void **state) +{ + cctest(64, 1000, 1024, 32); +} + +static void ocf_req_async_lock_rd_test05(void **state) +{ + cctest(rand() % 64, 1000, rand() % 1024, 32); +} + +int main(void) +{ + const struct CMUnitTest tests[] = { + cmocka_unit_test(ocf_req_async_lock_rd_test01), + cmocka_unit_test(ocf_req_async_lock_rd_test02), + cmocka_unit_test(ocf_req_async_lock_rd_test03), + cmocka_unit_test(ocf_req_async_lock_rd_test04), + cmocka_unit_test(ocf_req_async_lock_rd_test05) + }; + + print_message("Cacheline concurrency deadlock detection\n"); + + return cmocka_run_group_tests(tests, NULL, NULL); +}