diff --git a/src/cleaning/acp.c b/src/cleaning/acp.c index e8c17a4..6441749 100644 --- a/src/cleaning/acp.c +++ b/src/cleaning/acp.c @@ -396,9 +396,10 @@ static ocf_cache_line_t _acp_trylock_dirty(struct ocf_cache *cache, core_line); if (info.status == LOOKUP_HIT && - metadata_test_dirty(cache, info.coll_idx) && - ocf_cache_line_try_lock_rd(cache, info.coll_idx)) { - locked = true; + metadata_test_dirty(cache, info.coll_idx)) { + locked = ocf_cache_line_try_lock_rd( + cache->device->concurrency.cache_line, + info.coll_idx); } ocf_hb_cline_prot_unlock_rd(&cache->metadata.lock, lock_idx, core_id, @@ -472,7 +473,9 @@ static void _acp_flush_end(void *priv, int error) config = (void *)&cache->conf_meta->cleaning[ocf_cleaning_acp].data; for (i = 0; i < flush->size; i++) { - ocf_cache_line_unlock_rd(cache, flush->data[i].cache_line); + ocf_cache_line_unlock_rd( + cache->device->concurrency.cache_line, + flush->data[i].cache_line); ACP_DEBUG_END(acp, flush->data[i].cache_line); } diff --git a/src/cleaning/alru.c b/src/cleaning/alru.c index 97d3d48..dbea378 100644 --- a/src/cleaning/alru.c +++ b/src/cleaning/alru.c @@ -682,8 +682,11 @@ static bool block_is_busy(struct ocf_cache *cache, if (!cache->core[core_id].opened) return true; - if (ocf_cache_line_is_used(cache, cache_line)) + if (ocf_cache_line_is_used( + cache->device->concurrency.cache_line, + cache_line)) { return true; + } return false; } diff --git a/src/concurrency/ocf_cache_line_concurrency.c b/src/concurrency/ocf_cache_line_concurrency.c index 04154aa..5323aae 100644 --- a/src/concurrency/ocf_cache_line_concurrency.c +++ b/src/concurrency/ocf_cache_line_concurrency.c @@ -50,10 +50,12 @@ struct __waiters_list { }; struct ocf_cache_line_concurrency { + ocf_cache_t cache; env_mutex lock; env_atomic *access; env_atomic waiting; size_t access_limit; + ocf_cache_line_t num_clines; env_allocator *allocator; struct __waiters_list waiters_lsts[_WAITERS_LIST_ENTRIES]; @@ -66,7 +68,8 @@ struct ocf_cache_line_concurrency { #define ALLOCATOR_NAME_FMT "ocf_%s_cache_concurrency" #define ALLOCATOR_NAME_MAX (sizeof(ALLOCATOR_NAME_FMT) + OCF_CACHE_NAME_SIZE) -int ocf_cache_line_concurrency_init(struct ocf_cache *cache) +int ocf_cache_line_concurrency_init(struct ocf_cache_line_concurrency **self, + ocf_cache_t cache) { uint32_t i; int error = 0; @@ -75,8 +78,6 @@ int ocf_cache_line_concurrency_init(struct ocf_cache *cache) ocf_cache_line_t line_entries = ocf_metadata_collision_table_entries( cache); - ENV_BUG_ON(cache->device->concurrency.cache_line); - OCF_DEBUG_TRACE(cache); c = env_vzalloc(sizeof(*c)); @@ -85,14 +86,15 @@ int ocf_cache_line_concurrency_init(struct ocf_cache *cache) goto exit_err; } + c->cache = cache; + c->num_clines = line_entries; + error = env_mutex_init(&c->lock); if (error) { error = __LINE__; goto rwsem_err; } - cache->device->concurrency.cache_line = c; - OCF_REALLOC_INIT(&c->access, &c->access_limit); OCF_REALLOC_CP(&c->access, sizeof(c->access[0]), line_entries, &c->access_limit); @@ -124,6 +126,7 @@ int ocf_cache_line_concurrency_init(struct ocf_cache *cache) } } + *self = c; return 0; spinlock_err: @@ -146,24 +149,22 @@ exit_err: if (c) env_vfree(c); - cache->device->concurrency.cache_line = NULL; + *self = NULL; return -1; } /* * */ -void ocf_cache_line_concurrency_deinit(struct ocf_cache *cache) +void ocf_cache_line_concurrency_deinit(struct ocf_cache_line_concurrency **self) { + struct ocf_cache_line_concurrency *concurrency = *self; int i; - struct ocf_cache_line_concurrency *concurrency; - if (!cache->device->concurrency.cache_line) + if (!concurrency) return; - OCF_DEBUG_TRACE(cache); - - concurrency = cache->device->concurrency.cache_line; + OCF_DEBUG_TRACE(concurrency->cache); env_mutex_destroy(&concurrency->lock); @@ -178,10 +179,11 @@ void ocf_cache_line_concurrency_deinit(struct ocf_cache *cache) env_allocator_destroy(concurrency->allocator); env_vfree(concurrency); - cache->device->concurrency.cache_line = NULL; + + *self = NULL; } -size_t ocf_cache_line_concurrency_size_of(struct ocf_cache *cache) +size_t ocf_cache_line_concurrency_size_of(ocf_cache_t cache) { size_t size; @@ -377,12 +379,11 @@ static inline bool __try_lock_rd2rd(struct ocf_cache_line_concurrency *c, /* * */ -static void _req_on_lock(void *ctx, ocf_req_async_lock_cb cb, +static void _req_on_lock(struct ocf_cache_line_concurrency *c, + void *ctx, ocf_req_async_lock_cb cb, uint32_t ctx_id, ocf_cache_line_t line, int rw) { struct ocf_request *req = ctx; - struct ocf_cache_line_concurrency *c = req->cache->device->concurrency. - cache_line; if (rw == OCF_READ) req->map[ctx_id].rd_locked = true; @@ -415,7 +416,7 @@ static inline bool __lock_cache_line_wr(struct ocf_cache_line_concurrency *c, if (__try_lock_wr(c, line)) { /* lock was not owned by anyone */ - _req_on_lock(ctx, cb, ctx_id, line, OCF_WRITE); + _req_on_lock(c, ctx, cb, ctx_id, line, OCF_WRITE); return true; } @@ -447,7 +448,7 @@ unlock: __unlock_waiters_list(c, line, flags); if (!waiting) { - _req_on_lock(ctx, cb, ctx_id, line, OCF_WRITE); + _req_on_lock(c, ctx, cb, ctx_id, line, OCF_WRITE); env_allocator_del(c->allocator, waiter); } @@ -470,7 +471,7 @@ static inline bool __lock_cache_line_rd(struct ocf_cache_line_concurrency *c, if( __try_lock_rd_idle(c, line)) { /* lock was not owned by anyone */ - _req_on_lock(ctx, cb, ctx_id, line, OCF_READ); + _req_on_lock(c, ctx, cb, ctx_id, line, OCF_READ); return true; } @@ -507,7 +508,7 @@ unlock: __unlock_waiters_list(c, line, flags); if (!waiting) { - _req_on_lock(ctx, cb, ctx_id, line, OCF_READ); + _req_on_lock(c, ctx, cb, ctx_id, line, OCF_READ); env_allocator_del(c->allocator, waiter); } @@ -563,7 +564,7 @@ static inline void __unlock_cache_line_rd_common(struct ocf_cache_line_concurren exchanged = false; list_del(iter); - _req_on_lock(waiter->ctx, waiter->cb, waiter->ctx_id, + _req_on_lock(c, waiter->ctx, waiter->cb, waiter->ctx_id, line, waiter->rw); env_allocator_del(c->allocator, waiter); @@ -644,7 +645,7 @@ static inline void __unlock_cache_line_wr_common(struct ocf_cache_line_concurren exchanged = false; list_del(iter); - _req_on_lock(waiter->ctx, waiter->cb, waiter->ctx_id, line, + _req_on_lock(c, waiter->ctx, waiter->cb, waiter->ctx_id, line, waiter->rw); env_allocator_del(c->allocator, waiter); @@ -714,11 +715,10 @@ static inline void __remove_line_from_waiters_list(struct ocf_cache_line_concurr /* Try to read-lock request without adding waiters. Function should be called * under read lock, multiple threads may attempt to acquire the lock * concurrently. */ -static int _ocf_req_trylock_rd(struct ocf_request *req) +static int _ocf_req_trylock_rd(struct ocf_cache_line_concurrency *c, + struct ocf_request *req) { int32_t i; - struct ocf_cache_line_concurrency *c = req->cache->device->concurrency. - cache_line; ocf_cache_line_t line; int ret = OCF_LOCK_ACQUIRED; @@ -733,7 +733,7 @@ static int _ocf_req_trylock_rd(struct ocf_request *req) } line = req->map[i].coll_idx; - ENV_BUG_ON(line >= req->cache->device->collision_table_entries); + ENV_BUG_ON(line >= c->num_clines); ENV_BUG_ON(req->map[i].rd_locked); ENV_BUG_ON(req->map[i].wr_locked); @@ -768,11 +768,10 @@ static int _ocf_req_trylock_rd(struct ocf_request *req) * Read-lock request cache lines. Must be called under cacheline concurrency * write lock. */ -static int _ocf_req_lock_rd(struct ocf_request *req, ocf_req_async_lock_cb cb) +static int _ocf_req_lock_rd(struct ocf_cache_line_concurrency *c, + struct ocf_request *req, ocf_req_async_lock_cb cb) { int32_t i; - struct ocf_cache_line_concurrency *c = req->cache->device->concurrency. - cache_line; ocf_cache_line_t line; int ret = OCF_LOCK_NOT_ACQUIRED; @@ -791,7 +790,7 @@ static int _ocf_req_lock_rd(struct ocf_request *req, ocf_req_async_lock_cb cb) } line = req->map[i].coll_idx; - ENV_BUG_ON(line >= req->cache->device->collision_table_entries); + ENV_BUG_ON(line >= c->num_clines); ENV_BUG_ON(req->map[i].rd_locked); ENV_BUG_ON(req->map[i].wr_locked); @@ -821,17 +820,16 @@ err: } -int ocf_req_async_lock_rd(struct ocf_request *req, ocf_req_async_lock_cb cb) +int ocf_req_async_lock_rd(struct ocf_cache_line_concurrency *c, + struct ocf_request *req, ocf_req_async_lock_cb cb) { - struct ocf_cache_line_concurrency *c = - req->cache->device->concurrency.cache_line; int lock; - lock = _ocf_req_trylock_rd(req); + lock = _ocf_req_trylock_rd(c, req); if (lock != OCF_LOCK_ACQUIRED) { env_mutex_lock(&c->lock); - lock = _ocf_req_lock_rd(req, cb); + lock = _ocf_req_lock_rd(c, req, cb); env_mutex_unlock(&c->lock); } @@ -841,11 +839,10 @@ int ocf_req_async_lock_rd(struct ocf_request *req, ocf_req_async_lock_cb cb) /* Try to write-lock request without adding waiters. Function should be called * under read lock, multiple threads may attempt to acquire the lock * concurrently. */ -static int _ocf_req_trylock_wr(struct ocf_request *req) +static int _ocf_req_trylock_wr(struct ocf_cache_line_concurrency *c, + struct ocf_request *req) { int32_t i; - struct ocf_cache_line_concurrency *c = req->cache->device->concurrency. - cache_line; ocf_cache_line_t line; int ret = OCF_LOCK_ACQUIRED; @@ -858,7 +855,7 @@ static int _ocf_req_trylock_wr(struct ocf_request *req) } line = req->map[i].coll_idx; - ENV_BUG_ON(line >= req->cache->device->collision_table_entries); + ENV_BUG_ON(line >= c->num_clines); ENV_BUG_ON(req->map[i].rd_locked); ENV_BUG_ON(req->map[i].wr_locked); @@ -893,11 +890,10 @@ static int _ocf_req_trylock_wr(struct ocf_request *req) * Write-lock request cache lines. Must be called under cacheline concurrency * write lock. */ -static int _ocf_req_lock_wr(struct ocf_request *req, ocf_req_async_lock_cb cb) +static int _ocf_req_lock_wr(struct ocf_cache_line_concurrency *c, + struct ocf_request *req, ocf_req_async_lock_cb cb) { int32_t i; - struct ocf_cache_line_concurrency *c = req->cache->device->concurrency. - cache_line; ocf_cache_line_t line; int ret = OCF_LOCK_NOT_ACQUIRED; @@ -917,7 +913,7 @@ static int _ocf_req_lock_wr(struct ocf_request *req, ocf_req_async_lock_cb cb) } line = req->map[i].coll_idx; - ENV_BUG_ON(line >= req->cache->device->collision_table_entries); + ENV_BUG_ON(line >= c->num_clines); ENV_BUG_ON(req->map[i].rd_locked); ENV_BUG_ON(req->map[i].wr_locked); @@ -946,17 +942,16 @@ err: return ret; } -int ocf_req_async_lock_wr(struct ocf_request *req, ocf_req_async_lock_cb cb) +int ocf_req_async_lock_wr(struct ocf_cache_line_concurrency *c, + struct ocf_request *req, ocf_req_async_lock_cb cb) { - struct ocf_cache_line_concurrency *c = - req->cache->device->concurrency.cache_line; int lock; - lock = _ocf_req_trylock_wr(req); + lock = _ocf_req_trylock_wr(c, req); if (lock != OCF_LOCK_ACQUIRED) { env_mutex_lock(&c->lock); - lock = _ocf_req_lock_wr(req, cb); + lock = _ocf_req_lock_wr(c, req, cb); env_mutex_unlock(&c->lock); } @@ -967,9 +962,8 @@ int ocf_req_async_lock_wr(struct ocf_request *req, ocf_req_async_lock_cb cb) /* * */ -void ocf_req_unlock_rd(struct ocf_request *req) +void ocf_req_unlock_rd(struct ocf_cache_line_concurrency *c, struct ocf_request *req) { - struct ocf_cache_line_concurrency *c = req->cache->device->concurrency.cache_line; int32_t i; ocf_cache_line_t line; @@ -985,7 +979,7 @@ void ocf_req_unlock_rd(struct ocf_request *req) line = req->map[i].coll_idx; ENV_BUG_ON(!req->map[i].rd_locked); - ENV_BUG_ON(line >= req->cache->device->collision_table_entries); + ENV_BUG_ON(line >= c->num_clines); __unlock_cache_line_rd(c, line); req->map[i].rd_locked = false; @@ -995,9 +989,8 @@ void ocf_req_unlock_rd(struct ocf_request *req) /* * */ -void ocf_req_unlock_wr(struct ocf_request *req) +void ocf_req_unlock_wr(struct ocf_cache_line_concurrency *c, struct ocf_request *req) { - struct ocf_cache_line_concurrency *c = req->cache->device->concurrency.cache_line; int32_t i; ocf_cache_line_t line; @@ -1013,7 +1006,7 @@ void ocf_req_unlock_wr(struct ocf_request *req) line = req->map[i].coll_idx; ENV_BUG_ON(!req->map[i].wr_locked); - ENV_BUG_ON(line >= req->cache->device->collision_table_entries); + ENV_BUG_ON(line >= c->num_clines); __unlock_cache_line_wr(c, line); req->map[i].wr_locked = false; @@ -1023,9 +1016,8 @@ void ocf_req_unlock_wr(struct ocf_request *req) /* * */ -void ocf_req_unlock(struct ocf_request *req) +void ocf_req_unlock(struct ocf_cache_line_concurrency *c, struct ocf_request *req) { - struct ocf_cache_line_concurrency *c = req->cache->device->concurrency.cache_line; int32_t i; ocf_cache_line_t line; @@ -1039,7 +1031,7 @@ void ocf_req_unlock(struct ocf_request *req) } line = req->map[i].coll_idx; - ENV_BUG_ON(line >= req->cache->device->collision_table_entries); + ENV_BUG_ON(line >= c->num_clines); if (req->map[i].rd_locked && req->map[i].wr_locked) { ENV_BUG(); @@ -1058,11 +1050,9 @@ void ocf_req_unlock(struct ocf_request *req) /* * */ -void ocf_req_unlock_entry(struct ocf_cache *cache, +void ocf_req_unlock_entry(struct ocf_cache_line_concurrency *c, struct ocf_request *req, uint32_t entry) { - struct ocf_cache_line_concurrency *c = req->cache->device->concurrency.cache_line; - ENV_BUG_ON(req->map[entry].status == LOOKUP_MISS); if (req->map[entry].rd_locked && req->map[entry].wr_locked) { @@ -1081,17 +1071,15 @@ void ocf_req_unlock_entry(struct ocf_cache *cache, /* * */ -bool ocf_cache_line_is_used(struct ocf_cache *cache, +bool ocf_cache_line_is_used(struct ocf_cache_line_concurrency *c, ocf_cache_line_t line) { - struct ocf_cache_line_concurrency *c = cache->device->concurrency.cache_line; - - ENV_BUG_ON(line >= cache->device->collision_table_entries); + ENV_BUG_ON(line >= c->num_clines); if (env_atomic_read(&(c->access[line]))) return true; - if (ocf_cache_line_are_waiters(cache, line)) + if (ocf_cache_line_are_waiters(c, line)) return true; else return false; @@ -1100,14 +1088,13 @@ bool ocf_cache_line_is_used(struct ocf_cache *cache, /* * */ -bool ocf_cache_line_are_waiters(struct ocf_cache *cache, +bool ocf_cache_line_are_waiters(struct ocf_cache_line_concurrency *c, ocf_cache_line_t line) { - struct ocf_cache_line_concurrency *c = cache->device->concurrency.cache_line; bool are; unsigned long flags = 0; - ENV_BUG_ON(line >= cache->device->collision_table_entries); + ENV_BUG_ON(line >= c->num_clines); /* Lock waiters list */ __lock_waiters_list(c, line, flags); @@ -1122,42 +1109,35 @@ bool ocf_cache_line_are_waiters(struct ocf_cache *cache, /* * */ -uint32_t ocf_cache_line_concurrency_suspended_no(struct ocf_cache *cache) +uint32_t ocf_cache_line_concurrency_suspended_no(struct ocf_cache_line_concurrency *c) { - struct ocf_cache_line_concurrency *c = cache->device->concurrency.cache_line; - return env_atomic_read(&c->waiting); } -bool ocf_cache_line_try_lock_rd(struct ocf_cache *cache, ocf_cache_line_t line) +bool ocf_cache_line_try_lock_rd(struct ocf_cache_line_concurrency *c, + ocf_cache_line_t line) { - struct ocf_cache_line_concurrency *c = cache->device->concurrency.cache_line; return __try_lock_rd_idle(c, line); } /* * */ -void ocf_cache_line_unlock_rd(struct ocf_cache *cache, ocf_cache_line_t line) +void ocf_cache_line_unlock_rd(struct ocf_cache_line_concurrency *c, ocf_cache_line_t line) { - struct ocf_cache_line_concurrency *c = cache->device->concurrency.cache_line; - - OCF_DEBUG_RQ(cache, "Cache line = %u", line); - + OCF_DEBUG_RQ(c->cache, "Cache line = %u", line); __unlock_cache_line_rd(c, line); } -bool ocf_cache_line_try_lock_wr(struct ocf_cache *cache, ocf_cache_line_t line) +bool ocf_cache_line_try_lock_wr(struct ocf_cache_line_concurrency *c, + ocf_cache_line_t line) { - struct ocf_cache_line_concurrency *c = cache->device->concurrency.cache_line; return __try_lock_wr(c, line); } -void ocf_cache_line_unlock_wr(struct ocf_cache *cache, ocf_cache_line_t line) +void ocf_cache_line_unlock_wr(struct ocf_cache_line_concurrency *c, + ocf_cache_line_t line) { - struct ocf_cache_line_concurrency *c = cache->device->concurrency.cache_line; - - OCF_DEBUG_RQ(cache, "Cache line = %u", line); - + OCF_DEBUG_RQ(c->cache, "Cache line = %u", line); __unlock_cache_line_wr(c, line); } diff --git a/src/concurrency/ocf_cache_line_concurrency.h b/src/concurrency/ocf_cache_line_concurrency.h index 42a40cf..65ed263 100644 --- a/src/concurrency/ocf_cache_line_concurrency.h +++ b/src/concurrency/ocf_cache_line_concurrency.h @@ -19,27 +19,31 @@ struct ocf_cache_line_concurrency; /** * @brief Initialize OCF cache concurrency module * + * @param self - cacheline concurrency private data * @param cache - OCF cache instance + * @return 0 - Initialization successful, otherwise ERROR */ -int ocf_cache_line_concurrency_init(struct ocf_cache *cache); +int ocf_cache_line_concurrency_init(struct ocf_cache_line_concurrency **self, + struct ocf_cache *cache); /** * @biref De-Initialize OCF cache concurrency module * - * @param cache - OCF cache instance + * @param self - cacheline concurrency private data */ -void ocf_cache_line_concurrency_deinit(struct ocf_cache *cache); +void ocf_cache_line_concurrency_deinit( + struct ocf_cache_line_concurrency **self); /** * @brief Get number of waiting (suspended) OCF requests in due to cache * overlapping * - * @param cache - OCF cache instance + * @param c - cacheline concurrency private data * * @return Number of suspended OCF requests */ -uint32_t ocf_cache_line_concurrency_suspended_no(struct ocf_cache *cache); +uint32_t ocf_cache_line_concurrency_suspended_no(struct ocf_cache_line_concurrency *c); /** * @brief Return memory footprint conusmed by cache concurrency module @@ -48,7 +52,7 @@ uint32_t ocf_cache_line_concurrency_suspended_no(struct ocf_cache *cache); * * @return Memory footprint of cache concurrency module */ -size_t ocf_cache_line_concurrency_size_of(struct ocf_cache *cache); +size_t ocf_cache_line_concurrency_size_of(ocf_cache_t cache); /* async request cacheline lock acquisition callback */ typedef void (*ocf_req_async_lock_cb)(struct ocf_request *req); @@ -56,6 +60,7 @@ typedef void (*ocf_req_async_lock_cb)(struct ocf_request *req); /** * @brief Lock OCF request for write access (Lock all cache lines in map info) * + * @param c - cacheline concurrency private data * @param req - OCF request * @param cb - async lock acquisition callback * @@ -65,11 +70,13 @@ typedef void (*ocf_req_async_lock_cb)(struct ocf_request *req); * @retval OCF_LOCK_NOT_ACQUIRED - OCF request lock not acquired, request was * added into waiting list. When lock will be acquired @cb cllback be called */ -int ocf_req_async_lock_wr(struct ocf_request *req, ocf_req_async_lock_cb cb); +int ocf_req_async_lock_wr(struct ocf_cache_line_concurrency *c, + struct ocf_request *req, ocf_req_async_lock_cb cb); /** * @brief Lock OCF request for read access (Lock all cache lines in map info) * + * @param c - cacheline concurrency private data * @param req - OCF request * @param cb - async lock acquisition callback * @@ -79,28 +86,35 @@ int ocf_req_async_lock_wr(struct ocf_request *req, ocf_req_async_lock_cb cb); * @retval OCF_LOCK_NOT_ACQUIRED - OCF request lock not acquired, request was * added into waiting list. When lock will be acquired @cb callback be called */ -int ocf_req_async_lock_rd(struct ocf_request *req, ocf_req_async_lock_cb cb); +int ocf_req_async_lock_rd(struct ocf_cache_line_concurrency *c, + struct ocf_request *req, ocf_req_async_lock_cb cb); /** * @brief Unlock OCF request from write access * + * @param c - cacheline concurrency private data * @param req - OCF request */ -void ocf_req_unlock_wr(struct ocf_request *req); +void ocf_req_unlock_wr(struct ocf_cache_line_concurrency *c, + struct ocf_request *req); /** * @brief Unlock OCF request from read access * + * @param c - cacheline concurrency private data * @param req - OCF request */ -void ocf_req_unlock_rd(struct ocf_request *req); +void ocf_req_unlock_rd(struct ocf_cache_line_concurrency *c, + struct ocf_request *req); /** * @brief Unlock OCF request from read or write access * + * @param c - cacheline concurrency private data * @param req - OCF request */ -void ocf_req_unlock(struct ocf_request *req); +void ocf_req_unlock(struct ocf_cache_line_concurrency *c, + struct ocf_request *req); /** * @Check if cache line is used. @@ -116,30 +130,30 @@ void ocf_req_unlock(struct ocf_request *req); * @retval true - cache line is used * @retval false - cache line is not used */ -bool ocf_cache_line_is_used(struct ocf_cache *cache, +bool ocf_cache_line_is_used(struct ocf_cache_line_concurrency *c, ocf_cache_line_t line); /** * @brief Check if for specified cache line there are waiters * on the waiting list * - * @param cache - OCF cache instance + * @param c - cacheline concurrency private data * @param line - Cache line to be checked for waiters * * @retval true - there are waiters * @retval false - No waiters */ -bool ocf_cache_line_are_waiters(struct ocf_cache *cache, +bool ocf_cache_line_are_waiters(struct ocf_cache_line_concurrency *c, ocf_cache_line_t line); /** * @brief un_lock request map info entry from from write or read access. * - * @param cache - OCF cache instance + * @param c - cacheline concurrency private data * @param req - OCF request * @param entry - request map entry number */ -void ocf_req_unlock_entry(struct ocf_cache *cache, +void ocf_req_unlock_entry(struct ocf_cache_line_concurrency *c, struct ocf_request *req, uint32_t entry); /** @@ -148,36 +162,40 @@ void ocf_req_unlock_entry(struct ocf_cache *cache, * @param cache - OCF cache instance * @param line - Cache line to be unlocked */ -void ocf_cache_line_unlock_rd(struct ocf_cache *cache, ocf_cache_line_t line); +void ocf_cache_line_unlock_rd(struct ocf_cache_line_concurrency *c, + ocf_cache_line_t line); /** * @brief Attempt to lock cache line for read * - * @param cache - OCF cache instance + * @param c - cacheline concurrency private data * @param line - Cache line to be checked for waiters * * @retval true - read lock successfully acquired * @retval false - failed to acquire read lock */ -bool ocf_cache_line_try_lock_rd(struct ocf_cache *cache, ocf_cache_line_t line); +bool ocf_cache_line_try_lock_rd(struct ocf_cache_line_concurrency *c, + ocf_cache_line_t line); /** * @brief Release cache line write lock * - * @param cache - OCF cache instance + * @param c - cacheline concurrency private data * @param line - Cache line to be unlocked */ -void ocf_cache_line_unlock_wr(struct ocf_cache *cache, ocf_cache_line_t line); +void ocf_cache_line_unlock_wr(struct ocf_cache_line_concurrency *c, + ocf_cache_line_t line); /** * @brief Attempt to lock cache line for write * - * @param cache - OCF cache instance + * @param c - cacheline concurrency private data * @param line - Cache line to be checked for waiters * * @retval true - write lock successfully acquired * @retval false - failed to acquire write lock */ -bool ocf_cache_line_try_lock_wr(struct ocf_cache *cache, ocf_cache_line_t line); +bool ocf_cache_line_try_lock_wr(struct ocf_cache_line_concurrency *c, + ocf_cache_line_t line); #endif /* OCF_CONCURRENCY_H_ */ diff --git a/src/concurrency/ocf_concurrency.c b/src/concurrency/ocf_concurrency.c index 24ebc8e..7a45951 100644 --- a/src/concurrency/ocf_concurrency.c +++ b/src/concurrency/ocf_concurrency.c @@ -9,7 +9,9 @@ int ocf_concurrency_init(struct ocf_cache *cache) { int result = 0; - result = ocf_cache_line_concurrency_init(cache); + result = ocf_cache_line_concurrency_init( + &cache->device->concurrency.cache_line, + cache); if (result) ocf_concurrency_deinit(cache); @@ -19,6 +21,8 @@ int ocf_concurrency_init(struct ocf_cache *cache) void ocf_concurrency_deinit(struct ocf_cache *cache) { - ocf_cache_line_concurrency_deinit(cache); + ocf_cache_line_concurrency_deinit( + &cache->device->concurrency.cache_line); + } diff --git a/src/engine/engine_bf.c b/src/engine/engine_bf.c index afa59fa..79249ed 100644 --- a/src/engine/engine_bf.c +++ b/src/engine/engine_bf.c @@ -64,7 +64,7 @@ static void _ocf_backfill_complete(struct ocf_request *req, int error) ocf_core_stats_cache_error_update(req->core, OCF_WRITE); ocf_engine_invalidate(req); } else { - ocf_req_unlock(req); + ocf_req_unlock(cache->device->concurrency.cache_line, req); /* put the request at the last point of the completion path */ ocf_req_put(req); diff --git a/src/engine/engine_common.c b/src/engine/engine_common.c index db7569f..9401b6e 100644 --- a/src/engine/engine_common.c +++ b/src/engine/engine_common.c @@ -396,7 +396,8 @@ static void _ocf_engine_clean_end(void *private_data, int error) req->error |= error; /* End request and do not processing */ - ocf_req_unlock(req); + ocf_req_unlock(req->cache->device->concurrency.cache_line, + req); /* Complete request */ req->complete(req, error); @@ -414,12 +415,14 @@ static int lock_clines(struct ocf_request *req, const struct ocf_engine_callbacks *engine_cbs) { enum ocf_engine_lock_type lock_type = engine_cbs->get_lock_type(req); + struct ocf_cache_line_concurrency *c = + req->cache->device->concurrency.cache_line; switch (lock_type) { case ocf_engine_lock_write: - return ocf_req_async_lock_wr(req, engine_cbs->resume); + return ocf_req_async_lock_wr(c, req, engine_cbs->resume); case ocf_engine_lock_read: - return ocf_req_async_lock_rd(req, engine_cbs->resume); + return ocf_req_async_lock_rd(c, req, engine_cbs->resume); default: return OCF_LOCK_ACQUIRED; } @@ -703,7 +706,8 @@ static int _ocf_engine_refresh(struct ocf_request *req) req->complete(req, req->error); /* Release WRITE lock of request */ - ocf_req_unlock(req); + ocf_req_unlock(req->cache->device->concurrency.cache_line, + req); /* Release OCF request */ ocf_req_put(req); diff --git a/src/engine/engine_discard.c b/src/engine/engine_discard.c index a2d97c6..a5a8678 100644 --- a/src/engine/engine_discard.c +++ b/src/engine/engine_discard.c @@ -147,7 +147,7 @@ static void _ocf_discard_step_complete(struct ocf_request *req, int error) OCF_DEBUG_RQ(req, "Completion"); /* Release WRITE lock of request */ - ocf_req_unlock_wr(req); + ocf_req_unlock_wr(req->cache->device->concurrency.cache_line, req); if (req->error) { ocf_metadata_error(req->cache); @@ -235,7 +235,9 @@ static int _ocf_discard_step(struct ocf_request *req) if (ocf_engine_mapped_count(req)) { /* Some cache line are mapped, lock request for WRITE access */ - lock = ocf_req_async_lock_wr(req, _ocf_discard_on_resume); + lock = ocf_req_async_lock_wr( + cache->device->concurrency.cache_line, + req, _ocf_discard_on_resume); } else { lock = OCF_LOCK_ACQUIRED; } diff --git a/src/engine/engine_fast.c b/src/engine/engine_fast.c index abb9695..42e1047 100644 --- a/src/engine/engine_fast.c +++ b/src/engine/engine_fast.c @@ -46,7 +46,8 @@ static void _ocf_read_fast_complete(struct ocf_request *req, int error) ocf_core_stats_cache_error_update(req->core, OCF_READ); ocf_engine_push_req_front_pt(req); } else { - ocf_req_unlock(req); + ocf_req_unlock(req->cache->device->concurrency.cache_line, + req); /* Complete request */ req->complete(req, req->error); @@ -130,7 +131,9 @@ int ocf_read_fast(struct ocf_request *req) if (hit && part_has_space) { ocf_io_start(&req->ioi.io); - lock = ocf_req_async_lock_rd(req, ocf_engine_on_resume); + lock = ocf_req_async_lock_rd( + req->cache->device->concurrency.cache_line, + req, ocf_engine_on_resume); } ocf_hb_req_prot_unlock_rd(req); @@ -200,7 +203,9 @@ int ocf_write_fast(struct ocf_request *req) if (mapped && part_has_space) { ocf_io_start(&req->ioi.io); - lock = ocf_req_async_lock_wr(req, ocf_engine_on_resume); + lock = ocf_req_async_lock_wr( + req->cache->device->concurrency.cache_line, + req, ocf_engine_on_resume); } ocf_hb_req_prot_unlock_rd(req); diff --git a/src/engine/engine_inv.c b/src/engine/engine_inv.c index 0a15137..ee3af47 100644 --- a/src/engine/engine_inv.c +++ b/src/engine/engine_inv.c @@ -31,7 +31,7 @@ static void _ocf_invalidate_req(struct ocf_request *req, int error) if (req->error) ocf_engine_error(req, true, "Failed to flush metadata to cache"); - ocf_req_unlock(req); + ocf_req_unlock(req->cache->device->concurrency.cache_line, req); /* Put OCF request - decrease reference counter */ ocf_req_put(req); diff --git a/src/engine/engine_pt.c b/src/engine/engine_pt.c index 4ddb33b..3cb321d 100644 --- a/src/engine/engine_pt.c +++ b/src/engine/engine_pt.c @@ -34,7 +34,7 @@ static void _ocf_read_pt_complete(struct ocf_request *req, int error) /* Complete request */ req->complete(req, req->error); - ocf_req_unlock_rd(req); + ocf_req_unlock_rd(req->cache->device->concurrency.cache_line, req); /* Release OCF request */ ocf_req_put(req); @@ -127,7 +127,10 @@ int ocf_read_pt(struct ocf_request *req) /* There are mapped cache line, * lock request for READ access */ - lock = ocf_req_async_lock_rd(req, ocf_engine_on_resume); + lock = ocf_req_async_lock_rd( + req->cache->device->concurrency. + cache_line, + req, ocf_engine_on_resume); } else { /* No mapped cache lines, no need to get lock */ lock = OCF_LOCK_ACQUIRED; diff --git a/src/engine/engine_rd.c b/src/engine/engine_rd.c index b18369a..6a9f778 100644 --- a/src/engine/engine_rd.c +++ b/src/engine/engine_rd.c @@ -24,6 +24,9 @@ static void _ocf_read_generic_hit_complete(struct ocf_request *req, int error) { + struct ocf_cache_line_concurrency *c = + req->cache->device->concurrency.cache_line; + if (error) req->error |= error; @@ -41,8 +44,7 @@ static void _ocf_read_generic_hit_complete(struct ocf_request *req, int error) ocf_core_stats_cache_error_update(req->core, OCF_READ); ocf_engine_push_req_front_pt(req); } else { - - ocf_req_unlock(req); + ocf_req_unlock(c, req); /* Complete request */ req->complete(req, req->error); diff --git a/src/engine/engine_wb.c b/src/engine/engine_wb.c index 95bd8ee..70ab3cb 100644 --- a/src/engine/engine_wb.c +++ b/src/engine/engine_wb.c @@ -60,7 +60,7 @@ static void _ocf_write_wb_io_flush_metadata(struct ocf_request *req, int error) if (req->error) ocf_engine_error(req, true, "Failed to write data to cache"); - ocf_req_unlock_wr(req); + ocf_req_unlock_wr(req->cache->device->concurrency.cache_line, req); req->complete(req, req->error); diff --git a/src/engine/engine_wi.c b/src/engine/engine_wi.c index 52a9242..9b8d005 100644 --- a/src/engine/engine_wi.c +++ b/src/engine/engine_wi.c @@ -25,7 +25,7 @@ static const struct ocf_io_if _io_if_wi_update_metadata = { int _ocf_write_wi_next_pass(struct ocf_request *req) { - ocf_req_unlock_wr(req); + ocf_req_unlock_wr(req->cache->device->concurrency.cache_line, req); if (req->wi_second_pass) { req->complete(req, req->error); @@ -75,7 +75,7 @@ static void _ocf_write_wi_io_flush_metadata(struct ocf_request *req, int error) if (req->error) ocf_engine_error(req, true, "Failed to write data to cache"); - ocf_req_unlock_wr(req); + ocf_req_unlock_wr(req->cache->device->concurrency.cache_line, req); req->complete(req, req->error); @@ -128,7 +128,8 @@ static void _ocf_write_wi_core_complete(struct ocf_request *req, int error) OCF_DEBUG_RQ(req, "Completion"); if (req->error) { - ocf_req_unlock_wr(req); + ocf_req_unlock_wr(req->cache->device->concurrency.cache_line, + req); req->complete(req, req->error); @@ -198,7 +199,9 @@ int ocf_write_wi(struct ocf_request *req) if (ocf_engine_mapped_count(req)) { /* Some cache line are mapped, lock request for WRITE access */ - lock = ocf_req_async_lock_wr(req, _ocf_write_wi_on_resume); + lock = ocf_req_async_lock_wr( + req->cache->device->concurrency.cache_line, + req, _ocf_write_wi_on_resume); } else { lock = OCF_LOCK_ACQUIRED; } diff --git a/src/engine/engine_wo.c b/src/engine/engine_wo.c index 0b4a4bc..d210186 100644 --- a/src/engine/engine_wo.c +++ b/src/engine/engine_wo.c @@ -33,7 +33,7 @@ static void ocf_read_wo_cache_complete(struct ocf_request *req, int error) if (req->error) ocf_engine_error(req, true, "Failed to read data from cache"); - ocf_req_unlock_rd(req); + ocf_req_unlock_rd(req->cache->device->concurrency.cache_line, req); /* Complete request */ req->complete(req, req->error); @@ -169,7 +169,8 @@ static void _ocf_read_wo_core_complete(struct ocf_request *req, int error) if (!req->info.dirty_any || req->error) { OCF_DEBUG_RQ(req, "Completion"); req->complete(req, req->error); - ocf_req_unlock_rd(req); + ocf_req_unlock_rd(req->cache->device->concurrency.cache_line, + req); ocf_req_put(req); return; } @@ -236,7 +237,9 @@ int ocf_read_wo(struct ocf_request *req) /* There are mapped cache lines, * lock request for READ access */ - lock = ocf_req_async_lock_rd(req, ocf_engine_on_resume); + lock = ocf_req_async_lock_rd( + req->cache->device->concurrency.cache_line, + req, ocf_engine_on_resume); } ocf_hb_req_prot_unlock_rd(req); /*- END Metadata RD access -----------------*/ diff --git a/src/engine/engine_wt.c b/src/engine/engine_wt.c index d786a0f..a8eb087 100644 --- a/src/engine/engine_wt.c +++ b/src/engine/engine_wt.c @@ -34,7 +34,8 @@ static void _ocf_write_wt_req_complete(struct ocf_request *req) ocf_engine_invalidate(req); } else { /* Unlock reqest from WRITE access */ - ocf_req_unlock_wr(req); + ocf_req_unlock_wr(req->cache->device->concurrency.cache_line, + req); /* Complete request */ req->complete(req, req->info.core_error ? req->error : 0); diff --git a/src/engine/engine_zero.c b/src/engine/engine_zero.c index 2294064..4dd9c0a 100644 --- a/src/engine/engine_zero.c +++ b/src/engine/engine_zero.c @@ -31,7 +31,7 @@ static int ocf_zero_purge(struct ocf_request *req) ocf_hb_req_prot_unlock_wr(req); /*- END Metadata WR access ---------*/ } - ocf_req_unlock_wr(req); + ocf_req_unlock_wr(req->cache->device->concurrency.cache_line, req); req->complete(req, req->error); @@ -152,7 +152,9 @@ void ocf_engine_zero_line(struct ocf_request *req) req->io_if = &_io_if_ocf_zero_do; /* Some cache line are mapped, lock request for WRITE access */ - lock = ocf_req_async_lock_wr(req, ocf_engine_on_resume); + lock = ocf_req_async_lock_wr( + req->cache->device->concurrency.cache_line, + req, ocf_engine_on_resume); if (lock >= 0) { ENV_BUG_ON(lock != OCF_LOCK_ACQUIRED); diff --git a/src/eviction/lru.c b/src/eviction/lru.c index e63573c..c3bbc5a 100644 --- a/src/eviction/lru.c +++ b/src/eviction/lru.c @@ -357,7 +357,9 @@ static int evp_lru_clean_getter(ocf_cache_t cache, void *getter_context, break; /* Prevent evicting already locked items */ - if (ocf_cache_line_is_used(cache, cline)) { + if (ocf_cache_line_is_used( + cache->device->concurrency.cache_line, + cline)) { continue; } @@ -490,8 +492,11 @@ uint32_t evp_lru_req_clines(ocf_cache_t cache, ocf_queue_t io_queue, break; /* Prevent evicting already locked items */ - if (ocf_cache_line_is_used(cache, cline)) + if (ocf_cache_line_is_used( + cache->device->concurrency.cache_line, + cline)) { continue; + } ENV_BUG_ON(metadata_test_dirty(cache, cline)); diff --git a/src/metadata/metadata_misc.c b/src/metadata/metadata_misc.c index 62639d7..ff47f2e 100644 --- a/src/metadata/metadata_misc.c +++ b/src/metadata/metadata_misc.c @@ -51,6 +51,8 @@ int ocf_metadata_actor(struct ocf_cache *cache, ocf_cache_line_t i, next_i; uint64_t start_line, end_line; int ret = 0; + struct ocf_cache_line_concurrency *c = + cache->device->concurrency.cache_line; start_line = ocf_bytes_2_lines(cache, start_byte); end_line = ocf_bytes_2_lines(cache, end_byte); @@ -63,7 +65,7 @@ int ocf_metadata_actor(struct ocf_cache *cache, if (_is_cache_line_acting(cache, i, core_id, start_line, end_line)) { - if (ocf_cache_line_is_used(cache, i)) + if (ocf_cache_line_is_used(c, i)) ret = -OCF_ERR_AGAIN; else actor(cache, i); @@ -75,7 +77,7 @@ int ocf_metadata_actor(struct ocf_cache *cache, for (i = 0; i < cache->device->collision_table_entries; ++i) { if (_is_cache_line_acting(cache, i, core_id, start_line, end_line)) { - if (ocf_cache_line_is_used(cache, i)) + if (ocf_cache_line_is_used(c, i)) ret = -OCF_ERR_AGAIN; else actor(cache, i); diff --git a/src/mngt/ocf_mngt_common.c b/src/mngt/ocf_mngt_common.c index 8c63cdc..82a2873 100644 --- a/src/mngt/ocf_mngt_common.c +++ b/src/mngt/ocf_mngt_common.c @@ -80,14 +80,18 @@ void cache_mngt_core_deinit_attached_meta(ocf_core_t core) continue; } - if (!ocf_cache_line_try_lock_wr(cache, curr_cline)) + if (!ocf_cache_line_try_lock_wr( + cache->device->concurrency.cache_line, + curr_cline)) { break; + } if (metadata_test_dirty(cache, curr_cline)) ocf_purge_cleaning_policy(cache, curr_cline); ocf_metadata_sparse_cache_line(cache, curr_cline); - ocf_cache_line_unlock_wr(cache, curr_cline); + ocf_cache_line_unlock_wr(cache->device->concurrency.cache_line, + curr_cline); if (prev_cline != cache->device->collision_table_entries) curr_cline = ocf_metadata_get_collision_next(cache, prev_cline); diff --git a/src/utils/utils_cache_line.c b/src/utils/utils_cache_line.c index 02d2d36..0d3ab3d 100644 --- a/src/utils/utils_cache_line.c +++ b/src/utils/utils_cache_line.c @@ -43,7 +43,9 @@ static void __set_cache_line_invalid(struct ocf_cache *cache, uint8_t start_bit, * for this cache line which will use one, clear * only valid bits */ - if (!is_valid && !ocf_cache_line_are_waiters(cache, line)) { + if (!is_valid && !ocf_cache_line_are_waiters( + cache->device->concurrency.cache_line, + line)) { ocf_purge_eviction_policy(cache, line); ocf_metadata_remove_cache_line(cache, line); } diff --git a/src/utils/utils_cleaner.c b/src/utils/utils_cleaner.c index 2f536dc..5c07aa4 100644 --- a/src/utils/utils_cleaner.c +++ b/src/utils/utils_cleaner.c @@ -213,7 +213,9 @@ static int _ocf_cleaner_cache_line_lock(struct ocf_request *req) OCF_DEBUG_TRACE(req->cache); - return ocf_req_async_lock_rd(req, _ocf_cleaner_on_resume); + return ocf_req_async_lock_rd( + req->cache->device->concurrency.cache_line, + req, _ocf_cleaner_on_resume); } /* @@ -224,7 +226,8 @@ static void _ocf_cleaner_cache_line_unlock(struct ocf_request *req) { if (req->info.cleaner_cache_line_lock) { OCF_DEBUG_TRACE(req->cache); - ocf_req_unlock(req); + ocf_req_unlock(req->cache->device->concurrency.cache_line, + req); } }