diff --git a/src/concurrency/ocf_cache_line_concurrency.c b/src/concurrency/ocf_cache_line_concurrency.c index 572946d..ce5ed5d 100644 --- a/src/concurrency/ocf_cache_line_concurrency.c +++ b/src/concurrency/ocf_cache_line_concurrency.c @@ -9,8 +9,8 @@ #include "../utils/utils_alock.h" #include "../utils/utils_cache_line.h" -static bool ocf_cl_lock_line_needs_lock(struct ocf_request *req, - unsigned index) +static bool ocf_cl_lock_line_needs_lock(struct ocf_alock *alock, + struct ocf_request *req, unsigned index) { /* Remapped cachelines are assigned cacheline lock individually * during eviction @@ -19,14 +19,14 @@ static bool ocf_cl_lock_line_needs_lock(struct ocf_request *req, req->map[index].status != LOOKUP_REMAPPED; } -static bool ocf_cl_lock_line_is_acting(struct ocf_request *req, - unsigned index) +static bool ocf_cl_lock_line_is_acting(struct ocf_alock *alock, + struct ocf_request *req, unsigned index) { return req->map[index].status != LOOKUP_MISS; } -static bool ocf_cl_lock_line_is_locked(struct ocf_request *req, - unsigned index, int rw) +static bool ocf_cl_lock_line_is_locked(struct ocf_alock *alock, + struct ocf_request *req, unsigned index, int rw) { if (rw == OCF_WRITE) return req->map[index].wr_locked; @@ -34,8 +34,8 @@ static bool ocf_cl_lock_line_is_locked(struct ocf_request *req, return req->map[index].rd_locked; } -static void ocf_cl_lock_line_mark_locked(struct ocf_request *req, - unsigned index, int rw, bool locked) +static void ocf_cl_lock_line_mark_locked(struct ocf_alock *alock, + struct ocf_request *req, unsigned index, int rw, bool locked) { if (rw == OCF_WRITE) req->map[index].wr_locked = locked; @@ -43,7 +43,8 @@ static void ocf_cl_lock_line_mark_locked(struct ocf_request *req, req->map[index].rd_locked = locked; } -static ocf_cache_line_t ocf_cl_lock_line_get_entry(struct ocf_request *req, +static ocf_cache_line_t ocf_cl_lock_line_get_entry( + struct ocf_alock *alock, struct ocf_request *req, unsigned index) { return req->map[index].coll_idx; diff --git a/src/utils/utils_alock.c b/src/utils/utils_alock.c index 30b11e2..73ebc3f 100644 --- a/src/utils/utils_alock.c +++ b/src/utils/utils_alock.c @@ -347,7 +347,7 @@ static inline bool ocf_alock_lock_one_wr(struct ocf_alock *alock, if (ocf_alock_trylock_entry_wr(alock, entry)) { /* lock was not owned by anyone */ - cbs->line_mark_locked(req, idx, OCF_WRITE, true); + cbs->line_mark_locked(alock, req, idx, OCF_WRITE, true); ocf_alock_entry_locked(alock, req, cmpl); return true; } @@ -380,7 +380,7 @@ unlock: ocf_alock_waitlist_unlock(alock, entry, flags); if (!waiting) { - cbs->line_mark_locked(req, idx, OCF_WRITE, true); + cbs->line_mark_locked(alock, req, idx, OCF_WRITE, true); ocf_alock_entry_locked(alock, req, cmpl); env_allocator_del(alock->allocator, waiter); } @@ -405,7 +405,7 @@ static inline bool ocf_alock_lock_one_rd(struct ocf_alock *alock, if( ocf_alock_trylock_entry_rd_idle(alock, entry)) { /* lock was not owned by anyone */ - cbs->line_mark_locked(req, idx, OCF_READ, true); + cbs->line_mark_locked(alock, req, idx, OCF_READ, true); ocf_alock_entry_locked(alock, req, cmpl); return true; } @@ -443,7 +443,7 @@ unlock: ocf_alock_waitlist_unlock(alock, entry, flags); if (!waiting) { - cbs->line_mark_locked(req, idx, OCF_READ, true); + cbs->line_mark_locked(alock, req, idx, OCF_READ, true); ocf_alock_entry_locked(alock, req, cmpl); env_allocator_del(alock->allocator, waiter); } @@ -506,7 +506,7 @@ static inline void ocf_alock_unlock_one_rd_common(struct ocf_alock *alock, exchanged = false; list_del(iter); - cbs->line_mark_locked(waiter->req, waiter->idx, + cbs->line_mark_locked(alock, waiter->req, waiter->idx, waiter->rw, true); ocf_alock_entry_locked(alock, waiter->req, waiter->cmpl); @@ -599,7 +599,7 @@ static inline void ocf_alock_unlock_one_wr_common(struct ocf_alock *alock, exchanged = false; list_del(iter); - cbs->line_mark_locked(waiter->req, waiter->idx, + cbs->line_mark_locked(alock, waiter->req, waiter->idx, waiter->rw, true); ocf_alock_entry_locked(alock, waiter->req, waiter->cmpl); @@ -640,7 +640,7 @@ static inline void ocf_alock_waitlist_remove_entry(struct ocf_alock *alock, struct ocf_alock_lock_cbs *cbs, struct ocf_request *req, int i, int rw) { - ocf_cache_line_t entry = cbs->line_get_entry(req, i); + ocf_cache_line_t entry = cbs->line_get_entry(alock, req, i); uint32_t idx = _WAITERS_LIST_ITEM(entry); struct ocf_alock_waiters_list *lst = &alock->waiters_lsts[idx]; struct list_head *iter, *next; @@ -649,12 +649,12 @@ static inline void ocf_alock_waitlist_remove_entry(struct ocf_alock *alock, ocf_alock_waitlist_lock(alock, entry, flags); - if (cbs->line_is_locked(req, i, rw)) { + if (cbs->line_is_locked(alock, req, i, rw)) { if (rw == OCF_READ) ocf_alock_unlock_one_rd_common(alock, cbs, entry); else ocf_alock_unlock_one_wr_common(alock, cbs, entry); - cbs->line_mark_locked(req, i, rw, false); + cbs->line_mark_locked(alock, req, i, rw, false); } else { list_for_each_safe(iter, next, &lst->head) { waiter = list_entry(iter, struct ocf_alock_waiter, item); @@ -685,19 +685,19 @@ static int ocf_alock_lock_rd_fast(struct ocf_alock *alock, ENV_BUG_ON(env_atomic_read(&req->lock_remaining)); for (i = 0; i < req->core_line_count; i++) { - if (!cbs->line_needs_lock(req, i)) { + if (!cbs->line_needs_lock(alock, req, i)) { /* nothing to lock */ continue; } - entry = cbs->line_get_entry(req, i); + entry = cbs->line_get_entry(alock, req, i); ENV_BUG_ON(entry >= alock->num_entries); - ENV_BUG_ON(cbs->line_is_locked(req, i, OCF_READ)); - ENV_BUG_ON(cbs->line_is_locked(req, i, OCF_WRITE)); + ENV_BUG_ON(cbs->line_is_locked(alock, req, i, OCF_READ)); + ENV_BUG_ON(cbs->line_is_locked(alock, req, i, OCF_WRITE)); if( ocf_alock_trylock_entry_rd_idle(alock, entry)) { /* cache entry locked */ - cbs->line_mark_locked(req, i, OCF_READ, true); + cbs->line_mark_locked(alock, req, i, OCF_READ, true); } else { /* Not possible to lock all cachelines */ ret = OCF_LOCK_NOT_ACQUIRED; @@ -710,16 +710,16 @@ static int ocf_alock_lock_rd_fast(struct ocf_alock *alock, if (ret == OCF_LOCK_NOT_ACQUIRED) { /* Request is not locked, discard acquired locks */ for (; i >= 0; i--) { - if (!cbs->line_needs_lock(req, i)) { + if (!cbs->line_needs_lock(alock, req, i)) { /* nothing to discard */ continue; } - entry = cbs->line_get_entry(req, i); + entry = cbs->line_get_entry(alock, req, i); - if (cbs->line_is_locked(req, i, OCF_READ)) { + if (cbs->line_is_locked(alock, req, i, OCF_READ)) { ocf_alock_unlock_one_rd(alock, cbs, entry); - cbs->line_mark_locked(req, i, OCF_READ, false); + cbs->line_mark_locked(alock, req, i, OCF_READ, false); } } } @@ -746,16 +746,16 @@ static int ocf_alock_lock_rd_slow(struct ocf_alock *alock, env_atomic_inc(&req->lock_remaining); for (i = 0; i < req->core_line_count; i++) { - if (!cbs->line_needs_lock(req, i)) { + if (!cbs->line_needs_lock(alock, req, i)) { /* nothing to lock */ env_atomic_dec(&req->lock_remaining); continue; } - entry = cbs->line_get_entry(req, i); + entry = cbs->line_get_entry(alock, req, i); ENV_BUG_ON(entry >= alock->num_entries); - ENV_BUG_ON(cbs->line_is_locked(req, i, OCF_READ)); - ENV_BUG_ON(cbs->line_is_locked(req, i, OCF_WRITE)); + ENV_BUG_ON(cbs->line_is_locked(alock, req, i, OCF_READ)); + ENV_BUG_ON(cbs->line_is_locked(alock, req, i, OCF_WRITE)); if (!ocf_alock_lock_one_rd(alock, cbs, entry, cmpl, req, i)) { /* lock not acquired and not added to wait list */ @@ -773,7 +773,7 @@ static int ocf_alock_lock_rd_slow(struct ocf_alock *alock, err: for (; i >= 0; i--) { - if (!cbs->line_needs_lock(req, i)) + if (!cbs->line_needs_lock(alock, req, i)) continue; ocf_alock_waitlist_remove_entry(alock, cbs, req, i ,OCF_READ); @@ -816,19 +816,19 @@ static int ocf_alock_lock_wr_fast(struct ocf_alock *alock, ENV_BUG_ON(env_atomic_read(&req->lock_remaining)); for (i = 0; i < req->core_line_count; i++) { - if (!cbs->line_needs_lock(req, i)) { + if (!cbs->line_needs_lock(alock, req, i)) { /* nothing to lock */ continue; } - entry = cbs->line_get_entry(req, i); + entry = cbs->line_get_entry(alock, req, i); ENV_BUG_ON(entry >= alock->num_entries); - ENV_BUG_ON(cbs->line_is_locked(req, i, OCF_READ)); - ENV_BUG_ON(cbs->line_is_locked(req, i, OCF_WRITE)); + ENV_BUG_ON(cbs->line_is_locked(alock, req, i, OCF_READ)); + ENV_BUG_ON(cbs->line_is_locked(alock, req, i, OCF_WRITE)); if (ocf_alock_trylock_entry_wr(alock, entry)) { /* cache entry locked */ - cbs->line_mark_locked(req, i, OCF_WRITE, true); + cbs->line_mark_locked(alock, req, i, OCF_WRITE, true); } else { /* Not possible to lock all cachelines */ ret = OCF_LOCK_NOT_ACQUIRED; @@ -841,14 +841,14 @@ static int ocf_alock_lock_wr_fast(struct ocf_alock *alock, if (ret == OCF_LOCK_NOT_ACQUIRED) { /* Request is not locked, discard acquired locks */ for (; i >= 0; i--) { - if (!cbs->line_needs_lock(req, i)) + if (!cbs->line_needs_lock(alock, req, i)) continue; - entry = cbs->line_get_entry(req, i); + entry = cbs->line_get_entry(alock, req, i); - if (cbs->line_is_locked(req, i, OCF_WRITE)) { + if (cbs->line_is_locked(alock, req, i, OCF_WRITE)) { ocf_alock_unlock_one_wr(alock, cbs, entry); - cbs->line_mark_locked(req, i, OCF_WRITE, false); + cbs->line_mark_locked(alock, req, i, OCF_WRITE, false); } } } @@ -877,16 +877,16 @@ static int ocf_alock_lock_wr_slow(struct ocf_alock *alock, for (i = 0; i < req->core_line_count; i++) { - if (!cbs->line_needs_lock(req, i)) { + if (!cbs->line_needs_lock(alock, req, i)) { /* nothing to lock */ env_atomic_dec(&req->lock_remaining); continue; } - entry = cbs->line_get_entry(req, i); + entry = cbs->line_get_entry(alock, req, i); ENV_BUG_ON(entry >= alock->num_entries); - ENV_BUG_ON(cbs->line_is_locked(req, i, OCF_READ)); - ENV_BUG_ON(cbs->line_is_locked(req, i, OCF_WRITE)); + ENV_BUG_ON(cbs->line_is_locked(alock, req, i, OCF_READ)); + ENV_BUG_ON(cbs->line_is_locked(alock, req, i, OCF_WRITE)); if (!ocf_alock_lock_one_wr(alock, cbs, entry, cmpl, req, i)) { /* lock not acquired and not added to wait list */ @@ -904,7 +904,7 @@ static int ocf_alock_lock_wr_slow(struct ocf_alock *alock, err: for (; i >= 0; i--) { - if (!cbs->line_needs_lock(req, i)) + if (!cbs->line_needs_lock(alock, req, i)) continue; ocf_alock_waitlist_remove_entry(alock, cbs, req, i, OCF_WRITE); @@ -942,20 +942,20 @@ void ocf_alock_unlock_rd(struct ocf_alock *alock, OCF_DEBUG_RQ(req, "Unlock"); for (i = 0; i < req->core_line_count; i++) { - ENV_BUG_ON(cbs->line_is_locked(req, i, OCF_WRITE)); + ENV_BUG_ON(cbs->line_is_locked(alock, req, i, OCF_WRITE)); - if (!cbs->line_is_acting(req, i)) + if (!cbs->line_is_acting(alock, req, i)) continue; - if (!cbs->line_is_locked(req, i, OCF_READ)) + if (!cbs->line_is_locked(alock, req, i, OCF_READ)) continue; - entry = cbs->line_get_entry(req, i); + entry = cbs->line_get_entry(alock, req, i); ENV_BUG_ON(entry >= alock->num_entries); ocf_alock_unlock_one_rd(alock, cbs, entry); - cbs->line_mark_locked(req, i, OCF_READ, false); + cbs->line_mark_locked(alock, req, i, OCF_READ, false); } } @@ -969,20 +969,20 @@ void ocf_alock_unlock_wr(struct ocf_alock *alock, OCF_DEBUG_RQ(req, "Unlock"); for (i = 0; i < req->core_line_count; i++) { - ENV_BUG_ON(cbs->line_is_locked(req, i, OCF_READ)); + ENV_BUG_ON(cbs->line_is_locked(alock, req, i, OCF_READ)); - if (!cbs->line_is_acting(req, i)) + if (!cbs->line_is_acting(alock, req, i)) continue; - if (!cbs->line_is_locked(req, i, OCF_WRITE)) + if (!cbs->line_is_locked(alock, req, i, OCF_WRITE)) continue; - entry = cbs->line_get_entry(req, i); + entry = cbs->line_get_entry(alock, req, i); ENV_BUG_ON(entry >= alock->num_entries); ocf_alock_unlock_one_wr(alock, cbs, entry); - cbs->line_mark_locked(req, i, OCF_WRITE, false); + cbs->line_mark_locked(alock, req, i, OCF_WRITE, false); } } @@ -996,21 +996,21 @@ void ocf_alock_unlock(struct ocf_alock *alock, OCF_DEBUG_RQ(req, "Unlock"); for (i = 0; i < req->core_line_count; i++) { - if (!cbs->line_is_acting(req, i)) + if (!cbs->line_is_acting(alock, req, i)) continue; - entry = cbs->line_get_entry(req, i); + entry = cbs->line_get_entry(alock, req, i); ENV_BUG_ON(entry >= alock->num_entries); - if (cbs->line_is_locked(req, i, OCF_READ) && - cbs->line_is_locked(req, i, OCF_WRITE)) { + if (cbs->line_is_locked(alock, req, i, OCF_READ) && + cbs->line_is_locked(alock, req, i, OCF_WRITE)) { ENV_BUG(); - } else if (cbs->line_is_locked(req, i, OCF_READ)) { + } else if (cbs->line_is_locked(alock, req, i, OCF_READ)) { ocf_alock_unlock_one_rd(alock, cbs, entry); - cbs->line_mark_locked(req, i, OCF_READ, false); - } else if (cbs->line_is_locked(req, i, OCF_WRITE)) { + cbs->line_mark_locked(alock, req, i, OCF_READ, false); + } else if (cbs->line_is_locked(alock, req, i, OCF_WRITE)) { ocf_alock_unlock_one_wr(alock, cbs, entry); - cbs->line_mark_locked(req, i, OCF_WRITE, false); + cbs->line_mark_locked(alock, req, i, OCF_WRITE, false); } } } @@ -1019,19 +1019,19 @@ void ocf_alock_unlock_one(struct ocf_alock *alock, struct ocf_alock_lock_cbs *cbs, struct ocf_request *req, uint32_t idx) { - ocf_cache_line_t entry = cbs->line_get_entry(req, idx); + ocf_cache_line_t entry = cbs->line_get_entry(alock, req, idx); - ENV_BUG_ON(!cbs->line_is_acting(req, idx)); + ENV_BUG_ON(!cbs->line_is_acting(alock, req, idx)); - if (cbs->line_is_locked(req, idx, OCF_READ) && - cbs->line_is_locked(req, idx, OCF_WRITE)) { + if (cbs->line_is_locked(alock, req, idx, OCF_READ) && + cbs->line_is_locked(alock, req, idx, OCF_WRITE)) { ENV_BUG(); - } else if (cbs->line_is_locked(req, idx, OCF_READ)) { + } else if (cbs->line_is_locked(alock, req, idx, OCF_READ)) { ocf_alock_unlock_one_rd(alock, cbs, entry); - cbs->line_mark_locked(req, idx, OCF_READ, false); - } else if (cbs->line_is_locked(req, idx, OCF_WRITE)) { + cbs->line_mark_locked(alock, req, idx, OCF_READ, false); + } else if (cbs->line_is_locked(alock, req, idx, OCF_WRITE)) { ocf_alock_unlock_one_wr(alock, cbs, entry); - cbs->line_mark_locked(req, idx, OCF_WRITE, false); + cbs->line_mark_locked(alock, req, idx, OCF_WRITE, false); } else { ENV_BUG(); } diff --git a/src/utils/utils_alock.h b/src/utils/utils_alock.h index 0559ba9..d647abb 100644 --- a/src/utils/utils_alock.h +++ b/src/utils/utils_alock.h @@ -20,20 +20,21 @@ struct ocf_alock; /* async request cacheline lock acquisition callback */ typedef void (*ocf_req_async_lock_cb)(struct ocf_request *req); -typedef bool (*ocf_cl_lock_line_needs_lock_cb)(struct ocf_request *req, - unsigned index); +typedef bool (*ocf_cl_lock_line_needs_lock_cb)(struct ocf_alock *alock, + struct ocf_request *req, unsigned index); -typedef bool (*ocf_cl_lock_line_is_acting_cb)(struct ocf_request *req, - unsigned index); +typedef bool (*ocf_cl_lock_line_is_acting_cb)(struct ocf_alock *alock, + struct ocf_request *req, unsigned index); -typedef bool (*ocf_cl_lock_line_is_locked_cb)(struct ocf_request *req, - unsigned index, int rw); +typedef bool (*ocf_cl_lock_line_is_locked_cb)(struct ocf_alock *alock, + struct ocf_request *req, unsigned index, int rw); -typedef void (*ocf_cl_lock_line_mark_locked_cb)(struct ocf_request *req, - unsigned index, int rw, bool locked); +typedef void (*ocf_cl_lock_line_mark_locked_cb)(struct ocf_alock *alock, + struct ocf_request *req, unsigned index, int rw, bool locked); typedef ocf_cache_line_t (*ocf_cl_lock_line_get_entry_cb)( - struct ocf_request *req, unsigned index); + struct ocf_alock *alock, struct ocf_request *req, + unsigned index); struct ocf_alock_lock_cbs {