diff --git a/src/concurrency/ocf_cache_line_concurrency.c b/src/concurrency/ocf_cache_line_concurrency.c index 3babf48..1d07a45 100644 --- a/src/concurrency/ocf_cache_line_concurrency.c +++ b/src/concurrency/ocf_cache_line_concurrency.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ @@ -141,9 +141,17 @@ err: return ret; } +static uint32_t ocf_cl_lock_get_entries_count(struct ocf_alock *alock, + struct ocf_request *req) +{ + return req->core_line_count; +} + static struct ocf_alock_lock_cbs ocf_cline_conc_cbs = { .lock_entries_fast = ocf_cl_lock_line_fast, - .lock_entries_slow = ocf_cl_lock_line_slow + .lock_entries_slow = ocf_cl_lock_line_slow, + .get_entries_count = ocf_cl_lock_get_entries_count + }; bool ocf_cache_line_try_lock_rd(struct ocf_alock *alock, diff --git a/src/concurrency/ocf_mio_concurrency.c b/src/concurrency/ocf_mio_concurrency.c index aebc6ff..255ac6f 100644 --- a/src/concurrency/ocf_mio_concurrency.c +++ b/src/concurrency/ocf_mio_concurrency.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2021 Intel Corporation + * Copyright(c) 2021-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ @@ -98,9 +98,16 @@ err: return ret; } +static uint32_t ocf_mio_lock_get_entries_count(struct ocf_alock *alock, + struct ocf_request *req) +{ + return req->core_line_count; +} + static struct ocf_alock_lock_cbs ocf_mio_conc_cbs = { .lock_entries_fast = ocf_mio_lock_fast, - .lock_entries_slow = ocf_mio_lock_slow + .lock_entries_slow = ocf_mio_lock_slow, + .get_entries_count = ocf_mio_lock_get_entries_count }; int ocf_mio_async_lock(struct ocf_alock *alock, diff --git a/src/concurrency/ocf_pio_concurrency.c b/src/concurrency/ocf_pio_concurrency.c index c30490e..cf6b40c 100644 --- a/src/concurrency/ocf_pio_concurrency.c +++ b/src/concurrency/ocf_pio_concurrency.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2021 Intel Corporation + * Copyright(c) 2021-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ @@ -68,7 +68,7 @@ static int ocf_pio_lock_fast(struct ocf_alock *alock, for (i = 0; i < req->core_line_count; i++) { entry = ocf_pio_lock_get_entry(alock, req, i); - if (entry == OUT_OF_RANGE) + if (unlikely(entry == OUT_OF_RANGE)) continue; ENV_BUG_ON(ocf_alock_is_index_locked(alock, req, i)); @@ -87,7 +87,7 @@ static int ocf_pio_lock_fast(struct ocf_alock *alock, /* Request is not locked, discard acquired locks */ for (; i >= 0; i--) { entry = ocf_pio_lock_get_entry(alock, req, i); - if (entry == OUT_OF_RANGE) + if (unlikely(entry == OUT_OF_RANGE)) continue; if (ocf_alock_is_index_locked(alock, req, i)) { @@ -109,7 +109,7 @@ static int ocf_pio_lock_slow(struct ocf_alock *alock, for (i = 0; i < req->core_line_count; i++) { entry = ocf_pio_lock_get_entry(alock, req, i); - if (entry == OUT_OF_RANGE) + if (unlikely(entry == OUT_OF_RANGE)) continue; ENV_BUG_ON(ocf_alock_is_index_locked(alock, req, i)); @@ -127,7 +127,7 @@ static int ocf_pio_lock_slow(struct ocf_alock *alock, err: for (; i >= 0; i--) { entry = ocf_pio_lock_get_entry(alock, req, i); - if (entry == OUT_OF_RANGE) + if (unlikely(entry == OUT_OF_RANGE)) continue; ocf_alock_waitlist_remove_entry(alock, req, i, entry, OCF_WRITE); @@ -136,9 +136,26 @@ err: return ret; } +static uint32_t ocf_pio_lock_get_entries_count(struct ocf_alock *alock, + struct ocf_request *req) +{ + uint32_t i, count = 0; + ocf_cache_line_t entry; + + for (i = 0; i < req->core_line_count; i++) { + entry = ocf_pio_lock_get_entry(alock, req, i); + if (unlikely(entry == OUT_OF_RANGE)) + continue; + count++; + } + + return count; +} + static struct ocf_alock_lock_cbs ocf_pio_conc_cbs = { .lock_entries_fast = ocf_pio_lock_fast, - .lock_entries_slow = ocf_pio_lock_slow + .lock_entries_slow = ocf_pio_lock_slow, + .get_entries_count = ocf_pio_lock_get_entries_count }; int ocf_pio_async_lock(struct ocf_alock *alock, struct ocf_request *req, @@ -157,7 +174,7 @@ void ocf_pio_async_unlock(struct ocf_alock *alock, struct ocf_request *req) continue; entry = ocf_pio_lock_get_entry(alock, req, i); - if (entry == OUT_OF_RANGE) + if (unlikely(entry == OUT_OF_RANGE)) continue; ocf_alock_unlock_one_wr(alock, entry); diff --git a/src/utils/utils_alock.c b/src/utils/utils_alock.c index e36e9db..e118ac3 100644 --- a/src/utils/utils_alock.c +++ b/src/utils/utils_alock.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ @@ -697,6 +697,7 @@ int ocf_alock_lock_rd(struct ocf_alock *alock, struct ocf_request *req, ocf_req_async_lock_cb cmpl) { int lock, status; + uint32_t to_lock; ENV_BUG_ON(env_atomic_read(&req->lock_remaining)); req->alock_rw = OCF_READ; @@ -709,8 +710,10 @@ int ocf_alock_lock_rd(struct ocf_alock *alock, ENV_BUG_ON(env_atomic_read(&req->lock_remaining)); ENV_BUG_ON(!cmpl); + to_lock = alock->cbs->get_entries_count(alock, req); + env_atomic_inc(&alock->waiting); - env_atomic_set(&req->lock_remaining, req->core_line_count); + env_atomic_set(&req->lock_remaining, to_lock); env_atomic_inc(&req->lock_remaining); status = alock->cbs->lock_entries_slow(alock, req, OCF_READ, cmpl); @@ -734,6 +737,7 @@ int ocf_alock_lock_wr(struct ocf_alock *alock, struct ocf_request *req, ocf_req_async_lock_cb cmpl) { int lock, status; + uint32_t to_lock; ENV_BUG_ON(env_atomic_read(&req->lock_remaining)); req->alock_rw = OCF_WRITE; @@ -745,8 +749,10 @@ int ocf_alock_lock_wr(struct ocf_alock *alock, ENV_BUG_ON(env_atomic_read(&req->lock_remaining)); ENV_BUG_ON(!cmpl); + to_lock = alock->cbs->get_entries_count(alock, req); + env_atomic_inc(&alock->waiting); - env_atomic_set(&req->lock_remaining, req->core_line_count); + env_atomic_set(&req->lock_remaining, to_lock); env_atomic_inc(&req->lock_remaining); status = alock->cbs->lock_entries_slow(alock, req, OCF_WRITE, cmpl); diff --git a/src/utils/utils_alock.h b/src/utils/utils_alock.h index 4568808..4fb5fe2 100644 --- a/src/utils/utils_alock.h +++ b/src/utils/utils_alock.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ #ifndef OCF_UTILS_ALOCK_H_ @@ -26,10 +26,14 @@ typedef int (*ocf_cl_lock_fast)(struct ocf_alock *alock, typedef int (*ocf_cl_lock_slow)(struct ocf_alock *alock, struct ocf_request *req, int rw, ocf_req_async_lock_cb cmpl); +typedef uint32_t (*ocf_cl_lock_get_count)(struct ocf_alock *alock, + struct ocf_request *req); + struct ocf_alock_lock_cbs { ocf_cl_lock_fast lock_entries_fast; ocf_cl_lock_slow lock_entries_slow; + ocf_cl_lock_get_count get_entries_count; }; bool ocf_alock_trylock_one_rd(struct ocf_alock *alock, diff --git a/tests/unit/tests/concurrency/ocf_cache_line_concurrency.c/ocf_cache_line_concurrency.c b/tests/unit/tests/concurrency/ocf_cache_line_concurrency.c/ocf_cache_line_concurrency.c index 91cf79f..296fc00 100644 --- a/tests/unit/tests/concurrency/ocf_cache_line_concurrency.c/ocf_cache_line_concurrency.c +++ b/tests/unit/tests/concurrency/ocf_cache_line_concurrency.c/ocf_cache_line_concurrency.c @@ -1,3 +1,7 @@ +/* + * Copyright(c) 2021-2022 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause + */ /* * src/concurrency/ocf_cache_line_concurrency.c * ocf_req_async_lock_rd @@ -43,6 +47,7 @@ * ocf_cl_lock_line_get_entry * ocf_cl_lock_line_is_acting * ocf_cl_lock_line_slow + * ocf_cl_lock_get_entries_count * ocf_cl_lock_line_fast * */