From 60a6da7ee6d78e1579c65181a2983d0ae9dc2ebf Mon Sep 17 00:00:00 2001 From: Jan Musial Date: Mon, 16 May 2022 15:29:34 +0200 Subject: [PATCH 1/2] Extend alock API with entries_count method Right now alock assumes that number of locks taken will equal number of core lines. This is not the case in pio, where only parts of metadata are under locks. If pio request overlaps locked and not-locked metadata section it will have different core lines number and awaited locks number. To remedy this discrepancy additional method which gets count of locks that will be taken/waited on is added to alock API. Signed-off-by: Jan Musial --- src/concurrency/ocf_cache_line_concurrency.c | 12 +++++++++-- src/concurrency/ocf_mio_concurrency.c | 11 ++++++++-- src/concurrency/ocf_pio_concurrency.c | 21 +++++++++++++++++-- src/utils/utils_alock.c | 12 ++++++++--- src/utils/utils_alock.h | 6 +++++- .../ocf_cache_line_concurrency.c | 5 +++++ 6 files changed, 57 insertions(+), 10 deletions(-) diff --git a/src/concurrency/ocf_cache_line_concurrency.c b/src/concurrency/ocf_cache_line_concurrency.c index 3babf48..1d07a45 100644 --- a/src/concurrency/ocf_cache_line_concurrency.c +++ b/src/concurrency/ocf_cache_line_concurrency.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ @@ -141,9 +141,17 @@ err: return ret; } +static uint32_t ocf_cl_lock_get_entries_count(struct ocf_alock *alock, + struct ocf_request *req) +{ + return req->core_line_count; +} + static struct ocf_alock_lock_cbs ocf_cline_conc_cbs = { .lock_entries_fast = ocf_cl_lock_line_fast, - .lock_entries_slow = ocf_cl_lock_line_slow + .lock_entries_slow = ocf_cl_lock_line_slow, + .get_entries_count = ocf_cl_lock_get_entries_count + }; bool ocf_cache_line_try_lock_rd(struct ocf_alock *alock, diff --git a/src/concurrency/ocf_mio_concurrency.c b/src/concurrency/ocf_mio_concurrency.c index aebc6ff..255ac6f 100644 --- a/src/concurrency/ocf_mio_concurrency.c +++ b/src/concurrency/ocf_mio_concurrency.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2021 Intel Corporation + * Copyright(c) 2021-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ @@ -98,9 +98,16 @@ err: return ret; } +static uint32_t ocf_mio_lock_get_entries_count(struct ocf_alock *alock, + struct ocf_request *req) +{ + return req->core_line_count; +} + static struct ocf_alock_lock_cbs ocf_mio_conc_cbs = { .lock_entries_fast = ocf_mio_lock_fast, - .lock_entries_slow = ocf_mio_lock_slow + .lock_entries_slow = ocf_mio_lock_slow, + .get_entries_count = ocf_mio_lock_get_entries_count }; int ocf_mio_async_lock(struct ocf_alock *alock, diff --git a/src/concurrency/ocf_pio_concurrency.c b/src/concurrency/ocf_pio_concurrency.c index c30490e..b4015d6 100644 --- a/src/concurrency/ocf_pio_concurrency.c +++ b/src/concurrency/ocf_pio_concurrency.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2021 Intel Corporation + * Copyright(c) 2021-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ @@ -136,9 +136,26 @@ err: return ret; } +static uint32_t ocf_pio_lock_get_entries_count(struct ocf_alock *alock, + struct ocf_request *req) +{ + uint32_t i, count = 0; + ocf_cache_line_t entry; + + for (i = 0; i < req->core_line_count; i++) { + entry = ocf_pio_lock_get_entry(alock, req, i); + if (entry == OUT_OF_RANGE) + continue; + count++; + } + + return count; +} + static struct ocf_alock_lock_cbs ocf_pio_conc_cbs = { .lock_entries_fast = ocf_pio_lock_fast, - .lock_entries_slow = ocf_pio_lock_slow + .lock_entries_slow = ocf_pio_lock_slow, + .get_entries_count = ocf_pio_lock_get_entries_count }; int ocf_pio_async_lock(struct ocf_alock *alock, struct ocf_request *req, diff --git a/src/utils/utils_alock.c b/src/utils/utils_alock.c index e36e9db..e118ac3 100644 --- a/src/utils/utils_alock.c +++ b/src/utils/utils_alock.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ @@ -697,6 +697,7 @@ int ocf_alock_lock_rd(struct ocf_alock *alock, struct ocf_request *req, ocf_req_async_lock_cb cmpl) { int lock, status; + uint32_t to_lock; ENV_BUG_ON(env_atomic_read(&req->lock_remaining)); req->alock_rw = OCF_READ; @@ -709,8 +710,10 @@ int ocf_alock_lock_rd(struct ocf_alock *alock, ENV_BUG_ON(env_atomic_read(&req->lock_remaining)); ENV_BUG_ON(!cmpl); + to_lock = alock->cbs->get_entries_count(alock, req); + env_atomic_inc(&alock->waiting); - env_atomic_set(&req->lock_remaining, req->core_line_count); + env_atomic_set(&req->lock_remaining, to_lock); env_atomic_inc(&req->lock_remaining); status = alock->cbs->lock_entries_slow(alock, req, OCF_READ, cmpl); @@ -734,6 +737,7 @@ int ocf_alock_lock_wr(struct ocf_alock *alock, struct ocf_request *req, ocf_req_async_lock_cb cmpl) { int lock, status; + uint32_t to_lock; ENV_BUG_ON(env_atomic_read(&req->lock_remaining)); req->alock_rw = OCF_WRITE; @@ -745,8 +749,10 @@ int ocf_alock_lock_wr(struct ocf_alock *alock, ENV_BUG_ON(env_atomic_read(&req->lock_remaining)); ENV_BUG_ON(!cmpl); + to_lock = alock->cbs->get_entries_count(alock, req); + env_atomic_inc(&alock->waiting); - env_atomic_set(&req->lock_remaining, req->core_line_count); + env_atomic_set(&req->lock_remaining, to_lock); env_atomic_inc(&req->lock_remaining); status = alock->cbs->lock_entries_slow(alock, req, OCF_WRITE, cmpl); diff --git a/src/utils/utils_alock.h b/src/utils/utils_alock.h index 4568808..4fb5fe2 100644 --- a/src/utils/utils_alock.h +++ b/src/utils/utils_alock.h @@ -1,5 +1,5 @@ /* - * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2012-2022 Intel Corporation * SPDX-License-Identifier: BSD-3-Clause */ #ifndef OCF_UTILS_ALOCK_H_ @@ -26,10 +26,14 @@ typedef int (*ocf_cl_lock_fast)(struct ocf_alock *alock, typedef int (*ocf_cl_lock_slow)(struct ocf_alock *alock, struct ocf_request *req, int rw, ocf_req_async_lock_cb cmpl); +typedef uint32_t (*ocf_cl_lock_get_count)(struct ocf_alock *alock, + struct ocf_request *req); + struct ocf_alock_lock_cbs { ocf_cl_lock_fast lock_entries_fast; ocf_cl_lock_slow lock_entries_slow; + ocf_cl_lock_get_count get_entries_count; }; bool ocf_alock_trylock_one_rd(struct ocf_alock *alock, diff --git a/tests/unit/tests/concurrency/ocf_cache_line_concurrency.c/ocf_cache_line_concurrency.c b/tests/unit/tests/concurrency/ocf_cache_line_concurrency.c/ocf_cache_line_concurrency.c index 91cf79f..296fc00 100644 --- a/tests/unit/tests/concurrency/ocf_cache_line_concurrency.c/ocf_cache_line_concurrency.c +++ b/tests/unit/tests/concurrency/ocf_cache_line_concurrency.c/ocf_cache_line_concurrency.c @@ -1,3 +1,7 @@ +/* + * Copyright(c) 2021-2022 Intel Corporation + * SPDX-License-Identifier: BSD-3-Clause + */ /* * src/concurrency/ocf_cache_line_concurrency.c * ocf_req_async_lock_rd @@ -43,6 +47,7 @@ * ocf_cl_lock_line_get_entry * ocf_cl_lock_line_is_acting * ocf_cl_lock_line_slow + * ocf_cl_lock_get_entries_count * ocf_cl_lock_line_fast * */ From 6016a6f4c746b2dbd698722d5fcaed54ca7e4a36 Mon Sep 17 00:00:00 2001 From: Jan Musial Date: Wed, 18 May 2022 11:56:06 +0200 Subject: [PATCH 2/2] Mark unlikely branches in pio_concurrency Signed-off-by: Jan Musial --- src/concurrency/ocf_pio_concurrency.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/concurrency/ocf_pio_concurrency.c b/src/concurrency/ocf_pio_concurrency.c index b4015d6..cf6b40c 100644 --- a/src/concurrency/ocf_pio_concurrency.c +++ b/src/concurrency/ocf_pio_concurrency.c @@ -68,7 +68,7 @@ static int ocf_pio_lock_fast(struct ocf_alock *alock, for (i = 0; i < req->core_line_count; i++) { entry = ocf_pio_lock_get_entry(alock, req, i); - if (entry == OUT_OF_RANGE) + if (unlikely(entry == OUT_OF_RANGE)) continue; ENV_BUG_ON(ocf_alock_is_index_locked(alock, req, i)); @@ -87,7 +87,7 @@ static int ocf_pio_lock_fast(struct ocf_alock *alock, /* Request is not locked, discard acquired locks */ for (; i >= 0; i--) { entry = ocf_pio_lock_get_entry(alock, req, i); - if (entry == OUT_OF_RANGE) + if (unlikely(entry == OUT_OF_RANGE)) continue; if (ocf_alock_is_index_locked(alock, req, i)) { @@ -109,7 +109,7 @@ static int ocf_pio_lock_slow(struct ocf_alock *alock, for (i = 0; i < req->core_line_count; i++) { entry = ocf_pio_lock_get_entry(alock, req, i); - if (entry == OUT_OF_RANGE) + if (unlikely(entry == OUT_OF_RANGE)) continue; ENV_BUG_ON(ocf_alock_is_index_locked(alock, req, i)); @@ -127,7 +127,7 @@ static int ocf_pio_lock_slow(struct ocf_alock *alock, err: for (; i >= 0; i--) { entry = ocf_pio_lock_get_entry(alock, req, i); - if (entry == OUT_OF_RANGE) + if (unlikely(entry == OUT_OF_RANGE)) continue; ocf_alock_waitlist_remove_entry(alock, req, i, entry, OCF_WRITE); @@ -144,7 +144,7 @@ static uint32_t ocf_pio_lock_get_entries_count(struct ocf_alock *alock, for (i = 0; i < req->core_line_count; i++) { entry = ocf_pio_lock_get_entry(alock, req, i); - if (entry == OUT_OF_RANGE) + if (unlikely(entry == OUT_OF_RANGE)) continue; count++; } @@ -174,7 +174,7 @@ void ocf_pio_async_unlock(struct ocf_alock *alock, struct ocf_request *req) continue; entry = ocf_pio_lock_get_entry(alock, req, i); - if (entry == OUT_OF_RANGE) + if (unlikely(entry == OUT_OF_RANGE)) continue; ocf_alock_unlock_one_wr(alock, entry);