diff --git a/src/cleaning/acp.c b/src/cleaning/acp.c
index 0431367..1a3a56f 100644
--- a/src/cleaning/acp.c
+++ b/src/cleaning/acp.c
@@ -9,6 +9,7 @@
#include "../metadata/metadata.h"
#include "../utils/utils_cleaner.h"
#include "../utils/utils_cache_line.h"
+#include "../utils/utils_parallelize.h"
#include "../ocf_request.h"
#include "../cleaning/acp.h"
#include "../engine/engine_common.h"
@@ -270,8 +271,7 @@ void cleaning_policy_acp_setup(struct ocf_cache *cache)
config->flush_max_buffers = OCF_ACP_DEFAULT_FLUSH_MAX_BUFFERS;
}
-int cleaning_policy_acp_initialize(struct ocf_cache *cache,
- int init_metadata)
+int cleaning_policy_acp_init_common(ocf_cache_t cache)
{
struct acp_context *acp;
int err, i;
@@ -317,12 +317,206 @@ int cleaning_policy_acp_initialize(struct ocf_cache *cache,
}
}
+ return 0;
+}
+
+int cleaning_policy_acp_initialize(ocf_cache_t cache, int init_metadata)
+{
+ int result;
+
+ result = cleaning_policy_acp_init_common(cache);
+ if (result)
+ return result;
+
_acp_rebuild(cache);
ocf_kick_cleaner(cache);
return 0;
}
+#define OCF_ACP_RECOVERY_SHARDS_CNT 32
+
+struct ocf_acp_recovery_context {
+ ocf_cache_t cache;
+
+ struct {
+ uint16_t *chunk[OCF_CORE_MAX];
+ struct {
+ struct list_head chunk_list;
+ } bucket[ACP_MAX_BUCKETS];
+ } shard[OCF_ACP_RECOVERY_SHARDS_CNT];
+
+ ocf_cleaning_recovery_end_t cmpl;
+ void *priv;
+};
+
+static int ocf_acp_recovery_handle(ocf_parallelize_t parallelize,
+ void *priv, unsigned shard_id, unsigned shards_cnt)
+{
+ struct ocf_acp_recovery_context *context = priv;
+ ocf_cache_t cache = context->cache;
+ ocf_cache_line_t entries = cache->device->collision_table_entries;
+ ocf_cache_line_t cline, portion;
+ uint64_t begin, end;
+ struct acp_cleaning_policy_meta *acp_meta;
+ struct acp_chunk_info *chunk;
+ ocf_core_id_t core_id;
+ uint32_t step = 0;
+
+ portion = DIV_ROUND_UP((uint64_t)entries, shards_cnt);
+ begin = portion*shard_id;
+ end = OCF_MIN(portion*(shard_id + 1), entries);
+
+ for (cline = begin; cline < end; cline++) {
+ ocf_metadata_get_core_and_part_id(cache, cline, &core_id, NULL);
+
+ OCF_COND_RESCHED_DEFAULT(step);
+
+ if (core_id == OCF_CORE_MAX)
+ continue;
+
+ if (!metadata_test_dirty(cache, cline)) {
+ cleaning_policy_acp_init_cache_block(cache, cline);
+ continue;
+ }
+
+ acp_meta = _acp_meta_get(cache, cline);
+ acp_meta->dirty = 1;
+
+ chunk = _acp_get_chunk(cache, cline);
+ context->shard[shard_id].chunk[core_id][chunk->chunk_id]++;
+ }
+
+ return 0;
+}
+
+static void ocf_acp_recovery_chunk(struct ocf_acp_recovery_context *context,
+ struct acp_chunk_info *chunk)
+{
+ ocf_cache_t cache = context->cache;
+ struct acp_context *acp = _acp_get_ctx_from_cache(cache);
+ struct acp_bucket *bucket;
+ unsigned shard_id;
+ uint8_t bucket_id;
+
+ chunk->num_dirty = 0;
+ for (shard_id = 0; shard_id < OCF_ACP_RECOVERY_SHARDS_CNT; shard_id++) {
+ chunk->num_dirty += context->shard[shard_id]
+ .chunk[chunk->core_id][chunk->chunk_id];
+ }
+
+ for (bucket_id = 0; bucket_id < ACP_MAX_BUCKETS; bucket_id++) {
+ bucket = &acp->bucket_info[bucket_id];
+ if (chunk->num_dirty < bucket->threshold)
+ break;
+ }
+
+ bucket = &acp->bucket_info[--bucket_id];
+ chunk->bucket_id = bucket_id;
+
+ list_move_tail(&chunk->list, &bucket->chunk_list);
+}
+
+static void ocf_acp_recovery_finish(ocf_parallelize_t parallelize,
+ void *priv, int error)
+{
+ struct ocf_acp_recovery_context *context = priv;
+ ocf_cache_t cache = context->cache;
+ struct acp_context *acp = _acp_get_ctx_from_cache(cache);
+ ocf_core_id_t core_id;
+ ocf_core_t core;
+ uint64_t core_size;
+ uint64_t num_chunks;
+ uint64_t chunk_id;
+ uint32_t step = 0;
+
+ for_each_core(cache, core, core_id) {
+ core_size = core->conf_meta->length;
+ num_chunks = OCF_DIV_ROUND_UP(core_size, ACP_CHUNK_SIZE);
+
+ for (chunk_id = 0; chunk_id < num_chunks; chunk_id++) {
+ ocf_acp_recovery_chunk(context,
+ &acp->chunk_info[core_id][chunk_id]);
+ OCF_COND_RESCHED_DEFAULT(step);
+ }
+ }
+
+ ocf_cache_log(cache, log_info, "Finished rebuilding ACP metadata\n");
+
+ ocf_kick_cleaner(cache);
+
+ context->cmpl(context->priv, error);
+
+ for_each_core(cache, core, core_id) {
+ if (context->shard[0].chunk[core_id])
+ env_vfree(context->shard[0].chunk[core_id]);
+ }
+
+ ocf_parallelize_destroy(parallelize);
+}
+
+void cleaning_policy_acp_recovery(ocf_cache_t cache,
+ ocf_cleaning_recovery_end_t cmpl, void *priv)
+{
+ struct ocf_acp_recovery_context *context;
+ ocf_parallelize_t parallelize;
+ ocf_core_id_t core_id;
+ ocf_core_t core;
+ unsigned shards_cnt = OCF_ACP_RECOVERY_SHARDS_CNT;
+ unsigned shard_id;
+ uint64_t core_size;
+ uint64_t num_chunks;
+ uint16_t *chunks;
+ int result;
+
+ result = ocf_parallelize_create(¶llelize, cache,
+ OCF_ACP_RECOVERY_SHARDS_CNT, sizeof(*context),
+ ocf_acp_recovery_handle, ocf_acp_recovery_finish);
+ if (result) {
+ cmpl(priv, result);
+ return;
+ }
+
+ context = ocf_parallelize_get_priv(parallelize);
+ context->cache = cache;
+ context->cmpl = cmpl;
+ context->priv = priv;
+
+ for_each_core(cache, core, core_id) {
+ core_size = core->conf_meta->length;
+ num_chunks = OCF_DIV_ROUND_UP(core_size, ACP_CHUNK_SIZE);
+
+ chunks = env_vzalloc(sizeof(*chunks) * num_chunks * shards_cnt);
+ if (!chunks) {
+ result = -OCF_ERR_NO_MEM;
+ goto err;
+ }
+
+ for (shard_id = 0; shard_id < shards_cnt; shard_id++) {
+ context->shard[shard_id].chunk[core_id] =
+ &chunks[num_chunks * shard_id];
+ }
+ }
+
+ result = cleaning_policy_acp_init_common(cache);
+ if (result)
+ goto err;
+
+ ocf_parallelize_run(parallelize);
+
+ return;
+
+err:
+ for_each_core(cache, core, core_id) {
+ if (context->shard[0].chunk[core_id])
+ env_vfree(context->shard[0].chunk[core_id]);
+ }
+
+ ocf_parallelize_destroy(parallelize);
+
+ cmpl(priv, result);
+}
+
int cleaning_policy_acp_set_cleaning_param(ocf_cache_t cache,
uint32_t param_id, uint32_t param_value)
{
diff --git a/src/cleaning/acp.h b/src/cleaning/acp.h
index 01142f5..52e9fa1 100644
--- a/src/cleaning/acp.h
+++ b/src/cleaning/acp.h
@@ -12,6 +12,9 @@ void cleaning_policy_acp_setup(ocf_cache_t cache);
int cleaning_policy_acp_initialize(ocf_cache_t cache, int init_metadata);
+void cleaning_policy_acp_recovery(ocf_cache_t cache,
+ ocf_cleaning_recovery_end_t cmpl, void *priv);
+
void cleaning_policy_acp_deinitialize(ocf_cache_t cache);
void cleaning_policy_acp_perform_cleaning(ocf_cache_t cache,
diff --git a/src/cleaning/alru.c b/src/cleaning/alru.c
index 86ef0e4..4ca57b9 100644
--- a/src/cleaning/alru.c
+++ b/src/cleaning/alru.c
@@ -10,6 +10,7 @@
#include "../metadata/metadata.h"
#include "../utils/utils_cleaner.h"
#include "../utils/utils_user_part.h"
+#include "../utils/utils_parallelize.h"
#include "../utils/utils_realloc.h"
#include "../concurrency/ocf_cache_line_concurrency.h"
#include "../ocf_def_priv.h"
@@ -61,54 +62,63 @@ struct alru_context {
/* -- Start of ALRU functions -- */
-/* Adds the given collision_index to the _head_ of the ALRU list */
-static void add_alru_head(struct ocf_cache *cache, int partition_id,
- unsigned int collision_index)
+/* Appends given sublist to the _head_ of the ALRU list */
+static void append_alru_head(ocf_cache_t cache, ocf_part_id_t part_id,
+ ocf_cache_line_t head, ocf_cache_line_t tail)
{
- unsigned int curr_head_index;
- unsigned int collision_table_entries = cache->device->collision_table_entries;
- struct alru_cleaning_policy *part_alru = &cache->user_parts[partition_id]
- .clean_pol->policy.alru;
- struct alru_cleaning_policy_meta *alru;
+ ocf_cache_line_t terminator = cache->device->collision_table_entries;
+ struct alru_cleaning_policy *part_alru;
+ struct cleaning_policy_meta *meta;
+ struct alru_cleaning_policy_meta *old_head;
+ struct alru_cleaning_policy_meta *entry;
- ENV_BUG_ON(!(collision_index < collision_table_entries));
+ part_alru = &cache->user_parts[part_id].clean_pol->policy.alru;
- ENV_BUG_ON(env_atomic_read(&part_alru->size) < 0);
+ if (head == terminator && tail == terminator)
+ return;
- ENV_WARN_ON(!metadata_test_dirty(cache, collision_index));
- ENV_WARN_ON(!metadata_test_valid_any(cache, collision_index));
+ ENV_BUG_ON(head == terminator);
+ ENV_BUG_ON(tail == terminator);
- /* First node to be added/ */
- if (env_atomic_read(&part_alru->size) == 0) {
- part_alru->lru_head = collision_index;
- part_alru->lru_tail = collision_index;
-
- alru = &ocf_metadata_get_cleaning_policy(cache,
- collision_index)->meta.alru;
- alru->lru_next = collision_table_entries;
- alru->lru_prev = collision_table_entries;
- alru->timestamp = env_ticks_to_secs(
- env_get_tick_count());
+ if (part_alru->lru_head == terminator) {
+ part_alru->lru_head = head;
+ part_alru->lru_tail = tail;
} else {
- /* Not the first node to be added. */
+ meta = ocf_metadata_get_cleaning_policy(cache, part_alru->lru_head);
+ old_head = &meta->meta.alru;
+ old_head->lru_prev = tail;
- curr_head_index = part_alru->lru_head;
+ meta = ocf_metadata_get_cleaning_policy(cache, tail);
+ entry = &meta->meta.alru;
+ entry->lru_next = part_alru->lru_head;
- ENV_BUG_ON(!(curr_head_index < collision_table_entries));
-
- alru = &ocf_metadata_get_cleaning_policy(cache,
- collision_index)->meta.alru;
- alru->lru_next = curr_head_index;
- alru->lru_prev = collision_table_entries;
- alru->timestamp = env_ticks_to_secs(
- env_get_tick_count());
-
- alru = &ocf_metadata_get_cleaning_policy(cache,
- curr_head_index)->meta.alru;
- alru->lru_prev = collision_index;
-
- part_alru->lru_head = collision_index;
+ part_alru->lru_head = head;
}
+}
+
+/* Adds the given collision_index to the _head_ of the ALRU list */
+static void add_alru_head(ocf_cache_t cache, ocf_part_id_t part_id,
+ ocf_cache_line_t cline)
+{
+ ocf_cache_line_t terminator = cache->device->collision_table_entries;
+ struct alru_cleaning_policy *part_alru;
+ struct cleaning_policy_meta *meta;
+ struct alru_cleaning_policy_meta *entry;
+
+ ENV_BUG_ON(!(cline < terminator));
+
+ ENV_WARN_ON(!metadata_test_dirty(cache, cline));
+ ENV_WARN_ON(!metadata_test_valid_any(cache, cline));
+
+ part_alru = &cache->user_parts[part_id].clean_pol->policy.alru;
+
+ meta = ocf_metadata_get_cleaning_policy(cache, cline);
+ entry = &meta->meta.alru;
+ entry->lru_next = terminator;
+ entry->lru_prev = terminator;
+ entry->timestamp = env_ticks_to_secs(env_get_tick_count());
+
+ append_alru_head(cache, part_id, cline, cline);
env_atomic_inc(&part_alru->size);
}
@@ -375,24 +385,6 @@ static void _alru_rebuild(struct ocf_cache *cache)
}
}
-static int cleaning_policy_alru_initialize_part(struct ocf_cache *cache,
- struct ocf_user_part *user_part, int init_metadata)
-{
- struct alru_cleaning_policy *part_alru =
- &user_part->clean_pol->policy.alru;
-
- if (init_metadata) {
- /* ALRU initialization */
- env_atomic_set(&part_alru->size, 0);
- part_alru->lru_head = cache->device->collision_table_entries;
- part_alru->lru_tail = cache->device->collision_table_entries;
- }
-
- cache->device->runtime_meta->cleaning_thread_access = 0;
-
- return 0;
-}
-
void cleaning_policy_alru_setup(struct ocf_cache *cache)
{
struct alru_cleaning_policy_config *config;
@@ -405,10 +397,8 @@ void cleaning_policy_alru_setup(struct ocf_cache *cache)
config->activity_threshold = OCF_ALRU_DEFAULT_ACTIVITY_THRESHOLD;
}
-int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata)
+int cleaning_policy_alru_init_common(ocf_cache_t cache)
{
- struct ocf_user_part *user_part;
- ocf_part_id_t part_id;
struct alru_context *ctx;
int error = 0;
unsigned i;
@@ -432,13 +422,20 @@ int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata)
return error;
}
+ cache->device->runtime_meta->cleaning_thread_access = 0;
cache->cleaner.cleaning_policy_context = ctx;
- for_each_user_part(cache, user_part, part_id) {
- cleaning_policy_alru_initialize_part(cache,
- user_part, init_metadata);
- }
+ return 0;
+}
+
+int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata)
+{
+ int result;
+
+ result = cleaning_policy_alru_init_common(cache);
+ if (result)
+ return result;
if (init_metadata)
_alru_rebuild(cache);
@@ -448,6 +445,184 @@ int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata)
return 0;
}
+#define OCF_ALRU_RECOVERY_SHARDS_CNT 32
+
+struct ocf_alru_recovery_context {
+ ocf_cache_t cache;
+ struct {
+ struct {
+ ocf_cache_line_t head;
+ ocf_cache_line_t tail;
+ } part[OCF_USER_IO_CLASS_MAX];
+ } shard[OCF_ALRU_RECOVERY_SHARDS_CNT] __attribute__((aligned(64)));
+
+ ocf_cleaning_recovery_end_t cmpl;
+ void *priv;
+};
+
+static void add_alru_head_recovery(struct ocf_alru_recovery_context *context,
+ unsigned shard_id, ocf_core_id_t part_id,
+ ocf_cache_line_t cline)
+{
+ ocf_cache_t cache = context->cache;
+ ocf_cache_line_t curr_head, terminator;
+ struct cleaning_policy_meta *meta;
+ struct alru_cleaning_policy_meta *entry;
+ struct alru_cleaning_policy_meta *next;
+
+ terminator = ocf_metadata_collision_table_entries(cache);
+ curr_head = context->shard[shard_id].part[part_id].head;
+
+ meta = ocf_metadata_get_cleaning_policy(cache, cline);
+ entry = &meta->meta.alru;
+
+ if (curr_head == terminator) {
+ /* First node to be added/ */
+ entry->lru_next = terminator;
+ entry->lru_prev = terminator;
+ entry->timestamp = env_ticks_to_secs(env_get_tick_count());
+
+ context->shard[shard_id].part[part_id].head = cline;
+ context->shard[shard_id].part[part_id].tail = cline;
+ } else {
+ /* Not the first node to be added. */
+ entry->lru_next = curr_head;
+ entry->lru_prev = terminator;
+ entry->timestamp = env_ticks_to_secs(env_get_tick_count());
+
+ meta = ocf_metadata_get_cleaning_policy(cache, curr_head);
+ next = &meta->meta.alru;
+
+ next->lru_prev = cline;
+
+ context->shard[shard_id].part[part_id].head = cline;
+ }
+}
+
+static int ocf_alru_recovery_handle(ocf_parallelize_t parallelize,
+ void *priv, unsigned shard_id, unsigned shards_cnt)
+{
+ struct ocf_alru_recovery_context *context = priv;
+ ocf_cache_t cache = context->cache;
+ ocf_cache_line_t entries = cache->device->collision_table_entries;
+ ocf_cache_line_t terminator = entries;
+ unsigned part_size[OCF_USER_IO_CLASS_MAX] = {};
+ struct ocf_user_part *user_part;
+ struct alru_cleaning_policy *part_alru;
+ ocf_part_id_t part_id;
+ ocf_core_id_t core_id;
+ ocf_cache_line_t cline, portion;
+ uint64_t begin, end;
+ uint32_t step = 0;
+ int i;
+
+ portion = DIV_ROUND_UP((uint64_t)entries, shards_cnt);
+ begin = portion*shard_id;
+ end = OCF_MIN(portion*(shard_id + 1), entries);
+
+ for (i = 0; i < OCF_USER_IO_CLASS_MAX; i++) {
+ context->shard[shard_id].part[i].head = terminator;
+ context->shard[shard_id].part[i].tail = terminator;
+ }
+
+ for (cline = begin; cline < end; cline++) {
+ ocf_metadata_get_core_and_part_id(cache, cline,
+ &core_id, &part_id);
+
+ OCF_COND_RESCHED_DEFAULT(step);
+
+ if (core_id == OCF_CORE_MAX)
+ continue;
+
+ if (!metadata_test_dirty(cache, cline)) {
+ cleaning_policy_alru_init_cache_block(cache, cline);
+ } else {
+ add_alru_head_recovery(context, shard_id,
+ part_id, cline);
+ ++part_size[part_id];
+ }
+ }
+
+ for_each_user_part(cache, user_part, part_id) {
+ part_alru = &user_part->clean_pol->policy.alru;
+ env_atomic_add(part_size[part_id], &part_alru->size);
+ }
+
+ return 0;
+}
+
+static void ocf_alru_recovery_finish(ocf_parallelize_t parallelize,
+ void *priv, int error)
+{
+ struct ocf_alru_recovery_context *context = priv;
+ ocf_cache_t cache = context->cache;
+ ocf_part_id_t part_id;
+ ocf_cache_line_t head, tail;
+ unsigned shard;
+
+ if (error)
+ goto end;
+
+ for (part_id = 0; part_id < OCF_USER_IO_CLASS_MAX; part_id++) {
+ for (shard = 0; shard < OCF_ALRU_RECOVERY_SHARDS_CNT; shard++) {
+ head = context->shard[shard].part[part_id].head;
+ tail = context->shard[shard].part[part_id].tail;
+
+ append_alru_head(cache, part_id, head, tail);
+ }
+ }
+
+ ocf_kick_cleaner(cache);
+
+end:
+ context->cmpl(context->priv, error);
+
+ ocf_parallelize_destroy(parallelize);
+}
+
+void cleaning_policy_alru_recovery(ocf_cache_t cache,
+ ocf_cleaning_recovery_end_t cmpl, void *priv)
+{
+ struct ocf_alru_recovery_context *context;
+ ocf_parallelize_t parallelize;
+ struct alru_cleaning_policy *part_alru;
+ struct ocf_user_part *user_part;
+ ocf_part_id_t part_id;
+ int result;
+
+ result = ocf_parallelize_create(¶llelize, cache,
+ OCF_ALRU_RECOVERY_SHARDS_CNT, sizeof(*context),
+ ocf_alru_recovery_handle, ocf_alru_recovery_finish);
+ if (result) {
+ cmpl(priv, result);
+ return;
+ }
+
+
+ result = cleaning_policy_alru_init_common(cache);
+ if (result) {
+ ocf_parallelize_destroy(parallelize);
+ cmpl(priv, result);
+ return;
+ }
+
+ for_each_user_part(cache, user_part, part_id) {
+ /* ALRU initialization */
+ part_alru = &user_part->clean_pol->policy.alru;
+ env_atomic_set(&part_alru->size, 0);
+ part_alru->lru_head = cache->device->collision_table_entries;
+ part_alru->lru_tail = cache->device->collision_table_entries;
+ cache->device->runtime_meta->cleaning_thread_access = 0;
+ }
+
+ context = ocf_parallelize_get_priv(parallelize);
+ context->cache = cache;
+ context->cmpl = cmpl;
+ context->priv = priv;
+
+ ocf_parallelize_run(parallelize);
+}
+
void cleaning_policy_alru_deinitialize(struct ocf_cache *cache)
{
struct alru_context *alru = cache->cleaner.cleaning_policy_context;
diff --git a/src/cleaning/alru.h b/src/cleaning/alru.h
index c87dbe9..b72fa3a 100644
--- a/src/cleaning/alru.h
+++ b/src/cleaning/alru.h
@@ -11,6 +11,8 @@
void cleaning_policy_alru_setup(ocf_cache_t cache);
int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata);
+void cleaning_policy_alru_recovery(ocf_cache_t cache,
+ ocf_cleaning_recovery_end_t cmpl, void *priv);
void cleaning_policy_alru_deinitialize(ocf_cache_t cache);
void cleaning_policy_alru_init_cache_block(ocf_cache_t cache,
uint32_t cache_line);
diff --git a/src/cleaning/cleaning.h b/src/cleaning/cleaning.h
index c48d50e..f85c002 100644
--- a/src/cleaning/cleaning.h
+++ b/src/cleaning/cleaning.h
@@ -54,4 +54,6 @@ void ocf_kick_cleaner(ocf_cache_t cache);
void ocf_stop_cleaner(ocf_cache_t cache);
+typedef void (*ocf_cleaning_recovery_end_t)(void *priv, int error);
+
#endif
diff --git a/src/cleaning/cleaning_ops.h b/src/cleaning/cleaning_ops.h
index 72bfae8..134d324 100644
--- a/src/cleaning/cleaning_ops.h
+++ b/src/cleaning/cleaning_ops.h
@@ -14,6 +14,8 @@
struct cleaning_policy_ops {
void (*setup)(ocf_cache_t cache);
int (*initialize)(ocf_cache_t cache, int init_metadata);
+ void (*recovery)(ocf_cache_t cache,
+ ocf_cleaning_recovery_end_t cmpl, void *priv);
void (*deinitialize)(ocf_cache_t cache);
int (*add_core)(ocf_cache_t cache, ocf_core_id_t core_id);
void (*remove_core)(ocf_cache_t cache, ocf_core_id_t core_id);
@@ -43,6 +45,7 @@ static struct cleaning_policy_ops cleaning_policy_ops[ocf_cleaning_max] = {
.purge_range = cleaning_policy_alru_purge_range,
.set_hot_cache_line = cleaning_policy_alru_set_hot_cache_line,
.initialize = cleaning_policy_alru_initialize,
+ .recovery = cleaning_policy_alru_recovery,
.deinitialize = cleaning_policy_alru_deinitialize,
.set_cleaning_param = cleaning_policy_alru_set_cleaning_param,
.get_cleaning_param = cleaning_policy_alru_get_cleaning_param,
@@ -56,6 +59,7 @@ static struct cleaning_policy_ops cleaning_policy_ops[ocf_cleaning_max] = {
.purge_range = cleaning_policy_acp_purge_range,
.set_hot_cache_line = cleaning_policy_acp_set_hot_cache_line,
.initialize = cleaning_policy_acp_initialize,
+ .recovery = cleaning_policy_acp_recovery,
.deinitialize = cleaning_policy_acp_deinitialize,
.set_cleaning_param = cleaning_policy_acp_set_cleaning_param,
.get_cleaning_param = cleaning_policy_acp_get_cleaning_param,
@@ -87,6 +91,20 @@ static inline int ocf_cleaning_initialize(ocf_cache_t cache,
return cleaning_policy_ops[policy].initialize(cache, init_metadata);
}
+static inline void ocf_cleaning_recovery(ocf_cache_t cache,
+ ocf_cleaning_t policy,
+ ocf_cleaning_recovery_end_t cmpl, void *priv)
+{
+ ENV_BUG_ON(policy >= ocf_cleaning_max);
+
+ if (unlikely(!cleaning_policy_ops[policy].recovery)) {
+ cmpl(priv, 0);
+ return;
+ }
+
+ cleaning_policy_ops[policy].recovery(cache, cmpl, priv);
+}
+
static inline void ocf_cleaning_deinitialize(ocf_cache_t cache)
{
ocf_cleaning_t policy;
diff --git a/src/concurrency/ocf_metadata_concurrency.c b/src/concurrency/ocf_metadata_concurrency.c
index 8c190c2..9eae499 100644
--- a/src/concurrency/ocf_metadata_concurrency.c
+++ b/src/concurrency/ocf_metadata_concurrency.c
@@ -299,6 +299,18 @@ void ocf_hb_cline_naked_unlock_wr(struct ocf_metadata_lock *metadata_lock,
ocf_hb_id_naked_unlock(metadata_lock, hash, OCF_METADATA_WR);
}
+void ocf_hb_id_naked_lock_wr(struct ocf_metadata_lock *metadata_lock,
+ ocf_cache_line_t hash)
+{
+ ocf_hb_id_naked_lock(metadata_lock, hash, OCF_METADATA_WR);
+}
+
+void ocf_hb_id_naked_unlock_wr(struct ocf_metadata_lock *metadata_lock,
+ ocf_cache_line_t hash)
+{
+ ocf_hb_id_naked_unlock(metadata_lock, hash, OCF_METADATA_WR);
+}
+
/* common part of protected hash bucket lock routines */
static inline void ocf_hb_id_prot_lock_common(
struct ocf_metadata_lock *metadata_lock,
diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c
index 12a6c6f..ebc0761 100644
--- a/src/mngt/ocf_mngt_cache.c
+++ b/src/mngt/ocf_mngt_cache.c
@@ -17,10 +17,12 @@
#include "../utils/utils_cache_line.h"
#include "../utils/utils_io.h"
#include "../utils/utils_cache_line.h"
+#include "../utils/utils_parallelize.h"
#include "../utils/utils_pipeline.h"
#include "../utils/utils_refcnt.h"
#include "../utils/utils_async_lock.h"
#include "../concurrency/ocf_concurrency.h"
+#include "../concurrency/ocf_metadata_concurrency.h"
#include "../ocf_lru.h"
#include "../ocf_ctx_priv.h"
#include "../cleaning/cleaning.h"
@@ -215,14 +217,6 @@ static void __init_parts_attached(ocf_cache_t cache)
ocf_lru_init(cache, &cache->free);
}
-static void __populate_free(ocf_cache_t cache)
-{
- uint64_t free_clines = ocf_metadata_collision_table_entries(cache) -
- ocf_get_cache_occupancy(cache);
-
- ocf_lru_populate(cache, free_clines);
-}
-
static ocf_error_t __init_cleaning_policy(ocf_cache_t cache)
{
int i;
@@ -300,29 +294,6 @@ static void __reset_stats(ocf_cache_t cache)
}
}
-static ocf_error_t init_attached_data_structures(ocf_cache_t cache)
-{
- ocf_error_t result;
-
- /* Lock to ensure consistency */
-
- ocf_metadata_init_hash_table(cache);
- ocf_metadata_init_collision(cache);
- __init_parts_attached(cache);
- __populate_free(cache);
-
- result = __init_cleaning_policy(cache);
- if (result) {
- ocf_cache_log(cache, log_err,
- "Cannot initialize cleaning policy\n");
- return result;
- }
-
- __setup_promotion_policy(cache);
-
- return 0;
-}
-
static void init_attached_data_structures_recovery(ocf_cache_t cache,
bool init_collision)
{
@@ -487,28 +458,84 @@ err:
OCF_PL_FINISH_RET(pipeline, -OCF_ERR_START_CACHE_FAIL);
}
-static void _recovery_reset_cline_metadata(struct ocf_cache *cache,
- ocf_cache_line_t cline)
+typedef void (*ocf_mngt_rebuild_metadata_end_t)(void *priv, int error);
+
+/*
+ * IMPORTANT: This value must match number of LRU lists so that adding
+ * cache lines to the list can be implemented without locking (each shard
+ * owns it's own LRU list). Don't change this value unless you are really
+ * sure you know what you're doing.
+ */
+#define OCF_MNGT_REBUILD_METADATA_SHARDS_CNT OCF_NUM_LRU_LISTS
+
+struct ocf_mngt_rebuild_metadata_context {
+ ocf_cache_t cache;
+
+ struct {
+ env_atomic lines;
+ } core[OCF_CORE_MAX];
+
+ struct {
+ struct {
+ uint32_t lines;
+ } core[OCF_CORE_MAX];
+ } shard[OCF_MNGT_REBUILD_METADATA_SHARDS_CNT];
+
+ env_atomic free_lines;
+
+ ocf_mngt_rebuild_metadata_end_t cmpl;
+ void *priv;
+};
+
+static void ocf_mngt_cline_reset_metadata(ocf_cache_t cache,
+ ocf_cache_line_t cline, uint32_t lru_list)
{
ocf_metadata_set_core_info(cache, cline, OCF_CORE_MAX, ULLONG_MAX);
-
metadata_init_status_bits(cache, cline);
- ocf_cleaning_init_cache_block(cache, cline);
+ ocf_metadata_set_partition_id(cache, cline, PARTITION_FREELIST);
+
+ ocf_lru_add_free(cache, cline);
}
-static int _ocf_mngt_rebuild_metadata(ocf_cache_t cache)
+static void ocf_mngt_cline_rebuild_metadata(ocf_cache_t cache,
+ ocf_core_id_t core_id, uint64_t core_line,
+ ocf_cache_line_t cline)
{
- ocf_cache_line_t cline;
+ ocf_part_id_t part_id = PARTITION_DEFAULT;
+ ocf_cache_line_t hash_index;
+
+ ocf_metadata_set_partition_id(cache, cline, part_id);
+
+ hash_index = ocf_metadata_hash_func(cache, core_line, core_id);
+
+ ocf_hb_id_naked_lock_wr(&cache->metadata.lock, hash_index);
+ ocf_metadata_add_to_collision(cache, core_id, core_line, hash_index,
+ cline);
+ ocf_hb_id_naked_unlock_wr(&cache->metadata.lock, hash_index);
+
+ ocf_lru_init_cline(cache, cline);
+
+ ocf_lru_add(cache, cline);
+}
+
+static int ocf_mngt_rebuild_metadata_handle(ocf_parallelize_t parallelize,
+ void *priv, unsigned shard_id, unsigned shards_cnt)
+{
+ struct ocf_mngt_rebuild_metadata_context *context = priv;
+ ocf_cache_t cache = context->cache;
+ ocf_cache_line_t begin, increment, cline, free_lines;
+ ocf_core_t core;
ocf_core_id_t core_id;
uint64_t core_line;
unsigned char step = 0;
- const uint64_t collision_table_entries =
- ocf_metadata_collision_table_entries(cache);
+ const uint64_t entries = ocf_metadata_collision_table_entries(cache);
- ocf_metadata_start_exclusive_access(&cache->metadata.lock);
+ begin = shard_id;
+ increment = shards_cnt;
- for (cline = 0; cline < collision_table_entries; cline++) {
+ free_lines = 0;
+ for (cline = begin; cline < entries; cline += increment) {
bool any_valid = true;
OCF_COND_RESCHED(step, 128);
@@ -525,11 +552,12 @@ static int _ocf_mngt_rebuild_metadata(ocf_cache_t cache)
any_valid = metadata_clear_valid_if_clean(cache, cline);
if (!any_valid || core_id == OCF_CORE_MAX) {
/* Reset metadata for not mapped or clean cache line */
- _recovery_reset_cline_metadata(cache, cline);
+ ocf_mngt_cline_reset_metadata(cache, cline, shard_id);
+ free_lines++;
continue;
}
- if (!cache->core[core_id].added) {
+ if (!cache->core[core_id].conf_meta->valid) {
ocf_cache_log(cache, log_err, "Stale mapping in "
"on-disk metadata - refusing to "
"recover cache.\n");
@@ -537,58 +565,130 @@ static int _ocf_mngt_rebuild_metadata(ocf_cache_t cache)
}
/* Rebuild metadata for mapped cache line */
- ocf_cline_rebuild_metadata(cache, core_id, core_line, cline);
+ ocf_mngt_cline_rebuild_metadata(cache, core_id,
+ core_line, cline);
+
+ context->shard[shard_id].core[core_id].lines++;
}
- ocf_metadata_end_exclusive_access(&cache->metadata.lock);
+ for_each_core(cache, core, core_id) {
+ env_atomic_add(context->shard[shard_id].core[core_id].lines,
+ &context->core[core_id].lines);
+ }
+
+ env_atomic_add(free_lines, &context->free_lines);
return 0;
}
-static int _ocf_mngt_recovery_rebuild_metadata(ocf_cache_t cache)
+static void ocf_mngt_rebuild_metadata_finish(ocf_parallelize_t parallelize,
+ void *priv, int error)
{
- return _ocf_mngt_rebuild_metadata(cache);
+ struct ocf_mngt_rebuild_metadata_context *context = priv;
+ ocf_cache_t cache = context->cache;
+ ocf_part_id_t part_id = PARTITION_DEFAULT;
+ struct ocf_part_runtime *part;
+ ocf_core_t core;
+ ocf_core_id_t core_id;
+ uint32_t lines_total = 0;
+
+ for_each_core(cache, core, core_id) {
+ uint32_t lines = env_atomic_read(&context->core[core_id].lines);
+
+ env_atomic_set(&core->runtime_meta->cached_clines, lines);
+ env_atomic_set(&core->runtime_meta->
+ part_counters[part_id].cached_clines, lines);
+ env_atomic_set(&core->runtime_meta->dirty_clines, lines);
+ env_atomic_set(&core->runtime_meta->
+ part_counters[part_id].dirty_clines, lines);
+ if (lines) {
+ env_atomic64_set(&core->runtime_meta->dirty_since,
+ env_ticks_to_secs(env_get_tick_count()));
+ }
+
+ lines_total += lines;
+ }
+
+ part = cache->user_parts[part_id].part.runtime;
+ env_atomic_set(&part->curr_size, lines_total);
+
+ env_atomic_set(&cache->free.runtime->curr_size,
+ env_atomic_read(&context->free_lines));
+
+ context->cmpl(context->priv, error);
+
+ ocf_parallelize_destroy(parallelize);
}
-static inline ocf_error_t _ocf_init_cleaning_policy(ocf_cache_t cache,
- ocf_cleaning_t cleaning_policy,
- enum ocf_metadata_shutdown_status shutdown_status)
+static void ocf_mngt_rebuild_metadata(ocf_cache_t cache,
+ ocf_mngt_rebuild_metadata_end_t cmpl, void *priv)
{
- ocf_error_t result;
+ struct ocf_mngt_rebuild_metadata_context *context;
+ ocf_parallelize_t parallelize;
+ int result;
- if (shutdown_status == ocf_metadata_clean_shutdown)
- result = ocf_cleaning_initialize(cache, cleaning_policy, 0);
- else
- result = ocf_cleaning_initialize(cache, cleaning_policy, 1);
+ result = ocf_parallelize_create(¶llelize, cache,
+ OCF_MNGT_REBUILD_METADATA_SHARDS_CNT,
+ sizeof(*context), ocf_mngt_rebuild_metadata_handle,
+ ocf_mngt_rebuild_metadata_finish);
+ if (result) {
+ cmpl(priv, result);
+ return;
+ }
- if (result)
- ocf_cache_log(cache, log_err, "Cannot initialize cleaning policy\n");
+ context = ocf_parallelize_get_priv(parallelize);
+ context->cache = cache;
+ context->cmpl = cmpl;
+ context->priv = priv;
- return result;
+ ocf_parallelize_run(parallelize);
}
-static void _ocf_mngt_load_post_metadata_load(ocf_pipeline_t pipeline,
+static void _ocf_mngt_load_rebuild_metadata_complete(void *priv, int error)
+{
+ struct ocf_cache_attach_context *context = priv;
+
+ OCF_PL_NEXT_ON_SUCCESS_RET(context->pipeline, error);
+}
+
+static void _ocf_mngt_load_rebuild_metadata(ocf_pipeline_t pipeline,
+ void *priv, ocf_pipeline_arg_t arg)
+{
+ struct ocf_cache_attach_context *context = priv;
+ ocf_cache_t cache = context->cache;
+
+ if (context->metadata.shutdown_status != ocf_metadata_clean_shutdown) {
+ ocf_mngt_rebuild_metadata(cache,
+ _ocf_mngt_load_rebuild_metadata_complete,
+ context);
+ return;
+ }
+
+ ocf_pipeline_next(pipeline);
+}
+
+static void _ocf_mngt_cleaning_recovery_complete(void *priv, int error)
+{
+ struct ocf_cache_attach_context *context = priv;
+
+ OCF_PL_NEXT_ON_SUCCESS_RET(context->pipeline, error);
+}
+
+static void _ocf_mngt_load_init_cleaning(ocf_pipeline_t pipeline,
void *priv, ocf_pipeline_arg_t arg)
{
struct ocf_cache_attach_context *context = priv;
ocf_cache_t cache = context->cache;
ocf_error_t result;
- int ret;
- if (context->metadata.shutdown_status != ocf_metadata_clean_shutdown) {
- ret = _ocf_mngt_recovery_rebuild_metadata(cache);
- if (ret)
- OCF_PL_FINISH_RET(pipeline, ret);
- __populate_free(cache);
+ if (context->metadata.shutdown_status == ocf_metadata_clean_shutdown) {
+ result = ocf_cleaning_initialize(cache,
+ cache->cleaner.policy, 0);
+ OCF_PL_NEXT_ON_SUCCESS_RET(pipeline, result);
}
- result = _ocf_init_cleaning_policy(cache, cache->cleaner.policy,
- context->metadata.shutdown_status);
-
- if (result)
- OCF_PL_FINISH_RET(pipeline, result);
-
- ocf_pipeline_next(pipeline);
+ ocf_cleaning_recovery(cache, cache->cleaner.policy,
+ _ocf_mngt_cleaning_recovery_complete, context);
}
void _ocf_mngt_load_metadata_complete(void *priv, int error)
@@ -1108,16 +1208,51 @@ static void _ocf_mngt_attach_prepare_metadata(ocf_pipeline_t pipeline,
/**
* @brief initializing cache anew (not loading or recovering)
*/
-static void _ocf_mngt_attach_init_instance(ocf_pipeline_t pipeline,
+static void _ocf_mngt_attach_init_metadata(ocf_pipeline_t pipeline,
+ void *priv, ocf_pipeline_arg_t arg)
+{
+ struct ocf_cache_attach_context *context = priv;
+ ocf_cache_t cache = context->cache;
+
+ ocf_metadata_init_hash_table(cache);
+ ocf_metadata_init_collision(cache);
+ __init_parts_attached(cache);
+
+ ocf_pipeline_next(pipeline);
+}
+
+static void _ocf_mngt_attach_populate_free_complete(void *priv, int error)
+{
+ struct ocf_cache_attach_context *context = priv;
+
+ OCF_PL_NEXT_ON_SUCCESS_RET(context->pipeline, error);
+}
+
+static void _ocf_mngt_attach_populate_free(ocf_pipeline_t pipeline,
+ void *priv, ocf_pipeline_arg_t arg)
+{
+ struct ocf_cache_attach_context *context = priv;
+ ocf_cache_t cache = context->cache;
+
+ ocf_lru_populate(cache, _ocf_mngt_attach_populate_free_complete,
+ context);
+}
+
+static void _ocf_mngt_attach_init_services(ocf_pipeline_t pipeline,
void *priv, ocf_pipeline_arg_t arg)
{
struct ocf_cache_attach_context *context = priv;
ocf_cache_t cache = context->cache;
ocf_error_t result;
- result = init_attached_data_structures(cache);
- if (result)
+ result = __init_cleaning_policy(cache);
+ if (result) {
+ ocf_cache_log(cache, log_err,
+ "Cannot initialize cleaning policy\n");
OCF_PL_FINISH_RET(pipeline, result);
+ }
+
+ __setup_promotion_policy(cache);
/* In initial cache state there is no dirty data, so all dirty data is
considered to be flushed
@@ -1728,7 +1863,9 @@ struct ocf_pipeline_properties _ocf_mngt_cache_attach_pipeline_properties = {
OCF_PL_STEP(_ocf_mngt_test_volume),
OCF_PL_STEP(_ocf_mngt_init_cleaner),
OCF_PL_STEP(_ocf_mngt_init_promotion),
- OCF_PL_STEP(_ocf_mngt_attach_init_instance),
+ OCF_PL_STEP(_ocf_mngt_attach_init_metadata),
+ OCF_PL_STEP(_ocf_mngt_attach_populate_free),
+ OCF_PL_STEP(_ocf_mngt_attach_init_services),
OCF_PL_STEP(_ocf_mngt_zero_superblock),
OCF_PL_STEP(_ocf_mngt_attach_flush_metadata),
OCF_PL_STEP(_ocf_mngt_attach_discard),
@@ -1756,7 +1893,8 @@ struct ocf_pipeline_properties _ocf_mngt_cache_load_pipeline_properties = {
OCF_PL_STEP(_ocf_mngt_init_promotion),
OCF_PL_STEP(_ocf_mngt_load_add_cores),
OCF_PL_STEP(_ocf_mngt_load_metadata),
- OCF_PL_STEP(_ocf_mngt_load_post_metadata_load),
+ OCF_PL_STEP(_ocf_mngt_load_rebuild_metadata),
+ OCF_PL_STEP(_ocf_mngt_load_init_cleaning),
OCF_PL_STEP(_ocf_mngt_attach_shutdown_status),
OCF_PL_STEP(_ocf_mngt_attach_flush_metadata),
OCF_PL_STEP(_ocf_mngt_attach_shutdown_status),
@@ -2098,7 +2236,6 @@ static void _ocf_mngt_standby_init_structures_attach(ocf_pipeline_t pipeline,
ocf_cache_t cache = context->cache;
init_attached_data_structures_recovery(cache, true);
- __populate_free(cache);
ocf_pipeline_next(pipeline);
}
@@ -2128,37 +2265,6 @@ static void _ocf_mngt_standby_init_pio_concurrency(ocf_pipeline_t pipeline,
OCF_PL_NEXT_ON_SUCCESS_RET(context->pipeline, result);
}
-static void _ocf_mngt_standby_recovery(ocf_pipeline_t pipeline,
- void *priv, ocf_pipeline_arg_t arg)
-{
- struct ocf_cache_attach_context *context = priv;
- ocf_cache_t cache = context->cache;
- int ret;
-
- ret = _ocf_mngt_recovery_rebuild_metadata(cache);
- if (ret)
- OCF_PL_FINISH_RET(pipeline, ret);
- __populate_free(cache);
-
- ocf_pipeline_next(pipeline);
-}
-
-static void _ocf_mngt_standby_init_cleaning(ocf_pipeline_t pipeline,
- void *priv, ocf_pipeline_arg_t arg)
-{
- struct ocf_cache_attach_context *context = priv;
- ocf_cache_t cache = context->cache;
- ocf_error_t result;
-
- result = _ocf_init_cleaning_policy(cache, cache->cleaner.policy,
- context->metadata.shutdown_status);
-
- if (result)
- OCF_PL_FINISH_RET(pipeline, result);
-
- ocf_pipeline_next(pipeline);
-}
-
static void _ocf_mngt_standby_post_init(ocf_pipeline_t pipeline,
void *priv, ocf_pipeline_arg_t arg)
{
@@ -2185,7 +2291,8 @@ struct ocf_pipeline_properties _ocf_mngt_cache_standby_attach_pipeline_propertie
OCF_PL_STEP(_ocf_mngt_test_volume),
OCF_PL_STEP(_ocf_mngt_init_cleaner),
OCF_PL_STEP(_ocf_mngt_standby_init_structures_attach),
- OCF_PL_STEP(_ocf_mngt_standby_init_cleaning),
+ OCF_PL_STEP(_ocf_mngt_attach_populate_free),
+ OCF_PL_STEP(_ocf_mngt_load_init_cleaning),
OCF_PL_STEP(_ocf_mngt_standby_preapre_mempool),
OCF_PL_STEP(_ocf_mngt_standby_init_pio_concurrency),
OCF_PL_STEP(_ocf_mngt_zero_superblock),
@@ -2213,10 +2320,10 @@ struct ocf_pipeline_properties _ocf_mngt_cache_standby_load_pipeline_properties
OCF_PL_STEP(_ocf_mngt_load_metadata_recovery),
OCF_PL_STEP(_ocf_mngt_init_cleaner),
OCF_PL_STEP(_ocf_mngt_standby_init_structures_load),
- OCF_PL_STEP(_ocf_mngt_standby_init_cleaning),
+ OCF_PL_STEP(_ocf_mngt_load_init_cleaning),
OCF_PL_STEP(_ocf_mngt_standby_preapre_mempool),
OCF_PL_STEP(_ocf_mngt_standby_init_pio_concurrency),
- OCF_PL_STEP(_ocf_mngt_standby_recovery),
+ OCF_PL_STEP(_ocf_mngt_load_rebuild_metadata),
OCF_PL_STEP(_ocf_mngt_standby_post_init),
OCF_PL_STEP_TERMINATOR(),
},
@@ -2430,6 +2537,7 @@ struct ocf_pipeline_properties _ocf_mngt_cache_activate_pipeline_properties = {
.steps = {
OCF_PL_STEP(_ocf_mngt_copy_uuid_data),
OCF_PL_STEP(_ocf_mngt_activate_set_cache_device),
+ OCF_PL_STEP(_ocf_mngt_activate_init_properties),
OCF_PL_STEP(_ocf_mngt_activate_compare_superblock),
OCF_PL_STEP(_ocf_mngt_load_superblock),
OCF_PL_STEP(_ocf_mngt_activate_check_superblock),
diff --git a/src/ocf_lru.c b/src/ocf_lru.c
index ad87f1a..0062023 100644
--- a/src/ocf_lru.c
+++ b/src/ocf_lru.c
@@ -7,6 +7,8 @@
#include "ocf_lru.h"
#include "utils/utils_cleaner.h"
#include "utils/utils_cache_line.h"
+#include "utils/utils_generator.h"
+#include "utils/utils_parallelize.h"
#include "concurrency/ocf_concurrency.h"
#include "mngt/ocf_mngt_common.h"
#include "engine/engine_zero.h"
@@ -867,63 +869,91 @@ void ocf_lru_dirty_cline(ocf_cache_t cache, struct ocf_part *part,
OCF_METADATA_LRU_WR_UNLOCK(cline);
}
-static ocf_cache_line_t next_phys_invalid(ocf_cache_t cache,
- ocf_cache_line_t phys)
+struct ocf_lru_populate_context {
+ ocf_cache_t cache;
+ env_atomic curr_size;
+
+ ocf_lru_populate_end_t cmpl;
+ void *priv;
+};
+
+static int ocf_lru_populate_handle(ocf_parallelize_t parallelize,
+ void *priv, unsigned shard_id, unsigned shards_cnt)
{
- ocf_cache_line_t lg;
- ocf_cache_line_t collision_table_entries =
- ocf_metadata_collision_table_entries(cache);
-
- if (phys == collision_table_entries)
- return collision_table_entries;
-
- lg = ocf_metadata_map_phy2lg(cache, phys);
- while (metadata_test_valid_any(cache, lg)) {
- ++phys;
-
- if (phys == collision_table_entries)
- break;
-
- lg = ocf_metadata_map_phy2lg(cache, phys);
- }
-
- return phys;
-}
-
-/* put invalid cachelines on freelist partition lru list */
-void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines)
-{
- ocf_cache_line_t phys, cline;
- ocf_cache_line_t collision_table_entries =
- ocf_metadata_collision_table_entries(cache);
+ struct ocf_lru_populate_context *context = priv;
+ ocf_cache_t cache = context->cache;
+ ocf_cache_line_t cnt, cline;
+ ocf_cache_line_t entries = ocf_metadata_collision_table_entries(cache);
+ struct ocf_generator_bisect_state generator;
struct ocf_lru_list *list;
- unsigned lru_list;
- unsigned i;
+ unsigned lru_list = shard_id;
unsigned step = 0;
+ uint32_t portion, offset;
+ uint32_t i, idx;
- phys = 0;
- for (i = 0; i < num_free_clines; i++) {
- /* find first invalid cacheline */
- phys = next_phys_invalid(cache, phys);
- ENV_BUG_ON(phys == collision_table_entries);
- cline = ocf_metadata_map_phy2lg(cache, phys);
- ++phys;
+ portion = DIV_ROUND_UP((uint64_t)entries, shards_cnt);
+ offset = shard_id * portion / shards_cnt;
+ ocf_generator_bisect_init(&generator, portion, offset);
+
+ list = ocf_lru_get_list(&cache->free, lru_list, true);
+
+ cnt = 0;
+ for (i = 0; i < portion; i++) {
+ OCF_COND_RESCHED_DEFAULT(step);
+
+ idx = ocf_generator_bisect_next(&generator);
+ cline = idx * shards_cnt + shard_id;
+ if (cline >= entries)
+ continue;
ocf_metadata_set_partition_id(cache, cline, PARTITION_FREELIST);
- lru_list = (cline % OCF_NUM_LRU_LISTS);
- list = ocf_lru_get_list(&cache->free, lru_list, true);
+ add_lru_head_nobalance(cache, list, cline);
- add_lru_head(cache, list, cline);
-
- OCF_COND_RESCHED_DEFAULT(step);
+ cnt++;
}
- /* we should have reached the last invalid cache line */
- phys = next_phys_invalid(cache, phys);
- ENV_BUG_ON(phys != collision_table_entries);
+ env_atomic_add(cnt, &context->curr_size);
- env_atomic_set(&cache->free.runtime->curr_size, i);
+ return 0;
+}
+
+static void ocf_lru_populate_finish(ocf_parallelize_t parallelize,
+ void *priv, int error)
+{
+ struct ocf_lru_populate_context *context = priv;
+
+ env_atomic_set(&context->cache->free.runtime->curr_size,
+ env_atomic_read(&context->curr_size));
+
+ context->cmpl(context->priv, error);
+
+ ocf_parallelize_destroy(parallelize);
+}
+
+/* put invalid cachelines on freelist partition lru list */
+void ocf_lru_populate(ocf_cache_t cache,
+ ocf_lru_populate_end_t cmpl, void *priv)
+{
+ struct ocf_lru_populate_context *context;
+ ocf_parallelize_t parallelize;
+ int result;
+
+ result = ocf_parallelize_create(¶llelize, cache, OCF_NUM_LRU_LISTS,
+ sizeof(*context), ocf_lru_populate_handle,
+ ocf_lru_populate_finish);
+ if (result) {
+ cmpl(priv, result);
+ return;
+ }
+
+ context = ocf_parallelize_get_priv(parallelize);
+ context->cache = cache;
+ env_atomic_set(&context->curr_size, 0);
+ context->cmpl = cmpl;
+ context->priv = priv;
+
+ ocf_parallelize_run(parallelize);
}
static bool _is_cache_line_acting(struct ocf_cache *cache,
@@ -1027,3 +1057,12 @@ uint32_t ocf_lru_num_free(ocf_cache_t cache)
{
return env_atomic_read(&cache->free.runtime->curr_size);
}
+
+void ocf_lru_add_free(ocf_cache_t cache, ocf_cache_line_t cline)
+{
+ uint32_t lru_list = (cline % OCF_NUM_LRU_LISTS);
+ struct ocf_lru_list *list;
+
+ list = ocf_lru_get_list(&cache->free, lru_list, true);
+ add_lru_head_nobalance(cache, list, cline);
+}
diff --git a/src/ocf_lru.h b/src/ocf_lru.h
index a8b0abd..79fc559 100644
--- a/src/ocf_lru.h
+++ b/src/ocf_lru.h
@@ -30,11 +30,16 @@ void ocf_lru_clean(ocf_cache_t cache, struct ocf_user_part *user_part,
ocf_queue_t io_queue, uint32_t count);
void ocf_lru_repart(ocf_cache_t cache, ocf_cache_line_t cline,
struct ocf_part *src_upart, struct ocf_part *dst_upart);
+void ocf_lru_add_free(ocf_cache_t cache, ocf_cache_line_t cline);
uint32_t ocf_lru_num_free(ocf_cache_t cache);
-void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines);
struct ocf_lru_list *ocf_lru_get_list(struct ocf_part *part,
uint32_t lru_idx, bool clean);
void ocf_lru_remove_locked(ocf_cache_t cache, struct ocf_lru_list *list,
ocf_cache_line_t cline);
+typedef void (*ocf_lru_populate_end_t)(void *priv, int error);
+
+void ocf_lru_populate(ocf_cache_t cache,
+ ocf_lru_populate_end_t cmpl, void *priv);
+
#endif
diff --git a/src/utils/utils_generator.c b/src/utils/utils_generator.c
new file mode 100644
index 0000000..32b42a2
--- /dev/null
+++ b/src/utils/utils_generator.c
@@ -0,0 +1,187 @@
+#include "utils_generator.h"
+
+/**
+ * @brief Reverse bits of 32-bit value
+ *
+ * @param[in] x Value to be reversed
+ *
+ * @return Reversed value
+ */
+static inline uint32_t bitreverse32(register uint32_t x)
+{
+ x = (((x & 0xaaaaaaaa) >> 1) | ((x & 0x55555555) << 1));
+ x = (((x & 0xcccccccc) >> 2) | ((x & 0x33333333) << 2));
+ x = (((x & 0xf0f0f0f0) >> 4) | ((x & 0x0f0f0f0f) << 4));
+ x = (((x & 0xff00ff00) >> 8) | ((x & 0x00ff00ff) << 8));
+ return((x >> 16) | (x << 16));
+}
+
+/**
+ * @brief Initialize bisect generator state
+ *
+ * @param[in] generator Pointer to generator structure
+ * @param[in] limit Limit of the value returned by generator (maximum value
+ * returned by the generator is limit - 1)
+ * @param[in] offset Offset at which generator should start
+ *
+ * @return Reversed value
+ */
+void ocf_generator_bisect_init(
+ struct ocf_generator_bisect_state *generator,
+ uint32_t limit, uint32_t offset)
+{
+ unsigned clz;
+ uint32_t maplen;
+
+ clz = __builtin_clz(limit - 1);
+ maplen = 1 << (32 - clz);
+
+ generator->curr = (uint64_t)offset * maplen / limit;
+ generator->limit = limit;
+}
+
+/**
+ * @brief Generate next value of bisect generator
+ *
+ * This function calculates next value of the generator. The generator
+ * pattern is based on order of indexes in array visited with bisection
+ * algorithm, where always the left child is visited first at every depth.
+ * This can be imagined as a special implementation of BFS done on a full
+ * binary tree, where visiting nodes on each depth level is done the same
+ * order as original array (so the algorithm is recursive at each level).
+ *
+ * Example:
+ *
+ * 1. We generate array of all values for number of bits needed to express
+ * limit value - 1.
+ *
+ * For limit==14 (4 bits) it would be:
+ * [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+ *
+ * 2. We take first element of the array, and then build full binary tree
+ * from other elements (it should always be possible to build a full
+ * binary tree, as number of remaining elements is always 2^n-1).
+ *
+ * The full binary tree for example array looks like this:
+ *
+ * Depth 0 -> 8
+ * / \
+ * / \
+ * / \
+ * / \
+ * / \
+ * / \
+ * / \
+ * / \
+ * Depth 1 -> 4 12
+ * / \ / \
+ * / \ / \
+ * / \ / \
+ * Depth 2 -> 2 6 10 14
+ * / \ / \ / \ / \
+ * Depth 3 -> 1 3 5 7 9 11 13 15
+ *
+ * 3. We traverse the tree:
+ * a) If depth level has one element, we take it.
+ * b) If depth level has two elements, we take left and then right.
+ * c) If depth level has more than two elements, we repeat steps
+ * from 2 on array built from elements on that level left to right.
+ *
+ * At level 0 we take 8, at level 1 we take 4 and 12, and at level 3 we
+ * take 2 and build tree like this one:
+ *
+ * 10
+ * / \
+ * 6 14
+ *
+ * Then at level 0 of that tree we take 10, at level 1 we take 6 and 14,
+ * and then we go back the original tree.
+ *
+ * At level 3 we take 1 and we build another tree from remaining elements
+ * of that level:
+ *
+ * 9
+ * / \
+ * / \
+ * / \
+ * 5 13
+ * / \ / \
+ * 3 7 11 15
+ *
+ * Repeating step 3 on that tree we get elements 9, then 5 and 13, and then
+ * by running steps from 2 on the lowest level of that tree we get elements
+ * in following order: 3, 11, 7 and 15.
+ *
+ * So the entire sequence would be as follows:
+ * [0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15]
+ *
+ * This algorithm is however quite complex, and it can be simplified
+ * significantly thanks to properties of the result sequence. Note that
+ * when this sequence is written in binary it looks like this:
+ *
+ * 0 0000
+ * 8 1000
+ * 4 0100
+ * 12 1100
+ * 2 0010
+ * 10 1010
+ * 6 0110
+ * 14 1110
+ * 1 0001
+ * 9 1001
+ * 5 0101
+ * 13 1101
+ * 3 0011
+ * 11 1011
+ * 7 0111
+ * 15 1111
+ *
+ * So in its binary representation it looks like a mirror image of binary
+ * representation of the original sequence:
+ *
+ * 0 0000 0000 0
+ * 8 1000 0001 1
+ * 4 0100 0010 2
+ * 12 1100 0011 3
+ * 2 0010 0100 4
+ * 10 1010 0101 5
+ * 6 0110 0110 6
+ * 14 1110 0111 7
+ * 1 0001 1000 8
+ * 9 1001 1001 9
+ * 5 0101 1010 10
+ * 13 1101 1011 11
+ * 3 0011 1100 12
+ * 11 1011 1101 13
+ * 7 0111 1110 14
+ * 15 1111 1111 15
+ *
+ * With that knowledge we can easily calculate the next result value by just
+ * reversing order of bits for each value in original sequence.
+ *
+ * As a result we are left with sequence that contains all the numbers that
+ * can be expressed with number of bits reqiured to express limit - 1. The only
+ * thing we need to do at that point is to just filter out values that do not
+ * fit within the limit.
+ *
+ * @param[in] generator Pointer to generator structure
+ *
+ * @return Generated value
+ */
+uint32_t ocf_generator_bisect_next(
+ struct ocf_generator_bisect_state *generator)
+{
+ unsigned clz;
+ uint32_t maplen;
+ uint32_t value;
+
+ clz = __builtin_clz(generator->limit - 1);
+ maplen = 1 << (32 - clz);
+
+ do {
+ value = bitreverse32(generator->curr) >> clz;
+ generator->curr = (generator->curr + 1) % maplen;
+ } while (value >= generator->limit);
+
+ return value;
+}
diff --git a/src/utils/utils_generator.h b/src/utils/utils_generator.h
new file mode 100644
index 0000000..068602d
--- /dev/null
+++ b/src/utils/utils_generator.h
@@ -0,0 +1,23 @@
+/*
+ * Copyright(c) 2022 Intel Corporation
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef __UTILS_GENERATOR_H__
+#define __UTILS_GENERATOR_H__
+
+#include "ocf/ocf.h"
+
+struct ocf_generator_bisect_state {
+ uint32_t curr;
+ uint32_t limit;
+};
+
+void ocf_generator_bisect_init(
+ struct ocf_generator_bisect_state *generator,
+ uint32_t limit, uint32_t offset);
+
+uint32_t ocf_generator_bisect_next(
+ struct ocf_generator_bisect_state *generator);
+
+#endif /* __UTILS_GENERATOR_H__ */
diff --git a/src/utils/utils_parallelize.c b/src/utils/utils_parallelize.c
new file mode 100644
index 0000000..7c5b129
--- /dev/null
+++ b/src/utils/utils_parallelize.c
@@ -0,0 +1,145 @@
+/*
+ * Copyright(c) 2012-2021 Intel Corporation
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include "ocf/ocf.h"
+#include "../engine/cache_engine.h"
+#include "../engine/engine_common.h"
+#include "../ocf_request.h"
+#include "utils_parallelize.h"
+
+struct ocf_parallelize {
+ ocf_cache_t cache;
+ ocf_parallelize_handle_t handle;
+ ocf_parallelize_finish_t finish;
+ void *priv;
+
+ unsigned shards_cnt;
+ env_atomic remaining;
+ env_atomic error;
+
+ struct ocf_request *reqs[];
+};
+
+static int _ocf_parallelize_hndl(struct ocf_request *req)
+{
+ ocf_parallelize_t parallelize = req->priv;
+ ocf_parallelize_finish_t finish;
+ void *priv;
+ int error;
+
+ error = parallelize->handle(parallelize, parallelize->priv,
+ req->byte_position, parallelize->shards_cnt);
+
+ env_atomic_cmpxchg(¶llelize->error, 0, error);
+
+ if (env_atomic_dec_return(¶llelize->remaining))
+ return 0;
+
+ finish = parallelize->finish;
+ priv = parallelize->priv;
+ error = env_atomic_read(¶llelize->error);
+
+ finish(parallelize, priv, error);
+
+ return 0;
+}
+
+static const struct ocf_io_if _io_if_parallelize = {
+ .read = _ocf_parallelize_hndl,
+ .write = _ocf_parallelize_hndl,
+};
+
+int ocf_parallelize_create(ocf_parallelize_t *parallelize,
+ ocf_cache_t cache, unsigned shards_cnt, uint32_t priv_size,
+ ocf_parallelize_handle_t handle,
+ ocf_parallelize_finish_t finish)
+{
+ ocf_parallelize_t tmp_parallelize;
+ ocf_queue_t queue;
+ size_t prl_size;
+ unsigned queue_count = 0;
+ int result, i;
+
+ list_for_each_entry(queue, &cache->io_queues, list)
+ queue_count++;
+
+ if (shards_cnt == 0)
+ shards_cnt = queue_count;
+
+ prl_size = sizeof(*tmp_parallelize) +
+ shards_cnt * sizeof(*tmp_parallelize->reqs);
+
+ tmp_parallelize = env_vzalloc(prl_size + priv_size);
+ if (!tmp_parallelize)
+ return -OCF_ERR_NO_MEM;
+
+ if (priv_size > 0)
+ tmp_parallelize->priv = (void *)tmp_parallelize + prl_size;
+
+ tmp_parallelize->cache = cache;
+ tmp_parallelize->handle = handle;
+ tmp_parallelize->finish = finish;
+
+ tmp_parallelize->shards_cnt = shards_cnt;
+ env_atomic_set(&tmp_parallelize->remaining, shards_cnt);
+ env_atomic_set(&tmp_parallelize->error, 0);
+
+ for (i = 0; i < shards_cnt;) {
+ list_for_each_entry(queue, &cache->io_queues, list) {
+ if (i == shards_cnt)
+ break;
+ tmp_parallelize->reqs[i] = ocf_req_new(queue,
+ NULL, 0, 0, 0);
+ if (!tmp_parallelize->reqs[i]) {
+ result = -OCF_ERR_NO_MEM;
+ goto err_reqs;
+ }
+ tmp_parallelize->reqs[i]->info.internal = true;
+ tmp_parallelize->reqs[i]->io_if = &_io_if_parallelize;
+ tmp_parallelize->reqs[i]->byte_position = i;
+ tmp_parallelize->reqs[i]->priv = tmp_parallelize;
+ i++;
+ }
+ }
+
+ *parallelize = tmp_parallelize;
+
+ return 0;
+
+err_reqs:
+ while (i--)
+ ocf_req_put(tmp_parallelize->reqs[i]);
+ env_vfree(tmp_parallelize);
+
+ return result;
+}
+
+void ocf_parallelize_destroy(ocf_parallelize_t parallelize)
+{
+ int i;
+
+ for (i = 0; i < parallelize->shards_cnt; i++)
+ ocf_req_put(parallelize->reqs[i]);
+
+ env_vfree(parallelize);
+}
+
+void *ocf_parallelize_get_priv(ocf_parallelize_t parallelize)
+{
+ return parallelize->priv;
+}
+
+void ocf_parallelize_set_priv(ocf_parallelize_t parallelize, void *priv)
+{
+ parallelize->priv = priv;
+}
+
+void ocf_parallelize_run(ocf_parallelize_t parallelize)
+{
+ int i;
+
+ for (i = 0; i < parallelize->shards_cnt; i++)
+ ocf_engine_push_req_front(parallelize->reqs[i], false);
+}
diff --git a/src/utils/utils_parallelize.h b/src/utils/utils_parallelize.h
new file mode 100644
index 0000000..0066ef4
--- /dev/null
+++ b/src/utils/utils_parallelize.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright(c) 2022 Intel Corporation
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef __UTILS_PARALLELIZE_H__
+#define __UTILS_PARALLELIZE_H__
+
+#include "ocf/ocf.h"
+
+typedef struct ocf_parallelize *ocf_parallelize_t;
+
+typedef int (*ocf_parallelize_handle_t)(ocf_parallelize_t parallelize,
+ void *priv, unsigned shard_id, unsigned shards_cnt);
+
+typedef void (*ocf_parallelize_finish_t)(ocf_parallelize_t parallelize,
+ void *priv, int error);
+
+int ocf_parallelize_create(ocf_parallelize_t *parallelize,
+ ocf_cache_t cache, unsigned shards_cnt, uint32_t priv_size,
+ ocf_parallelize_handle_t handle,
+ ocf_parallelize_finish_t finish);
+
+void ocf_parallelize_destroy(ocf_parallelize_t parallelize);
+
+void ocf_parallelize_set_priv(ocf_parallelize_t parallelize, void *priv);
+
+void *ocf_parallelize_get_priv(ocf_parallelize_t parallelize);
+
+void ocf_parallelize_run(ocf_parallelize_t parallelize);
+
+#endif /* __UTILS_PARALLELIZE_H__ */
diff --git a/tests/unit/tests/cleaning/alru.c/cleaning_policy_alru_initialize_part_test.c b/tests/unit/tests/cleaning/alru.c/cleaning_policy_alru_initialize_part_test.c
deleted file mode 100644
index a2c3823..0000000
--- a/tests/unit/tests/cleaning/alru.c/cleaning_policy_alru_initialize_part_test.c
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Copyright(c) 2012-2021 Intel Corporation
- * SPDX-License-Identifier: BSD-3-Clause
- */
-/*
-src/cleaning/alru.c
-cleaning_policy_alru_initialize_part
-
-
-*/
-
-#undef static
-#undef inline
-/*
- * This headers must be in test source file. It's important that cmocka.h is
- * last.
- */
-#include
-#include
-#include
-#include
-#include "print_desc.h"
-
-/*
- * Headers from tested target.
- */
-#include "ocf/ocf.h"
-#include "../ocf_cache_priv.h"
-#include "cleaning.h"
-#include "alru.h"
-#include "../metadata/metadata.h"
-#include "../utils/utils_cleaner.h"
-#include "../utils/utils_user_part.h"
-#include "../utils/utils_realloc.h"
-#include "../concurrency/ocf_cache_line_concurrency.h"
-#include "../ocf_def_priv.h"
-
-#include "cleaning/alru.c/cleaning_policy_alru_initialize_part_test_generated_wraps.c"
-
-
-static void cleaning_policy_alru_initialize_test01(void **state)
-{
- int result;
- struct ocf_cache *cache;
- ocf_part_id_t part_id = 0;
-
- int collision_table_entries = 900729;
-
- print_test_description("Check if all variables are set correctly");
-
- cache = test_malloc(sizeof(*cache));
- cache->user_parts[part_id].part.runtime = test_malloc(sizeof(struct ocf_part_runtime));
- cache->user_parts[part_id].clean_pol = test_malloc(sizeof(*cache->user_parts[part_id].clean_pol));
- cache->user_parts[part_id].part.id = part_id;
- cache->device = test_malloc(sizeof(struct ocf_cache_device));
- cache->device->runtime_meta = test_malloc(sizeof(struct ocf_superblock_runtime));
-
- cache->device->collision_table_entries = collision_table_entries;
-
- result = cleaning_policy_alru_initialize_part(cache, &cache->user_parts[part_id], 1, 1);
-
- assert_int_equal(result, 0);
-
- assert_int_equal(env_atomic_read(&cache->user_parts[part_id].clean_pol->policy.alru.size), 0);
- assert_int_equal(cache->user_parts[part_id].clean_pol->policy.alru.lru_head, collision_table_entries);
- assert_int_equal(cache->user_parts[part_id].clean_pol->policy.alru.lru_tail, collision_table_entries);
-
- assert_int_equal(cache->device->runtime_meta->cleaning_thread_access, 0);
-
- test_free(cache->device->runtime_meta);
- test_free(cache->device);
- test_free(cache->user_parts[part_id].clean_pol);
- test_free(cache->user_parts[part_id].part.runtime);
- test_free(cache);
-}
-
-static void cleaning_policy_alru_initialize_test02(void **state)
-{
- int result;
- struct ocf_cache *cache;
- ocf_part_id_t part_id = 0;
-
- uint32_t collision_table_entries = 900729;
-
- print_test_description("Check if only appropirate variables are changed");
-
- cache = test_malloc(sizeof(*cache));
- cache->user_parts[part_id].part.runtime = test_malloc(sizeof(struct ocf_part_runtime));
- cache->user_parts[part_id].clean_pol = test_malloc(sizeof(*cache->user_parts[part_id].clean_pol));
- cache->device = test_malloc(sizeof(struct ocf_cache_device));
- cache->device->runtime_meta = test_malloc(sizeof(struct ocf_superblock_runtime));
-
- env_atomic_set(&cache->user_parts[part_id].clean_pol->policy.alru.size, 1);
- cache->user_parts[part_id].clean_pol->policy.alru.lru_head = -collision_table_entries;
- cache->user_parts[part_id].clean_pol->policy.alru.lru_tail = -collision_table_entries;
-
- result = cleaning_policy_alru_initialize_part(cache, &cache->user_parts[part_id], 0, 0);
-
- assert_int_equal(result, 0);
-
- assert_int_equal(env_atomic_read(&cache->user_parts[part_id].clean_pol->policy.alru.size), 1);
- assert_int_equal(cache->user_parts[part_id].clean_pol->policy.alru.lru_head, -collision_table_entries);
- assert_int_equal(cache->user_parts[part_id].clean_pol->policy.alru.lru_tail, -collision_table_entries);
-
- assert_int_equal(cache->device->runtime_meta->cleaning_thread_access, 0);
-
- test_free(cache->device->runtime_meta);
- test_free(cache->device);
- test_free(cache->user_parts[part_id].clean_pol);
- test_free(cache->user_parts[part_id].part.runtime);
- test_free(cache);
-}
-
-/*
- * Main function. It runs tests.
- */
-int main(void)
-{
- const struct CMUnitTest tests[] = {
- cmocka_unit_test(cleaning_policy_alru_initialize_test01),
- cmocka_unit_test(cleaning_policy_alru_initialize_test02)
- };
-
- print_message("Unit test of alru.c\n");
-
- return cmocka_run_group_tests(tests, NULL, NULL);
-}
-
-
-
diff --git a/tests/unit/tests/utils/utils_generator.c/utils_generator_bisect.c b/tests/unit/tests/utils/utils_generator.c/utils_generator_bisect.c
new file mode 100644
index 0000000..0f9e853
--- /dev/null
+++ b/tests/unit/tests/utils/utils_generator.c/utils_generator_bisect.c
@@ -0,0 +1,214 @@
+/*
+ * Copyright(c) 2022 Intel Corporation
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/*
+ * src/utils/utils_generator.c
+ * ocf_generator_bisect_next
+ *
+ * bitreverse32
+ * ocf_generator_bisect_init
+ *
+ */
+
+#undef static
+
+#undef inline
+
+
+#include
+#include
+#include
+#include
+#include "print_desc.h"
+
+#include "utils_generator.h"
+
+#include "utils/utils_generator.c/utils_generator_bisect_generated_wraps.c"
+
+static void ocf_generator_bisect_test01(void **state)
+{
+ struct {
+ uint32_t values[16];
+ uint32_t limit;
+ } expected_output[] = {
+ {
+ .values = { 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15 },
+ .limit = 16,
+ },
+ {
+ .values = { 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7 },
+ .limit = 15,
+ },
+ {
+ .values = { 0, 8, 4, 12, 2, 10, 6, 1, 9, 5, 13, 3, 11, 7 },
+ .limit = 14,
+ },
+ {
+ .values = { 0, 8, 4, 12, 2, 10, 6, 1, 9, 5, 3, 11, 7 },
+ .limit = 13,
+ },
+ {
+ .values = { 0, 8, 4, 2, 10, 6, 1, 9, 5, 3, 11, 7 },
+ .limit = 12,
+ },
+ {
+ .values = { 0, 8, 4, 2, 10, 6, 1, 9, 5, 3, 7 },
+ .limit = 11,
+ },
+ {
+ .values = { 0, 8, 4, 2, 6, 1, 9, 5, 3, 7 },
+ .limit = 10,
+ },
+ {
+ .values = { 0, 8, 4, 2, 6, 1, 5, 3, 7 },
+ .limit = 9,
+ },
+ {
+ .values = { 0, 4, 2, 6, 1, 5, 3, 7 },
+ .limit = 8,
+ },
+ {
+ .values = { 0, 4, 2, 6, 1, 5, 3 },
+ .limit = 7,
+ },
+ {
+ .values = { 0, 4, 2, 1, 5, 3 },
+ .limit = 6,
+ },
+ {
+ .values = { 0, 4, 2, 1, 3 },
+ .limit = 5,
+ },
+ {
+ .values = { 0, 2, 1, 3 },
+ .limit = 4,
+ },
+ {
+ .values = { 0, 2, 1 },
+ .limit = 3,
+ },
+ {
+ .values = { 0, 1 },
+ .limit = 2,
+ },
+ {
+ .values = { 0 },
+ .limit = 1,
+ },
+ };
+ struct ocf_generator_bisect_state generator;
+ uint32_t value;
+ int i, j;
+
+ print_test_description("Check if sequence order is correct");
+
+ for (i = 0; i < sizeof(expected_output)/sizeof(*expected_output); i++) {
+ ocf_generator_bisect_init(&generator,
+ expected_output[i].limit, 0);
+
+ for (j = 0; j < expected_output[i].limit; j++) {
+ value = ocf_generator_bisect_next(&generator);
+ assert_int_equal(value, expected_output[i].values[j]);
+ }
+ }
+}
+
+static void ocf_generator_bisect_test02(void **state)
+{
+ struct {
+ uint32_t values[16];
+ uint32_t limit;
+ uint32_t offset;
+ } expected_output[] = {
+ {
+ .values = { 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15, 0 },
+ .limit = 16,
+ .offset = 1,
+ },
+ {
+ .values = { 15, 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7 },
+ .limit = 16,
+ .offset = 15,
+ },
+ {
+ .values = { 1, 9, 5, 13, 3, 11, 7, 15, 0, 8, 4, 12, 2, 10, 6, 14 },
+ .limit = 16,
+ .offset = 8,
+ },
+ {
+ .values = { 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 0 },
+ .limit = 15,
+ .offset = 1,
+ },
+ {
+ .values = { 7, 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11 },
+ .limit = 15,
+ .offset = 14,
+ },
+ {
+ .values = { 1, 9, 5, 13, 3, 11, 7, 0, 8, 4, 12, 2, 10, 6, 14 },
+ .limit = 15,
+ .offset = 8,
+ },
+ {
+ .values = { 8, 4, 2, 10, 6, 1, 9, 5, 3, 11, 7, 0 },
+ .limit = 12,
+ .offset = 1,
+ },
+ {
+ .values = { 7, 0, 8, 4, 2, 10, 6, 1, 9, 5, 3, 11 },
+ .limit = 12,
+ .offset = 11,
+ },
+ {
+ .values = { 1, 9, 5, 3, 11, 7, 0, 8, 4, 2, 10, 6 },
+ .limit = 12,
+ .offset = 6,
+ },
+ {
+ .values = { 8, 4, 2, 6, 1, 5, 3, 7, 0 },
+ .limit = 9,
+ .offset = 1,
+ },
+ {
+ .values = { 7, 0, 8, 4, 2, 6, 1, 5, 3 },
+ .limit = 9,
+ .offset = 8,
+ },
+ {
+ .values = { 1, 5, 3, 7, 0, 8, 4, 2, 6 },
+ .limit = 9,
+ .offset = 5,
+ },
+ };
+ struct ocf_generator_bisect_state generator;
+ uint32_t value;
+ int i, j;
+
+ print_test_description("Check if offset works correctly");
+
+ for (i = 0; i < sizeof(expected_output)/sizeof(*expected_output); i++) {
+ ocf_generator_bisect_init(&generator,
+ expected_output[i].limit,
+ expected_output[i].offset);
+
+ for (j = 0; j < expected_output[i].limit; j++) {
+ value = ocf_generator_bisect_next(&generator);
+ assert_int_equal(value, expected_output[i].values[j]);
+ }
+ }
+}
+
+int main(void)
+{
+ const struct CMUnitTest tests[] = {
+ cmocka_unit_test(ocf_generator_bisect_test01),
+ cmocka_unit_test(ocf_generator_bisect_test02),
+ };
+
+ print_message("Unit tests for generator bisect\n");
+
+ return cmocka_run_group_tests(tests, NULL, NULL);
+}