Parallelize ALRU recovery

Signed-off-by: Robert Baldyga <robert.baldyga@intel.com>

src/cleaning/acp.c

@@ -323,6 +323,15 @@ int cleaning_policy_acp_initialize(struct ocf_cache *cache,
 	return 0;
 }
 
+void cleaning_policy_acp_recovery(ocf_cache_t cache,
+                ocf_cleaning_recovery_end_t cmpl, void *priv)
+{
+	int result;
+
+	result = cleaning_policy_acp_initialize(cache, 1);
+	cmpl(priv, result);
+}
+
 int cleaning_policy_acp_set_cleaning_param(ocf_cache_t cache,
 		uint32_t param_id, uint32_t param_value)
 {

src/cleaning/acp.h

@@ -12,6 +12,9 @@ void cleaning_policy_acp_setup(ocf_cache_t cache);
 
 int cleaning_policy_acp_initialize(ocf_cache_t cache, int init_metadata);
 
+void cleaning_policy_acp_recovery(ocf_cache_t cache,
+                ocf_cleaning_recovery_end_t cmpl, void *priv);
+
 void cleaning_policy_acp_deinitialize(ocf_cache_t cache);
 
 void cleaning_policy_acp_perform_cleaning(ocf_cache_t cache,

src/cleaning/alru.c

@@ -10,6 +10,7 @@
 #include "../metadata/metadata.h"
 #include "../utils/utils_cleaner.h"
 #include "../utils/utils_user_part.h"
+#include "../utils/utils_parallelize.h"
 #include "../utils/utils_realloc.h"
 #include "../concurrency/ocf_cache_line_concurrency.h"
 #include "../ocf_def_priv.h"
@@ -61,54 +62,63 @@ struct alru_context {
 
 /* -- Start of ALRU functions -- */
 
-/* Adds the given collision_index to the _head_ of the ALRU list */
-static void add_alru_head(struct ocf_cache *cache, int partition_id,
-		unsigned int collision_index)
+/* Appends given sublist to the _head_ of the ALRU list */
+static void append_alru_head(ocf_cache_t cache, ocf_part_id_t part_id,
+		ocf_cache_line_t head, ocf_cache_line_t tail)
 {
-	unsigned int curr_head_index;
-	unsigned int collision_table_entries = cache->device->collision_table_entries;
-	struct alru_cleaning_policy *part_alru = &cache->user_parts[partition_id]
-			.clean_pol->policy.alru;
-	struct alru_cleaning_policy_meta *alru;
+	ocf_cache_line_t terminator = cache->device->collision_table_entries;
+	struct alru_cleaning_policy *part_alru;
+	struct cleaning_policy_meta *meta;
+	struct alru_cleaning_policy_meta *old_head;
+	struct alru_cleaning_policy_meta *entry;
 
-	ENV_BUG_ON(!(collision_index < collision_table_entries));
+	part_alru = &cache->user_parts[part_id].clean_pol->policy.alru;
 
-	ENV_BUG_ON(env_atomic_read(&part_alru->size) < 0);
+	if (head == terminator && tail == terminator)
+		return;
 
-	ENV_WARN_ON(!metadata_test_dirty(cache, collision_index));
-	ENV_WARN_ON(!metadata_test_valid_any(cache, collision_index));
+	ENV_BUG_ON(head == terminator);
+	ENV_BUG_ON(tail == terminator);
 
-	/* First node to be added/ */
-	if (env_atomic_read(&part_alru->size) == 0) {
-		part_alru->lru_head = collision_index;
-		part_alru->lru_tail = collision_index;
-
-		alru = &ocf_metadata_get_cleaning_policy(cache,
-				collision_index)->meta.alru;
-		alru->lru_next = collision_table_entries;
-		alru->lru_prev = collision_table_entries;
-		alru->timestamp = env_ticks_to_secs(
-			env_get_tick_count());
+	if (part_alru->lru_head == terminator) {
+		part_alru->lru_head = head;
+		part_alru->lru_tail = tail;
 	} else {
-		/* Not the first node to be added. */
+		meta = ocf_metadata_get_cleaning_policy(cache, part_alru->lru_head);
+		old_head = &meta->meta.alru;
+		old_head->lru_prev = tail;
 
-		curr_head_index = part_alru->lru_head;
+		meta = ocf_metadata_get_cleaning_policy(cache, tail);
+		entry = &meta->meta.alru;
+		entry->lru_next = part_alru->lru_head;
 
-		ENV_BUG_ON(!(curr_head_index < collision_table_entries));
-
-		alru = &ocf_metadata_get_cleaning_policy(cache,
-				collision_index)->meta.alru;
-		alru->lru_next = curr_head_index;
-		alru->lru_prev = collision_table_entries;
-		alru->timestamp = env_ticks_to_secs(
-			env_get_tick_count());
-
-		alru = &ocf_metadata_get_cleaning_policy(cache,
-				curr_head_index)->meta.alru;
-		alru->lru_prev = collision_index;
-
-		part_alru->lru_head = collision_index;
+		part_alru->lru_head = head;
 	}
+}
+
+/* Adds the given collision_index to the _head_ of the ALRU list */
+static void add_alru_head(ocf_cache_t cache, ocf_part_id_t part_id,
+		ocf_cache_line_t cline)
+{
+	ocf_cache_line_t terminator = cache->device->collision_table_entries;
+	struct alru_cleaning_policy *part_alru;
+	struct cleaning_policy_meta *meta;
+	struct alru_cleaning_policy_meta *entry;
+
+	ENV_BUG_ON(!(cline < terminator));
+
+	ENV_WARN_ON(!metadata_test_dirty(cache, cline));
+	ENV_WARN_ON(!metadata_test_valid_any(cache, cline));
+
+	part_alru = &cache->user_parts[part_id].clean_pol->policy.alru;
+
+	meta = ocf_metadata_get_cleaning_policy(cache, cline);
+	entry = &meta->meta.alru;
+	entry->lru_next = terminator;
+	entry->lru_prev = terminator;
+	entry->timestamp = env_ticks_to_secs(env_get_tick_count());
+
+	append_alru_head(cache, part_id, cline, cline);
 
 	env_atomic_inc(&part_alru->size);
 }
@@ -375,24 +385,6 @@ static void _alru_rebuild(struct ocf_cache *cache)
 	}
 }
 
-static int cleaning_policy_alru_initialize_part(struct ocf_cache *cache,
-		struct ocf_user_part *user_part, int init_metadata)
-{
-	struct alru_cleaning_policy *part_alru =
-		&user_part->clean_pol->policy.alru;
-
-	if (init_metadata) {
-		/* ALRU initialization */
-		env_atomic_set(&part_alru->size, 0);
-		part_alru->lru_head = cache->device->collision_table_entries;
-		part_alru->lru_tail = cache->device->collision_table_entries;
-	}
-
-	cache->device->runtime_meta->cleaning_thread_access = 0;
-
-	return 0;
-}
-
 void cleaning_policy_alru_setup(struct ocf_cache *cache)
 {
 	struct alru_cleaning_policy_config *config;
@@ -405,10 +397,8 @@ void cleaning_policy_alru_setup(struct ocf_cache *cache)
 	config->activity_threshold = OCF_ALRU_DEFAULT_ACTIVITY_THRESHOLD;
 }
 
-int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata)
+int cleaning_policy_alru_init_common(ocf_cache_t cache)
 {
-	struct ocf_user_part *user_part;
-	ocf_part_id_t part_id;
 	struct alru_context *ctx;
 	int error = 0;
 	unsigned i;
@@ -432,13 +422,20 @@ int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata)
 		return error;
 	}
 
+	cache->device->runtime_meta->cleaning_thread_access = 0;
 
 	cache->cleaner.cleaning_policy_context = ctx;
 
-	for_each_user_part(cache, user_part, part_id) {
-		cleaning_policy_alru_initialize_part(cache,
-				user_part, init_metadata);
-	}
+	return 0;
+}
+
+int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata)
+{
+	int result;
+
+	result = cleaning_policy_alru_init_common(cache);
+	if (result)
+		return result;
 
 	if (init_metadata)
 		_alru_rebuild(cache);
@@ -448,6 +445,184 @@ int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata)
 	return 0;
 }
 
+#define OCF_ALRU_RECOVERY_SHARDS_CNT 32
+
+struct ocf_alru_recovery_context {
+	ocf_cache_t cache;
+	struct {
+		struct {
+			ocf_cache_line_t head;
+			ocf_cache_line_t tail;
+		} part[OCF_USER_IO_CLASS_MAX];
+	} shard[OCF_ALRU_RECOVERY_SHARDS_CNT] __attribute__((aligned(64)));
+
+	ocf_cleaning_recovery_end_t cmpl;
+	void *priv;
+};
+
+static void add_alru_head_recovery(struct ocf_alru_recovery_context *context,
+		unsigned shard_id, ocf_core_id_t part_id,
+		ocf_cache_line_t cline)
+{
+	ocf_cache_t cache = context->cache;
+	ocf_cache_line_t curr_head, terminator;
+	struct cleaning_policy_meta *meta;
+	struct alru_cleaning_policy_meta *entry;
+	struct alru_cleaning_policy_meta *next;
+
+	terminator = ocf_metadata_collision_table_entries(cache);
+	curr_head = context->shard[shard_id].part[part_id].head;
+
+	meta = ocf_metadata_get_cleaning_policy(cache, cline);
+	entry = &meta->meta.alru;
+
+	if (curr_head == terminator) {
+		/* First node to be added/ */
+		entry->lru_next = terminator;
+		entry->lru_prev = terminator;
+		entry->timestamp = env_ticks_to_secs(env_get_tick_count());
+
+		context->shard[shard_id].part[part_id].head = cline;
+		context->shard[shard_id].part[part_id].tail = cline;
+	} else {
+		/* Not the first node to be added. */
+		entry->lru_next = curr_head;
+		entry->lru_prev = terminator;
+		entry->timestamp = env_ticks_to_secs(env_get_tick_count());
+
+		meta = ocf_metadata_get_cleaning_policy(cache, curr_head);
+		next = &meta->meta.alru;
+
+		next->lru_prev = cline;
+
+		context->shard[shard_id].part[part_id].head = cline;
+	}
+}
+
+static int ocf_alru_recovery_handle(ocf_parallelize_t parallelize,
+		void *priv, unsigned shard_id, unsigned shards_cnt)
+{
+	struct ocf_alru_recovery_context *context = priv;
+	ocf_cache_t cache = context->cache;
+	ocf_cache_line_t entries = cache->device->collision_table_entries;
+	ocf_cache_line_t terminator = entries;
+	unsigned part_size[OCF_USER_IO_CLASS_MAX] = {};
+	struct ocf_user_part *user_part;
+	struct alru_cleaning_policy *part_alru;
+	ocf_part_id_t part_id;
+	ocf_core_id_t core_id;
+	ocf_cache_line_t cline, portion;
+	uint64_t begin, end;
+	uint32_t step = 0;
+	int i;
+
+	portion = DIV_ROUND_UP((uint64_t)entries, shards_cnt);
+	begin = portion*shard_id;
+	end = OCF_MIN(portion*(shard_id + 1), entries);
+
+	for (i = 0; i < OCF_USER_IO_CLASS_MAX; i++) {
+		context->shard[shard_id].part[i].head = terminator;
+		context->shard[shard_id].part[i].tail = terminator;
+	}
+
+	for (cline = begin; cline < end; cline++) {
+		ocf_metadata_get_core_and_part_id(cache, cline,
+				&core_id, &part_id);
+
+		OCF_COND_RESCHED_DEFAULT(step);
+
+		if (core_id == OCF_CORE_MAX)
+			continue;
+
+		if (!metadata_test_dirty(cache, cline)) {
+			cleaning_policy_alru_init_cache_block(cache, cline);
+		} else {
+			add_alru_head_recovery(context, shard_id,
+					part_id, cline);
+			++part_size[part_id];
+		}
+	}
+
+	for_each_user_part(cache, user_part, part_id) {
+		part_alru = &user_part->clean_pol->policy.alru;
+		env_atomic_add(part_size[part_id], &part_alru->size);
+	}
+
+	return 0;
+}
+
+static void ocf_alru_recovery_finish(ocf_parallelize_t parallelize,
+		void *priv, int error)
+{
+	struct ocf_alru_recovery_context *context = priv;
+	ocf_cache_t cache = context->cache;
+	ocf_part_id_t part_id;
+	ocf_cache_line_t head, tail;
+	unsigned shard;
+
+	if (error)
+		goto end;
+
+	for (part_id = 0; part_id < OCF_USER_IO_CLASS_MAX; part_id++) {
+		for (shard = 0; shard < OCF_ALRU_RECOVERY_SHARDS_CNT; shard++) {
+			head = context->shard[shard].part[part_id].head;
+			tail = context->shard[shard].part[part_id].tail;
+
+			append_alru_head(cache, part_id, head, tail);
+		}
+	}
+
+	ocf_kick_cleaner(cache);
+
+end:
+	context->cmpl(context->priv, error);
+
+	ocf_parallelize_destroy(parallelize);
+}
+
+void cleaning_policy_alru_recovery(ocf_cache_t cache,
+		ocf_cleaning_recovery_end_t cmpl, void *priv)
+{
+	struct ocf_alru_recovery_context *context;
+	ocf_parallelize_t parallelize;
+	struct alru_cleaning_policy *part_alru;
+	struct ocf_user_part *user_part;
+	ocf_part_id_t part_id;
+	int result;
+
+	result = ocf_parallelize_create(&parallelize, cache,
+			OCF_ALRU_RECOVERY_SHARDS_CNT, sizeof(*context),
+			ocf_alru_recovery_handle, ocf_alru_recovery_finish);
+	if (result) {
+		cmpl(priv, result);
+		return;
+	}
+
+
+	result = cleaning_policy_alru_init_common(cache);
+	if (result) {
+		ocf_parallelize_destroy(parallelize);
+		cmpl(priv, result);
+		return;
+	}
+
+	for_each_user_part(cache, user_part, part_id) {
+		/* ALRU initialization */
+		part_alru = &user_part->clean_pol->policy.alru;
+		env_atomic_set(&part_alru->size, 0);
+		part_alru->lru_head = cache->device->collision_table_entries;
+		part_alru->lru_tail = cache->device->collision_table_entries;
+		cache->device->runtime_meta->cleaning_thread_access = 0;
+	}
+
+	context = ocf_parallelize_get_priv(parallelize);
+	context->cache = cache;
+	context->cmpl = cmpl;
+	context->priv = priv;
+
+	ocf_parallelize_run(parallelize);
+}
+
 void cleaning_policy_alru_deinitialize(struct ocf_cache *cache)
 {
 	struct alru_context *alru = cache->cleaner.cleaning_policy_context;

src/cleaning/alru.h

@@ -11,6 +11,8 @@
 
 void cleaning_policy_alru_setup(ocf_cache_t cache);
 int cleaning_policy_alru_initialize(ocf_cache_t cache, int init_metadata);
+void cleaning_policy_alru_recovery(ocf_cache_t cache,
+                ocf_cleaning_recovery_end_t cmpl, void *priv);
 void cleaning_policy_alru_deinitialize(ocf_cache_t cache);
 void cleaning_policy_alru_init_cache_block(ocf_cache_t cache,
 		uint32_t cache_line);

src/cleaning/cleaning.h

@@ -54,4 +54,6 @@ void ocf_kick_cleaner(ocf_cache_t cache);
 
 void ocf_stop_cleaner(ocf_cache_t cache);
 
+typedef void (*ocf_cleaning_recovery_end_t)(void *priv, int error);
+
 #endif

src/cleaning/cleaning_ops.h

@@ -14,6 +14,8 @@
 struct cleaning_policy_ops {
 	void (*setup)(ocf_cache_t cache);
 	int (*initialize)(ocf_cache_t cache, int init_metadata);
+	void (*recovery)(ocf_cache_t cache,
+			ocf_cleaning_recovery_end_t cmpl, void *priv);
 	void (*deinitialize)(ocf_cache_t cache);
 	int (*add_core)(ocf_cache_t cache, ocf_core_id_t core_id);
 	void (*remove_core)(ocf_cache_t cache, ocf_core_id_t core_id);
@@ -43,6 +45,7 @@ static struct cleaning_policy_ops cleaning_policy_ops[ocf_cleaning_max] = {
 		.purge_range = cleaning_policy_alru_purge_range,
 		.set_hot_cache_line = cleaning_policy_alru_set_hot_cache_line,
 		.initialize = cleaning_policy_alru_initialize,
+		.recovery = cleaning_policy_alru_recovery,
 		.deinitialize = cleaning_policy_alru_deinitialize,
 		.set_cleaning_param = cleaning_policy_alru_set_cleaning_param,
 		.get_cleaning_param = cleaning_policy_alru_get_cleaning_param,
@@ -56,6 +59,7 @@ static struct cleaning_policy_ops cleaning_policy_ops[ocf_cleaning_max] = {
 		.purge_range = cleaning_policy_acp_purge_range,
 		.set_hot_cache_line = cleaning_policy_acp_set_hot_cache_line,
 		.initialize = cleaning_policy_acp_initialize,
+		.recovery = cleaning_policy_acp_recovery,
 		.deinitialize = cleaning_policy_acp_deinitialize,
 		.set_cleaning_param = cleaning_policy_acp_set_cleaning_param,
 		.get_cleaning_param = cleaning_policy_acp_get_cleaning_param,
@@ -87,6 +91,20 @@ static inline int ocf_cleaning_initialize(ocf_cache_t cache,
 	return cleaning_policy_ops[policy].initialize(cache, init_metadata);
 }
 
+static inline void ocf_cleaning_recovery(ocf_cache_t cache,
+		ocf_cleaning_t policy,
+		ocf_cleaning_recovery_end_t cmpl, void *priv)
+{
+	ENV_BUG_ON(policy >= ocf_cleaning_max);
+
+	if (unlikely(!cleaning_policy_ops[policy].recovery)) {
+		cmpl(priv, 0);
+		return;
+	}
+
+	cleaning_policy_ops[policy].recovery(cache, cmpl, priv);
+}
+
 static inline void ocf_cleaning_deinitialize(ocf_cache_t cache)
 {
 	ocf_cleaning_t policy;