diff --git a/src/eviction/lru.c b/src/eviction/lru.c index d3529cf..f5f00bb 100644 --- a/src/eviction/lru.c +++ b/src/eviction/lru.c @@ -15,176 +15,115 @@ #define OCF_EVICTION_MAX_SCAN 1024 -/* -- Start of LRU functions --*/ - -/* Returns 1 if the given collision_index is the _head_ of - * the LRU list, 0 otherwise. - */ -/* static inline int is_lru_head(unsigned collision_index) { - * return collision_index == lru_list.lru_head; - * } - */ - -#define is_lru_head(x) (x == collision_table_entries) -#define is_lru_tail(x) (x == collision_table_entries) - -/* Sets the given collision_index as the new _head_ of the LRU list. */ -static inline void update_lru_head(ocf_cache_t cache, - int partition_id, unsigned int collision_index, - int cline_dirty) -{ - struct ocf_user_part *part = &cache->user_parts[partition_id]; - - - if (cline_dirty) - part->runtime->eviction.policy.lru.dirty_head = collision_index; - else - part->runtime->eviction.policy.lru.clean_head = collision_index; -} - -/* Sets the given collision_index as the new _tail_ of the LRU list. */ -static inline void update_lru_tail(ocf_cache_t cache, - int partition_id, unsigned int collision_index, - int cline_dirty) -{ - struct ocf_user_part *part = &cache->user_parts[partition_id]; - - if (cline_dirty) - part->runtime->eviction.policy.lru.dirty_tail = collision_index; - else - part->runtime->eviction.policy.lru.clean_tail = collision_index; -} - -/* Sets the given collision_index as the new _head_ and _tail_ of - * the LRU list. - */ -static inline void update_lru_head_tail(ocf_cache_t cache, - int partition_id, unsigned int collision_index, int cline_dirty) -{ - update_lru_head(cache, partition_id, collision_index, cline_dirty); - update_lru_tail(cache, partition_id, collision_index, cline_dirty); -} +static const ocf_cache_line_t end_marker = (ocf_cache_line_t)-1; /* Adds the given collision_index to the _head_ of the LRU list */ -static void add_lru_head(ocf_cache_t cache, int partition_id, - unsigned int collision_index, int cline_dirty) -{ - unsigned int curr_head_index; - unsigned int collision_table_entries = - cache->device->collision_table_entries; - struct ocf_user_part *part = &cache->user_parts[partition_id]; - union eviction_policy_meta eviction; +static void add_lru_head(ocf_cache_t cache, + struct ocf_lru_list *list, + unsigned int collision_index) - ENV_BUG_ON(!(collision_index < collision_table_entries)); +{ + union eviction_policy_meta eviction; + struct lru_eviction_policy_meta *node; + unsigned int curr_head_index; + + ENV_BUG_ON(collision_index == end_marker); ocf_metadata_get_evicition_policy(cache, collision_index, &eviction); + node = &eviction.lru; /* First node to be added/ */ - if ((cline_dirty && !part->runtime->eviction.policy.lru.has_dirty_nodes) || - (!cline_dirty && !part->runtime->eviction.policy.lru.has_clean_nodes)) { - update_lru_head_tail(cache, partition_id, collision_index, cline_dirty); + if (!list->num_nodes) { + list->head = collision_index; + list->tail = collision_index; - eviction.lru.next = collision_table_entries; - eviction.lru.prev = collision_table_entries; + node->next = end_marker; + node->prev = end_marker; - if (cline_dirty) - part->runtime->eviction.policy.lru.has_dirty_nodes = 1; - else - part->runtime->eviction.policy.lru.has_clean_nodes = 1; + list->num_nodes = 1; ocf_metadata_set_evicition_policy(cache, collision_index, &eviction); } else { - union eviction_policy_meta eviction_curr; + union eviction_policy_meta eviction_curr_head; + struct lru_eviction_policy_meta *curr_head; /* Not the first node to be added. */ - curr_head_index = cline_dirty ? - part->runtime->eviction.policy.lru.dirty_head : - part->runtime->eviction.policy.lru.clean_head; + curr_head_index = list->head; - ENV_BUG_ON(!(curr_head_index < collision_table_entries)); + ENV_BUG_ON(curr_head_index == end_marker); ocf_metadata_get_evicition_policy(cache, curr_head_index, - &eviction_curr); + &eviction_curr_head); + curr_head = &eviction_curr_head.lru; - eviction.lru.next = curr_head_index; - eviction.lru.prev = collision_table_entries; - eviction_curr.lru.prev = collision_index; + node->next = curr_head_index; + node->prev = end_marker; + curr_head->prev = collision_index; - update_lru_head(cache, partition_id, collision_index, cline_dirty); + list->head = collision_index; + + ++list->num_nodes; ocf_metadata_set_evicition_policy(cache, curr_head_index, - &eviction_curr); + &eviction_curr_head); ocf_metadata_set_evicition_policy(cache, collision_index, &eviction); } } /* Deletes the node with the given collision_index from the lru list */ -static void remove_lru_list(ocf_cache_t cache, int partition_id, - unsigned int collision_index, int cline_dirty) +static void remove_lru_list(ocf_cache_t cache, + struct ocf_lru_list *list, + unsigned int collision_index) { - int is_clean_head = 0, is_clean_tail = 0, is_dirty_head = 0, is_dirty_tail = 0; + int is_head = 0, is_tail = 0; uint32_t prev_lru_node, next_lru_node; - uint32_t collision_table_entries = cache->device->collision_table_entries; - struct ocf_user_part *part = &cache->user_parts[partition_id]; union eviction_policy_meta eviction; + struct lru_eviction_policy_meta *node; - ENV_BUG_ON(!(collision_index < collision_table_entries)); + ENV_BUG_ON(collision_index == end_marker); ocf_metadata_get_evicition_policy(cache, collision_index, &eviction); + node = &eviction.lru; - /* Find out if this node is LRU _head_ or LRU _tail_ */ - if (part->runtime->eviction.policy.lru.clean_head == collision_index) - is_clean_head = 1; - if (part->runtime->eviction.policy.lru.dirty_head == collision_index) - is_dirty_head = 1; - if (part->runtime->eviction.policy.lru.clean_tail == collision_index) - is_clean_tail = 1; - if (part->runtime->eviction.policy.lru.dirty_tail == collision_index) - is_dirty_tail = 1; - ENV_BUG_ON((is_clean_tail || is_clean_head) && (is_dirty_tail || is_dirty_head)); + is_head = (list->head == collision_index); + is_tail = (list->tail == collision_index); /* Set prev and next (even if not existent) */ - next_lru_node = eviction.lru.next; - prev_lru_node = eviction.lru.prev; + next_lru_node = node->next; + prev_lru_node = node->prev; /* Case 1: If we are head AND tail, there is only one node. * So unlink node and set that there is no node left in the list. */ - if ((is_clean_head && is_clean_tail) || (is_dirty_head && is_dirty_tail)) { - eviction.lru.next = collision_table_entries; - eviction.lru.prev = collision_table_entries; - - update_lru_head_tail(cache, partition_id, collision_table_entries, cline_dirty); - - if (cline_dirty) - part->runtime->eviction.policy.lru.has_dirty_nodes = 0; - else - part->runtime->eviction.policy.lru.has_clean_nodes = 0; + if (is_head && is_tail) { + node->next = end_marker; + node->prev = end_marker; ocf_metadata_set_evicition_policy(cache, collision_index, &eviction); - update_lru_head_tail(cache, partition_id, - collision_table_entries, cline_dirty); + list->head = end_marker; + list->tail = end_marker; } /* Case 2: else if this collision_index is LRU head, but not tail, * update head and return */ - else if ((!is_clean_tail && is_clean_head) || (!is_dirty_tail && is_dirty_head)) { + else if (is_head) { union eviction_policy_meta eviction_next; + struct lru_eviction_policy_meta *next_node; - ENV_BUG_ON(!(next_lru_node < collision_table_entries)); + ENV_BUG_ON(next_lru_node == end_marker); ocf_metadata_get_evicition_policy(cache, next_lru_node, &eviction_next); + next_node = &eviction_next.lru; - update_lru_head(cache, partition_id, next_lru_node, cline_dirty); - - eviction.lru.next = collision_table_entries; - eviction_next.lru.prev = collision_table_entries; + list->head = next_lru_node; + node->next = end_marker; + next_node->prev = end_marker; ocf_metadata_set_evicition_policy(cache, collision_index, &eviction); @@ -196,18 +135,20 @@ static void remove_lru_list(ocf_cache_t cache, int partition_id, /* Case 3: else if this collision_index is LRU tail, but not head, * update tail and return */ - else if ((is_clean_tail && !is_clean_head) || (is_dirty_tail && !is_dirty_head)) { + else if (is_tail) { union eviction_policy_meta eviction_prev; + struct lru_eviction_policy_meta *prev_node; - ENV_BUG_ON(!(prev_lru_node < collision_table_entries)); + ENV_BUG_ON(prev_lru_node == end_marker); - update_lru_tail(cache, partition_id, prev_lru_node, cline_dirty); + list->tail = prev_lru_node; ocf_metadata_get_evicition_policy(cache, prev_lru_node, &eviction_prev); + prev_node = &eviction_prev.lru; - eviction.lru.prev = collision_table_entries; - eviction_prev.lru.next = collision_table_entries; + node->prev = end_marker; + prev_node->next = end_marker; ocf_metadata_set_evicition_policy(cache, collision_index, &eviction); @@ -222,22 +163,26 @@ static void remove_lru_list(ocf_cache_t cache, int partition_id, else { union eviction_policy_meta eviction_prev; union eviction_policy_meta eviction_next; + struct lru_eviction_policy_meta *prev_node; + struct lru_eviction_policy_meta *next_node; - ENV_BUG_ON(!(next_lru_node < collision_table_entries)); - ENV_BUG_ON(!(prev_lru_node < collision_table_entries)); + ENV_BUG_ON(next_lru_node == end_marker); + ENV_BUG_ON(prev_lru_node == end_marker); ocf_metadata_get_evicition_policy(cache, next_lru_node, &eviction_next); + next_node = &eviction_next.lru; ocf_metadata_get_evicition_policy(cache, prev_lru_node, &eviction_prev); + prev_node = &eviction_prev.lru; /* Update prev and next nodes */ - eviction_prev.lru.next = eviction.lru.next; - eviction_next.lru.prev = eviction.lru.prev; + prev_node->next = node->next; + next_node->prev = node->prev; /* Update the given node */ - eviction.lru.next = collision_table_entries; - eviction.lru.prev = collision_table_entries; + node->next = end_marker; + node->prev = end_marker; ocf_metadata_set_evicition_policy(cache, collision_index, &eviction); @@ -246,6 +191,8 @@ static void remove_lru_list(ocf_cache_t cache, int partition_id, ocf_metadata_set_evicition_policy(cache, prev_lru_node, &eviction_prev); } + + --list->num_nodes; } /*-- End of LRU functions*/ @@ -253,11 +200,13 @@ static void remove_lru_list(ocf_cache_t cache, int partition_id, void evp_lru_init_cline(ocf_cache_t cache, ocf_cache_line_t cline) { union eviction_policy_meta eviction; + struct lru_eviction_policy_meta *node; ocf_metadata_get_evicition_policy(cache, cline, &eviction); + node = &eviction.lru; - eviction.lru.prev = cache->device->collision_table_entries; - eviction.lru.next = cache->device->collision_table_entries; + node->prev = end_marker; + node->next = end_marker; ocf_metadata_set_evicition_policy(cache, cline, &eviction); } @@ -267,8 +216,14 @@ void evp_lru_init_cline(ocf_cache_t cache, ocf_cache_line_t cline) void evp_lru_rm_cline(ocf_cache_t cache, ocf_cache_line_t cline) { ocf_part_id_t part_id = ocf_metadata_get_partition_id(cache, cline); + struct ocf_user_part *part = &cache->user_parts[part_id]; + struct ocf_lru_list *list; - remove_lru_list(cache, part_id, cline, metadata_test_dirty(cache, cline)); + list = metadata_test_dirty(cache, cline) ? + &part->runtime->eviction.policy.lru.dirty : + &part->runtime->eviction.policy.lru.clean; + + remove_lru_list(cache, list, cline); } static void evp_lru_clean_end(void *private_data, int error) @@ -285,7 +240,7 @@ static int evp_lru_clean_getter(ocf_cache_t cache, struct ocf_cleaner_attribs *attribs = getter_context; ocf_cache_line_t prev_cline, curr_cline = attribs->getter_item; - while (curr_cline < cache->device->collision_table_entries) { + while (curr_cline != end_marker) { ocf_metadata_get_evicition_policy(cache, curr_cline, &eviction); prev_cline = eviction.lru.prev; @@ -320,7 +275,7 @@ static void evp_lru_clean(ocf_cache_t cache, ocf_queue_t io_queue, .getter = evp_lru_clean_getter, .getter_context = &attribs, - .getter_item = part->runtime->eviction.policy.lru.dirty_tail, + .getter_item = part->runtime->eviction.policy.lru.dirty.tail, .count = count > 32 ? 32 : count, @@ -402,15 +357,13 @@ uint32_t evp_lru_req_clines(ocf_cache_t cache, ocf_queue_t io_queue, return 0; i = 0; - curr_cline = part->runtime->eviction.policy.lru.clean_tail; + curr_cline = part->runtime->eviction.policy.lru.clean.tail; /* Find cachelines to be evicted. */ while (i < cline_no) { - ENV_BUG_ON(curr_cline > cache->device->collision_table_entries); - if (!evp_lru_can_evict(cache)) break; - if (curr_cline == cache->device->collision_table_entries) + if (curr_cline == end_marker) break; ocf_metadata_get_evicition_policy(cache, curr_cline, @@ -447,8 +400,8 @@ uint32_t evp_lru_req_clines(ocf_cache_t cache, ocf_queue_t io_queue, curr_cline = prev_cline; } - if (i < cline_no && part->runtime->eviction.policy.lru.dirty_tail != - cache->device->collision_table_entries) { + if (i < cline_no && part->runtime->eviction.policy.lru.dirty.tail != + end_marker) { evp_lru_clean(cache, io_queue, part_id, cline_no - i); } @@ -461,62 +414,78 @@ void evp_lru_hot_cline(ocf_cache_t cache, ocf_cache_line_t cline) { ocf_part_id_t part_id = ocf_metadata_get_partition_id(cache, cline); struct ocf_user_part *part = &cache->user_parts[part_id]; - - uint32_t prev_lru_node, next_lru_node; - uint32_t collision_table_entries = cache->device->collision_table_entries; union eviction_policy_meta eviction; - + struct lru_eviction_policy_meta *node; int cline_dirty; + struct ocf_lru_list *list; ocf_metadata_get_evicition_policy(cache, cline, &eviction); - - next_lru_node = eviction.lru.next; - prev_lru_node = eviction.lru.prev; + node = &eviction.lru; cline_dirty = metadata_test_dirty(cache, cline); + list = cline_dirty ? + &part->runtime->eviction.policy.lru.dirty : + &part->runtime->eviction.policy.lru.clean; - if ((next_lru_node != collision_table_entries) || - (prev_lru_node != collision_table_entries) || - ((part->runtime->eviction.policy.lru.clean_head == cline) && - (part->runtime->eviction.policy.lru.clean_tail == cline)) || - ((part->runtime->eviction.policy.lru.dirty_head == cline) && - (part->runtime->eviction.policy.lru.dirty_tail == cline))) { - remove_lru_list(cache, part_id, cline, cline_dirty); + if (node->next != end_marker || + node->prev != end_marker || + list->head == cline || list->tail == cline) { + remove_lru_list(cache, list, cline); } /* Update LRU */ - add_lru_head(cache, part_id, cline, cline_dirty); + add_lru_head(cache, list, cline); +} + +static inline void _lru_init(struct ocf_lru_list *list) +{ + list->num_nodes = 0; + list->head = end_marker; + list->tail = end_marker; } void evp_lru_init_evp(ocf_cache_t cache, ocf_part_id_t part_id) { - unsigned int collision_table_entries = - cache->device->collision_table_entries; struct ocf_user_part *part = &cache->user_parts[part_id]; + struct ocf_lru_list *clean_list; + struct ocf_lru_list *dirty_list; - part->runtime->eviction.policy.lru.has_clean_nodes = 0; - part->runtime->eviction.policy.lru.has_dirty_nodes = 0; - part->runtime->eviction.policy.lru.clean_head = collision_table_entries; - part->runtime->eviction.policy.lru.clean_tail = collision_table_entries; - part->runtime->eviction.policy.lru.dirty_head = collision_table_entries; - part->runtime->eviction.policy.lru.dirty_tail = collision_table_entries; + clean_list = &part->runtime->eviction.policy.lru.clean; + dirty_list = &part->runtime->eviction.policy.lru.dirty; + + _lru_init(clean_list); + _lru_init(dirty_list); } void evp_lru_clean_cline(ocf_cache_t cache, ocf_part_id_t part_id, uint32_t cline) { + struct ocf_user_part *part = &cache->user_parts[part_id]; + struct ocf_lru_list *clean_list; + struct ocf_lru_list *dirty_list; + + clean_list = &part->runtime->eviction.policy.lru.clean; + dirty_list = &part->runtime->eviction.policy.lru.dirty; + OCF_METADATA_EVICTION_LOCK(); - remove_lru_list(cache, part_id, cline, 1); - add_lru_head(cache, part_id, cline, 0); + remove_lru_list(cache, dirty_list, cline); + add_lru_head(cache, clean_list, cline); OCF_METADATA_EVICTION_UNLOCK(); } void evp_lru_dirty_cline(ocf_cache_t cache, ocf_part_id_t part_id, uint32_t cline) { + struct ocf_user_part *part = &cache->user_parts[part_id]; + struct ocf_lru_list *clean_list; + struct ocf_lru_list *dirty_list; + + clean_list = &part->runtime->eviction.policy.lru.clean; + dirty_list = &part->runtime->eviction.policy.lru.dirty; + OCF_METADATA_EVICTION_LOCK(); - remove_lru_list(cache, part_id, cline, 0); - add_lru_head(cache, part_id, cline, 1); + remove_lru_list(cache, clean_list, cline); + add_lru_head(cache, dirty_list, cline); OCF_METADATA_EVICTION_UNLOCK(); } diff --git a/src/eviction/lru_structs.h b/src/eviction/lru_structs.h index 520a03b..9c2b529 100644 --- a/src/eviction/lru_structs.h +++ b/src/eviction/lru_structs.h @@ -12,13 +12,15 @@ struct lru_eviction_policy_meta { uint32_t next; } __attribute__((packed)); +struct ocf_lru_list { + uint32_t num_nodes; + uint32_t head; + uint32_t tail; +}; + struct lru_eviction_policy { - int has_clean_nodes; - int has_dirty_nodes; - uint32_t dirty_head; - uint32_t dirty_tail; - uint32_t clean_head; - uint32_t clean_tail; + struct ocf_lru_list clean; + struct ocf_lru_list dirty; }; #endif