github/ocf
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Per-execution-context freelists

Browse Source 
Global free cacheline list is divided into a set of freelists, one
per execution context. When attempting to map addres to cache, first
the freelist for current execution context is considered (fast path).
If current execution context freelist is empty (fast path failure),
mapping function attempts to get freelist from other execution context
list (slow path).

The purpose of this change is improve concurrency in freelist access.
It is part of fine granularity metadata lock implementation.

Signed-off-by: Adam Rutkowski <adam.j.rutkowski@intel.com>
This commit is contained in:
Adam Rutkowski 2019-08-26 14:54:19 -04:00
parent f892bb962d
commit 13cf871a13
21 changed files with 545 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
env/posix/ocf_env.h vendored
View File

@ -653,5 +653,4 @@ unsigned env_get_execution_context(void);
void env_put_execution_context(unsigned ctx);
unsigned env_get_execution_context_count(void);
#endif /* __OCF_ENV_H__ */

3
src/engine/cache_engine.c
View File

@ -169,7 +169,8 @@ static inline bool ocf_seq_cutoff_is_on(ocf_cache_t cache)
if (!ocf_cache_is_device_attached(cache))
return false;
return (cache->device->freelist_part->curr_size <= SEQ_CUTOFF_FULL_MARGIN);
return (ocf_freelist_num_free(cache->freelist) <=
SEQ_CUTOFF_FULL_MARGIN);
}
bool ocf_seq_cutoff_check(ocf_core_t core, uint32_t dir, uint64_t addr,

11
src/engine/engine_common.c
View File

@ -7,6 +7,7 @@
#include "../ocf_priv.h"
#include "../ocf_cache_priv.h"
#include "../ocf_queue_priv.h"
#include "../ocf_freelist.h"
#include "engine_common.h"
#define OCF_ENGINE_DEBUG_IO_NAME "common"
#include "engine_debug.h"
@ -250,19 +251,11 @@ static void ocf_engine_map_cache_line(struct ocf_request *req,
ocf_part_id_t part_id = req->part_id;
ocf_cleaning_t clean_policy_type;
if (cache->device->freelist_part->curr_size == 0) {
if (!ocf_freelist_get_cache_line(cache->freelist, cache_line)) {
req->info.mapping_error = 1;
return;
}
*cache_line = cache->device->freelist_part->head;
/* add_to_collision_list changes .next_col and other fields for entry
* so updated last_cache_line_give must be updated before calling it.
*/
ocf_metadata_remove_from_free_list(cache, *cache_line);
ocf_metadata_add_to_partition(cache, part_id, *cache_line);
/* Add the block to the corresponding collision list */

6
src/eviction/eviction.c
View File

@ -107,11 +107,13 @@ int space_managment_evict_do(struct ocf_cache *cache,
struct ocf_request *req, uint32_t evict_cline_no)
{
uint32_t evicted;
uint32_t free;
if (evict_cline_no <= cache->device->freelist_part->curr_size)
free = ocf_freelist_num_free(cache->freelist);
if (evict_cline_no <= free)
return LOOKUP_MAPPED;
evict_cline_no -= cache->device->freelist_part->curr_size;
evict_cline_no -= free;
evicted = ocf_evict_do(cache, req->io_queue, evict_cline_no,
req->part_id);

3
src/eviction/eviction.h
View File

@ -32,8 +32,7 @@ union eviction_policy_meta {
* set core_id to -2 to purge the whole cache partition
*/
struct eviction_policy_ops {
void (*init_cline)(ocf_cache_t cache,
ocf_cache_line_t cline);
void (*init_cline)(ocf_cache_t cache, ocf_cache_line_t cline);
void (*rm_cline)(ocf_cache_t cache,
ocf_cache_line_t cline);
bool (*can_evict)(ocf_cache_t cache);

3
src/eviction/lru.h
View File

@ -8,8 +8,7 @@
#include "eviction.h"
#include "lru_structs.h"
void evp_lru_init_cline(struct ocf_cache *cache,
ocf_cache_line_t cline);
void evp_lru_init_cline(struct ocf_cache *cache, ocf_cache_line_t cline);
void evp_lru_rm_cline(struct ocf_cache *cache, ocf_cache_line_t cline);
bool evp_lru_can_evict(struct ocf_cache *cache);
uint32_t evp_lru_req_clines(struct ocf_cache *cache, ocf_queue_t io_queue,

12
src/metadata/metadata.c
View File

@ -53,18 +53,18 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache, uint64_t device_siz
cache_line_size, layout);
}
void ocf_metadata_init_freelist_partition(struct ocf_cache *cache)
{
OCF_DEBUG_TRACE(cache);
cache->metadata.iface.init_freelist(cache);
}
void ocf_metadata_init_hash_table(struct ocf_cache *cache)
{
OCF_DEBUG_TRACE(cache);
cache->metadata.iface.init_hash_table(cache);
}
void ocf_metadata_init_collision(struct ocf_cache *cache)
{
OCF_DEBUG_TRACE(cache);
cache->metadata.iface.init_collision(cache);
}
void ocf_metadata_deinit(struct ocf_cache *cache)
{
OCF_DEBUG_TRACE(cache);

13
src/metadata/metadata.h
View File

@ -60,6 +60,13 @@ void ocf_metadata_init_freelist_partition(struct ocf_cache *cache);
*/
void ocf_metadata_init_hash_table(struct ocf_cache *cache);
/**
* @brief Initialize collision table
*
* @param cache - Cache instance
*/
void ocf_metadata_init_collision(struct ocf_cache *cache);
/**
* @brief De-Initialize metadata
*
@ -207,4 +214,10 @@ typedef void (*ocf_metadata_load_properties_end_t)(void *priv, int error,
void ocf_metadata_load_properties(ocf_volume_t volume,
ocf_metadata_load_properties_end_t cmpl, void *priv);
static inline ocf_cache_line_t ocf_metadata_collision_table_entries(
struct ocf_cache *cache)
{
return cache->device->collision_table_entries;
}
#endif /* METADATA_H_ */

42
src/metadata/metadata_hash.c
View File

@ -13,6 +13,7 @@
#include "../utils/utils_pipeline.h"
#include "../ocf_def_priv.h"
#include "../ocf_priv.h"
#include "../ocf_freelist.h"
#define OCF_METADATA_HASH_DEBUG 0
@ -1025,14 +1026,10 @@ finalize:
}
static inline void _ocf_init_collision_entry(struct ocf_cache *cache,
ocf_cache_line_t idx, ocf_cache_line_t next,
ocf_cache_line_t prev)
ocf_cache_line_t idx)
{
ocf_cache_line_t invalid_idx = cache->device->collision_table_entries;
ocf_part_id_t invalid_part_id = PARTITION_INVALID;
ocf_metadata_set_partition_info(cache, idx,
invalid_part_id, next, prev);
ocf_metadata_set_collision_info(cache, idx, invalid_idx, invalid_idx);
ocf_metadata_set_core_info(cache, idx,
OCF_CORE_MAX, ULONG_MAX);
@ -1040,36 +1037,17 @@ static inline void _ocf_init_collision_entry(struct ocf_cache *cache,
}
/*
* Modified initialization of freelist partition
* Initialize collision table
*/
static void ocf_metadata_hash_init_freelist(struct ocf_cache *cache)
static void ocf_metadata_hash_init_collision(struct ocf_cache *cache)
{
uint32_t step = 0;
unsigned int i;
ocf_cache_line_t prev, next;
ocf_cache_line_t idx;
ocf_cache_line_t collision_table_entries =
cache->device->collision_table_entries;
prev = collision_table_entries;
idx = 0;
for (i = 0; i < cache->device->collision_table_entries - 1; i++) {
next = ocf_metadata_map_phy2lg(cache, i + 1);
_ocf_init_collision_entry(cache, idx, next, prev);
prev = idx;
idx = next;
OCF_COND_RESCHED_DEFAULT(step);
for (i = 0; i < cache->device->collision_table_entries; i++) {
_ocf_init_collision_entry(cache, i);
}
_ocf_init_collision_entry(cache, idx, collision_table_entries, prev);
/* Initialize freelist partition */
cache->device->freelist_part->head = 0;
cache->device->freelist_part->tail = idx;
cache->device->freelist_part->curr_size = cache->device->
collision_table_entries;
}
/*
* Initialize hash table
*/
@ -1850,8 +1828,6 @@ static void _recovery_rebuild_cline_metadata(ocf_cache_t cache,
part_id = PARTITION_DEFAULT;
ocf_metadata_remove_from_free_list(cache, cache_line);
ocf_metadata_add_to_partition(cache, part_id, cache_line);
hash_index = ocf_metadata_hash_func(cache, core_line, core_id);
@ -1917,10 +1893,12 @@ static void _recovery_rebuild_metadata(ocf_pipeline_t pipeline,
ocf_core_id_t core_id;
uint64_t core_line;
unsigned char step = 0;
const uint64_t collision_table_entries =
ocf_metadata_collision_table_entries(cache);
OCF_METADATA_LOCK_WR();
for (cline = 0; cline < cache->device->collision_table_entries; cline++) {
for (cline = 0; cline < collision_table_entries; cline++) {
ocf_metadata_get_core_info(cache, cline, &core_id, &core_line);
if (core_id != OCF_CORE_MAX &&
(!dirty_only || metadata_test_dirty(cache,
@ -2648,7 +2626,7 @@ static const struct ocf_metadata_iface metadata_hash_iface = {
.init_variable_size = ocf_metadata_hash_init_variable_size,
.deinit_variable_size = ocf_metadata_hash_deinit_variable_size,
.init_hash_table = ocf_metadata_hash_init_hash_table,
.init_freelist = ocf_metadata_hash_init_freelist,
.init_collision = ocf_metadata_hash_init_collision,
.layout_iface = NULL,
.pages = ocf_metadata_hash_pages,

3
src/metadata/metadata_misc.c
View File

@ -5,6 +5,7 @@
#include "ocf/ocf.h"
#include "metadata.h"
#include "../ocf_freelist.h"
#include "../utils/utils_cache_line.h"
static bool _is_cache_line_acting(struct ocf_cache *cache,
@ -100,7 +101,7 @@ void ocf_metadata_sparse_cache_line(struct ocf_cache *cache,
ocf_metadata_remove_from_partition(cache, partition_id, cache_line);
ocf_metadata_add_to_free_list(cache, cache_line);
ocf_freelist_put_cache_line(cache->freelist, cache_line);
}
static void _ocf_metadata_sparse_cache_line(struct ocf_cache *cache,

93
src/metadata/metadata_partition.c
View File

@ -16,99 +16,6 @@ static void update_partition_head(struct ocf_cache *cache,
part->runtime->head = line;
}
void ocf_metadata_remove_from_free_list(struct ocf_cache *cache,
ocf_cache_line_t cline)
{
struct ocf_part *free_list = cache->device->freelist_part;
int is_head, is_tail;
ocf_part_id_t invalid_part_id = PARTITION_INVALID;
ocf_cache_line_t prev, next;
ocf_cache_line_t line_entries = cache->device->collision_table_entries;
ENV_BUG_ON(cline >= line_entries);
/* Get Partition info */
ocf_metadata_get_partition_info(cache, cline, NULL, &next, &prev);
/* Find out if this node is Partition _head_ */
is_head = (prev == line_entries);
is_tail = (next == line_entries);
/* Case 1: If we are head and there is only one node. So unlink node
* and set that there is no node left in the list.
*/
if (is_head && (free_list->curr_size == 1)) {
ocf_metadata_set_partition_info(cache, cline, invalid_part_id,
line_entries, line_entries);
free_list->head = line_entries;
free_list->tail = line_entries;
} else if (is_head) {
/* Case 2: else if this collision_index is partition list head,
* but many nodes, update head and return
*/
ENV_BUG_ON(next >= line_entries);
free_list->head = next;
ocf_metadata_set_partition_prev(cache, next, line_entries);
ocf_metadata_set_partition_next(cache, cline, line_entries);
} else if (is_tail) {
/* Case 3: else if this cline is partition list tail */
ENV_BUG_ON(prev >= line_entries);
free_list->tail = prev;
ocf_metadata_set_partition_prev(cache, cline, line_entries);
ocf_metadata_set_partition_next(cache, prev, line_entries);
} else {
/* Case 4: else this collision_index is a middle node.
* There is no change to the head and the tail pointers.
*/
ENV_BUG_ON(next >= line_entries || prev >= line_entries);
/* Update prev and next nodes */
ocf_metadata_set_partition_prev(cache, next, prev);
ocf_metadata_set_partition_next(cache, prev, next);
/* Update the given node */
ocf_metadata_set_partition_info(cache, cline, invalid_part_id,
line_entries, line_entries);
}
free_list->curr_size--;
}
void ocf_metadata_add_to_free_list(struct ocf_cache *cache,
ocf_cache_line_t line)
{
struct ocf_part *free_list = cache->device->freelist_part;
ocf_cache_line_t tail;
ocf_cache_line_t line_entries = cache->device->collision_table_entries;
ocf_part_id_t invalid_part_id = PARTITION_INVALID;
ENV_BUG_ON(line >= line_entries);
if (free_list->curr_size == 0) {
free_list->head = line;
free_list->tail = line;
ocf_metadata_set_partition_info(cache, line, invalid_part_id,
line_entries, line_entries);
} else {
tail = free_list->tail;
ENV_BUG_ON(tail >= line_entries);
ocf_metadata_set_partition_info(cache, line, invalid_part_id,
line_entries, tail);
ocf_metadata_set_partition_next(cache, tail, line);
free_list->tail = line;
}
free_list->curr_size++;
}
/* Adds the given collision_index to the _head_ of the Partition list */
void ocf_metadata_add_to_partition(struct ocf_cache *cache,
ocf_part_id_t part_id, ocf_cache_line_t line)

6
src/metadata/metadata_partition.h
View File

@ -75,12 +75,6 @@ static inline void ocf_metadata_set_partition_info(
next_line, prev_line);
}
void ocf_metadata_add_to_free_list(struct ocf_cache *cache,
ocf_cache_line_t cline);
void ocf_metadata_remove_from_free_list(struct ocf_cache *cache,
ocf_cache_line_t cline);
void ocf_metadata_add_to_partition(struct ocf_cache *cache,
ocf_part_id_t part_id, ocf_cache_line_t line);

6
src/metadata/metadata_partition_structs.h
View File

@ -10,12 +10,6 @@
#include "../cleaning/cleaning.h"
#include "../eviction/eviction.h"
struct ocf_part {
ocf_cache_line_t head;
ocf_cache_line_t tail;
uint32_t curr_size;
};
struct ocf_user_part_config {
char name[OCF_IO_CLASS_NAME_MAX];
uint32_t min_size;

14
src/metadata/metadata_structs.h
View File

@ -124,13 +124,6 @@ struct ocf_metadata_iface {
struct ocf_volume_uuid *uuid, uint32_t count,
ocf_metadata_query_cores_end_t cmpl, void *priv);
/**
* @brief Initialize freelist partition
*
* @param cache - Cache instance
*/
void (*init_freelist)(struct ocf_cache *cache);
/**
* @brief Metadata cache line location on pages interface
@ -144,6 +137,13 @@ struct ocf_metadata_iface {
*/
void (*init_hash_table)(struct ocf_cache *cache);
/**
* @brief Initialize collision table
*
* @param cache - Cache instance
*/
void (*init_collision)(struct ocf_cache *cache);
/**
* @brief De-Initialize metadata
*

2
src/metadata/metadata_superblock.h
View File

@ -59,8 +59,6 @@ struct ocf_superblock_config {
* @brief OCF cache metadata runtime superblock
*/
struct ocf_superblock_runtime {
struct ocf_part freelist_part;
uint32_t cleaning_thread_access;
};

44
src/mngt/ocf_mngt_cache.c
View File

@ -21,6 +21,7 @@
#include "../concurrency/ocf_concurrency.h"
#include "../eviction/ops.h"
#include "../ocf_ctx_priv.h"
#include "../ocf_freelist.h"
#include "../cleaning/cleaning.h"
#define OCF_ASSERT_PLUGGED(cache) ENV_BUG_ON(!(cache)->device)
@ -117,6 +118,8 @@ struct ocf_cache_attach_context {
* load or recovery
*/
bool freelist_inited : 1;
bool concurrency_inited : 1;
} flags;
@ -156,18 +159,6 @@ struct ocf_cache_attach_context {
ocf_pipeline_t pipeline;
};
static void __init_hash_table(ocf_cache_t cache)
{
/* Initialize hash table*/
ocf_metadata_init_hash_table(cache);
}
static void __init_freelist(ocf_cache_t cache)
{
/* Initialize free list partition*/
ocf_metadata_init_freelist_partition(cache);
}
static void __init_partitions(ocf_cache_t cache)
{
ocf_part_id_t i_part;
@ -204,6 +195,14 @@ static void __init_partitions_attached(ocf_cache_t cache)
}
}
static void __init_freelist(ocf_cache_t cache)
{
uint64_t free_clines = ocf_metadata_collision_table_entries(cache) -
ocf_get_cache_occupancy(cache);
ocf_freelist_populate(cache->freelist, free_clines);
}
static ocf_error_t __init_cleaning_policy(ocf_cache_t cache)
{
ocf_cleaning_t cleaning_policy = ocf_cleaning_default;
@ -295,9 +294,10 @@ static ocf_error_t init_attached_data_structures(ocf_cache_t cache,
/* Lock to ensure consistency */
OCF_METADATA_LOCK_WR();
__init_hash_table(cache);
__init_freelist(cache);
ocf_metadata_init_hash_table(cache);
ocf_metadata_init_collision(cache);
__init_partitions_attached(cache);
__init_freelist(cache);
result = __init_cleaning_policy(cache);
if (result) {
@ -325,8 +325,8 @@ static ocf_error_t init_attached_data_structures(ocf_cache_t cache,
static void init_attached_data_structures_recovery(ocf_cache_t cache)
{
OCF_METADATA_LOCK_WR();
__init_hash_table(cache);
__init_freelist(cache);
ocf_metadata_init_hash_table(cache);
ocf_metadata_init_collision(cache);
__init_partitions_attached(cache);
__reset_stats(cache);
__init_metadata_version(cache);
@ -469,6 +469,8 @@ void _ocf_mngt_init_instance_load_complete(void *priv, int error)
OCF_PL_FINISH_RET(context->pipeline, -OCF_ERR_START_CACHE_FAIL);
}
__init_freelist(cache);
result = __init_promotion_policy(cache);
if (result) {
ocf_cache_log(cache, log_err,
@ -985,8 +987,10 @@ static void _ocf_mngt_attach_prepare_metadata(ocf_pipeline_t pipeline,
OCF_PL_FINISH_RET(context->pipeline, -OCF_ERR_START_CACHE_FAIL);
}
cache->device->freelist_part = &cache->device->runtime_meta->freelist_part;
cache->freelist = ocf_freelist_init(cache);
if (!cache->freelist)
OCF_PL_FINISH_RET(context->pipeline, -OCF_ERR_START_CACHE_FAIL);
context->flags.freelist_inited = true;
ret = ocf_concurrency_init(cache);
if (ret)
@ -1143,6 +1147,9 @@ static void _ocf_mngt_attach_handle_error(
if (context->flags.concurrency_inited)
ocf_concurrency_deinit(cache);
if (context->flags.freelist_inited)
ocf_freelist_deinit(cache->freelist);
if (context->flags.volume_inited)
ocf_volume_deinit(&cache->device->volume);
@ -1725,6 +1732,7 @@ static void _ocf_mngt_cache_unplug_complete(void *priv, int error)
ocf_metadata_deinit_variable_size(cache);
ocf_concurrency_deinit(cache);
ocf_freelist_deinit(cache->freelist);
ocf_volume_deinit(&cache->device->volume);

17
src/ocf_cache_priv.h
View File

@ -21,6 +21,7 @@
#include "ocf_logger_priv.h"
#include "ocf/ocf_trace.h"
#include "promotion/promotion.h"
#include "ocf_freelist.h"
#define DIRTY_FLUSHED 1
#define DIRTY_NOT_FLUSHED 0
@ -82,8 +83,6 @@ struct ocf_cache_device {
uint64_t metadata_offset;
struct ocf_part *freelist_part;
struct {
struct ocf_cache_line_concurrency *cache_line;
} concurrency;
@ -110,6 +109,8 @@ struct ocf_cache {
struct ocf_metadata metadata;
ocf_freelist_t freelist;
ocf_eviction_t eviction_policy_init;
struct {
@ -199,4 +200,16 @@ static inline ocf_core_t ocf_cache_get_core(ocf_cache_t cache,
#define ocf_cache_log_rl(cache) \
ocf_log_rl(ocf_cache_get_ctx(cache))
static inline uint64_t ocf_get_cache_occupancy(ocf_cache_t cache)
{
uint64_t result = 0;
ocf_core_t core;
ocf_core_id_t core_id;
for_each_core(cache, core, core_id)
result += env_atomic_read(&core->runtime_meta->cached_clines);
return result;
}
#endif /* __OCF_CACHE_PRIV_H__ */

420
src/ocf_freelist.c Normal file
View File

@ -0,0 +1,420 @@
/*
* Copyright(c) 2019-2019 Intel Corporation
* SPDX-License-Identifier: BSD-3-Clause-Clear
*/
#include "ocf/ocf.h"
#include "metadata/metadata.h"
struct ocf_part {
ocf_cache_line_t head;
ocf_cache_line_t tail;
env_atomic64 curr_size;
};
struct ocf_freelist {
/* parent cache */
struct ocf_cache *cache;
/* partition list array */
struct ocf_part *part;
/* freelist lock array */
env_spinlock *lock;
/* number of free lists */
uint32_t count;
/* next slowpath victim idx */
env_atomic slowpath_victim_idx;
/* total number of free lines */
env_atomic64 total_free;
};
static void ocf_freelist_lock(ocf_freelist_t freelist, uint32_t ctx)
{
env_spinlock_lock(&freelist->lock[ctx]);
}
static int ocf_freelist_trylock(ocf_freelist_t freelist, uint32_t ctx)
{
return env_spinlock_trylock(&freelist->lock[ctx]);
}
static void ocf_freelist_unlock(ocf_freelist_t freelist, uint32_t ctx)
{
env_spinlock_unlock(&freelist->lock[ctx]);
}
/* Sets the given collision_index as the new _head_ of the Partition list. */
static void _ocf_freelist_remove_cache_line(ocf_freelist_t freelist,
uint32_t ctx, ocf_cache_line_t cline)
{
struct ocf_cache *cache = freelist->cache;
struct ocf_part *freelist_part = &freelist->part[ctx];
int is_head, is_tail;
ocf_part_id_t invalid_part_id = PARTITION_INVALID;
ocf_cache_line_t prev, next;
ocf_cache_line_t line_entries = ocf_metadata_collision_table_entries(
freelist->cache);
uint32_t free;
ENV_BUG_ON(cline >= line_entries);
/* Get Partition info */
ocf_metadata_get_partition_info(cache, cline, NULL, &next, &prev);
/* Find out if this node is Partition _head_ */
is_head = (prev == line_entries);
is_tail = (next == line_entries);
free = env_atomic64_read(&freelist_part->curr_size);
/* Case 1: If we are head and there is only one node. So unlink node
* and set that there is no node left in the list.
*/
if (is_head && free == 1) {
ocf_metadata_set_partition_info(cache, cline, invalid_part_id,
line_entries, line_entries);
freelist_part->head = line_entries;
freelist_part->tail = line_entries;
} else if (is_head) {
/* Case 2: else if this collision_index is partition list head,
* but many nodes, update head and return
*/
ENV_BUG_ON(next >= line_entries);
freelist_part->head = next;
ocf_metadata_set_partition_prev(cache, next, line_entries);
ocf_metadata_set_partition_next(cache, cline, line_entries);
} else if (is_tail) {
/* Case 3: else if this cline is partition list tail */
ENV_BUG_ON(prev >= line_entries);
freelist_part->tail = prev;
ocf_metadata_set_partition_prev(cache, cline, line_entries);
ocf_metadata_set_partition_next(cache, prev, line_entries);
} else {
/* Case 4: else this collision_index is a middle node.
* There is no change to the head and the tail pointers.
*/
ENV_BUG_ON(next >= line_entries || prev >= line_entries);
/* Update prev and next nodes */
ocf_metadata_set_partition_prev(cache, next, prev);
ocf_metadata_set_partition_next(cache, prev, next);
/* Update the given node */
ocf_metadata_set_partition_info(cache, cline, invalid_part_id,
line_entries, line_entries);
}
env_atomic64_dec(&freelist_part->curr_size);
env_atomic64_dec(&freelist->total_free);
}
static ocf_cache_line_t next_phys_invalid(ocf_cache_t cache,
ocf_cache_line_t phys)
{
ocf_cache_line_t lg;
ocf_cache_line_t collision_table_entries =
ocf_metadata_collision_table_entries(cache);
if (phys == collision_table_entries)
return collision_table_entries;
lg = ocf_metadata_map_phy2lg(cache, phys);
while (metadata_test_valid_any(cache, lg)) {
++phys;
if (phys == collision_table_entries)
break;
lg = ocf_metadata_map_phy2lg(cache, phys);
}
return phys;
}
/* Assign unused cachelines to freelist */
void ocf_freelist_populate(ocf_freelist_t freelist,
ocf_cache_line_t num_free_clines)
{
unsigned step = 0;
ocf_cache_t cache = freelist->cache;
unsigned num_freelists = freelist->count;
ocf_cache_line_t prev, next, idx;
ocf_cache_line_t phys;
ocf_cache_line_t collision_table_entries =
ocf_metadata_collision_table_entries(cache);
unsigned freelist_idx;
uint64_t freelist_size;
phys = 0;
for (freelist_idx = 0; freelist_idx < num_freelists; freelist_idx++)
{
/* calculate current freelist size */
freelist_size = num_free_clines / num_freelists;
if (freelist_idx < (num_free_clines % num_freelists))
++freelist_size;
env_atomic64_set(&freelist->part[freelist_idx].curr_size,
freelist_size);
if (!freelist_size) {
/* init empty freelist and move to next one */
freelist->part[freelist_idx].head =
collision_table_entries;
freelist->part[freelist_idx].tail =
collision_table_entries;
continue;
}
/* find first invalid cacheline */
phys = next_phys_invalid(cache, phys);
ENV_BUG_ON(phys == collision_table_entries);
idx = ocf_metadata_map_phy2lg(cache, phys);
++phys;
/* store freelist head */
freelist->part[freelist_idx].head = idx;
/* link freelist elements using partition list */
prev = collision_table_entries;
while (--freelist_size) {
phys = next_phys_invalid(cache, phys);
ENV_BUG_ON(phys == collision_table_entries);
next = ocf_metadata_map_phy2lg(cache, phys);
++phys;
ocf_metadata_set_partition_info(cache, idx,
PARTITION_INVALID, next, prev);
prev = idx;
idx = next;
OCF_COND_RESCHED_DEFAULT(step);
}
/* terminate partition list */
ocf_metadata_set_partition_info(cache, idx, PARTITION_INVALID,
collision_table_entries, prev);
/* store freelist tail */
freelist->part[freelist_idx].tail = idx;
}
/* we should have reached the last invalid cache line */
phys = next_phys_invalid(cache, phys);
ENV_BUG_ON(phys != collision_table_entries);
env_atomic64_set(&freelist->total_free, num_free_clines);
}
static void ocf_freelist_add_cache_line(ocf_freelist_t freelist,
uint32_t ctx, ocf_cache_line_t line)
{
struct ocf_cache *cache = freelist->cache;
struct ocf_part *freelist_part = &freelist->part[ctx];
ocf_cache_line_t tail;
ocf_cache_line_t line_entries = ocf_metadata_collision_table_entries(
freelist->cache);
ocf_part_id_t invalid_part_id = PARTITION_INVALID;
ENV_BUG_ON(line >= line_entries);
if (env_atomic64_read(&freelist_part->curr_size) == 0) {
freelist_part->head = line;
freelist_part->tail = line;
ocf_metadata_set_partition_info(cache, line, invalid_part_id,
line_entries, line_entries);
} else {
tail = freelist_part->tail;
ENV_BUG_ON(tail >= line_entries);
ocf_metadata_set_partition_info(cache, line, invalid_part_id,
line_entries, tail);
ocf_metadata_set_partition_next(cache, tail, line);
freelist_part->tail = line;
}
env_atomic64_inc(&freelist_part->curr_size);
env_atomic64_inc(&freelist->total_free);
}
typedef enum {
OCF_FREELIST_ERR_NOLOCK = 1,
OCF_FREELIST_ERR_LIST_EMPTY,
} ocf_freelist_get_err_t;
static ocf_freelist_get_err_t ocf_freelist_get_cache_line_ctx(
ocf_freelist_t freelist, uint32_t ctx, bool can_wait,
ocf_cache_line_t *cline)
{
if (env_atomic64_read(&freelist->part[ctx].curr_size) == 0)
return -OCF_FREELIST_ERR_LIST_EMPTY;
if (!can_wait && ocf_freelist_trylock(freelist, ctx))
return -OCF_FREELIST_ERR_NOLOCK;
if (can_wait)
ocf_freelist_lock(freelist, ctx);
if (env_atomic64_read(&freelist->part[ctx].curr_size) == 0) {
ocf_freelist_unlock(freelist, ctx);
return -OCF_FREELIST_ERR_LIST_EMPTY;
}
*cline = freelist->part[ctx].head;
_ocf_freelist_remove_cache_line(freelist, ctx, *cline);
ocf_freelist_unlock(freelist, ctx);
return 0;
}
static int get_next_victim_freelist(ocf_freelist_t freelist)
{
int ctx, next;
do {
ctx = env_atomic_read(&freelist->slowpath_victim_idx);
next = (ctx + 1) % freelist->count;
} while (ctx != env_atomic_cmpxchg(&freelist->slowpath_victim_idx, ctx,
next));
return ctx;
}
static bool ocf_freelist_get_cache_line_slow(ocf_freelist_t freelist,
ocf_cache_line_t *cline)
{
int i, ctx;
int err;
bool lock_err;
/* try slowpath without waiting on lock */
lock_err = false;
for (i = 0; i < freelist->count; i++) {
ctx = get_next_victim_freelist(freelist);
err = ocf_freelist_get_cache_line_ctx(freelist, ctx, false,
cline);
if (!err)
return true;
if (err == -OCF_FREELIST_ERR_NOLOCK)
lock_err = true;
}
if (!lock_err) {
/* Slowpath failed due to empty freelists - no point in
* iterating through contexts to attempt slowpath with full
* lock */
return false;
}
/* slow path with waiting on lock */
for (i = 0; i < freelist->count; i++) {
ctx = get_next_victim_freelist(freelist);
if (!ocf_freelist_get_cache_line_ctx(freelist, ctx, true,
cline)) {
return true;
}
}
return false;
}
static bool ocf_freelist_get_cache_line_fast(ocf_freelist_t freelist,
ocf_cache_line_t *cline)
{
bool ret;
uint32_t ctx = env_get_execution_context();
ret = !ocf_freelist_get_cache_line_ctx(freelist, ctx, false, cline);
env_put_execution_context(ctx);
return ret;
}
bool ocf_freelist_get_cache_line(ocf_freelist_t freelist,
ocf_cache_line_t *cline)
{
if (env_atomic64_read(&freelist->total_free) == 0)
return false;
if (!ocf_freelist_get_cache_line_fast(freelist, cline))
return ocf_freelist_get_cache_line_slow(freelist, cline);
return true;
}
void ocf_freelist_put_cache_line(ocf_freelist_t freelist,
ocf_cache_line_t cline)
{
uint32_t ctx = env_get_execution_context();
ocf_freelist_lock(freelist, ctx);
ocf_freelist_add_cache_line(freelist, ctx, cline);
ocf_freelist_unlock(freelist, ctx);
env_put_execution_context(ctx);
}
ocf_freelist_t ocf_freelist_init(struct ocf_cache *cache)
{
uint32_t num;
int i;
ocf_freelist_t freelist;
ocf_cache_line_t line_entries = ocf_metadata_collision_table_entries(
cache);
freelist = env_vzalloc(sizeof(*freelist));
if (!freelist)
return NULL;
num = env_get_execution_context_count();
freelist->cache = cache;
freelist->count = num;
env_atomic64_set(&freelist->total_free, 0);
freelist->lock = env_vzalloc(sizeof(freelist->lock[0]) * num);
freelist->part = env_vzalloc(sizeof(freelist->part[0]) * num);
if (!freelist->lock || !freelist->part) {
env_vfree(freelist->lock);
env_vfree(freelist->part);
env_vfree(freelist);
return NULL;
}
for (i = 0; i < num; i++) {
env_spinlock_init(&freelist->lock[i]);
freelist->part[i].head = line_entries;
freelist->part[i].tail = line_entries;
env_atomic64_set(&freelist->part[i].curr_size, 0);
}
return freelist;
}
void ocf_freelist_deinit(ocf_freelist_t freelist)
{
int i;
for (i = 0; i < freelist->count; i++)
env_spinlock_destroy(&freelist->lock[i]);
env_vfree(freelist->lock);
env_vfree(freelist->part);
env_vfree(freelist);
}
ocf_cache_line_t ocf_freelist_num_free(ocf_freelist_t freelist)
{
return env_atomic64_read(&freelist->total_free);
}

34
src/ocf_freelist.h Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright(c) 2019-2019 Intel Corporation
* SPDX-License-Identifier: BSD-3-Clause-Clear
*/
#ifndef __OCF_FREELIST_H__
#define __OCF_FREELIST_H__
#include "ocf_cache_priv.h"
struct ocf_freelist;
typedef struct ocf_freelist *ocf_freelist_t;
/* Init / deinit freelist runtime structures */
ocf_freelist_t ocf_freelist_init(struct ocf_cache *cache);
void ocf_freelist_deinit(ocf_freelist_t freelist);
/* Assign unused cachelines to freelist */
void ocf_freelist_populate(ocf_freelist_t freelist,
ocf_cache_line_t num_free_clines);
/* Get cacheline from freelist */
bool ocf_freelist_get_cache_line(ocf_freelist_t freelist,
ocf_cache_line_t *cline);
/* Put cacheline back to freelist */
void ocf_freelist_put_cache_line(ocf_freelist_t freelist,
ocf_cache_line_t cline);
/* Return total number of free cachelines */
ocf_cache_line_t ocf_freelist_num_free(ocf_freelist_t freelist);
#endif /* __OCF_FREELIST_H__ */

14
src/ocf_stats_builder.c
View File

@ -44,18 +44,6 @@ static uint64_t _bytes4k(uint64_t bytes)
return (bytes + 4095UL) >> 12;
}
static uint64_t _get_cache_occupancy(ocf_cache_t cache)
{
uint64_t result = 0;
ocf_core_t core;
ocf_core_id_t core_id;
for_each_core(cache, core, core_id)
result += env_atomic_read(&core->runtime_meta->cached_clines);
return result;
}
static void _set(struct ocf_stat *stat, uint64_t value, uint64_t denominator)
{
stat->value = value;
@ -169,7 +157,7 @@ int ocf_stats_collect_core(ocf_core_t core,
cache = ocf_core_get_cache(core);
cache_line_size = ocf_cache_get_line_size(cache);
cache_size = cache->conf_meta->cachelines;
cache_occupancy = _get_cache_occupancy(cache);
cache_occupancy = ocf_get_cache_occupancy(cache);
_ocf_stats_zero(usage);
_ocf_stats_zero(req);

2
src/promotion/nhit/nhit.c
View File

@ -181,7 +181,7 @@ bool nhit_req_should_promote(ocf_promotion_policy_t policy,
uint64_t core_line;
uint64_t occupied_cachelines =
ocf_metadata_get_cachelines_count(policy->owner) -
policy->owner->device->freelist_part->curr_size;
ocf_freelist_num_free(policy->owner->freelist);
if (occupied_cachelines > env_atomic64_read(&ctx->trigger_threshold))
return true;