github/open-cas-linux
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Include mpool in env and use new allocator API

Browse Source 
Signed-off-by: Jan Musial <jan.musial@intel.com>
This commit is contained in:
Jan Musial 2021-02-25 09:46:34 +01:00
parent bcc55f4f3c
commit f1a19363e5
5 changed files with 107 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
modules/cas_cache/context.c
View File

@ -11,7 +11,7 @@
#include "utils/utils_mpool.h" #include "utils/utils_mpool.h"
#include "threads.h" #include "threads.h"
struct cas_mpool *cas_bvec_pool; struct env_mpool *cas_bvec_pool;
struct cas_reserve_pool *cas_bvec_pages_rpool; struct cas_reserve_pool *cas_bvec_pages_rpool;
@ -87,7 +87,7 @@ ctx_data_t *__cas_ctx_data_alloc(uint32_t pages, bool zalloc)
struct page *page = NULL; struct page *page = NULL;
int cpu; int cpu;
data = cas_mpool_new(cas_bvec_pool, pages); data = env_mpool_new(cas_bvec_pool, pages);
if (!data) { if (!data) {
CAS_PRINT_RL(KERN_ERR "Couldn't allocate BIO vector.\n"); CAS_PRINT_RL(KERN_ERR "Couldn't allocate BIO vector.\n");
@ -133,7 +133,7 @@ ctx_data_t *__cas_ctx_data_alloc(uint32_t pages, bool zalloc)
} }
} }
cas_mpool_del(cas_bvec_pool, data, pages); env_mpool_del(cas_bvec_pool, data, pages);
data = NULL; data = NULL;
} else { } else {
/* Initialize iterator */ /* Initialize iterator */
@ -175,7 +175,7 @@ void cas_ctx_data_free(ctx_data_t *ctx_data)
__free_page(page); __free_page(page);
} }
cas_mpool_del(cas_bvec_pool, data, data->size); env_mpool_del(cas_bvec_pool, data, data->size);
} }
static int _cas_ctx_data_mlock(ctx_data_t *ctx_data) static int _cas_ctx_data_mlock(ctx_data_t *ctx_data)
@ -426,15 +426,15 @@ static const struct ocf_ctx_config ctx_cfg = {
int cas_initialize_context(void) int cas_initialize_context(void)
{ {
struct blk_data data;
int ret; int ret;
ret = ocf_ctx_create(&cas_ctx, &ctx_cfg); ret = ocf_ctx_create(&cas_ctx, &ctx_cfg);
if (ret < 0) if (ret < 0)
return ret; return ret;
cas_bvec_pool = cas_mpool_create(sizeof(data), sizeof(data.vec[0]), cas_bvec_pool = env_mpool_create(sizeof(struct blk_data),
GFP_NOIO, 7, "cas_biovec"); sizeof(struct bio_vec), GFP_NOIO, 7, true, NULL,
"cas_biovec");
if (!cas_bvec_pool) { if (!cas_bvec_pool) {
printk(KERN_ERR "Cannot create BIO vector memory pool\n"); printk(KERN_ERR "Cannot create BIO vector memory pool\n");
@ -472,7 +472,7 @@ int cas_initialize_context(void)
err_rpool: err_rpool:
cas_rpool_destroy(cas_bvec_pages_rpool, _cas_free_page_rpool, NULL); cas_rpool_destroy(cas_bvec_pages_rpool, _cas_free_page_rpool, NULL);
err_mpool: err_mpool:
cas_mpool_destroy(cas_bvec_pool); env_mpool_destroy(cas_bvec_pool);
err_ctx: err_ctx:
ocf_ctx_put(cas_ctx); ocf_ctx_put(cas_ctx);
@ -482,7 +482,7 @@ err_ctx:
void cas_cleanup_context(void) void cas_cleanup_context(void)
{ {
cas_garbage_collector_deinit(); cas_garbage_collector_deinit();
cas_mpool_destroy(cas_bvec_pool); env_mpool_destroy(cas_bvec_pool);
cas_rpool_destroy(cas_bvec_pages_rpool, _cas_free_page_rpool, NULL); cas_rpool_destroy(cas_bvec_pages_rpool, _cas_free_page_rpool, NULL);
ocf_ctx_put(cas_ctx); ocf_ctx_put(cas_ctx);
@ -495,7 +495,7 @@ void cas_cleanup_context(void)
*/ */
struct blk_data *cas_alloc_blk_data(uint32_t size, gfp_t flags) struct blk_data *cas_alloc_blk_data(uint32_t size, gfp_t flags)
{ {
struct blk_data *data = cas_mpool_new_f(cas_bvec_pool, size, flags); struct blk_data *data = env_mpool_new_f(cas_bvec_pool, size, flags);
if (data) if (data)
data->size = size; data->size = size;
@ -511,6 +511,6 @@ void cas_free_blk_data(struct blk_data *data)
if (!data) if (!data)
return; return;
cas_mpool_del(cas_bvec_pool, data, data->size); env_mpool_del(cas_bvec_pool, data, data->size);
} }

1
modules/cas_cache/ocf_env.h
View File

@ -10,6 +10,7 @@
#include "linux_kernel_version.h" #include "linux_kernel_version.h"
#include "utils/utils_gc.h" #include "utils/utils_gc.h"
#include "ocf/ocf_err.h" #include "ocf/ocf_err.h"
#include "utils/utils_mpool.h"
/* linux sector 512-bytes */ /* linux sector 512-bytes */
#define ENV_SECTOR_SHIFT 9 #define ENV_SECTOR_SHIFT 9

86
modules/cas_cache/utils/utils_mpool.c
View File

@ -3,34 +3,60 @@
* SPDX-License-Identifier: BSD-3-Clause-Clear * SPDX-License-Identifier: BSD-3-Clause-Clear
*/ */
#include "ocf_env.h"
#include "utils_mpool.h" #include "utils_mpool.h"
#include "ocf_env.h"
struct env_mpool {
int mpool_max;
/*!< Max mpool allocation order */
struct cas_mpool *cas_mpool_create(uint32_t hdr_size, uint32_t size, int flags, env_allocator *allocator[env_mpool_max];
int mpool_max, const char *name_perfix) /*!< OS handle to memory pool */
uint32_t hdr_size;
/*!< Data header size (constant allocation part) */
uint32_t elem_size;
/*!< Per element size increment (variable allocation part) */
bool fallback;
/*!< Should mpool fallback to vmalloc */
int flags;
/*!< Allocation flags */
};
struct env_mpool *env_mpool_create(uint32_t hdr_size, uint32_t elem_size,
int flags, int mpool_max, bool fallback,
const uint32_t limits[env_mpool_max],
const char *name_perfix)
{ {
uint32_t i; uint32_t i;
char name[ALLOCATOR_NAME_MAX] = { '\0' }; char name[MPOOL_ALLOCATOR_NAME_MAX] = { '\0' };
int result; int result;
struct cas_mpool *mpool; struct env_mpool *mpool;
size_t size;
mpool = env_zalloc(sizeof(*mpool), ENV_MEM_NORMAL); mpool = env_zalloc(sizeof(*mpool), ENV_MEM_NORMAL);
if (!mpool) if (!mpool)
return NULL; return NULL;
mpool->item_size = size;
mpool->hdr_size = hdr_size;
mpool->flags = flags; mpool->flags = flags;
mpool->fallback = fallback;
mpool->mpool_max = mpool_max;
mpool->hdr_size = hdr_size;
mpool->elem_size = elem_size;
for (i = 0; i < min(cas_mpool_max, mpool_max + 1); i++) { for (i = 0; i < min(env_mpool_max, mpool_max + 1); i++) {
result = snprintf(name, sizeof(name), "%s_%u", name_perfix, result = snprintf(name, sizeof(name), "%s_%u", name_perfix,
(1 << i)); (1 << i));
if (result < 0 || result >= sizeof(name)) if (result < 0 || result >= sizeof(name))
goto err; goto err;
mpool->allocator[i] = env_allocator_create( size = hdr_size + (elem_size * (1 << i));
hdr_size + (size * (1 << i)), name);
mpool->allocator[i] = env_allocator_create_extended(
size, name, limits ? limits[i] : -1);
if (!mpool->allocator[i]) if (!mpool->allocator[i])
goto err; goto err;
@ -39,16 +65,16 @@ struct cas_mpool *cas_mpool_create(uint32_t hdr_size, uint32_t size, int flags,
return mpool; return mpool;
err: err:
cas_mpool_destroy(mpool); env_mpool_destroy(mpool);
return NULL; return NULL;
} }
void cas_mpool_destroy(struct cas_mpool *mallocator) void env_mpool_destroy(struct env_mpool *mallocator)
{ {
if (mallocator) { if (mallocator) {
uint32_t i; uint32_t i;
for (i = 0; i < cas_mpool_max; i++) for (i = 0; i < env_mpool_max; i++)
if (mallocator->allocator[i]) if (mallocator->allocator[i])
env_allocator_destroy(mallocator->allocator[i]); env_allocator_destroy(mallocator->allocator[i]);
@ -56,38 +82,38 @@ void cas_mpool_destroy(struct cas_mpool *mallocator)
} }
} }
static env_allocator *cas_mpool_get_allocator( static env_allocator *env_mpool_get_allocator(
struct cas_mpool *mallocator, uint32_t count) struct env_mpool *mallocator, uint32_t count)
{ {
unsigned int idx; unsigned int idx;
if (unlikely(count == 0)) if (unlikely(count == 0))
return cas_mpool_1; return env_mpool_1;
idx = 31 - __builtin_clz(count); idx = 31 - __builtin_clz(count);
if (__builtin_ffs(count) <= idx) if (__builtin_ffs(count) <= idx)
idx++; idx++;
if (idx >= cas_mpool_max) if (idx >= env_mpool_max || idx > mallocator->mpool_max)
return NULL; return NULL;
return mallocator->allocator[idx]; return mallocator->allocator[idx];
} }
void *cas_mpool_new_f(struct cas_mpool *mpool, uint32_t count, int flags) void *env_mpool_new_f(struct env_mpool *mpool, uint32_t count, int flags)
{ {
unsigned long size;
void *items = NULL; void *items = NULL;
env_allocator *allocator; env_allocator *allocator;
size_t size = mpool->hdr_size + (mpool->elem_size * count);
allocator = cas_mpool_get_allocator(mpool, count); allocator = env_mpool_get_allocator(mpool, count);
if (allocator) { if (allocator) {
items = env_allocator_new(allocator); items = env_allocator_new(allocator);
} else { } else if(mpool->fallback) {
size = mpool->hdr_size + (mpool->item_size * count); items = cas_vmalloc(size,
items = cas_vmalloc(size, flags | __GFP_ZERO | __GFP_HIGHMEM); flags | __GFP_ZERO | __GFP_HIGHMEM);
} }
#ifdef ZERO_OR_NULL_PTR #ifdef ZERO_OR_NULL_PTR
@ -98,20 +124,24 @@ void *cas_mpool_new_f(struct cas_mpool *mpool, uint32_t count, int flags)
return items; return items;
} }
void *cas_mpool_new(struct cas_mpool *mpool, uint32_t count) void *env_mpool_new(struct env_mpool *mpool, uint32_t count)
{ {
return cas_mpool_new_f(mpool, count, mpool->flags); return env_mpool_new_f(mpool, count, mpool->flags);
} }
void cas_mpool_del(struct cas_mpool *mpool, bool env_mpool_del(struct env_mpool *mpool,
void *items, uint32_t count) void *items, uint32_t count)
{ {
env_allocator *allocator; env_allocator *allocator;
allocator = cas_mpool_get_allocator(mpool, count); allocator = env_mpool_get_allocator(mpool, count);
if (allocator) if (allocator)
env_allocator_del(allocator, items); env_allocator_del(allocator, items);
else else if (mpool->fallback)
cas_vfree(items); cas_vfree(items);
else
return false;
return true;
} }

60
modules/cas_cache/utils/utils_mpool.h
View File

@ -6,55 +6,51 @@
#ifndef UTILS_MPOOL_H_ #ifndef UTILS_MPOOL_H_
#define UTILS_MPOOL_H_ #define UTILS_MPOOL_H_
#define ALLOCATOR_NAME_MAX 128 #include <linux/types.h>
#define MPOOL_ALLOCATOR_NAME_MAX 128
enum { enum {
cas_mpool_1, env_mpool_1,
cas_mpool_2, env_mpool_2,
cas_mpool_4, env_mpool_4,
cas_mpool_8, env_mpool_8,
cas_mpool_16, env_mpool_16,
cas_mpool_32, env_mpool_32,
cas_mpool_64, env_mpool_64,
cas_mpool_128, env_mpool_128,
cas_mpool_max env_mpool_max
}; };
struct cas_mpool { struct env_mpool;
uint32_t item_size;
/*!< Size of specific item of memory pool */
uint32_t hdr_size;
/*!< Header size before items */
env_allocator *allocator[cas_mpool_max];
/*!< OS handle to memory pool */
int flags;
/*!< Allocation flags */
};
/** /**
* @brief Create CAS memory pool * @brief Create CAS memory pool
* *
* @param hdr_size Header size before array of items * @param hdr_size size of constant allocation part
* @param size Size of particular item * @param elem_size size increment for each element
* @param flags Allocation flags * @param flags Allocation flags
* @param mpool_max Maximal allocator size (power of two) * @param mpool_max Maximal allocator size (power of two)
* @param fallback Should allocations fall back to vmalloc if allocator fails
* @param limits Array of rpool preallocation limits per each mpool allocation
* order or NULL if defaults are to be used. Array should have
* mpool_max elements
* @param name_prefix Format name prefix * @param name_prefix Format name prefix
* *
* @return CAS memory pool * @return CAS memory pool
*/ */
struct cas_mpool *cas_mpool_create(uint32_t hdr_size, uint32_t size, int flags, struct env_mpool *env_mpool_create(uint32_t hdr_size, uint32_t elem_size,
int mpool_max, const char *name_perfix); int flags, int mpool_max, bool fallback,
const uint32_t limits[env_mpool_max],
const char *name_perfix);
/** /**
* @brief Destroy existing memory pool * @brief Destroy existing memory pool
* *
* @param mpool memory pool * @param mpool memory pool
*/ */
void cas_mpool_destroy(struct cas_mpool *mpool); void env_mpool_destroy(struct env_mpool *mpool);
/** /**
* @brief Allocate new items of memory pool * @brief Allocate new items of memory pool
@ -67,7 +63,7 @@ void cas_mpool_destroy(struct cas_mpool *mpool);
* *
* @return Pointer to the new items * @return Pointer to the new items
*/ */
void *cas_mpool_new(struct cas_mpool *mpool, uint32_t count); void *env_mpool_new(struct env_mpool *mpool, uint32_t count);
/** /**
* @brief Allocate new items of memory pool with specified allocation flag * @brief Allocate new items of memory pool with specified allocation flag
@ -78,7 +74,7 @@ void *cas_mpool_new(struct cas_mpool *mpool, uint32_t count);
* *
* @return Pointer to the new items * @return Pointer to the new items
*/ */
void *cas_mpool_new_f(struct cas_mpool *mpool, uint32_t count, int flags); void *env_mpool_new_f(struct env_mpool *mpool, uint32_t count, int flags);
/** /**
* @brief Free existing items of memory pool * @brief Free existing items of memory pool
@ -86,7 +82,9 @@ void *cas_mpool_new_f(struct cas_mpool *mpool, uint32_t count, int flags);
* @param mpool CAS memory pool reference * @param mpool CAS memory pool reference
* @param items Items to be freed * @param items Items to be freed
* @param count - Count of elements to be free * @param count - Count of elements to be free
*
* @return Allocation was freed
*/ */
void cas_mpool_del(struct cas_mpool *mpool, void *items, uint32_t count); bool env_mpool_del(struct env_mpool *mpool, void *items, uint32_t count);
#endif /* UTILS_MPOOL_H_ */ #endif /* UTILS_MPOOL_H_ */

16
modules/cas_cache/volume/vol_atomic_dev_bottom.c
View File

@ -61,7 +61,7 @@ struct cas_atomic_io {
struct bio_vec_iter iter; struct bio_vec_iter iter;
}; };
static struct cas_mpool *atomic_io_allocator; static struct env_mpool *atomic_io_allocator;
static inline uint32_t cas_atomic_max_io_sectors(void) static inline uint32_t cas_atomic_max_io_sectors(void)
{ {
@ -96,7 +96,7 @@ static void cas_atomic_dealloc(struct cas_atomic_io *atomics)
} }
} }
cas_mpool_del(atomic_io_allocator, atomics, atomics->count); env_mpool_del(atomic_io_allocator, atomics, atomics->count);
} }
static struct cas_atomic_io *cas_atomic_alloc(int dir, struct ocf_io *io, bool write_zero) static struct cas_atomic_io *cas_atomic_alloc(int dir, struct ocf_io *io, bool write_zero)
@ -138,7 +138,7 @@ static struct cas_atomic_io *cas_atomic_alloc(int dir, struct ocf_io *io, bool w
/* Get number of IOs to be issued */ /* Get number of IOs to be issued */
ios_count = DIV_ROUND_UP(bytes, max_io_size); ios_count = DIV_ROUND_UP(bytes, max_io_size);
atoms = cas_mpool_new(atomic_io_allocator, atoms = env_mpool_new(atomic_io_allocator,
ios_count); ios_count);
if (!atoms) if (!atoms)
return NULL; return NULL;
@ -793,7 +793,7 @@ void cas_atomic_submit_discard(struct ocf_io *io)
} }
/* Allocate and setup control structure. */ /* Allocate and setup control structure. */
atom = cas_mpool_new(atomic_io_allocator, 1); atom = env_mpool_new(atomic_io_allocator, 1);
if (!atom) { if (!atom) {
CAS_PRINT_RL(KERN_ERR "Couldn't allocate memory for IO ctrl\n"); CAS_PRINT_RL(KERN_ERR "Couldn't allocate memory for IO ctrl\n");
io->end(io, -ENOMEM); io->end(io, -ENOMEM);
@ -854,7 +854,7 @@ void cas_atomic_submit_flush(struct ocf_io *io)
} }
/* Allocate and setup control structure. */ /* Allocate and setup control structure. */
atom = cas_mpool_new(atomic_io_allocator, 1); atom = env_mpool_new(atomic_io_allocator, 1);
if (!atom) { if (!atom) {
CAS_PRINT_RL(KERN_ERR "Couldn't allocate memory for IO ctrl\n"); CAS_PRINT_RL(KERN_ERR "Couldn't allocate memory for IO ctrl\n");
io->end(io, -ENOMEM); io->end(io, -ENOMEM);
@ -1101,7 +1101,7 @@ error:
static void atomic_dev_deinit(void) static void atomic_dev_deinit(void)
{ {
if (atomic_io_allocator) { if (atomic_io_allocator) {
cas_mpool_destroy(atomic_io_allocator); env_mpool_destroy(atomic_io_allocator);
atomic_io_allocator = NULL; atomic_io_allocator = NULL;
} }
} }
@ -1141,8 +1141,8 @@ int atomic_dev_init(void)
if (ret < 0) if (ret < 0)
return -EINVAL; return -EINVAL;
atomic_io_allocator = cas_mpool_create(0, sizeof(struct cas_atomic_io), atomic_io_allocator = env_mpool_create(0, sizeof(struct cas_atomic_io),
GFP_NOIO, 1, "cas_atomic_io"); GFP_NOIO, 1, true, NULL, "cas_atomic_io");
if (!atomic_io_allocator) { if (!atomic_io_allocator) {
ocf_ctx_unregister_volume_type(cas_ctx, ATOMIC_DEVICE_VOLUME); ocf_ctx_unregister_volume_type(cas_ctx, ATOMIC_DEVICE_VOLUME);