Merge pull request #712 from imjfckm/allocators-revamp

Allocators revamp
This commit is contained in:
Michał Mielewczyk 2021-03-05 15:57:54 +01:00 committed by GitHub
commit ac63045117
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 135 additions and 96 deletions

View File

@ -11,7 +11,7 @@
#include "utils/utils_mpool.h"
#include "threads.h"
struct cas_mpool *cas_bvec_pool;
struct env_mpool *cas_bvec_pool;
struct cas_reserve_pool *cas_bvec_pages_rpool;
@ -87,7 +87,7 @@ ctx_data_t *__cas_ctx_data_alloc(uint32_t pages, bool zalloc)
struct page *page = NULL;
int cpu;
data = cas_mpool_new(cas_bvec_pool, pages);
data = env_mpool_new(cas_bvec_pool, pages);
if (!data) {
CAS_PRINT_RL(KERN_ERR "Couldn't allocate BIO vector.\n");
@ -133,7 +133,7 @@ ctx_data_t *__cas_ctx_data_alloc(uint32_t pages, bool zalloc)
}
}
cas_mpool_del(cas_bvec_pool, data, pages);
env_mpool_del(cas_bvec_pool, data, pages);
data = NULL;
} else {
/* Initialize iterator */
@ -175,7 +175,7 @@ void cas_ctx_data_free(ctx_data_t *ctx_data)
__free_page(page);
}
cas_mpool_del(cas_bvec_pool, data, data->size);
env_mpool_del(cas_bvec_pool, data, data->size);
}
static int _cas_ctx_data_mlock(ctx_data_t *ctx_data)
@ -426,15 +426,15 @@ static const struct ocf_ctx_config ctx_cfg = {
int cas_initialize_context(void)
{
struct blk_data data;
int ret;
ret = ocf_ctx_create(&cas_ctx, &ctx_cfg);
if (ret < 0)
return ret;
cas_bvec_pool = cas_mpool_create(sizeof(data), sizeof(data.vec[0]),
GFP_NOIO, 7, "cas_biovec");
cas_bvec_pool = env_mpool_create(sizeof(struct blk_data),
sizeof(struct bio_vec), GFP_NOIO, 7, true, NULL,
"cas_biovec");
if (!cas_bvec_pool) {
printk(KERN_ERR "Cannot create BIO vector memory pool\n");
@ -472,7 +472,7 @@ int cas_initialize_context(void)
err_rpool:
cas_rpool_destroy(cas_bvec_pages_rpool, _cas_free_page_rpool, NULL);
err_mpool:
cas_mpool_destroy(cas_bvec_pool);
env_mpool_destroy(cas_bvec_pool);
err_ctx:
ocf_ctx_put(cas_ctx);
@ -482,7 +482,7 @@ err_ctx:
void cas_cleanup_context(void)
{
cas_garbage_collector_deinit();
cas_mpool_destroy(cas_bvec_pool);
env_mpool_destroy(cas_bvec_pool);
cas_rpool_destroy(cas_bvec_pages_rpool, _cas_free_page_rpool, NULL);
ocf_ctx_put(cas_ctx);
@ -495,7 +495,7 @@ void cas_cleanup_context(void)
*/
struct blk_data *cas_alloc_blk_data(uint32_t size, gfp_t flags)
{
struct blk_data *data = cas_mpool_new_f(cas_bvec_pool, size, flags);
struct blk_data *data = env_mpool_new_f(cas_bvec_pool, size, flags);
if (data)
data->size = size;
@ -511,6 +511,6 @@ void cas_free_blk_data(struct blk_data *data)
if (!data)
return;
cas_mpool_del(cas_bvec_pool, data, data->size);
env_mpool_del(cas_bvec_pool, data, data->size);
}

View File

@ -45,7 +45,9 @@ void *env_allocator_new(env_allocator *allocator)
struct _env_allocator_item *item = NULL;
int cpu;
item = cas_rpool_try_get(allocator->rpool, &cpu);
if (allocator->rpool)
item = cas_rpool_try_get(allocator->rpool, &cpu);
if (item) {
memset(item->data, 0, allocator->item_size -
sizeof(struct _env_allocator_item));
@ -91,7 +93,8 @@ static void env_allocator_del_rpool(void *allocator_ctx, void *_item)
#define ENV_ALLOCATOR_NAME_MAX 128
env_allocator *env_allocator_create(uint32_t size, const char *name)
env_allocator *env_allocator_create_extended(uint32_t size, const char *name,
int rpool_limit)
{
int error = -1;
bool retry = true;
@ -112,13 +115,6 @@ env_allocator *env_allocator_create(uint32_t size, const char *name)
}
allocator->item_size = size + sizeof(struct _env_allocator_item);
if (allocator->item_size > PAGE_SIZE) {
printk(KERN_WARNING "Creating allocator with item size"
" greater than 4096B");
ENV_WARN(true, OCF_PREFIX_SHORT" Creating allocator"
" with item size greater than 4096B\n");
}
allocator->name = kstrdup(name, ENV_MEM_NORMAL);
if (!allocator->name) {
@ -156,13 +152,17 @@ RETRY:
#endif
/* Initialize reserve pool handler per cpu */
if (rpool_limit < 0)
rpool_limit = CAS_ALLOC_ALLOCATOR_LIMIT;
allocator->rpool = cas_rpool_create(CAS_ALLOC_ALLOCATOR_LIMIT,
allocator->name, allocator->item_size, env_allocator_new_rpool,
env_allocator_del_rpool, allocator);
if (!allocator->rpool) {
error = __LINE__;
goto err;
if (rpool_limit > 0) {
allocator->rpool = cas_rpool_create(rpool_limit,
allocator->name, allocator->item_size, env_allocator_new_rpool,
env_allocator_del_rpool, allocator);
if (!allocator->rpool) {
error = __LINE__;
goto err;
}
}
return allocator;
@ -174,6 +174,11 @@ err:
return NULL;
}
env_allocator *env_allocator_create(uint32_t size, const char *name)
{
return env_allocator_create_extended(size, name, -1);
}
void env_allocator_del(env_allocator *allocator, void *obj)
{
struct _env_allocator_item *item =
@ -195,9 +200,11 @@ void env_allocator_del(env_allocator *allocator, void *obj)
void env_allocator_destroy(env_allocator *allocator)
{
if (allocator) {
cas_rpool_destroy(allocator->rpool, env_allocator_del_rpool,
allocator);
allocator->rpool = NULL;
if (allocator->rpool) {
cas_rpool_destroy(allocator->rpool, env_allocator_del_rpool,
allocator);
allocator->rpool = NULL;
}
if (atomic_read(&allocator->count)) {
printk(KERN_CRIT "Not all object deallocated\n");

View File

@ -10,6 +10,7 @@
#include "linux_kernel_version.h"
#include "utils/utils_gc.h"
#include "ocf/ocf_err.h"
#include "utils/utils_mpool.h"
/* linux sector 512-bytes */
#define ENV_SECTOR_SHIFT 9
@ -78,6 +79,9 @@ static inline void env_secure_free(const void *ptr, size_t size)
typedef struct _env_allocator env_allocator;
env_allocator *env_allocator_create_extended(uint32_t size, const char *name,
int rpool_limit);
env_allocator *env_allocator_create(uint32_t size, const char *name);
void env_allocator_destroy(env_allocator *allocator);

View File

@ -3,34 +3,60 @@
* SPDX-License-Identifier: BSD-3-Clause-Clear
*/
#include "ocf_env.h"
#include "utils_mpool.h"
#include "ocf_env.h"
struct env_mpool {
int mpool_max;
/*!< Max mpool allocation order */
struct cas_mpool *cas_mpool_create(uint32_t hdr_size, uint32_t size, int flags,
int mpool_max, const char *name_perfix)
env_allocator *allocator[env_mpool_max];
/*!< OS handle to memory pool */
uint32_t hdr_size;
/*!< Data header size (constant allocation part) */
uint32_t elem_size;
/*!< Per element size increment (variable allocation part) */
bool fallback;
/*!< Should mpool fallback to vmalloc */
int flags;
/*!< Allocation flags */
};
struct env_mpool *env_mpool_create(uint32_t hdr_size, uint32_t elem_size,
int flags, int mpool_max, bool fallback,
const uint32_t limits[env_mpool_max],
const char *name_perfix)
{
uint32_t i;
char name[ALLOCATOR_NAME_MAX] = { '\0' };
char name[MPOOL_ALLOCATOR_NAME_MAX] = { '\0' };
int result;
struct cas_mpool *mpool;
struct env_mpool *mpool;
size_t size;
mpool = env_zalloc(sizeof(*mpool), ENV_MEM_NORMAL);
if (!mpool)
return NULL;
mpool->item_size = size;
mpool->hdr_size = hdr_size;
mpool->flags = flags;
mpool->fallback = fallback;
mpool->mpool_max = mpool_max;
mpool->hdr_size = hdr_size;
mpool->elem_size = elem_size;
for (i = 0; i < min(cas_mpool_max, mpool_max + 1); i++) {
for (i = 0; i < min(env_mpool_max, mpool_max + 1); i++) {
result = snprintf(name, sizeof(name), "%s_%u", name_perfix,
(1 << i));
if (result < 0 || result >= sizeof(name))
goto err;
mpool->allocator[i] = env_allocator_create(
hdr_size + (size * (1 << i)), name);
size = hdr_size + (elem_size * (1 << i));
mpool->allocator[i] = env_allocator_create_extended(
size, name, limits ? limits[i] : -1);
if (!mpool->allocator[i])
goto err;
@ -39,16 +65,16 @@ struct cas_mpool *cas_mpool_create(uint32_t hdr_size, uint32_t size, int flags,
return mpool;
err:
cas_mpool_destroy(mpool);
env_mpool_destroy(mpool);
return NULL;
}
void cas_mpool_destroy(struct cas_mpool *mallocator)
void env_mpool_destroy(struct env_mpool *mallocator)
{
if (mallocator) {
uint32_t i;
for (i = 0; i < cas_mpool_max; i++)
for (i = 0; i < env_mpool_max; i++)
if (mallocator->allocator[i])
env_allocator_destroy(mallocator->allocator[i]);
@ -56,38 +82,38 @@ void cas_mpool_destroy(struct cas_mpool *mallocator)
}
}
static env_allocator *cas_mpool_get_allocator(
struct cas_mpool *mallocator, uint32_t count)
static env_allocator *env_mpool_get_allocator(
struct env_mpool *mallocator, uint32_t count)
{
unsigned int idx;
if (unlikely(count == 0))
return cas_mpool_1;
return env_mpool_1;
idx = 31 - __builtin_clz(count);
if (__builtin_ffs(count) <= idx)
idx++;
if (idx >= cas_mpool_max)
if (idx >= env_mpool_max || idx > mallocator->mpool_max)
return NULL;
return mallocator->allocator[idx];
}
void *cas_mpool_new_f(struct cas_mpool *mpool, uint32_t count, int flags)
void *env_mpool_new_f(struct env_mpool *mpool, uint32_t count, int flags)
{
unsigned long size;
void *items = NULL;
env_allocator *allocator;
size_t size = mpool->hdr_size + (mpool->elem_size * count);
allocator = cas_mpool_get_allocator(mpool, count);
allocator = env_mpool_get_allocator(mpool, count);
if (allocator) {
items = env_allocator_new(allocator);
} else {
size = mpool->hdr_size + (mpool->item_size * count);
items = cas_vmalloc(size, flags | __GFP_ZERO | __GFP_HIGHMEM);
} else if(mpool->fallback) {
items = cas_vmalloc(size,
flags | __GFP_ZERO | __GFP_HIGHMEM);
}
#ifdef ZERO_OR_NULL_PTR
@ -98,20 +124,24 @@ void *cas_mpool_new_f(struct cas_mpool *mpool, uint32_t count, int flags)
return items;
}
void *cas_mpool_new(struct cas_mpool *mpool, uint32_t count)
void *env_mpool_new(struct env_mpool *mpool, uint32_t count)
{
return cas_mpool_new_f(mpool, count, mpool->flags);
return env_mpool_new_f(mpool, count, mpool->flags);
}
void cas_mpool_del(struct cas_mpool *mpool,
bool env_mpool_del(struct env_mpool *mpool,
void *items, uint32_t count)
{
env_allocator *allocator;
allocator = cas_mpool_get_allocator(mpool, count);
allocator = env_mpool_get_allocator(mpool, count);
if (allocator)
env_allocator_del(allocator, items);
else
else if (mpool->fallback)
cas_vfree(items);
else
return false;
return true;
}

View File

@ -6,55 +6,51 @@
#ifndef UTILS_MPOOL_H_
#define UTILS_MPOOL_H_
#define ALLOCATOR_NAME_MAX 128
#include <linux/types.h>
#define MPOOL_ALLOCATOR_NAME_MAX 128
enum {
cas_mpool_1,
cas_mpool_2,
cas_mpool_4,
cas_mpool_8,
cas_mpool_16,
cas_mpool_32,
cas_mpool_64,
cas_mpool_128,
env_mpool_1,
env_mpool_2,
env_mpool_4,
env_mpool_8,
env_mpool_16,
env_mpool_32,
env_mpool_64,
env_mpool_128,
cas_mpool_max
env_mpool_max
};
struct cas_mpool {
uint32_t item_size;
/*!< Size of specific item of memory pool */
uint32_t hdr_size;
/*!< Header size before items */
env_allocator *allocator[cas_mpool_max];
/*!< OS handle to memory pool */
int flags;
/*!< Allocation flags */
};
struct env_mpool;
/**
* @brief Create CAS memory pool
*
* @param hdr_size Header size before array of items
* @param size Size of particular item
* @param hdr_size size of constant allocation part
* @param elem_size size increment for each element
* @param flags Allocation flags
* @param mpool_max Maximal allocator size (power of two)
* @param fallback Should allocations fall back to vmalloc if allocator fails
* @param limits Array of rpool preallocation limits per each mpool allocation
* order or NULL if defaults are to be used. Array should have
* mpool_max elements
* @param name_prefix Format name prefix
*
* @return CAS memory pool
*/
struct cas_mpool *cas_mpool_create(uint32_t hdr_size, uint32_t size, int flags,
int mpool_max, const char *name_perfix);
struct env_mpool *env_mpool_create(uint32_t hdr_size, uint32_t elem_size,
int flags, int mpool_max, bool fallback,
const uint32_t limits[env_mpool_max],
const char *name_perfix);
/**
* @brief Destroy existing memory pool
*
* @param mpool memory pool
*/
void cas_mpool_destroy(struct cas_mpool *mpool);
void env_mpool_destroy(struct env_mpool *mpool);
/**
* @brief Allocate new items of memory pool
@ -67,7 +63,7 @@ void cas_mpool_destroy(struct cas_mpool *mpool);
*
* @return Pointer to the new items
*/
void *cas_mpool_new(struct cas_mpool *mpool, uint32_t count);
void *env_mpool_new(struct env_mpool *mpool, uint32_t count);
/**
* @brief Allocate new items of memory pool with specified allocation flag
@ -78,7 +74,7 @@ void *cas_mpool_new(struct cas_mpool *mpool, uint32_t count);
*
* @return Pointer to the new items
*/
void *cas_mpool_new_f(struct cas_mpool *mpool, uint32_t count, int flags);
void *env_mpool_new_f(struct env_mpool *mpool, uint32_t count, int flags);
/**
* @brief Free existing items of memory pool
@ -86,7 +82,9 @@ void *cas_mpool_new_f(struct cas_mpool *mpool, uint32_t count, int flags);
* @param mpool CAS memory pool reference
* @param items Items to be freed
* @param count - Count of elements to be free
*
* @return Allocation was freed
*/
void cas_mpool_del(struct cas_mpool *mpool, void *items, uint32_t count);
bool env_mpool_del(struct env_mpool *mpool, void *items, uint32_t count);
#endif /* UTILS_MPOOL_H_ */

View File

@ -61,7 +61,7 @@ struct cas_atomic_io {
struct bio_vec_iter iter;
};
static struct cas_mpool *atomic_io_allocator;
static struct env_mpool *atomic_io_allocator;
static inline uint32_t cas_atomic_max_io_sectors(void)
{
@ -96,7 +96,7 @@ static void cas_atomic_dealloc(struct cas_atomic_io *atomics)
}
}
cas_mpool_del(atomic_io_allocator, atomics, atomics->count);
env_mpool_del(atomic_io_allocator, atomics, atomics->count);
}
static struct cas_atomic_io *cas_atomic_alloc(int dir, struct ocf_io *io, bool write_zero)
@ -138,7 +138,7 @@ static struct cas_atomic_io *cas_atomic_alloc(int dir, struct ocf_io *io, bool w
/* Get number of IOs to be issued */
ios_count = DIV_ROUND_UP(bytes, max_io_size);
atoms = cas_mpool_new(atomic_io_allocator,
atoms = env_mpool_new(atomic_io_allocator,
ios_count);
if (!atoms)
return NULL;
@ -793,7 +793,7 @@ void cas_atomic_submit_discard(struct ocf_io *io)
}
/* Allocate and setup control structure. */
atom = cas_mpool_new(atomic_io_allocator, 1);
atom = env_mpool_new(atomic_io_allocator, 1);
if (!atom) {
CAS_PRINT_RL(KERN_ERR "Couldn't allocate memory for IO ctrl\n");
io->end(io, -ENOMEM);
@ -854,7 +854,7 @@ void cas_atomic_submit_flush(struct ocf_io *io)
}
/* Allocate and setup control structure. */
atom = cas_mpool_new(atomic_io_allocator, 1);
atom = env_mpool_new(atomic_io_allocator, 1);
if (!atom) {
CAS_PRINT_RL(KERN_ERR "Couldn't allocate memory for IO ctrl\n");
io->end(io, -ENOMEM);
@ -1101,7 +1101,7 @@ error:
static void atomic_dev_deinit(void)
{
if (atomic_io_allocator) {
cas_mpool_destroy(atomic_io_allocator);
env_mpool_destroy(atomic_io_allocator);
atomic_io_allocator = NULL;
}
}
@ -1141,8 +1141,8 @@ int atomic_dev_init(void)
if (ret < 0)
return -EINVAL;
atomic_io_allocator = cas_mpool_create(0, sizeof(struct cas_atomic_io),
GFP_NOIO, 1, "cas_atomic_io");
atomic_io_allocator = env_mpool_create(0, sizeof(struct cas_atomic_io),
GFP_NOIO, 1, true, NULL, "cas_atomic_io");
if (!atomic_io_allocator) {
ocf_ctx_unregister_volume_type(cas_ctx, ATOMIC_DEVICE_VOLUME);