diff --git a/modules/cas_cache/context.c b/modules/cas_cache/context.c index 60fc3d8..b287625 100644 --- a/modules/cas_cache/context.c +++ b/modules/cas_cache/context.c @@ -11,7 +11,7 @@ #include "utils/utils_mpool.h" #include "threads.h" -struct cas_mpool *cas_bvec_pool; +struct env_mpool *cas_bvec_pool; struct cas_reserve_pool *cas_bvec_pages_rpool; @@ -87,7 +87,7 @@ ctx_data_t *__cas_ctx_data_alloc(uint32_t pages, bool zalloc) struct page *page = NULL; int cpu; - data = cas_mpool_new(cas_bvec_pool, pages); + data = env_mpool_new(cas_bvec_pool, pages); if (!data) { CAS_PRINT_RL(KERN_ERR "Couldn't allocate BIO vector.\n"); @@ -133,7 +133,7 @@ ctx_data_t *__cas_ctx_data_alloc(uint32_t pages, bool zalloc) } } - cas_mpool_del(cas_bvec_pool, data, pages); + env_mpool_del(cas_bvec_pool, data, pages); data = NULL; } else { /* Initialize iterator */ @@ -175,7 +175,7 @@ void cas_ctx_data_free(ctx_data_t *ctx_data) __free_page(page); } - cas_mpool_del(cas_bvec_pool, data, data->size); + env_mpool_del(cas_bvec_pool, data, data->size); } static int _cas_ctx_data_mlock(ctx_data_t *ctx_data) @@ -426,15 +426,15 @@ static const struct ocf_ctx_config ctx_cfg = { int cas_initialize_context(void) { - struct blk_data data; int ret; ret = ocf_ctx_create(&cas_ctx, &ctx_cfg); if (ret < 0) return ret; - cas_bvec_pool = cas_mpool_create(sizeof(data), sizeof(data.vec[0]), - GFP_NOIO, 7, "cas_biovec"); + cas_bvec_pool = env_mpool_create(sizeof(struct blk_data), + sizeof(struct bio_vec), GFP_NOIO, 7, true, NULL, + "cas_biovec"); if (!cas_bvec_pool) { printk(KERN_ERR "Cannot create BIO vector memory pool\n"); @@ -472,7 +472,7 @@ int cas_initialize_context(void) err_rpool: cas_rpool_destroy(cas_bvec_pages_rpool, _cas_free_page_rpool, NULL); err_mpool: - cas_mpool_destroy(cas_bvec_pool); + env_mpool_destroy(cas_bvec_pool); err_ctx: ocf_ctx_put(cas_ctx); @@ -482,7 +482,7 @@ err_ctx: void cas_cleanup_context(void) { cas_garbage_collector_deinit(); - cas_mpool_destroy(cas_bvec_pool); + env_mpool_destroy(cas_bvec_pool); cas_rpool_destroy(cas_bvec_pages_rpool, _cas_free_page_rpool, NULL); ocf_ctx_put(cas_ctx); @@ -495,7 +495,7 @@ void cas_cleanup_context(void) */ struct blk_data *cas_alloc_blk_data(uint32_t size, gfp_t flags) { - struct blk_data *data = cas_mpool_new_f(cas_bvec_pool, size, flags); + struct blk_data *data = env_mpool_new_f(cas_bvec_pool, size, flags); if (data) data->size = size; @@ -511,6 +511,6 @@ void cas_free_blk_data(struct blk_data *data) if (!data) return; - cas_mpool_del(cas_bvec_pool, data, data->size); + env_mpool_del(cas_bvec_pool, data, data->size); } diff --git a/modules/cas_cache/ocf_env.h b/modules/cas_cache/ocf_env.h index 04c8822..46721d2 100644 --- a/modules/cas_cache/ocf_env.h +++ b/modules/cas_cache/ocf_env.h @@ -10,6 +10,7 @@ #include "linux_kernel_version.h" #include "utils/utils_gc.h" #include "ocf/ocf_err.h" +#include "utils/utils_mpool.h" /* linux sector 512-bytes */ #define ENV_SECTOR_SHIFT 9 diff --git a/modules/cas_cache/utils/utils_mpool.c b/modules/cas_cache/utils/utils_mpool.c index e094d41..34b0ec0 100644 --- a/modules/cas_cache/utils/utils_mpool.c +++ b/modules/cas_cache/utils/utils_mpool.c @@ -3,34 +3,60 @@ * SPDX-License-Identifier: BSD-3-Clause-Clear */ -#include "ocf_env.h" #include "utils_mpool.h" +#include "ocf_env.h" +struct env_mpool { + int mpool_max; + /*!< Max mpool allocation order */ -struct cas_mpool *cas_mpool_create(uint32_t hdr_size, uint32_t size, int flags, - int mpool_max, const char *name_perfix) + env_allocator *allocator[env_mpool_max]; + /*!< OS handle to memory pool */ + + uint32_t hdr_size; + /*!< Data header size (constant allocation part) */ + + uint32_t elem_size; + /*!< Per element size increment (variable allocation part) */ + + bool fallback; + /*!< Should mpool fallback to vmalloc */ + + int flags; + /*!< Allocation flags */ +}; + +struct env_mpool *env_mpool_create(uint32_t hdr_size, uint32_t elem_size, + int flags, int mpool_max, bool fallback, + const uint32_t limits[env_mpool_max], + const char *name_perfix) { uint32_t i; - char name[ALLOCATOR_NAME_MAX] = { '\0' }; + char name[MPOOL_ALLOCATOR_NAME_MAX] = { '\0' }; int result; - struct cas_mpool *mpool; + struct env_mpool *mpool; + size_t size; mpool = env_zalloc(sizeof(*mpool), ENV_MEM_NORMAL); if (!mpool) return NULL; - mpool->item_size = size; - mpool->hdr_size = hdr_size; mpool->flags = flags; + mpool->fallback = fallback; + mpool->mpool_max = mpool_max; + mpool->hdr_size = hdr_size; + mpool->elem_size = elem_size; - for (i = 0; i < min(cas_mpool_max, mpool_max + 1); i++) { + for (i = 0; i < min(env_mpool_max, mpool_max + 1); i++) { result = snprintf(name, sizeof(name), "%s_%u", name_perfix, (1 << i)); if (result < 0 || result >= sizeof(name)) goto err; - mpool->allocator[i] = env_allocator_create( - hdr_size + (size * (1 << i)), name); + size = hdr_size + (elem_size * (1 << i)); + + mpool->allocator[i] = env_allocator_create_extended( + size, name, limits ? limits[i] : -1); if (!mpool->allocator[i]) goto err; @@ -39,16 +65,16 @@ struct cas_mpool *cas_mpool_create(uint32_t hdr_size, uint32_t size, int flags, return mpool; err: - cas_mpool_destroy(mpool); + env_mpool_destroy(mpool); return NULL; } -void cas_mpool_destroy(struct cas_mpool *mallocator) +void env_mpool_destroy(struct env_mpool *mallocator) { if (mallocator) { uint32_t i; - for (i = 0; i < cas_mpool_max; i++) + for (i = 0; i < env_mpool_max; i++) if (mallocator->allocator[i]) env_allocator_destroy(mallocator->allocator[i]); @@ -56,38 +82,38 @@ void cas_mpool_destroy(struct cas_mpool *mallocator) } } -static env_allocator *cas_mpool_get_allocator( - struct cas_mpool *mallocator, uint32_t count) +static env_allocator *env_mpool_get_allocator( + struct env_mpool *mallocator, uint32_t count) { unsigned int idx; if (unlikely(count == 0)) - return cas_mpool_1; + return env_mpool_1; idx = 31 - __builtin_clz(count); if (__builtin_ffs(count) <= idx) idx++; - if (idx >= cas_mpool_max) + if (idx >= env_mpool_max || idx > mallocator->mpool_max) return NULL; return mallocator->allocator[idx]; } -void *cas_mpool_new_f(struct cas_mpool *mpool, uint32_t count, int flags) +void *env_mpool_new_f(struct env_mpool *mpool, uint32_t count, int flags) { - unsigned long size; void *items = NULL; env_allocator *allocator; + size_t size = mpool->hdr_size + (mpool->elem_size * count); - allocator = cas_mpool_get_allocator(mpool, count); + allocator = env_mpool_get_allocator(mpool, count); if (allocator) { items = env_allocator_new(allocator); - } else { - size = mpool->hdr_size + (mpool->item_size * count); - items = cas_vmalloc(size, flags | __GFP_ZERO | __GFP_HIGHMEM); + } else if(mpool->fallback) { + items = cas_vmalloc(size, + flags | __GFP_ZERO | __GFP_HIGHMEM); } #ifdef ZERO_OR_NULL_PTR @@ -98,20 +124,24 @@ void *cas_mpool_new_f(struct cas_mpool *mpool, uint32_t count, int flags) return items; } -void *cas_mpool_new(struct cas_mpool *mpool, uint32_t count) +void *env_mpool_new(struct env_mpool *mpool, uint32_t count) { - return cas_mpool_new_f(mpool, count, mpool->flags); + return env_mpool_new_f(mpool, count, mpool->flags); } -void cas_mpool_del(struct cas_mpool *mpool, +bool env_mpool_del(struct env_mpool *mpool, void *items, uint32_t count) { env_allocator *allocator; - allocator = cas_mpool_get_allocator(mpool, count); + allocator = env_mpool_get_allocator(mpool, count); if (allocator) env_allocator_del(allocator, items); - else + else if (mpool->fallback) cas_vfree(items); + else + return false; + + return true; } diff --git a/modules/cas_cache/utils/utils_mpool.h b/modules/cas_cache/utils/utils_mpool.h index 796f6a0..e1506ca 100644 --- a/modules/cas_cache/utils/utils_mpool.h +++ b/modules/cas_cache/utils/utils_mpool.h @@ -6,55 +6,51 @@ #ifndef UTILS_MPOOL_H_ #define UTILS_MPOOL_H_ -#define ALLOCATOR_NAME_MAX 128 +#include + +#define MPOOL_ALLOCATOR_NAME_MAX 128 enum { - cas_mpool_1, - cas_mpool_2, - cas_mpool_4, - cas_mpool_8, - cas_mpool_16, - cas_mpool_32, - cas_mpool_64, - cas_mpool_128, + env_mpool_1, + env_mpool_2, + env_mpool_4, + env_mpool_8, + env_mpool_16, + env_mpool_32, + env_mpool_64, + env_mpool_128, - cas_mpool_max + env_mpool_max }; -struct cas_mpool { - uint32_t item_size; - /*!< Size of specific item of memory pool */ - - uint32_t hdr_size; - /*!< Header size before items */ - - env_allocator *allocator[cas_mpool_max]; - /*!< OS handle to memory pool */ - - int flags; - /*!< Allocation flags */ -}; +struct env_mpool; /** * @brief Create CAS memory pool * - * @param hdr_size Header size before array of items - * @param size Size of particular item + * @param hdr_size size of constant allocation part + * @param elem_size size increment for each element * @param flags Allocation flags * @param mpool_max Maximal allocator size (power of two) + * @param fallback Should allocations fall back to vmalloc if allocator fails + * @param limits Array of rpool preallocation limits per each mpool allocation + * order or NULL if defaults are to be used. Array should have + * mpool_max elements * @param name_prefix Format name prefix * * @return CAS memory pool */ -struct cas_mpool *cas_mpool_create(uint32_t hdr_size, uint32_t size, int flags, - int mpool_max, const char *name_perfix); +struct env_mpool *env_mpool_create(uint32_t hdr_size, uint32_t elem_size, + int flags, int mpool_max, bool fallback, + const uint32_t limits[env_mpool_max], + const char *name_perfix); /** * @brief Destroy existing memory pool * * @param mpool memory pool */ -void cas_mpool_destroy(struct cas_mpool *mpool); +void env_mpool_destroy(struct env_mpool *mpool); /** * @brief Allocate new items of memory pool @@ -67,7 +63,7 @@ void cas_mpool_destroy(struct cas_mpool *mpool); * * @return Pointer to the new items */ -void *cas_mpool_new(struct cas_mpool *mpool, uint32_t count); +void *env_mpool_new(struct env_mpool *mpool, uint32_t count); /** * @brief Allocate new items of memory pool with specified allocation flag @@ -78,7 +74,7 @@ void *cas_mpool_new(struct cas_mpool *mpool, uint32_t count); * * @return Pointer to the new items */ -void *cas_mpool_new_f(struct cas_mpool *mpool, uint32_t count, int flags); +void *env_mpool_new_f(struct env_mpool *mpool, uint32_t count, int flags); /** * @brief Free existing items of memory pool @@ -86,7 +82,9 @@ void *cas_mpool_new_f(struct cas_mpool *mpool, uint32_t count, int flags); * @param mpool CAS memory pool reference * @param items Items to be freed * @param count - Count of elements to be free + * + * @return Allocation was freed */ -void cas_mpool_del(struct cas_mpool *mpool, void *items, uint32_t count); +bool env_mpool_del(struct env_mpool *mpool, void *items, uint32_t count); #endif /* UTILS_MPOOL_H_ */ diff --git a/modules/cas_cache/volume/vol_atomic_dev_bottom.c b/modules/cas_cache/volume/vol_atomic_dev_bottom.c index c0552b2..376ca91 100644 --- a/modules/cas_cache/volume/vol_atomic_dev_bottom.c +++ b/modules/cas_cache/volume/vol_atomic_dev_bottom.c @@ -61,7 +61,7 @@ struct cas_atomic_io { struct bio_vec_iter iter; }; -static struct cas_mpool *atomic_io_allocator; +static struct env_mpool *atomic_io_allocator; static inline uint32_t cas_atomic_max_io_sectors(void) { @@ -96,7 +96,7 @@ static void cas_atomic_dealloc(struct cas_atomic_io *atomics) } } - cas_mpool_del(atomic_io_allocator, atomics, atomics->count); + env_mpool_del(atomic_io_allocator, atomics, atomics->count); } static struct cas_atomic_io *cas_atomic_alloc(int dir, struct ocf_io *io, bool write_zero) @@ -138,7 +138,7 @@ static struct cas_atomic_io *cas_atomic_alloc(int dir, struct ocf_io *io, bool w /* Get number of IOs to be issued */ ios_count = DIV_ROUND_UP(bytes, max_io_size); - atoms = cas_mpool_new(atomic_io_allocator, + atoms = env_mpool_new(atomic_io_allocator, ios_count); if (!atoms) return NULL; @@ -793,7 +793,7 @@ void cas_atomic_submit_discard(struct ocf_io *io) } /* Allocate and setup control structure. */ - atom = cas_mpool_new(atomic_io_allocator, 1); + atom = env_mpool_new(atomic_io_allocator, 1); if (!atom) { CAS_PRINT_RL(KERN_ERR "Couldn't allocate memory for IO ctrl\n"); io->end(io, -ENOMEM); @@ -854,7 +854,7 @@ void cas_atomic_submit_flush(struct ocf_io *io) } /* Allocate and setup control structure. */ - atom = cas_mpool_new(atomic_io_allocator, 1); + atom = env_mpool_new(atomic_io_allocator, 1); if (!atom) { CAS_PRINT_RL(KERN_ERR "Couldn't allocate memory for IO ctrl\n"); io->end(io, -ENOMEM); @@ -1101,7 +1101,7 @@ error: static void atomic_dev_deinit(void) { if (atomic_io_allocator) { - cas_mpool_destroy(atomic_io_allocator); + env_mpool_destroy(atomic_io_allocator); atomic_io_allocator = NULL; } } @@ -1141,8 +1141,8 @@ int atomic_dev_init(void) if (ret < 0) return -EINVAL; - atomic_io_allocator = cas_mpool_create(0, sizeof(struct cas_atomic_io), - GFP_NOIO, 1, "cas_atomic_io"); + atomic_io_allocator = env_mpool_create(0, sizeof(struct cas_atomic_io), + GFP_NOIO, 1, true, NULL, "cas_atomic_io"); if (!atomic_io_allocator) { ocf_ctx_unregister_volume_type(cas_ctx, ATOMIC_DEVICE_VOLUME);