From da04add8d2842408adadf727f7aa7483f7a22c58 Mon Sep 17 00:00:00 2001 From: Michal Mielewczyk Date: Mon, 26 Apr 2021 15:41:28 +0200 Subject: [PATCH] bdev/ocf: add mempools to ocf env OCF requires separate mempool for each request size from rage 4k-128k. Mempools are meant for both IO and OCF's internal requests. Signed-off-by: Michal Mielewczyk Change-Id: I3b37d287bbd6f963a24007c8485002f414fb8a7e Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7774 Tested-by: SPDK CI Jenkins Community-CI: Broadcom CI Reviewed-by: Tomasz Zawadzki Reviewed-by: Jim Harris --- lib/env_ocf/mpool.c | 171 ++++++++++++++++++++++++++++++++++++++++++ lib/env_ocf/mpool.h | 64 ++++++++++++++++ lib/env_ocf/ocf_env.c | 12 ++- lib/env_ocf/ocf_env.h | 5 ++ 4 files changed, 250 insertions(+), 2 deletions(-) create mode 100644 lib/env_ocf/mpool.c create mode 100644 lib/env_ocf/mpool.h diff --git a/lib/env_ocf/mpool.c b/lib/env_ocf/mpool.c new file mode 100644 index 000000000..97ac2a779 --- /dev/null +++ b/lib/env_ocf/mpool.c @@ -0,0 +1,171 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/env.h" +#include "ocf_env.h" + +#include "mpool.h" + +#define MEMPOOL_SIZE 24 + +struct env_mpool { + env_allocator *allocator[env_mpool_max]; + /* Handles to memory pools */ + + uint32_t hdr_size; + /* Data header size (constant allocation part) */ + + uint32_t elem_size; + /* Per element size increment (variable allocation part) */ + + uint32_t mpool_max; + /* Max mpool allocation order */ + + bool fallback; + /* Fallback to vmalloc */ +}; + +struct env_mpool *env_mpool_create(uint32_t hdr_size, uint32_t elem_size, + int flags, int mpool_max, bool fallback, + const uint32_t limits[env_mpool_max], + const char *name_perfix) +{ + int i; + char name[MEMPOOL_SIZE] = {}; + int ret; + int size; + + struct env_mpool *mpool = env_zalloc(sizeof(struct env_mpool), ENV_MEM_NOIO); + if (!mpool) { + return NULL; + } + + mpool->hdr_size = hdr_size; + mpool->elem_size = elem_size; + mpool->mpool_max = mpool_max; + mpool->fallback = fallback; + + for (i = 0; i < min(env_mpool_max, mpool_max + 1); i++) { + ret = snprintf(name, sizeof(name), "%s_%u", name, (1 << i)); + if (ret < 0 || ret >= (int)sizeof(name)) { + goto err; + } + + size = hdr_size + (elem_size * (1 << i)); + + mpool->allocator[i] = env_allocator_create_extended(size, name, + limits ? limits[i] : -1); + + if (!mpool->allocator[i]) { + goto err; + } + } + + return mpool; + +err: + env_mpool_destroy(mpool); + return NULL; +} + +void env_mpool_destroy(struct env_mpool *mpool) +{ + if (mpool) { + int i; + + for (i = 0; i < env_mpool_max; i++) { + if (mpool->allocator[i]) { + env_allocator_destroy(mpool->allocator[i]); + } + } + + env_free(mpool); + } +} + +static env_allocator *env_mpool_get_allocator(struct env_mpool *mpool, + uint32_t count) +{ + unsigned int idx; + + if (unlikely(count == 0)) { + return mpool->allocator[env_mpool_1]; + } + + idx = 31 - __builtin_clz(count); + + if (__builtin_ffs(count) <= idx) { + idx++; + } + + if (idx >= env_mpool_max || idx > mpool->mpool_max) { + return NULL; + } + + return mpool->allocator[idx]; +} + +void *env_mpool_new(struct env_mpool *mpool, uint32_t count) +{ + void *items = NULL; + env_allocator *allocator; + size_t size = mpool->hdr_size + (mpool->elem_size * count); + + allocator = env_mpool_get_allocator(mpool, count); + + if (allocator) { + items = env_allocator_new(allocator); + } else if (mpool->fallback) { + items = env_vmalloc(size); + } + + return items; +} + +bool env_mpool_del(struct env_mpool *mpool, + void *items, uint32_t count) +{ + env_allocator *allocator; + + allocator = env_mpool_get_allocator(mpool, count); + + if (allocator) { + env_allocator_del(allocator, items); + } else if (mpool->fallback) { + env_vfree(items); + } else { + return false; + } + + return true; +} diff --git a/lib/env_ocf/mpool.h b/lib/env_ocf/mpool.h new file mode 100644 index 000000000..b5ddb368e --- /dev/null +++ b/lib/env_ocf/mpool.h @@ -0,0 +1,64 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef OCF_MPOOL_H +#define OCF_MPOOL_H + +enum { + env_mpool_1, + env_mpool_2, + env_mpool_4, + env_mpool_8, + env_mpool_16, + env_mpool_32, + env_mpool_64, + env_mpool_128, + + env_mpool_max +}; + +struct env_mpool; + +struct env_mpool *env_mpool_create(uint32_t hdr_size, uint32_t elem_size, + int flags, int mpool_max, bool fallback, + const uint32_t limits[env_mpool_max], + const char *name_perfix); + +void env_mpool_destroy(struct env_mpool *mpools); + +void *env_mpool_new(struct env_mpool *mpool, uint32_t count); + +bool env_mpool_del(struct env_mpool *mpool, + void *items, uint32_t count); + +#endif diff --git a/lib/env_ocf/ocf_env.c b/lib/env_ocf/ocf_env.c index c25aec90c..721138dd6 100644 --- a/lib/env_ocf/ocf_env.c +++ b/lib/env_ocf/ocf_env.c @@ -45,6 +45,7 @@ * for every request it sends and recieves */ #define ENV_ALLOCATOR_NBUFS 32767 +#define GET_ELEMENTS_COUNT(_limit) (_limit < 0 ? ENV_ALLOCATOR_NBUFS : _limit) /* Use unique index for env allocators */ static env_atomic g_env_allocator_index = 0; @@ -63,6 +64,12 @@ env_allocator_new(env_allocator *allocator) env_allocator * env_allocator_create(uint32_t size, const char *name) +{ + return env_allocator_create_extended(size, name, -1); +} + +env_allocator * +env_allocator_create_extended(uint32_t size, const char *name, int limit) { env_allocator *allocator; char qualified_name[128] = {0}; @@ -75,7 +82,7 @@ env_allocator_create(uint32_t size, const char *name) } allocator->mempool = spdk_mempool_create(qualified_name, - ENV_ALLOCATOR_NBUFS, size, + GET_ELEMENTS_COUNT(limit), size, SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, SPDK_ENV_SOCKET_ID_ANY); @@ -86,6 +93,7 @@ env_allocator_create(uint32_t size, const char *name) } allocator->element_size = size; + allocator->element_count = GET_ELEMENTS_COUNT(limit); return allocator; } @@ -100,7 +108,7 @@ void env_allocator_destroy(env_allocator *allocator) { if (allocator) { - if (ENV_ALLOCATOR_NBUFS - spdk_mempool_count(allocator->mempool)) { + if (allocator->element_count - spdk_mempool_count(allocator->mempool)) { SPDK_ERRLOG("Not all objects deallocated\n"); assert(false); } diff --git a/lib/env_ocf/ocf_env.h b/lib/env_ocf/ocf_env.h index 4484954de..abbe7173e 100644 --- a/lib/env_ocf/ocf_env.h +++ b/lib/env_ocf/ocf_env.h @@ -54,6 +54,8 @@ #include "ocf_env_list.h" #include "ocf/ocf_err.h" +#include "mpool.h" + typedef uint8_t u8; typedef uint16_t u16; typedef uint32_t u32; @@ -181,8 +183,11 @@ static inline uint64_t env_get_free_memory(void) typedef struct { struct spdk_mempool *mempool; size_t element_size; + size_t element_count; } env_allocator; +env_allocator *env_allocator_create_extended(uint32_t size, const char *name, int limit); + env_allocator *env_allocator_create(uint32_t size, const char *name); void env_allocator_destroy(env_allocator *allocator);