From c39647df83e4be9bcc49025132c48bf2414ef8b1 Mon Sep 17 00:00:00 2001 From: Rafal Stefanowski Date: Thu, 23 Dec 2021 16:14:46 +0100 Subject: [PATCH] bdev/ocf: Improve OCF mpools - Reduce the size of initial memory needed by OCF. Number of allocator buffers equal to 16383 is tested to work on 24 caches running IO of io_size=512 and io_depth=512, which should be more than enough for any real life scenario. This reduces initial OCF memory usage from 726 MiB to 392 MiB. - Fix string handling for the name of the mempool. Signed-off-by: Rafal Stefanowski Change-Id: I40063ab1897c479c25904ae4096c5dae3351f73b Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10843 Tested-by: SPDK CI Jenkins Community-CI: Mellanox Build Bot Community-CI: Broadcom CI Reviewed-by: Tomasz Zawadzki Reviewed-by: Ben Walker --- lib/env_ocf/mpool.c | 8 +++----- lib/env_ocf/ocf_env.c | 15 ++++++++++++--- lib/env_ocf/ocf_env.h | 2 +- module/bdev/ocf/volume.c | 2 +- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/lib/env_ocf/mpool.c b/lib/env_ocf/mpool.c index 7395a4fd2..adc1abce7 100644 --- a/lib/env_ocf/mpool.c +++ b/lib/env_ocf/mpool.c @@ -36,8 +36,6 @@ #include "mpool.h" -#define MEMPOOL_SIZE 24 - struct env_mpool { env_allocator *allocator[env_mpool_max]; /* Handles to memory pools */ @@ -58,10 +56,10 @@ struct env_mpool { struct env_mpool *env_mpool_create(uint32_t hdr_size, uint32_t elem_size, int flags, int mpool_max, bool fallback, const uint32_t limits[env_mpool_max], - const char *name_perfix, bool zero) + const char *name_prefix, bool zero) { int i; - char name[MEMPOOL_SIZE] = {}; + char name[OCF_ALLOCATOR_NAME_MAX] = {}; int ret; int size; @@ -76,7 +74,7 @@ struct env_mpool *env_mpool_create(uint32_t hdr_size, uint32_t elem_size, mpool->fallback = fallback; for (i = 0; i < min(env_mpool_max, mpool_max + 1); i++) { - ret = snprintf(name, sizeof(name), "%s_%u", name, (1 << i)); + ret = snprintf(name, sizeof(name), "%s_%u", name_prefix, (1 << i)); if (ret < 0 || ret >= (int)sizeof(name)) { goto err; } diff --git a/lib/env_ocf/ocf_env.c b/lib/env_ocf/ocf_env.c index 12ce322ad..075588a4f 100644 --- a/lib/env_ocf/ocf_env.c +++ b/lib/env_ocf/ocf_env.c @@ -43,8 +43,16 @@ * in itself depends on the workload * It is a big number because OCF uses allocators * for every request it sends and receives + * + * The value of 16383 is tested to work on 24 caches + * running IO of io_size=512 and io_depth=512, which + * should be more than enough for any real life scenario. + * Increase this value if needed. It will result in + * more memory being used initially on SPDK app start, + * when compiled with OCF support. */ -#define ENV_ALLOCATOR_NBUFS 32767 +#define ENV_ALLOCATOR_NBUFS 16383 + #define GET_ELEMENTS_COUNT(_limit) (_limit < 0 ? ENV_ALLOCATOR_NBUFS : _limit) /* Use unique index for env allocators */ @@ -76,9 +84,10 @@ env_allocator * env_allocator_create_extended(uint32_t size, const char *name, int limit, bool zero) { env_allocator *allocator; - char qualified_name[128] = {0}; + char qualified_name[OCF_ALLOCATOR_NAME_MAX] = {0}; - snprintf(qualified_name, 128, "ocf_env_%d", env_atomic_inc_return(&g_env_allocator_index)); + snprintf(qualified_name, OCF_ALLOCATOR_NAME_MAX, "ocf_env_%d:%s", + env_atomic_inc_return(&g_env_allocator_index), name); allocator = calloc(1, sizeof(*allocator)); if (!allocator) { diff --git a/lib/env_ocf/ocf_env.h b/lib/env_ocf/ocf_env.h index e7851e489..d5efd1483 100644 --- a/lib/env_ocf/ocf_env.h +++ b/lib/env_ocf/ocf_env.h @@ -178,7 +178,7 @@ static inline uint64_t env_get_free_memory(void) /* *** ALLOCATOR *** */ -#define OCF_ALLOCATOR_NAME_MAX 128 +#define OCF_ALLOCATOR_NAME_MAX 24 typedef struct { struct spdk_mempool *mempool; diff --git a/module/bdev/ocf/volume.c b/module/bdev/ocf/volume.c index 648109764..028db40f3 100644 --- a/module/bdev/ocf/volume.c +++ b/module/bdev/ocf/volume.c @@ -407,7 +407,7 @@ vbdev_ocf_volume_get_max_io_size(ocf_volume_t volume) } static struct ocf_volume_properties vbdev_volume_props = { - .name = "SPDK block device", + .name = "SPDK_block_device", .io_priv_size = sizeof(struct ocf_io_ctx), .volume_priv_size = sizeof(struct vbdev_ocf_base *), .caps = {