From e189949f38e3dacbe5c4ccf496ed99e05fdad696 Mon Sep 17 00:00:00 2001 From: Yue-Zhu Date: Mon, 21 Nov 2022 15:10:58 -0500 Subject: [PATCH] app: enable configurable msg_mempool_size for reactor This patch adds "--msg-mempool-size" option for spdk app to allow reactors' msg_mempool_size being configurable via commond line. We tested the rbd_bdev performance for Ceph CTX sharing with high RBD volume count via bdevperf. When testing with 256 volumes and limited Ceph CTX (e.g., 2 Ceph ctx for 256 volumes, which are created though bdev_rbd_register_cluster), error message "the *ERROR*: msg could not be allocated error message" keeps showing and the bdev_perf program hangs. We found the issue from the limited msg_mempool_size size, which is hardcoded by SPDK_DEFAULT_MSG_MEMPOOL_SIZE in thread.h. Therefore, we enable the "--msg-mempool-size" option to allow configurable msg_mempool_size. Signed-off-by: Yue-Zhu Change-Id: I54db7fd46247b2f18112bb994ecce6f4b7e5bf9c Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15552 Reviewed-by: Shuhei Matsumoto Reviewed-by: Aleksey Marchuk Reviewed-by: Jim Harris Tested-by: SPDK CI Jenkins Community-CI: Mellanox Build Bot --- lib/event/app.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/event/app.c b/lib/event/app.c index 2d8d28ca6..e34a5a536 100644 --- a/lib/event/app.c +++ b/lib/event/app.c @@ -128,6 +128,8 @@ static const struct option g_cmdline_options[] = { {"rpcs-allowed", required_argument, NULL, RPCS_ALLOWED_OPT_IDX}, #define ENV_VF_TOKEN_OPT_IDX 269 {"vfio-vf-token", required_argument, NULL, ENV_VF_TOKEN_OPT_IDX}, +#define MSG_MEMPOOL_SIZE_OPT_IDX 270 + {"msg-mempool-size", required_argument, NULL, MSG_MEMPOOL_SIZE_OPT_IDX}, }; static void @@ -860,6 +862,8 @@ usage(void (*app_usage)(void)) } printf(" --disable-cpumask-locks Disable CPU core lock files.\n"); printf(" --silence-noticelog disable notice level logging to stderr\n"); + printf(" --msg-mempool-size global message memory pool size in count (default: %d)\n", + SPDK_DEFAULT_MSG_MEMPOOL_SIZE); printf(" -u, --no-pci disable PCI access\n"); printf(" --wait-for-rpc wait for RPCs to initialize subsystems\n"); printf(" --max-delay maximum reactor delay (in microseconds)\n"); @@ -1036,6 +1040,16 @@ spdk_app_parse_args(int argc, char **argv, struct spdk_app_opts *opts, opts->mem_size = (int) mem_size_mb; break; } + case MSG_MEMPOOL_SIZE_OPT_IDX: + tmp = spdk_strtol(optarg, 10); + if (tmp <= 0) { + SPDK_ERRLOG("Invalid message memory pool size %s\n", optarg); + goto out; + } + + opts->msg_mempool_size = (size_t)tmp; + break; + case NO_PCI_OPT_IDX: opts->no_pci = true; break;