From 1d4628efc9eb993432fd660e1f4f5cbd03c2e8f8 Mon Sep 17 00:00:00 2001 From: John Kariuki Date: Tue, 6 Dec 2022 15:52:16 -0700 Subject: [PATCH] lib/idxd: change max idxd completions processed This patch fixes issue # 2809, by changing the max completions processed per poll. A new parameter called IDXD_MAX_COMPLETIONS is used to set maximum completions processed per poll to 128 because we observed performance degradation on a system with 16 NVMe SSDs at a queue depth of 64 per SSD. When using DSA to compute the data digest, the target application can issue upto 1024(16x64) request to compute data digest concurrently to DSA. Limiting the maximum completions processed per poll to 32 using DESC_PER_BATCH cause up to 43% IOPS degradation. Use IDXD_MAX_COMPLETIONS to control the number of completions proccessed per poll in spdk_idxd_process_event based on your workload. For example, if your application is issuing 1000s of concurrent request to DSA you might want to set IDXD_MAX_COMPLETIONS to a value higher than 128. Change-Id: I2a1db993283a83a20266f40dac851728d63e6127 Signed-off-by: John Kariuki Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15801 Reviewed-by: Jim Harris Reviewed-by: Changpeng Liu Tested-by: SPDK CI Jenkins --- lib/idxd/idxd.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/idxd/idxd.c b/lib/idxd/idxd.c index 6c106b009..c633e708c 100644 --- a/lib/idxd/idxd.c +++ b/lib/idxd/idxd.c @@ -19,6 +19,9 @@ #define USERSPACE_DRIVER_NAME "user" #define KERNEL_DRIVER_NAME "kernel" +/* The max number of completions processed per poll */ +#define IDXD_MAX_COMPLETIONS 128 + static STAILQ_HEAD(, spdk_idxd_impl) g_idxd_impls = STAILQ_HEAD_INITIALIZER(g_idxd_impls); static struct spdk_idxd_impl *g_idxd_impl; @@ -1442,7 +1445,7 @@ spdk_idxd_process_events(struct spdk_idxd_io_channel *chan) /* reset the status */ status = 0; /* break the processing loop to prevent from starving the rest of the system */ - if (rc > DESC_PER_BATCH) { + if (rc > IDXD_MAX_COMPLETIONS) { break; } }