From 02f448d86ed81a602f48e9d6f96908c32b2f3757 Mon Sep 17 00:00:00 2001 From: Konrad Sztyber Date: Thu, 16 Dec 2021 11:36:57 +0100 Subject: [PATCH] bdev/malloc: complete requests through poller Requests that are completed immediately (i.e. those not using the accel engine) are now queued and their completion is delayed to the completion poller. It ensures that they're not completed from the context of a submission, which gets rid of an spdk_thread_send_msg() call. It significantly improves performance on some workloads. For instance, 4k zcopy reads (queue depth 128) on an malloc bdev exposed through NVMe/TCP went from 204k IOPS to 485k IOPS. Signed-off-by: Konrad Sztyber Change-Id: I196f55fc07d167f1ed117d2430e9c37f9d05f70d Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10805 Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins Reviewed-by: Aleksey Marchuk Reviewed-by: Ben Walker Reviewed-by: Jim Harris --- module/bdev/malloc/bdev_malloc.c | 33 +++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/module/bdev/malloc/bdev_malloc.c b/module/bdev/malloc/bdev_malloc.c index 908c6de02..49e641dd0 100644 --- a/module/bdev/malloc/bdev_malloc.c +++ b/module/bdev/malloc/bdev_malloc.c @@ -83,6 +83,14 @@ malloc_done(void *ref, int status) } } +static void +malloc_complete_task(struct malloc_task *task, struct malloc_channel *mch, + enum spdk_bdev_io_status status) +{ + task->status = status; + TAILQ_INSERT_TAIL(&mch->completed_tasks, task, tailq); +} + static TAILQ_HEAD(, malloc_disk) g_malloc_disks = TAILQ_HEAD_INITIALIZER(g_malloc_disks); int malloc_disk_count = 0; @@ -225,9 +233,8 @@ bdev_malloc_unmap(struct malloc_disk *mdisk, byte_count, malloc_done, task); } -static int _bdev_malloc_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) +static int _bdev_malloc_submit_request(struct malloc_channel *mch, struct spdk_bdev_io *bdev_io) { - struct malloc_channel *mch = spdk_io_channel_get_ctx(ch); uint32_t block_size = bdev_io->bdev->blocklen; switch (bdev_io->type) { @@ -238,7 +245,8 @@ static int _bdev_malloc_submit_request(struct spdk_io_channel *ch, struct spdk_b ((struct malloc_disk *)bdev_io->bdev->ctxt)->malloc_buf + bdev_io->u.bdev.offset_blocks * block_size; bdev_io->u.bdev.iovs[0].iov_len = bdev_io->u.bdev.num_blocks * block_size; - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); + malloc_complete_task((struct malloc_task *)bdev_io->driver_ctx, mch, + SPDK_BDEV_IO_STATUS_SUCCESS); return 0; } @@ -262,11 +270,13 @@ static int _bdev_malloc_submit_request(struct spdk_io_channel *ch, struct spdk_b return 0; case SPDK_BDEV_IO_TYPE_RESET: - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); + malloc_complete_task((struct malloc_task *)bdev_io->driver_ctx, mch, + SPDK_BDEV_IO_STATUS_SUCCESS); return 0; case SPDK_BDEV_IO_TYPE_FLUSH: - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); + malloc_complete_task((struct malloc_task *)bdev_io->driver_ctx, mch, + SPDK_BDEV_IO_STATUS_SUCCESS); return 0; case SPDK_BDEV_IO_TYPE_UNMAP: @@ -295,10 +305,12 @@ static int _bdev_malloc_submit_request(struct spdk_io_channel *ch, struct spdk_b spdk_bdev_io_set_buf(bdev_io, buf, len); } - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); + malloc_complete_task((struct malloc_task *)bdev_io->driver_ctx, mch, + SPDK_BDEV_IO_STATUS_SUCCESS); return 0; case SPDK_BDEV_IO_TYPE_ABORT: - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + malloc_complete_task((struct malloc_task *)bdev_io->driver_ctx, mch, + SPDK_BDEV_IO_STATUS_FAILED); return 0; default: return -1; @@ -308,8 +320,11 @@ static int _bdev_malloc_submit_request(struct spdk_io_channel *ch, struct spdk_b static void bdev_malloc_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) { - if (_bdev_malloc_submit_request(ch, bdev_io) != 0) { - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + struct malloc_channel *mch = spdk_io_channel_get_ctx(ch); + + if (_bdev_malloc_submit_request(mch, bdev_io) != 0) { + malloc_complete_task((struct malloc_task *)bdev_io->driver_ctx, mch, + SPDK_BDEV_IO_STATUS_FAILED); } }