From c1b0b339cf55828dd176c29ce05b039f65858d4f Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Mon, 7 Mar 2022 10:38:34 +0900 Subject: [PATCH] bdev/nvme: Refactor create/destroy_ctrlr_channel_cb() The following patches will have the following changes. Add nvme_qpair object and move qpair and poll_group pointers and the io_path_list list from nvme_ctrlr_channel to nvme_qpair. nvme_qpair is allocated dynamically when creating nvme_ctrlr_channel, and nvme_ctrlr_channel points to nvme_qpair. qpair is disconnected asynchronously and nvme_ctrlr_channel is deleted asynchronously. To make the following patches simpler, refactor two functions, bdev_nvme_create_ctrlr_channel_cb() and bdev_nvme_destroy_ctrlr_channel_cb(). The details are as follows. Factor out nvme_qpair_create() from bdev_nvme_create_ctrlr_channel_cb() and factor out nvme_qpair_delete() from bdev_nvme_destroy_ctrlr_channel_cb(). Then reorder a few operation in these. Additionally, reorder a operation in _bdev_nvme_add_io_path(). Change-Id: Idf0328fa77a54f40fe52ca72c3842dde82d55972 Signed-off-by: Shuhei Matsumoto Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11831 Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Jim Harris --- module/bdev/nvme/bdev_nvme.c | 46 ++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/module/bdev/nvme/bdev_nvme.c b/module/bdev/nvme/bdev_nvme.c index ec2abbd27..7d5f6a5ab 100644 --- a/module/bdev/nvme/bdev_nvme.c +++ b/module/bdev/nvme/bdev_nvme.c @@ -587,6 +587,8 @@ _bdev_nvme_add_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ return -ENOMEM; } + io_path->nvme_ns = nvme_ns; + ch = spdk_get_io_channel(nvme_ns->ctrlr); if (ch == NULL) { free(io_path); @@ -597,8 +599,6 @@ _bdev_nvme_add_io_path(struct nvme_bdev_channel *nbdev_ch, struct nvme_ns *nvme_ io_path->ctrlr_ch = spdk_io_channel_get_ctx(ch); TAILQ_INSERT_TAIL(&io_path->ctrlr_ch->io_path_list, io_path, tailq); - io_path->nvme_ns = nvme_ns; - io_path->nbdev_ch = nbdev_ch; STAILQ_INSERT_TAIL(&nbdev_ch->io_path_list, io_path, stailq); @@ -2041,20 +2041,19 @@ bdev_nvme_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) } static int -bdev_nvme_create_ctrlr_channel_cb(void *io_device, void *ctx_buf) +nvme_qpair_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ctrlr_channel *ctrlr_ch) { - struct nvme_ctrlr *nvme_ctrlr = io_device; - struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf; struct spdk_io_channel *pg_ch; int rc; + TAILQ_INIT(&ctrlr_ch->io_path_list); + pg_ch = spdk_get_io_channel(&g_nvme_bdev_ctrlrs); if (!pg_ch) { return -1; } ctrlr_ch->group = spdk_io_channel_get_ctx(pg_ch); - TAILQ_INSERT_TAIL(&ctrlr_ch->group->ctrlr_ch_list, ctrlr_ch, tailq); #ifdef SPDK_CONFIG_VTUNE ctrlr_ch->group->collect_spin_stat = true; @@ -2062,26 +2061,41 @@ bdev_nvme_create_ctrlr_channel_cb(void *io_device, void *ctx_buf) ctrlr_ch->group->collect_spin_stat = false; #endif - TAILQ_INIT(&ctrlr_ch->pending_resets); - TAILQ_INIT(&ctrlr_ch->io_path_list); - rc = bdev_nvme_create_qpair(ctrlr_ch); if (rc != 0) { /* nvme ctrlr can't create IO qpair during reset. In that case ctrlr_ch->qpair * pointer will be NULL and IO qpair will be created when reset completes. * If the user submits IO requests during reset, they will be queued and resubmitted later */ if (!nvme_ctrlr->resetting) { - goto err_qpair; + spdk_put_io_channel(pg_ch); + return rc; } } + TAILQ_INSERT_TAIL(&ctrlr_ch->group->ctrlr_ch_list, ctrlr_ch, tailq); + return 0; +} + +static int +bdev_nvme_create_ctrlr_channel_cb(void *io_device, void *ctx_buf) +{ + struct nvme_ctrlr *nvme_ctrlr = io_device; + struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf; + + TAILQ_INIT(&ctrlr_ch->pending_resets); + + return nvme_qpair_create(nvme_ctrlr, ctrlr_ch); +} + +static void +nvme_qpair_delete(struct nvme_ctrlr_channel *ctrlr_ch) +{ + assert(ctrlr_ch->group != NULL); -err_qpair: TAILQ_REMOVE(&ctrlr_ch->group->ctrlr_ch_list, ctrlr_ch, tailq); - spdk_put_io_channel(pg_ch); - return rc; + spdk_put_io_channel(spdk_io_channel_from_ctx(ctrlr_ch->group)); } static void @@ -2089,8 +2103,6 @@ bdev_nvme_destroy_ctrlr_channel_cb(void *io_device, void *ctx_buf) { struct nvme_ctrlr_channel *ctrlr_ch = ctx_buf; - assert(ctrlr_ch->group != NULL); - _bdev_nvme_clear_io_path_cache(ctrlr_ch); if (ctrlr_ch->qpair != NULL) { @@ -2098,9 +2110,7 @@ bdev_nvme_destroy_ctrlr_channel_cb(void *io_device, void *ctx_buf) ctrlr_ch->qpair = NULL; } - TAILQ_REMOVE(&ctrlr_ch->group->ctrlr_ch_list, ctrlr_ch, tailq); - - spdk_put_io_channel(spdk_io_channel_from_ctx(ctrlr_ch->group)); + nvme_qpair_delete(ctrlr_ch); } static void