From 9c64110700fb2aecee3a49450ba29a7d62a98cd2 Mon Sep 17 00:00:00 2001 From: Maciej Szwed Date: Tue, 15 Oct 2019 13:03:36 +0200 Subject: [PATCH] bdev/nvme: Change NVMe controller unregister flow This patch moves around code related to unregister flow. This is a preparation for upcoming changes. It also changes IO device for NVMe bdev to nvme_bdev_ctrlr to make code clearer. Signed-off-by: Maciej Szwed Change-Id: Ic97a5b1973923a0cf44ed6c2d51b707dd7628d2c Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/468980 Tested-by: SPDK CI Jenkins Reviewed-by: Alexey Marchuk Reviewed-by: Ben Walker Reviewed-by: Jim Harris Reviewed-by: Shuhei Matsumoto Community-CI: Broadcom SPDK FC-NVMe CI --- module/bdev/nvme/bdev_nvme.c | 45 ++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/module/bdev/nvme/bdev_nvme.c b/module/bdev/nvme/bdev_nvme.c index 550a335fd..b0248271a 100644 --- a/module/bdev/nvme/bdev_nvme.c +++ b/module/bdev/nvme/bdev_nvme.c @@ -216,9 +216,16 @@ bdev_nvme_poll_adminq(void *arg) static void bdev_nvme_unregister_cb(void *io_device) { - struct spdk_nvme_ctrlr *ctrlr = io_device; + struct nvme_bdev_ctrlr *nvme_bdev_ctrlr = io_device; - spdk_nvme_detach(ctrlr); + pthread_mutex_lock(&g_bdev_nvme_mutex); + TAILQ_REMOVE(&g_nvme_bdev_ctrlrs, nvme_bdev_ctrlr, tailq); + pthread_mutex_unlock(&g_bdev_nvme_mutex); + spdk_nvme_detach(nvme_bdev_ctrlr->ctrlr); + spdk_poller_unregister(&nvme_bdev_ctrlr->adminq_timer_poller); + free(nvme_bdev_ctrlr->name); + free(nvme_bdev_ctrlr->bdevs); + free(nvme_bdev_ctrlr); } static void @@ -234,14 +241,8 @@ bdev_nvme_ctrlr_destruct(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr) spdk_opal_close(nvme_bdev_ctrlr->opal_dev); nvme_bdev_ctrlr->opal_dev = NULL; } - pthread_mutex_lock(&g_bdev_nvme_mutex); - TAILQ_REMOVE(&g_nvme_bdev_ctrlrs, nvme_bdev_ctrlr, tailq); - pthread_mutex_unlock(&g_bdev_nvme_mutex); - spdk_io_device_unregister(nvme_bdev_ctrlr->ctrlr, bdev_nvme_unregister_cb); - spdk_poller_unregister(&nvme_bdev_ctrlr->adminq_timer_poller); - free(nvme_bdev_ctrlr->name); - free(nvme_bdev_ctrlr->bdevs); - free(nvme_bdev_ctrlr); + + spdk_io_device_unregister(nvme_bdev_ctrlr, bdev_nvme_unregister_cb); } static int @@ -288,15 +289,15 @@ _bdev_nvme_reset_done(struct spdk_io_channel_iter *i, int status) static void _bdev_nvme_reset_create_qpair(struct spdk_io_channel_iter *i) { - struct spdk_nvme_ctrlr *ctrlr = spdk_io_channel_iter_get_io_device(i); + struct nvme_bdev_ctrlr *nvme_bdev_ctrlr = spdk_io_channel_iter_get_io_device(i); struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); struct nvme_io_channel *nvme_ch = spdk_io_channel_get_ctx(_ch); struct spdk_nvme_io_qpair_opts opts; - spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts)); + spdk_nvme_ctrlr_get_default_io_qpair_opts(nvme_bdev_ctrlr->ctrlr, &opts, sizeof(opts)); opts.delay_pcie_doorbell = true; - nvme_ch->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, &opts, sizeof(opts)); + nvme_ch->qpair = spdk_nvme_ctrlr_alloc_io_qpair(nvme_bdev_ctrlr->ctrlr, &opts, sizeof(opts)); if (!nvme_ch->qpair) { spdk_for_each_channel_continue(i, -1); return; @@ -308,7 +309,7 @@ _bdev_nvme_reset_create_qpair(struct spdk_io_channel_iter *i) static void _bdev_nvme_reset(struct spdk_io_channel_iter *i, int status) { - struct spdk_nvme_ctrlr *ctrlr = spdk_io_channel_iter_get_io_device(i); + struct nvme_bdev_ctrlr *nvme_bdev_ctrlr = spdk_io_channel_iter_get_io_device(i); struct nvme_bdev_io *bio = spdk_io_channel_iter_get_ctx(i); int rc; @@ -317,14 +318,14 @@ _bdev_nvme_reset(struct spdk_io_channel_iter *i, int status) return; } - rc = spdk_nvme_ctrlr_reset(ctrlr); + rc = spdk_nvme_ctrlr_reset(nvme_bdev_ctrlr->ctrlr); if (rc != 0) { spdk_bdev_io_complete(spdk_bdev_io_from_ctx(bio), SPDK_BDEV_IO_STATUS_FAILED); return; } /* Recreate all of the I/O queue pairs */ - spdk_for_each_channel(ctrlr, + spdk_for_each_channel(nvme_bdev_ctrlr, _bdev_nvme_reset_create_qpair, bio, _bdev_nvme_reset_done); @@ -351,7 +352,7 @@ static int bdev_nvme_reset(struct nvme_bdev *nbdev, struct nvme_bdev_io *bio) { /* First, delete all NVMe I/O queue pairs. */ - spdk_for_each_channel(nbdev->nvme_bdev_ctrlr->ctrlr, + spdk_for_each_channel(nbdev->nvme_bdev_ctrlr, _bdev_nvme_reset_destroy_qpair, bio, _bdev_nvme_reset); @@ -538,7 +539,7 @@ bdev_nvme_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) static int bdev_nvme_create_cb(void *io_device, void *ctx_buf) { - struct spdk_nvme_ctrlr *ctrlr = io_device; + struct nvme_bdev_ctrlr *nvme_bdev_ctrlr = io_device; struct nvme_io_channel *ch = ctx_buf; struct spdk_nvme_io_qpair_opts opts; @@ -548,12 +549,12 @@ bdev_nvme_create_cb(void *io_device, void *ctx_buf) ch->collect_spin_stat = false; #endif - spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts)); + spdk_nvme_ctrlr_get_default_io_qpair_opts(nvme_bdev_ctrlr->ctrlr, &opts, sizeof(opts)); opts.delay_pcie_doorbell = true; opts.io_queue_requests = spdk_max(g_opts.io_queue_requests, opts.io_queue_requests); g_opts.io_queue_requests = opts.io_queue_requests; - ch->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, &opts, sizeof(opts)); + ch->qpair = spdk_nvme_ctrlr_alloc_io_qpair(nvme_bdev_ctrlr->ctrlr, &opts, sizeof(opts)); if (ch->qpair == NULL) { return -1; @@ -577,7 +578,7 @@ bdev_nvme_get_io_channel(void *ctx) { struct nvme_bdev *nvme_bdev = ctx; - return spdk_get_io_channel(nvme_bdev->nvme_bdev_ctrlr->ctrlr); + return spdk_get_io_channel(nvme_bdev->nvme_bdev_ctrlr); } static int @@ -1004,7 +1005,7 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, } nvme_bdev_ctrlr->prchk_flags = prchk_flags; - spdk_io_device_register(ctrlr, bdev_nvme_create_cb, bdev_nvme_destroy_cb, + spdk_io_device_register(nvme_bdev_ctrlr, bdev_nvme_create_cb, bdev_nvme_destroy_cb, sizeof(struct nvme_io_channel), name);