From 6152d5bf8fdfb5e09008561b52e9bb82292695d0 Mon Sep 17 00:00:00 2001 From: Cunyin Chang Date: Tue, 20 Dec 2016 11:41:01 +0800 Subject: [PATCH] nvme: move hot-plug logic to nvme pcie layer. Change-Id: I2042b34e4284a9c59aa3092ccd061c075748880c Signed-off-by: Cunyin Chang --- lib/nvme/nvme.c | 58 ------------------- lib/nvme/nvme_pcie.c | 56 ++++++++++++++++++ test/lib/nvme/unit/nvme_c/nvme_ut.c | 12 ---- .../nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c | 20 ------- 4 files changed, 56 insertions(+), 90 deletions(-) diff --git a/lib/nvme/nvme.c b/lib/nvme/nvme.c index 20336ccf5..3b0652ff3 100644 --- a/lib/nvme/nvme.c +++ b/lib/nvme/nvme.c @@ -33,7 +33,6 @@ #include "spdk/nvmf_spec.h" #include "nvme_internal.h" -#include "nvme_uevent.h" #define SPDK_NVME_DRIVER_NAME "spdk_nvme_driver" @@ -41,8 +40,6 @@ struct nvme_driver *g_spdk_nvme_driver; int32_t spdk_nvme_retry_count; -static int hotplug_fd = -1; - int spdk_nvme_detach(struct spdk_nvme_ctrlr *ctrlr) { @@ -389,51 +386,6 @@ nvme_init_controllers(void *cb_ctx, spdk_nvme_attach_cb attach_cb) return rc; } -static int -nvme_hotplug_monitor(void *cb_ctx, spdk_nvme_probe_cb probe_cb, - spdk_nvme_remove_cb remove_cb) -{ - struct spdk_nvme_ctrlr *ctrlr; - struct spdk_uevent event; - struct spdk_pci_addr pci_addr; - - while (spdk_get_uevent(hotplug_fd, &event) > 0) { - if (event.subsystem == SPDK_NVME_UEVENT_SUBSYSTEM_UIO) { - if (event.action == SPDK_NVME_UEVENT_ADD) { - SPDK_TRACELOG(SPDK_TRACE_NVME, "add nvme address: %s\n", - event.traddr); - if (spdk_process_is_primary()) { - if (!spdk_pci_addr_parse(&pci_addr, event.traddr)) { - nvme_transport_ctrlr_attach(SPDK_NVME_TRANSPORT_PCIE, probe_cb, cb_ctx, &pci_addr); - } - } - } else if (event.action == SPDK_NVME_UEVENT_REMOVE) { - bool in_list = false; - - TAILQ_FOREACH(ctrlr, &g_spdk_nvme_driver->attached_ctrlrs, tailq) { - if (strcmp(event.traddr, ctrlr->trid.traddr) == 0) { - in_list = true; - break; - } - } - if (in_list == false) { - return 0; - } - SPDK_TRACELOG(SPDK_TRACE_NVME, "remove nvme address: %s\n", - event.traddr); - - nvme_ctrlr_fail(ctrlr, true); - - /* get the user app to clean up and stop I/O */ - if (remove_cb) { - remove_cb(cb_ctx, ctrlr); - } - } - } - } - return 0; -} - int spdk_nvme_probe(const struct spdk_nvme_transport_id *trid, void *cb_ctx, spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb attach_cb, @@ -460,16 +412,6 @@ spdk_nvme_probe(const struct spdk_nvme_transport_id *trid, void *cb_ctx, } nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock); - if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) { - if (hotplug_fd < 0) { - hotplug_fd = spdk_uevent_connect(); - if (hotplug_fd < 0) { - SPDK_ERRLOG("Failed to open uevent netlink socket\n"); - } - } else { - nvme_hotplug_monitor(cb_ctx, probe_cb, remove_cb); - } - } nvme_transport_ctrlr_scan(trid, cb_ctx, probe_cb, remove_cb); diff --git a/lib/nvme/nvme_pcie.c b/lib/nvme/nvme_pcie.c index 34c2fb605..abef82a73 100644 --- a/lib/nvme/nvme_pcie.c +++ b/lib/nvme/nvme_pcie.c @@ -41,6 +41,7 @@ #include #include "nvme_internal.h" +#include "nvme_uevent.h" #define NVME_ADMIN_ENTRIES (128) #define NVME_ADMIN_TRACKERS (64) @@ -186,6 +187,7 @@ static int nvme_pcie_qpair_destroy(struct spdk_nvme_qpair *qpair); __thread struct nvme_pcie_ctrlr *g_thread_mmio_ctrlr = NULL; static volatile uint16_t g_signal_lock; static bool g_sigset = false; +static int hotplug_fd = -1; static void nvme_sigbus_fault_sighandler(int signum, siginfo_t *info, void *ctx) @@ -226,6 +228,51 @@ nvme_pcie_ctrlr_setup_signal(void) sigaction(SIGBUS, &sa, NULL); } +static int +_nvme_pcie_hotplug_monitor(void *cb_ctx, spdk_nvme_probe_cb probe_cb, + spdk_nvme_remove_cb remove_cb) +{ + struct spdk_nvme_ctrlr *ctrlr; + struct spdk_uevent event; + struct spdk_pci_addr pci_addr; + + while (spdk_get_uevent(hotplug_fd, &event) > 0) { + if (event.subsystem == SPDK_NVME_UEVENT_SUBSYSTEM_UIO) { + if (event.action == SPDK_NVME_UEVENT_ADD) { + SPDK_TRACELOG(SPDK_TRACE_NVME, "add nvme address: %s\n", + event.traddr); + if (spdk_process_is_primary()) { + if (!spdk_pci_addr_parse(&pci_addr, event.traddr)) { + nvme_transport_ctrlr_attach(SPDK_NVME_TRANSPORT_PCIE, probe_cb, cb_ctx, &pci_addr); + } + } + } else if (event.action == SPDK_NVME_UEVENT_REMOVE) { + bool in_list = false; + + TAILQ_FOREACH(ctrlr, &g_spdk_nvme_driver->attached_ctrlrs, tailq) { + if (strcmp(event.traddr, ctrlr->trid.traddr) == 0) { + in_list = true; + break; + } + } + if (in_list == false) { + return 0; + } + SPDK_TRACELOG(SPDK_TRACE_NVME, "remove nvme address: %s\n", + event.traddr); + + nvme_ctrlr_fail(ctrlr, true); + + /* get the user app to clean up and stop I/O */ + if (remove_cb) { + remove_cb(cb_ctx, ctrlr); + } + } + } + } + return 0; +} + static inline struct nvme_pcie_ctrlr * nvme_pcie_ctrlr(struct spdk_nvme_ctrlr *ctrlr) { @@ -579,6 +626,15 @@ nvme_pcie_ctrlr_scan(const struct spdk_nvme_transport_id *trid, enum_ctx.probe_cb = probe_cb; enum_ctx.cb_ctx = cb_ctx; + if (hotplug_fd < 0) { + hotplug_fd = spdk_uevent_connect(); + if (hotplug_fd < 0) { + SPDK_ERRLOG("Failed to open uevent netlink socket\n"); + } + } else { + _nvme_pcie_hotplug_monitor(cb_ctx, probe_cb, remove_cb); + } + return spdk_pci_nvme_enumerate(pcie_nvme_enum_cb, &enum_ctx); } diff --git a/test/lib/nvme/unit/nvme_c/nvme_ut.c b/test/lib/nvme/unit/nvme_c/nvme_ut.c index 73155f949..3c5c77b0a 100644 --- a/test/lib/nvme/unit/nvme_c/nvme_ut.c +++ b/test/lib/nvme/unit/nvme_c/nvme_ut.c @@ -113,18 +113,6 @@ nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove) { } -int -spdk_uevent_connect(void) -{ - return 0; -} - -int -spdk_get_uevent(int fd, struct spdk_uevent *uevent) -{ - return 0; -} - void spdk_nvme_ctrlr_opts_set_defaults(struct spdk_nvme_ctrlr_opts *opts) { diff --git a/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c b/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c index d954894ae..9352c8a39 100644 --- a/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c +++ b/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c @@ -46,14 +46,6 @@ struct nvme_driver *g_spdk_nvme_driver = &_g_nvme_driver; struct nvme_request *g_request = NULL; -struct spdk_uevent; - -int -spdk_uevent_connect(void); - -int -spdk_get_uevent(int fd, struct spdk_uevent *uevent); - int spdk_pci_nvme_enumerate(spdk_pci_enum_cb enum_cb, void *enum_ctx) { @@ -110,18 +102,6 @@ nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove) { } -int -spdk_uevent_connect(void) -{ - return 0; -} - -int -spdk_get_uevent(int fd, struct spdk_uevent *uevent) -{ - return 0; -} - struct spdk_pci_addr spdk_pci_device_get_addr(struct spdk_pci_device *pci_dev) {