nvme: move hot-plug logic to nvme pcie layer.

Change-Id: I2042b34e4284a9c59aa3092ccd061c075748880c
Signed-off-by: Cunyin Chang <cunyin.chang@intel.com>

lib/nvme/nvme.c

@@ -33,7 +33,6 @@
 
 #include "spdk/nvmf_spec.h"
 #include "nvme_internal.h"
-#include "nvme_uevent.h"
 
 #define SPDK_NVME_DRIVER_NAME "spdk_nvme_driver"
 
@@ -41,8 +40,6 @@ struct nvme_driver	*g_spdk_nvme_driver;
 
 int32_t			spdk_nvme_retry_count;
 
-static int		hotplug_fd = -1;
-
 int
 spdk_nvme_detach(struct spdk_nvme_ctrlr *ctrlr)
 {
@@ -389,51 +386,6 @@ nvme_init_controllers(void *cb_ctx, spdk_nvme_attach_cb attach_cb)
 	return rc;
 }
 
-static int
-nvme_hotplug_monitor(void *cb_ctx, spdk_nvme_probe_cb probe_cb,
-		     spdk_nvme_remove_cb remove_cb)
-{
-	struct spdk_nvme_ctrlr *ctrlr;
-	struct spdk_uevent event;
-	struct spdk_pci_addr pci_addr;
-
-	while (spdk_get_uevent(hotplug_fd, &event) > 0) {
-		if (event.subsystem == SPDK_NVME_UEVENT_SUBSYSTEM_UIO) {
-			if (event.action == SPDK_NVME_UEVENT_ADD) {
-				SPDK_TRACELOG(SPDK_TRACE_NVME, "add nvme address: %s\n",
-					      event.traddr);
-				if (spdk_process_is_primary()) {
-					if (!spdk_pci_addr_parse(&pci_addr, event.traddr)) {
-						nvme_transport_ctrlr_attach(SPDK_NVME_TRANSPORT_PCIE, probe_cb, cb_ctx, &pci_addr);
-					}
-				}
-			} else if (event.action == SPDK_NVME_UEVENT_REMOVE) {
-				bool in_list = false;
-
-				TAILQ_FOREACH(ctrlr, &g_spdk_nvme_driver->attached_ctrlrs, tailq) {
-					if (strcmp(event.traddr, ctrlr->trid.traddr) == 0) {
-						in_list = true;
-						break;
-					}
-				}
-				if (in_list == false) {
-					return 0;
-				}
-				SPDK_TRACELOG(SPDK_TRACE_NVME, "remove nvme address: %s\n",
-					      event.traddr);
-
-				nvme_ctrlr_fail(ctrlr, true);
-
-				/* get the user app to clean up and stop I/O */
-				if (remove_cb) {
-					remove_cb(cb_ctx, ctrlr);
-				}
-			}
-		}
-	}
-	return 0;
-}
-
 int
 spdk_nvme_probe(const struct spdk_nvme_transport_id *trid, void *cb_ctx,
 		spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb attach_cb,
@@ -460,16 +412,6 @@ spdk_nvme_probe(const struct spdk_nvme_transport_id *trid, void *cb_ctx,
 	}
 
 	nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock);
-	if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {
-		if (hotplug_fd < 0) {
-			hotplug_fd = spdk_uevent_connect();
-			if (hotplug_fd < 0) {
-				SPDK_ERRLOG("Failed to open uevent netlink socket\n");
-			}
-		} else {
-			nvme_hotplug_monitor(cb_ctx, probe_cb, remove_cb);
-		}
-	}
 
 	nvme_transport_ctrlr_scan(trid, cb_ctx, probe_cb, remove_cb);
 

lib/nvme/nvme_pcie.c

@@ -41,6 +41,7 @@
 #include <sys/types.h>
 
 #include "nvme_internal.h"
+#include "nvme_uevent.h"
 
 #define NVME_ADMIN_ENTRIES	(128)
 #define NVME_ADMIN_TRACKERS	(64)
@@ -186,6 +187,7 @@ static int nvme_pcie_qpair_destroy(struct spdk_nvme_qpair *qpair);
 __thread struct nvme_pcie_ctrlr *g_thread_mmio_ctrlr = NULL;
 static volatile uint16_t g_signal_lock;
 static bool g_sigset = false;
+static int hotplug_fd = -1;
 
 static void
 nvme_sigbus_fault_sighandler(int signum, siginfo_t *info, void *ctx)
@@ -226,6 +228,51 @@ nvme_pcie_ctrlr_setup_signal(void)
 	sigaction(SIGBUS, &sa, NULL);
 }
 
+static int
+_nvme_pcie_hotplug_monitor(void *cb_ctx, spdk_nvme_probe_cb probe_cb,
+			   spdk_nvme_remove_cb remove_cb)
+{
+	struct spdk_nvme_ctrlr *ctrlr;
+	struct spdk_uevent event;
+	struct spdk_pci_addr pci_addr;
+
+	while (spdk_get_uevent(hotplug_fd, &event) > 0) {
+		if (event.subsystem == SPDK_NVME_UEVENT_SUBSYSTEM_UIO) {
+			if (event.action == SPDK_NVME_UEVENT_ADD) {
+				SPDK_TRACELOG(SPDK_TRACE_NVME, "add nvme address: %s\n",
+					      event.traddr);
+				if (spdk_process_is_primary()) {
+					if (!spdk_pci_addr_parse(&pci_addr, event.traddr)) {
+						nvme_transport_ctrlr_attach(SPDK_NVME_TRANSPORT_PCIE, probe_cb, cb_ctx, &pci_addr);
+					}
+				}
+			} else if (event.action == SPDK_NVME_UEVENT_REMOVE) {
+				bool in_list = false;
+
+				TAILQ_FOREACH(ctrlr, &g_spdk_nvme_driver->attached_ctrlrs, tailq) {
+					if (strcmp(event.traddr, ctrlr->trid.traddr) == 0) {
+						in_list = true;
+						break;
+					}
+				}
+				if (in_list == false) {
+					return 0;
+				}
+				SPDK_TRACELOG(SPDK_TRACE_NVME, "remove nvme address: %s\n",
+					      event.traddr);
+
+				nvme_ctrlr_fail(ctrlr, true);
+
+				/* get the user app to clean up and stop I/O */
+				if (remove_cb) {
+					remove_cb(cb_ctx, ctrlr);
+				}
+			}
+		}
+	}
+	return 0;
+}
+
 static inline struct nvme_pcie_ctrlr *
 nvme_pcie_ctrlr(struct spdk_nvme_ctrlr *ctrlr)
 {
@@ -579,6 +626,15 @@ nvme_pcie_ctrlr_scan(const struct spdk_nvme_transport_id *trid,
 	enum_ctx.probe_cb = probe_cb;
 	enum_ctx.cb_ctx = cb_ctx;
 
+	if (hotplug_fd < 0) {
+		hotplug_fd = spdk_uevent_connect();
+		if (hotplug_fd < 0) {
+			SPDK_ERRLOG("Failed to open uevent netlink socket\n");
+		}
+	} else {
+		_nvme_pcie_hotplug_monitor(cb_ctx, probe_cb, remove_cb);
+	}
+
 	return spdk_pci_nvme_enumerate(pcie_nvme_enum_cb, &enum_ctx);
 }
 

test/lib/nvme/unit/nvme_c/nvme_ut.c

@@ -113,18 +113,6 @@ nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove)
 {
 }
 
-int
-spdk_uevent_connect(void)
-{
-	return 0;
-}
-
-int
-spdk_get_uevent(int fd, struct spdk_uevent *uevent)
-{
-	return 0;
-}
-
 void
 spdk_nvme_ctrlr_opts_set_defaults(struct spdk_nvme_ctrlr_opts *opts)
 {

test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c

@@ -46,14 +46,6 @@ struct nvme_driver *g_spdk_nvme_driver = &_g_nvme_driver;
 
 struct nvme_request *g_request = NULL;
 
-struct spdk_uevent;
-
-int
-spdk_uevent_connect(void);
-
-int
-spdk_get_uevent(int fd, struct spdk_uevent *uevent);
-
 int
 spdk_pci_nvme_enumerate(spdk_pci_enum_cb enum_cb, void *enum_ctx)
 {
@@ -110,18 +102,6 @@ nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove)
 {
 }
 
-int
-spdk_uevent_connect(void)
-{
-	return 0;
-}
-
-int
-spdk_get_uevent(int fd, struct spdk_uevent *uevent)
-{
-	return 0;
-}
-
 struct spdk_pci_addr
 spdk_pci_device_get_addr(struct spdk_pci_device *pci_dev)
 {