nvme: Add support for hotplug.

Change-Id: Iac504ce15e4ea3100e5afa31764fcfff7f979dbb
Signed-off-by: Cunyin Chang <cunyin.chang@intel.com>

lib/nvme/nvme.c

@@ -33,9 +33,11 @@
 
 #include "spdk/nvmf_spec.h"
 #include "nvme_internal.h"
+#include "nvme_uevent.h"
 
 struct nvme_driver _g_nvme_driver = {
 	.lock = PTHREAD_MUTEX_INITIALIZER,
+	.hotplug_fd = -1,
 	.init_ctrlrs = TAILQ_HEAD_INITIALIZER(_g_nvme_driver.init_ctrlrs),
 	.attached_ctrlrs = TAILQ_HEAD_INITIALIZER(_g_nvme_driver.attached_ctrlrs),
 	.request_mempool = NULL,
@@ -254,51 +256,14 @@ nvme_probe_one(enum spdk_nvme_transport transport, spdk_nvme_probe_cb probe_cb,
 }
 
 static int
-_spdk_nvme_probe(const struct spdk_nvme_discover_info *info, void *cb_ctx,
+nvme_init_controllers(void *cb_ctx, spdk_nvme_attach_cb attach_cb)
-		 spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb attach_cb,
-		 spdk_nvme_remove_cb remove_cb)
 {
-	int rc, start_rc;
+	int rc = 0;
+	int start_rc;
 	struct spdk_nvme_ctrlr *ctrlr, *ctrlr_tmp;
-	enum spdk_nvme_transport transport;
-
-	if (!spdk_process_is_primary()) {
-		while (g_spdk_nvme_driver->initialized == false) {
-			usleep(200 * 1000);
-		}
-	}
 
 	pthread_mutex_lock(&g_spdk_nvme_driver->lock);
 
-	if (g_spdk_nvme_driver->request_mempool == NULL) {
-		g_spdk_nvme_driver->request_mempool = spdk_mempool_create("nvme_request", 8192,
-						      sizeof(struct nvme_request), -1);
-		if (g_spdk_nvme_driver->request_mempool == NULL) {
-			SPDK_ERRLOG("Unable to allocate pool of requests\n");
-			pthread_mutex_unlock(&g_spdk_nvme_driver->lock);
-			return -1;
-		}
-	}
-
-	if (!info) {
-		transport = SPDK_NVME_TRANSPORT_PCIE;
-	} else {
-		if (!spdk_nvme_transport_available(info->trtype)) {
-			SPDK_ERRLOG("NVMe over Fabrics trtype %u not available\n", info->trtype);
-			pthread_mutex_unlock(&g_spdk_nvme_driver->lock);
-			return -1;
-		}
-
-		transport = (uint8_t)info->trtype;
-	}
-
-	rc = nvme_transport_ctrlr_scan(transport, probe_cb, cb_ctx, (void *)info, NULL);
-
-	/*
-	 * Keep going even if one or more nvme_attach() calls failed,
-	 *  but maintain the value of rc to signal errors when we return.
-	 */
-
 	/* Initialize all new controllers in the init_ctrlrs list in parallel. */
 	while (!TAILQ_EMPTY(&g_spdk_nvme_driver->init_ctrlrs)) {
 		TAILQ_FOREACH_SAFE(ctrlr, &g_spdk_nvme_driver->init_ctrlrs, tailq, ctrlr_tmp) {
@@ -349,8 +314,75 @@ _spdk_nvme_probe(const struct spdk_nvme_discover_info *info, void *cb_ctx,
 		}
 	}
 
-	g_spdk_nvme_driver->initialized = true;
+	pthread_mutex_unlock(&g_spdk_nvme_driver->lock);
+	return rc;
+}
 
+static int
+nvme_attach_one(void *cb_ctx, spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb attach_cb,
+		struct spdk_pci_addr *pci_address)
+{
+	nvme_transport_ctrlr_scan(SPDK_NVME_TRANSPORT_PCIE, probe_cb, cb_ctx, NULL, pci_address);
+	return nvme_init_controllers(cb_ctx, attach_cb);
+}
+
+static int
+_spdk_nvme_probe(const struct spdk_nvme_discover_info *info, void *cb_ctx,
+		 spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb attach_cb,
+		 spdk_nvme_remove_cb remove_cb)
+{
+	int rc;
+	enum spdk_nvme_transport transport;
+
+	if (!spdk_process_is_primary()) {
+		while (g_spdk_nvme_driver->initialized == false) {
+			usleep(200 * 1000);
+		}
+	}
+
+	pthread_mutex_lock(&g_spdk_nvme_driver->lock);
+
+	if (g_spdk_nvme_driver->hotplug_fd < 0) {
+		g_spdk_nvme_driver->hotplug_fd = spdk_uevent_connect();
+		if (g_spdk_nvme_driver->hotplug_fd < 0) {
+			SPDK_ERRLOG("Failed to open uevent netlink socket\n");
+		}
+	}
+
+	if (g_spdk_nvme_driver->request_mempool == NULL) {
+		g_spdk_nvme_driver->request_mempool = spdk_mempool_create("nvme_request", 8192,
+						      sizeof(struct nvme_request), -1);
+		if (g_spdk_nvme_driver->request_mempool == NULL) {
+			SPDK_ERRLOG("Unable to allocate pool of requests\n");
+			pthread_mutex_unlock(&g_spdk_nvme_driver->lock);
+			return -1;
+		}
+	}
+
+	if (!info) {
+		transport = SPDK_NVME_TRANSPORT_PCIE;
+	} else {
+		if (!spdk_nvme_transport_available(info->trtype)) {
+			SPDK_ERRLOG("NVMe over Fabrics trtype %u not available\n", info->trtype);
+			pthread_mutex_unlock(&g_spdk_nvme_driver->lock);
+			return -1;
+		}
+
+		transport = (uint8_t)info->trtype;
+	}
+
+	nvme_transport_ctrlr_scan(transport, probe_cb, cb_ctx, (void *)info, NULL);
+
+	pthread_mutex_unlock(&g_spdk_nvme_driver->lock);
+	/*
+	 * Keep going even if one or more nvme_attach() calls failed,
+	 *  but maintain the value of rc to signal errors when we return.
+	 */
+
+	rc = nvme_init_controllers(cb_ctx, attach_cb);
+
+	pthread_mutex_lock(&g_spdk_nvme_driver->lock);
+	g_spdk_nvme_driver->initialized = true;
 	pthread_mutex_unlock(&g_spdk_nvme_driver->lock);
 	return rc;
 }
@@ -367,11 +399,66 @@ int spdk_nvme_discover(const struct spdk_nvme_discover_info *info, void *cb_ctx,
 	return _spdk_nvme_probe(info, cb_ctx, probe_cb, attach_cb, remove_cb);
 }
 
+static int
+nvme_hotplug_monitor(void *cb_ctx, spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb attach_cb,
+		     spdk_nvme_remove_cb remove_cb)
+{
+	int rc = 0;
+	struct spdk_nvme_ctrlr *ctrlr;
+	struct spdk_uevent event;
+	struct nvme_driver *nvme_driver = g_spdk_nvme_driver;
+
+	while (spdk_get_uevent(nvme_driver->hotplug_fd, &event) > 0) {
+		if (event.subsystem == SPDK_NVME_UEVENT_SUBSYSTEM_UIO) {
+			if (event.action == SPDK_NVME_UEVENT_ADD) {
+				SPDK_TRACELOG(SPDK_TRACE_NVME, "add nvme address: %04x:%02x:%02x.%u\n",
+					      event.pci_addr.domain, event.pci_addr.bus, event.pci_addr.dev, event.pci_addr.func);
+				if (spdk_process_is_primary()) {
+					nvme_attach_one(cb_ctx, probe_cb, attach_cb, &event.pci_addr);
+				}
+			} else if (event.action == SPDK_NVME_UEVENT_REMOVE) {
+				bool in_list = false;
+
+				TAILQ_FOREACH(ctrlr, &g_spdk_nvme_driver->attached_ctrlrs, tailq) {
+					if (spdk_pci_addr_compare(&event.pci_addr, &ctrlr->probe_info.pci_addr) == 0) {
+						in_list = true;
+						break;
+					}
+				}
+				if (in_list == false) {
+					return 0;
+				}
+				SPDK_TRACELOG(SPDK_TRACE_NVME, "remove nvme address: %04x:%02x:%02x.%u\n",
+					      event.pci_addr.domain, event.pci_addr.bus, event.pci_addr.dev, event.pci_addr.func);
+
+				nvme_ctrlr_fail(ctrlr, true);
+
+				/* get the user app to clean up and stop I/O */
+				if (remove_cb) {
+					remove_cb(cb_ctx, ctrlr);
+				}
+				if (spdk_process_is_primary()) {
+					rc = spdk_nvme_detach(ctrlr);
+					if (rc) {
+						SPDK_ERRLOG("Failed to hot detach nvme address: %04x:%04x:%04x.%u\n",
+							    event.pci_addr.domain, event.pci_addr.bus, event.pci_addr.dev,
+							    event.pci_addr.func);
+					}
+				}
+			}
+		}
+	}
+	return 0;
+}
+
 int
 spdk_nvme_probe(void *cb_ctx, spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb attach_cb,
 		spdk_nvme_remove_cb remove_cb)
 {
+	if (g_spdk_nvme_driver->hotplug_fd < 0) {
 		return _spdk_nvme_probe(NULL, cb_ctx, probe_cb, attach_cb, remove_cb);
+	} else {
+		return nvme_hotplug_monitor(cb_ctx, probe_cb, attach_cb, remove_cb);
+	}
 }
-
 SPDK_LOG_REGISTER_TRACE_FLAG("nvme", SPDK_TRACE_NVME)

lib/nvme/nvme_ctrlr.c

@@ -289,11 +289,14 @@ nvme_ctrlr_set_supported_features(struct spdk_nvme_ctrlr *ctrlr)
 	}
 }
 
-static void
+void
-nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr)
+nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove)
 {
 	struct spdk_nvme_qpair *qpair;
 
+	if (hot_remove) {
+		ctrlr->is_removed = true;
+	}
 	ctrlr->is_failed = true;
 	nvme_qpair_fail(ctrlr->adminq);
 	TAILQ_FOREACH(qpair, &ctrlr->active_io_qpairs, tailq) {
@@ -308,6 +311,10 @@ nvme_ctrlr_shutdown(struct spdk_nvme_ctrlr *ctrlr)
 	union spdk_nvme_csts_register	csts;
 	int				ms_waited = 0;
 
+	if (ctrlr->is_removed) {
+		return;
+	}
+
 	if (nvme_ctrlr_get_cc(ctrlr, &cc)) {
 		SPDK_ERRLOG("get_cc() failed\n");
 		return;
@@ -471,7 +478,7 @@ spdk_nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr)
 	while (ctrlr->state != NVME_CTRLR_STATE_READY) {
 		if (nvme_ctrlr_process_init(ctrlr) != 0) {
 			SPDK_ERRLOG("%s: controller reinitialization failed\n", __func__);
-			nvme_ctrlr_fail(ctrlr);
+			nvme_ctrlr_fail(ctrlr, false);
 			rc = -1;
 			break;
 		}
@@ -481,7 +488,7 @@ spdk_nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr)
 		/* Reinitialize qpairs */
 		TAILQ_FOREACH(qpair, &ctrlr->active_io_qpairs, tailq) {
 			if (nvme_transport_ctrlr_reinit_io_qpair(ctrlr, qpair) != 0) {
-				nvme_ctrlr_fail(ctrlr);
+				nvme_ctrlr_fail(ctrlr, false);
 				rc = -1;
 			}
 		}
@@ -1013,7 +1020,7 @@ nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr)
 	if (nvme_ctrlr_get_cc(ctrlr, &cc) ||
 	    nvme_ctrlr_get_csts(ctrlr, &csts)) {
 		SPDK_ERRLOG("get registers failed\n");
-		nvme_ctrlr_fail(ctrlr);
+		nvme_ctrlr_fail(ctrlr, false);
 		return -EIO;
 	}
 
@@ -1044,7 +1051,7 @@ nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr)
 			cc.bits.en = 0;
 			if (nvme_ctrlr_set_cc(ctrlr, &cc)) {
 				SPDK_ERRLOG("set_cc() failed\n");
-				nvme_ctrlr_fail(ctrlr);
+				nvme_ctrlr_fail(ctrlr, false);
 				return -EIO;
 			}
 			nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, ready_timeout_in_ms);
@@ -1087,7 +1094,7 @@ nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr)
 			cc.bits.en = 0;
 			if (nvme_ctrlr_set_cc(ctrlr, &cc)) {
 				SPDK_ERRLOG("set_cc() failed\n");
-				nvme_ctrlr_fail(ctrlr);
+				nvme_ctrlr_fail(ctrlr, false);
 				return -EIO;
 			}
 			nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, ready_timeout_in_ms);
@@ -1121,14 +1128,14 @@ nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr)
 
 	default:
 		assert(0);
-		nvme_ctrlr_fail(ctrlr);
+		nvme_ctrlr_fail(ctrlr, false);
 		return -1;
 	}
 
 	if (ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE &&
 	    spdk_get_ticks() > ctrlr->state_timeout_tsc) {
 		SPDK_ERRLOG("Initialization timed out in state %d\n", ctrlr->state);
-		nvme_ctrlr_fail(ctrlr);
+		nvme_ctrlr_fail(ctrlr, false);
 		return -1;
 	}
 

lib/nvme/nvme_internal.h

@@ -341,6 +341,8 @@ struct spdk_nvme_ctrlr {
 
 	uint32_t			num_ns;
 
+	bool				is_removed;
+
 	bool				is_resetting;
 
 	bool				is_failed;
@@ -413,6 +415,7 @@ struct spdk_nvme_ctrlr {
 
 struct nvme_driver {
 	pthread_mutex_t			lock;
+	int				hotplug_fd;
 	TAILQ_HEAD(, spdk_nvme_ctrlr)	init_ctrlrs;
 	TAILQ_HEAD(, spdk_nvme_ctrlr)	attached_ctrlrs;
 	struct spdk_mempool		*request_mempool;
@@ -496,6 +499,7 @@ int	nvme_probe_one(enum spdk_nvme_transport type, spdk_nvme_probe_cb probe_cb, v
 
 int	nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr);
 void	nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr);
+void	nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove);
 int	nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr);
 int	nvme_ctrlr_start(struct spdk_nvme_ctrlr *ctrlr);
 

lib/nvme/nvme_pcie.c

@@ -483,6 +483,10 @@ nvme_pcie_ctrlr_free_bars(struct nvme_pcie_ctrlr *pctrlr)
 	int rc = 0;
 	void *addr = (void *)pctrlr->regs;
 
+	if (pctrlr->ctrlr.is_removed) {
+		return rc;
+	}
+
 	rc = nvme_pcie_ctrlr_unmap_cmb(pctrlr);
 	if (rc != 0) {
 		SPDK_ERRLOG("nvme_ctrlr_unmap_cmb failed with error code %d\n", rc);
@@ -577,6 +581,7 @@ struct spdk_nvme_ctrlr *nvme_pcie_ctrlr_construct(enum spdk_nvme_transport trans
 	}
 
 	pctrlr->is_remapped = false;
+	pctrlr->ctrlr.is_removed = false;
 	pctrlr->ctrlr.transport = SPDK_NVME_TRANSPORT_PCIE;
 	pctrlr->devhandle = devhandle;
 	pctrlr->ctrlr.opts = *opts;
@@ -1343,6 +1348,10 @@ nvme_pcie_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_
 
 	assert(ctrlr != NULL);
 
+	if (ctrlr->is_removed) {
+		goto free;
+	}
+
 	/* Delete the I/O submission queue and then the completion queue */
 
 	status.done = false;
@@ -1369,8 +1378,8 @@ nvme_pcie_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_
 		return -1;
 	}
 
+free:
 	nvme_pcie_qpair_destroy(qpair);
-
 	return 0;
 }
 

test/lib/nvme/unit/nvme_c/nvme_ut.c

@@ -101,6 +101,23 @@ nvme_ctrlr_start(struct spdk_nvme_ctrlr *ctrlr)
 	return 0;
 }
 
+void
+nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove)
+{
+}
+
+int
+spdk_uevent_connect(void)
+{
+	return 0;
+}
+
+int
+spdk_get_uevent(int fd, struct spdk_uevent *uevent)
+{
+	return 0;
+}
+
 void
 spdk_nvme_ctrlr_opts_set_defaults(struct spdk_nvme_ctrlr_opts *opts)
 {

test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c

@@ -1169,7 +1169,7 @@ test_nvme_ctrlr_fail(void)
 	struct spdk_nvme_ctrlr	ctrlr = {};
 
 	ctrlr.opts.num_io_queues = 0;
-	nvme_ctrlr_fail(&ctrlr);
+	nvme_ctrlr_fail(&ctrlr, false);
 
 	CU_ASSERT(ctrlr.is_failed == true);
 }

test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c

@@ -39,6 +39,14 @@
 
 struct nvme_request *g_request = NULL;
 
+struct spdk_uevent;
+
+int
+spdk_uevent_connect(void);
+
+int
+spdk_get_uevent(int fd, struct spdk_uevent *uevent);
+
 int
 spdk_pci_enumerate(enum spdk_pci_device_type type,
 		   spdk_pci_enum_cb enum_cb,
@@ -93,6 +101,23 @@ nvme_ctrlr_start(struct spdk_nvme_ctrlr *ctrlr)
 	return 0;
 }
 
+void
+nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove)
+{
+}
+
+int
+spdk_uevent_connect(void)
+{
+	return 0;
+}
+
+int
+spdk_get_uevent(int fd, struct spdk_uevent *uevent)
+{
+	return 0;
+}
+
 struct spdk_pci_addr
 spdk_pci_device_get_addr(struct spdk_pci_device *pci_dev)
 {