nbd: Use async manner to stop nbd device.

Purpose: In order to free all the allocated resources.

Our current code uses sync behaviour, and it will not wait for all
the resources are freed if there is active I/Os, and thus we will
not free some resources, e.g., some fds will not be closed.

Signed-off-by: Ziye Yang <ziye.yang@intel.com>
Change-Id: Iaf9a606da2049ffd0096860c46d89d094038a5ff
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5601
Community-CI: Broadcom CI
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Sun Zhenyuan <sunzhenyuan@baidu.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: <dongx.yi@intel.com>

CHANGELOG.md

@@ -29,6 +29,11 @@ An `opts_size`element was added in the `spdk_app_opts` structure
 to solve the ABI compatiblity issue between different SPDK version. An `opts_size`
 parameter is added into `spdk_app_opts_init` function.
 
+### nbd
+
+Change the return type of function `spdk_nbd_stop` from void to int. And update the
+`spdk_nbd_fini` with two parameters to make its behavior from sync to async.
+
 ### nvmf
 
 Broadcom FC LLD driver and SPDK NVMe-oF FC transport consolidated one LLD API,

include/spdk/nbd.h

@@ -45,6 +45,7 @@ extern "C" {
 struct spdk_bdev;
 struct spdk_nbd_disk;
 struct spdk_json_write_ctx;
+typedef void (*spdk_nbd_fini_cb)(void *arg);
 
 /**
  * Initialize the network block device layer.
@@ -55,8 +56,11 @@ int spdk_nbd_init(void);
 
 /**
  * Stop and close all the running network block devices.
+ *
+ * \param cb_fn Callback to be always called.
+ * \param cb_arg Passed to cb_fn.
  */
-void spdk_nbd_fini(void);
+void spdk_nbd_fini(spdk_nbd_fini_cb cb_fn, void *cb_arg);
 
 /**
  * Called when an NBD device has been started.
@@ -80,8 +84,10 @@ void spdk_nbd_start(const char *bdev_name, const char *nbd_path,
  * Stop the running network block device safely.
  *
  * \param nbd A pointer to the network block device to stop.
+ *
+ * \return 0 on success.
  */
-void spdk_nbd_stop(struct spdk_nbd_disk *nbd);
+int spdk_nbd_stop(struct spdk_nbd_disk *nbd);
 
 /**
  * Get the local filesystem path used for the network block device.

lib/nbd/Makefile

@@ -34,7 +34,7 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-SO_VER := 2
+SO_VER := 3
 SO_MINOR := 0
 
 LIBNAME = nbd

lib/nbd/nbd.c

@@ -120,6 +120,8 @@ struct spdk_nbd_disk_globals {
 };
 
 static struct spdk_nbd_disk_globals g_spdk_nbd;
+static spdk_nbd_fini_cb g_fini_cb_fn;
+static void *g_fini_cb_arg;
 
 static int
 nbd_submit_bdev_io(struct spdk_nbd_disk *nbd, struct nbd_io *io);
@@ -132,10 +134,11 @@ spdk_nbd_init(void)
 	return 0;
 }
 
-void
+static void
-spdk_nbd_fini(void)
+_nbd_fini(void *arg1)
 {
 	struct spdk_nbd_disk *nbd_idx, *nbd_tmp;
+	int rc = 0;
 
 	/*
 	 * Stop running spdk_nbd_disk.
@@ -144,8 +147,26 @@ spdk_nbd_fini(void)
 	 * remove nbd from TAILQ.
 	 */
 	TAILQ_FOREACH_SAFE(nbd_idx, &g_spdk_nbd.disk_head, tailq, nbd_tmp) {
-		spdk_nbd_stop(nbd_idx);
+		rc = spdk_nbd_stop(nbd_idx);
+		if (rc) {
+			break;
+		}
 	}
+
+	if (!rc) {
+		g_fini_cb_fn(g_fini_cb_arg);
+	} else {
+		spdk_thread_send_msg(spdk_get_thread(), _nbd_fini, NULL);
+	}
+}
+
+void
+spdk_nbd_fini(spdk_nbd_fini_cb cb_fn, void *cb_arg)
+{
+	g_fini_cb_fn = cb_fn;
+	g_fini_cb_arg = cb_arg;
+
+	_nbd_fini(NULL);
 }
 
 static int
@@ -388,11 +409,13 @@ _nbd_stop(struct spdk_nbd_disk *nbd)
 	free(nbd);
 }
 
-void
+int
 spdk_nbd_stop(struct spdk_nbd_disk *nbd)
 {
+	int rc = 0;
+
 	if (nbd == NULL) {
-		return;
+		return rc;
 	}
 
 	nbd->state = NBD_DISK_STATE_HARDDISC;
@@ -400,9 +423,13 @@ spdk_nbd_stop(struct spdk_nbd_disk *nbd)
 	/*
 	 * Stop action should be called only after all nbd_io are executed.
 	 */
-	if (!nbd_cleanup_io(nbd)) {
+
+	rc = nbd_cleanup_io(nbd);
+	if (!rc) {
 		_nbd_stop(nbd);
 	}
+
+	return rc;
 }
 
 static int64_t

module/event/subsystems/nbd/nbd.c

@@ -47,11 +47,16 @@ nbd_subsystem_init(void)
 	spdk_subsystem_init_next(rc);
 }
 
+static void
+nbd_subsystem_fini_done(void *arg)
+{
+	spdk_subsystem_fini_next();
+}
+
 static void
 nbd_subsystem_fini(void)
 {
-	spdk_nbd_fini();
+	spdk_nbd_fini(nbd_subsystem_fini_done, NULL);
-	spdk_subsystem_fini_next();
 }
 
 static void