event/vhost: separate vhost subsystem to scsi and blk

Separate out SCSI and BLK vhost subsystems to later add
virtio_blk transport abstraction.

This allows for further changes to the vhost_blk, not
affecting vhost_scsi.

Signed-off-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Change-Id: Id1ecfeafeb936809a479a43c321e13f75cb3d5ad
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9539
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Konrad Sztyber <konrad.sztyber@intel.com>

app/spdk_tgt/Makefile

@@ -50,7 +50,7 @@ endif
 ifeq ($(OS),Linux)
 SPDK_LIB_LIST += event_nbd
 ifeq ($(CONFIG_VHOST),y)
-SPDK_LIB_LIST += event_vhost
+SPDK_LIB_LIST += event_vhost_blk event_vhost_scsi
 endif
 endif
 

app/vhost/Makefile

@@ -39,7 +39,7 @@ APP = vhost
 
 C_SRCS := vhost.c
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event event_vhost event_nbd
+SPDK_LIB_LIST = $(ALL_MODULES_LIST) event event_vhost_blk event_vhost_scsi event_nbd
 
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 SPDK_LIB_LIST += env_dpdk_rpc

examples/interrupt_tgt/Makefile

@@ -42,7 +42,7 @@ C_SRCS := interrupt_tgt.c
 SPDK_LIB_LIST = $(INTR_BLOCKDEV_MODULES_LIST) event event_bdev conf
 
 SPDK_LIB_LIST += event_nbd
-SPDK_LIB_LIST += event_vhost
+SPDK_LIB_LIST += event_vhost_blk event_vhost_scsi
 
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 SPDK_LIB_LIST += env_dpdk_rpc

include/spdk/vhost.h

@@ -75,22 +75,42 @@ int spdk_vhost_set_socket_path(const char *basename);
  *
  * \param init_cb Function to be called when the initialization is complete.
  */
-void spdk_vhost_init(spdk_vhost_init_cb init_cb);
+void spdk_vhost_scsi_init(spdk_vhost_init_cb init_cb);
 
 /**
  * Clean up the environment of vhost.
  *
  * \param fini_cb Function to be called when the cleanup is complete.
  */
-void spdk_vhost_fini(spdk_vhost_fini_cb fini_cb);
+void spdk_vhost_scsi_fini(spdk_vhost_fini_cb fini_cb);
-
 
 /**
  * Write vhost subsystem configuration into provided JSON context.
  *
  * \param w JSON write context
  */
-void spdk_vhost_config_json(struct spdk_json_write_ctx *w);
+void spdk_vhost_scsi_config_json(struct spdk_json_write_ctx *w);
+
+/**
+ * Init vhost environment.
+ *
+ * \param init_cb Function to be called when the initialization is complete.
+ */
+void spdk_vhost_blk_init(spdk_vhost_init_cb init_cb);
+
+/**
+ * Clean up the environment of vhost.
+ *
+ * \param fini_cb Function to be called when the cleanup is complete.
+ */
+void spdk_vhost_blk_fini(spdk_vhost_fini_cb fini_cb);
+
+/**
+ * Write vhost subsystem configuration into provided JSON context.
+ *
+ * \param w JSON write context
+ */
+void spdk_vhost_blk_config_json(struct spdk_json_write_ctx *w);
 
 /**
  * Deinit vhost application. This is called once by SPDK app layer.

lib/vhost/rte_vhost_user.c

@@ -1839,12 +1839,19 @@ vhost_user_init(void)
 	return 0;
 }
 
+static void
+vhost_user_session_shutdown_on_init(void *vhost_cb)
+{
+	spdk_vhost_fini_cb fn = vhost_cb;
+
+	fn();
+}
+
 static void *
-vhost_user_session_shutdown(void *arg)
+vhost_user_session_shutdown(void *vhost_cb)
 {
 	struct spdk_vhost_dev *vdev = NULL;
 	struct spdk_vhost_session *vsession;
-	vhost_fini_cb vhost_cb = arg;
 
 	for (vdev = spdk_vhost_dev_next(NULL); vdev != NULL;
 	     vdev = spdk_vhost_dev_next(vdev)) {
@@ -1860,18 +1867,18 @@ vhost_user_session_shutdown(void *arg)
 	}
 
 	SPDK_INFOLOG(vhost, "Exiting\n");
-	spdk_thread_send_msg(g_vhost_user_init_thread, vhost_cb, NULL);
+	spdk_thread_send_msg(g_vhost_user_init_thread, vhost_user_session_shutdown_on_init, vhost_cb);
 	return NULL;
 }
 
 void
-vhost_user_fini(vhost_fini_cb vhost_cb)
+vhost_user_fini(spdk_vhost_fini_cb vhost_cb)
 {
 	pthread_t tid;
 	int rc;
 
 	if (!g_vhost_user_started) {
-		vhost_cb(NULL);
+		vhost_cb();
 		return;
 	}
 

lib/vhost/spdk_vhost.map

@@ -3,9 +3,12 @@
 
 	# public functions
 	spdk_vhost_set_socket_path;
-	spdk_vhost_init;
+	spdk_vhost_scsi_init;
-	spdk_vhost_fini;
+	spdk_vhost_scsi_fini;
-	spdk_vhost_config_json;
+	spdk_vhost_scsi_config_json;
+	spdk_vhost_blk_init;
+	spdk_vhost_blk_fini;
+	spdk_vhost_blk_config_json;
 	spdk_vhost_shutdown_cb;
 	spdk_vhost_lock;
 	spdk_vhost_trylock;

lib/vhost/vhost.c

@@ -221,7 +221,7 @@ spdk_vhost_unlock(void)
 }
 
 void
-spdk_vhost_init(spdk_vhost_init_cb init_cb)
+spdk_vhost_scsi_init(spdk_vhost_init_cb init_cb)
 {
 	uint32_t i;
 	int ret = 0;
@@ -242,7 +242,7 @@ spdk_vhost_init(spdk_vhost_init_cb init_cb)
 static spdk_vhost_fini_cb g_fini_cb;
 
 static void
-vhost_fini(void *arg1)
+vhost_fini(void)
 {
 	struct spdk_vhost_dev *vdev, *tmp;
 
@@ -260,7 +260,34 @@ vhost_fini(void *arg1)
 }
 
 void
-spdk_vhost_fini(spdk_vhost_fini_cb fini_cb)
+spdk_vhost_blk_init(spdk_vhost_init_cb init_cb)
+{
+	uint32_t i;
+	int ret = 0;
+
+	ret = vhost_user_init();
+	if (ret != 0) {
+		goto out;
+	}
+
+	spdk_cpuset_zero(&g_vhost_core_mask);
+	SPDK_ENV_FOREACH_CORE(i) {
+		spdk_cpuset_set_cpu(&g_vhost_core_mask, i, true);
+	}
+out:
+	init_cb(ret);
+}
+
+void
+spdk_vhost_scsi_fini(spdk_vhost_fini_cb fini_cb)
+{
+	g_fini_cb = fini_cb;
+
+	vhost_user_fini(vhost_fini);
+}
+
+void
+spdk_vhost_blk_fini(spdk_vhost_fini_cb fini_cb)
 {
 	g_fini_cb = fini_cb;
 
@@ -291,7 +318,7 @@ vhost_user_config_json(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *
 }
 
 void
-spdk_vhost_config_json(struct spdk_json_write_ctx *w)
+spdk_vhost_scsi_config_json(struct spdk_json_write_ctx *w)
 {
 	struct spdk_vhost_dev *vdev;
 
@@ -300,7 +327,28 @@ spdk_vhost_config_json(struct spdk_json_write_ctx *w)
 	spdk_vhost_lock();
 	for (vdev = spdk_vhost_dev_next(NULL); vdev != NULL;
 	     vdev = spdk_vhost_dev_next(vdev)) {
-		vhost_user_config_json(vdev, w);
+		if (vdev->backend->type == VHOST_BACKEND_SCSI) {
+			vhost_user_config_json(vdev, w);
+		}
+	}
+	spdk_vhost_unlock();
+
+	spdk_json_write_array_end(w);
+}
+
+void
+spdk_vhost_blk_config_json(struct spdk_json_write_ctx *w)
+{
+	struct spdk_vhost_dev *vdev;
+
+	spdk_json_write_array_begin(w);
+
+	spdk_vhost_lock();
+	for (vdev = spdk_vhost_dev_next(NULL); vdev != NULL;
+	     vdev = spdk_vhost_dev_next(vdev)) {
+		if (vdev->backend->type == VHOST_BACKEND_BLK) {
+			vhost_user_config_json(vdev, w);
+		}
 	}
 	spdk_vhost_unlock();
 

lib/vhost/vhost_internal.h

@@ -531,8 +531,7 @@ int vhost_user_dev_register(struct spdk_vhost_dev *vdev, const char *name,
 			    struct spdk_cpuset *cpumask, const struct spdk_vhost_user_dev_backend *user_backend);
 int vhost_user_dev_unregister(struct spdk_vhost_dev *vdev);
 int vhost_user_init(void);
-typedef void (*vhost_fini_cb)(void *ctx);
+void vhost_user_fini(spdk_vhost_fini_cb vhost_cb);
-void vhost_user_fini(vhost_fini_cb vhost_cb);
 
 struct spdk_vhost_blk_task {
 	struct spdk_bdev_io *bdev_io;

lib/vhost/vhost_scsi.c

@@ -885,7 +885,6 @@ spdk_vhost_scsi_dev_construct(const char *name, const char *cpumask)
 	rc = vhost_dev_register(&svdev->vdev, name, cpumask,
 				&spdk_vhost_scsi_device_backend,
 				&spdk_vhost_scsi_user_device_backend);
-
 	if (rc) {
 		free(svdev);
 		spdk_vhost_unlock();

mk/spdk.lib_deps.mk

@@ -183,5 +183,6 @@ DEPDIRS-event_nvmf := init nvmf event_bdev event_scheduler event_sock thread log
 DEPDIRS-event_scsi := init scsi event_bdev
 
 DEPDIRS-event_iscsi := init iscsi event_scheduler event_scsi event_sock
-DEPDIRS-event_vhost := init vhost event_scheduler event_scsi
+DEPDIRS-event_vhost_blk := init vhost
+DEPDIRS-event_vhost_scsi := init vhost event_scheduler event_scsi
 DEPDIRS-event_sock := init sock

module/event/subsystems/Makefile

@@ -40,7 +40,7 @@ ifeq ($(OS),Linux)
 DIRS-y += nbd
 endif
 
-DIRS-$(CONFIG_VHOST) += vhost
+DIRS-$(CONFIG_VHOST) += vhost_blk vhost_scsi
 
 # These dependencies are not based specifically on symbols, but rather
 # the subsystem dependency tree defined within the event subsystem C files
@@ -51,7 +51,7 @@ DEPDIRS-iscsi := scsi
 DEPDIRS-nbd := bdev
 DEPDIRS-nvmf := bdev
 DEPDIRS-scsi := bdev
-DEPDIRS-vhost := scsi
+DEPDIRS-vhost_scsi := scsi
 
 .PHONY: all clean $(DIRS-y)
 

module/event/subsystems/vhost_blk/Makefile

@@ -34,11 +34,11 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-SO_VER := 4
+SO_VER := 1
 SO_MINOR := 0
 
-C_SRCS = vhost.c
+C_SRCS = vhost_blk.c
-LIBNAME = event_vhost
+LIBNAME = event_vhost_blk
 
 SPDK_MAP_FILE = $(SPDK_ROOT_DIR)/mk/spdk_blank.map
 

module/event/subsystems/vhost_blk/vhost_blk.c

@@ -38,35 +38,35 @@
 #include "spdk_internal/init.h"
 
 static void
-vhost_subsystem_init_done(int rc)
+vhost_blk_subsystem_init_done(int rc)
 {
 	spdk_subsystem_init_next(rc);
 }
 
 static void
-vhost_subsystem_init(void)
+vhost_blk_subsystem_init(void)
 {
-	spdk_vhost_init(vhost_subsystem_init_done);
+	spdk_vhost_blk_init(vhost_blk_subsystem_init_done);
 }
 
 static void
-vhost_subsystem_fini_done(void)
+vhost_blk_subsystem_fini_done(void)
 {
 	spdk_subsystem_fini_next();
 }
 
 static void
-vhost_subsystem_fini(void)
+vhost_blk_subsystem_fini(void)
 {
-	spdk_vhost_fini(vhost_subsystem_fini_done);
+	spdk_vhost_blk_fini(vhost_blk_subsystem_fini_done);
 }
 
-static struct spdk_subsystem g_spdk_subsystem_vhost = {
+static struct spdk_subsystem g_spdk_subsystem_vhost_blk = {
-	.name = "vhost",
+	.name = "vhost_blk",
-	.init = vhost_subsystem_init,
+	.init = vhost_blk_subsystem_init,
-	.fini = vhost_subsystem_fini,
+	.fini = vhost_blk_subsystem_fini,
-	.write_config_json = spdk_vhost_config_json,
+	.write_config_json = spdk_vhost_blk_config_json,
 };
 
-SPDK_SUBSYSTEM_REGISTER(g_spdk_subsystem_vhost);
+SPDK_SUBSYSTEM_REGISTER(g_spdk_subsystem_vhost_blk);
-SPDK_SUBSYSTEM_DEPEND(vhost, scsi)
+SPDK_SUBSYSTEM_DEPEND(vhost_blk, bdev)

module/event/subsystems/vhost_scsi/Makefile

@@ -0,0 +1,45 @@
+#
+#  BSD LICENSE
+#
+#  Copyright (c) Intel Corporation.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+SO_VER := 1
+SO_MINOR := 0
+
+C_SRCS = vhost_scsi.c
+LIBNAME = event_vhost_scsi
+
+SPDK_MAP_FILE = $(SPDK_ROOT_DIR)/mk/spdk_blank.map
+
+include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

module/event/subsystems/vhost_scsi/vhost_scsi.c

@@ -0,0 +1,72 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/vhost.h"
+
+#include "spdk_internal/init.h"
+
+static void
+vhost_scsi_subsystem_init_done(int rc)
+{
+	spdk_subsystem_init_next(rc);
+}
+
+static void
+vhost_scsi_subsystem_init(void)
+{
+	spdk_vhost_scsi_init(vhost_scsi_subsystem_init_done);
+}
+
+static void
+vhost_scsi_subsystem_fini_done(void)
+{
+	spdk_subsystem_fini_next();
+}
+
+static void
+vhost_scsi_subsystem_fini(void)
+{
+	spdk_vhost_scsi_fini(vhost_scsi_subsystem_fini_done);
+}
+
+static struct spdk_subsystem g_spdk_subsystem_vhost_scsi = {
+	.name = "vhost_scsi",
+	.init = vhost_scsi_subsystem_init,
+	.fini = vhost_scsi_subsystem_fini,
+	.write_config_json = spdk_vhost_scsi_config_json,
+};
+
+SPDK_SUBSYSTEM_REGISTER(g_spdk_subsystem_vhost_scsi);
+SPDK_SUBSYSTEM_DEPEND(vhost_scsi, scsi)

python/spdk/spdkcli/ui_root.py

@@ -37,7 +37,7 @@ class UIRoot(UINode):
         self._children = set([])
         UIBdevs(self)
         UILvolStores(self)
-        if self.has_subsystem("vhost"):
+        if self.has_subsystem("vhost_scsi") or self.has_subsystem("vhost_blk"):
             UIVhosts(self)
         if self.has_subsystem("nvmf"):
             UINVMf(self)

test/json_config/clear_config.py

@@ -139,7 +139,7 @@ def clear_interface_subsystem(args, interface_config):
     pass
 
 
-def clear_vhost_subsystem(args, vhost_config):
+def clear_vhost_scsi_subsystem(args, vhost_config):
     for vhost in reversed(vhost_config):
         if 'method' in vhost:
             method = vhost['method']
@@ -147,8 +147,15 @@ def clear_vhost_subsystem(args, vhost_config):
                 args.client.call("vhost_scsi_controller_remove_target",
                                  {"ctrlr": vhost['params']['ctrlr'],
                                   "scsi_target_num": vhost['params']['scsi_target_num']})
-            elif method in ['vhost_create_scsi_controller', 'vhost_create_blk_controller',
+            elif method in ['vhost_create_scsi_controller']:
-                            'vhost_create_nvme_controller']:
+                args.client.call("vhost_delete_controller", {'ctrlr': vhost['params']['ctrlr']})
+
+
+def clear_vhost_blk_subsystem(args, vhost_config):
+    for vhost in reversed(vhost_config):
+        if 'method' in vhost:
+            method = vhost['method']
+            if method in ['vhost_create_blk_controller']:
                 args.client.call("vhost_delete_controller", {'ctrlr': vhost['params']['ctrlr']})
 
 

test/spdkcli/vhost.sh

@@ -74,7 +74,8 @@ timing_exit spdkcli_check_match
 timing_enter spdkcli_save_config
 $spdkcli_job "'save_config $testdir/config.json'
 'save_subsystem_config $testdir/config_bdev.json bdev'
-'save_subsystem_config $testdir/config_vhost.json vhost'
+'save_subsystem_config $testdir/config_vhost_scsi.json vhost_scsi'
+'save_subsystem_config $testdir/config_vhost_blk.json vhost_blk'
 "
 timing_exit spdkcli_save_config
 
@@ -130,7 +131,8 @@ $spdk_clear_config_py clear_config
 # FIXME: remove this sleep when NVMe driver will be fixed to wait for reset to complete
 sleep 2
 $spdkcli_job "'load_subsystem_config $testdir/config_bdev.json'
-'load_subsystem_config $testdir/config_vhost.json'
+'load_subsystem_config $testdir/config_vhost_scsi.json'
+'load_subsystem_config $testdir/config_vhost_blk.json'
 '/lvol_stores create lvs0 Malloc0' 'lvs0' True
 '/lvol_stores create lvs1 Malloc5' 'lvs1' True
 '/bdevs/logical_volume create lvol0 16 lvs0' 'lvs0/lvol0' True
@@ -140,7 +142,8 @@ check_match
 $spdk_clear_config_py clear_config
 rm -f $testdir/config.json
 rm -f $testdir/config_bdev.json
-rm -f $testdir/config_vhost.json
+rm -f $testdir/config_vhost_scsi.json
+rm -f $testdir/config_vhost_blk.json
 rm -f "$sample_aio" "$sample_aio2"
 timing_exit spdkcli_load_config