lib/vhost: remove dependency on lib/event

In current implementation, lib/vhost assumes that it always runs
together with lib/event, and would call lib/event's functions in
vhost.c. This is not necessary and make program unable to
create/destroy vhost module without init/fini the whole spdk env. It
would cause problems when program runs with vhost and other spdk
components together.

In this patch, we remove the dependency of lib/vhost on lib/event by
adding a global vairable g_vhost_core_mask so that it could handle
core mask by itself.

Signed-off-by: Kyle Zhang <kyle@smartx.com>
Change-Id: I38ceb92ac39b6980955346fda41e968aaead863d
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1204
Community-CI: Mellanox Build Bot
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

lib/vhost/vhost.c

@@ -42,6 +42,8 @@
 #include "spdk/vhost.h"
 #include "vhost_internal.h"
 
+static struct spdk_cpuset g_vhost_core_mask;
+
 /* Path to folder where character device will be created. Can be set by user. */
 static char dev_dirname[PATH_MAX] = "";
 
@@ -782,25 +784,47 @@ vhost_parse_core_mask(const char *mask, struct spdk_cpuset *cpumask)
 	}
 
 	if (mask == NULL) {
-		spdk_cpuset_copy(cpumask, spdk_app_get_core_mask());
+		spdk_cpuset_copy(cpumask, &g_vhost_core_mask);
 		return 0;
 	}
 
-	rc = spdk_app_parse_core_mask(mask, cpumask);
+	rc = spdk_cpuset_parse(cpumask, mask);
 	if (rc < 0) {
 		SPDK_ERRLOG("invalid cpumask %s\n", mask);
 		return -1;
 	}
 
+	spdk_cpuset_and(cpumask, &g_vhost_core_mask);
+
 	if (spdk_cpuset_count(cpumask) == 0) {
-		SPDK_ERRLOG("no cpu is selected among reactor mask(=%s)\n",
+		SPDK_ERRLOG("no cpu is selected among core mask(=%s)\n",
-			    spdk_cpuset_fmt(spdk_app_get_core_mask()));
+			    spdk_cpuset_fmt(&g_vhost_core_mask));
 		return -1;
 	}
 
 	return 0;
 }
 
+static void
+vhost_setup_core_mask(void *ctx)
+{
+	struct spdk_thread *thread = spdk_get_thread();
+	spdk_cpuset_or(&g_vhost_core_mask, spdk_thread_get_cpumask(thread));
+}
+
+static void
+vhost_setup_core_mask_done(void *ctx)
+{
+	spdk_vhost_init_cb init_cb = ctx;
+
+	if (spdk_cpuset_count(&g_vhost_core_mask) == 0) {
+		init_cb(-ECHILD);
+		return;
+	}
+
+	init_cb(0);
+}
+
 static void
 vhost_dev_thread_exit(void *arg1)
 {
@@ -822,8 +846,8 @@ vhost_dev_register(struct spdk_vhost_dev *vdev, const char *name, const char *ma
 	}
 
 	if (vhost_parse_core_mask(mask_str, &cpumask) != 0) {
-		SPDK_ERRLOG("cpumask %s is invalid (app mask is 0x%s)\n",
+		SPDK_ERRLOG("cpumask %s is invalid (core mask is 0x%s)\n",
-			    mask_str, spdk_cpuset_fmt(spdk_app_get_core_mask()));
+			    mask_str, spdk_cpuset_fmt(&g_vhost_core_mask));
 		return -EINVAL;
 	}
 
@@ -1502,6 +1526,13 @@ spdk_vhost_init(spdk_vhost_init_cb init_cb)
 	}
 #endif
 
+	spdk_cpuset_zero(&g_vhost_core_mask);
+
+	/* iterate threads instead of using SPDK_ENV_FOREACH_CORE to ensure that threads are really
+	 * created.
+	 */
+	spdk_for_each_thread(vhost_setup_core_mask, init_cb, vhost_setup_core_mask_done);
+	return;
 out:
 	init_cb(ret);
 }
@@ -1521,6 +1552,8 @@ _spdk_vhost_fini(void *arg1)
 	}
 	spdk_vhost_unlock();
 
+	spdk_cpuset_zero(&g_vhost_core_mask);
+
 	/* All devices are removed now. */
 	sem_destroy(&g_dpdk_sem);
 

lib/vhost/vhost_internal.h

@@ -41,7 +41,6 @@
 
 #include "spdk_internal/vhost_user.h"
 #include "spdk_internal/log.h"
-#include "spdk/event.h"
 #include "spdk/util.h"
 #include "spdk/rpc.h"
 #include "spdk/config.h"

mk/spdk.lib_deps.mk

@@ -79,7 +79,7 @@ DEPDIRS-nvmf := log sock util nvme thread $(JSON_LIBS) trace bdev
 DEPDIRS-scsi := log util thread $(JSON_LIBS) trace bdev
 
 DEPDIRS-iscsi := log sock util conf thread $(JSON_LIBS) trace event scsi
-DEPDIRS-vhost = log util conf thread $(JSON_LIBS) bdev event scsi
+DEPDIRS-vhost = log util conf thread $(JSON_LIBS) bdev scsi
 ifeq ($(CONFIG_VHOST_INTERNAL_LIB),y)
 DEPDIRS-vhost += rte_vhost
 endif

test/unit/lib/vhost/vhost.c/vhost_ut.c

@@ -57,33 +57,6 @@ DEFINE_STUB(rte_vhost_vring_call, int, (int vid, uint16_t vring_idx), 0);
 DEFINE_STUB_V(rte_vhost_log_used_vring, (int vid, uint16_t vring_idx,
 		uint64_t offset, uint64_t len));
 
-static struct spdk_cpuset *g_app_core_mask;
-struct spdk_cpuset *spdk_app_get_core_mask(void)
-{
-	if (g_app_core_mask == NULL) {
-		g_app_core_mask = spdk_cpuset_alloc();
-		spdk_cpuset_set_cpu(g_app_core_mask, 0, true);
-	}
-	return g_app_core_mask;
-}
-
-int
-spdk_app_parse_core_mask(const char *mask, struct spdk_cpuset *cpumask)
-{
-	int ret;
-	struct spdk_cpuset *validmask;
-
-	ret = spdk_cpuset_parse(cpumask, mask);
-	if (ret < 0) {
-		return ret;
-	}
-
-	validmask = spdk_app_get_core_mask();
-	spdk_cpuset_and(cpumask, validmask);
-
-	return 0;
-}
-
 DEFINE_STUB(rte_vhost_get_mem_table, int, (int vid, struct rte_vhost_memory **mem), 0);
 DEFINE_STUB(rte_vhost_get_negotiated_features, int, (int vid, uint64_t *features), 0);
 DEFINE_STUB(rte_vhost_get_vhost_vring, int,
@@ -196,6 +169,8 @@ desc_to_iov_test(void)
 	struct vring_desc desc;
 	int rc;
 
+	spdk_cpuset_set_cpu(&g_vhost_core_mask, 0, true);
+
 	rc = alloc_vdev(&vdev, "vdev_name_0", "0x1");
 	SPDK_CU_ASSERT_FATAL(rc == 0 && vdev);
 	start_vdev(vdev);
@@ -277,7 +252,7 @@ create_controller_test(void)
 	int ret;
 	char long_name[PATH_MAX];
 
-	/* NOTE: spdk_app_get_core_mask stub always sets coremask 0x01 */
+	spdk_cpuset_set_cpu(&g_vhost_core_mask, 0, true);
 
 	/* Create device with no name */
 	ret = alloc_vdev(&vdev, NULL, "0x1");