configure: add option not to use the internal rte_vhost copy

It's disabled by default, so no functionality is changed yet.
The intention is to use the upstream rte_vhost from DPDK,
which - starting from DPDK 19.05 - is finally capable of
running with storage device backends.

SPDK still requires a lot of changes in order to support
that upstream version, but the most fundamental change is
dropping vhost-nvme support. It'll remain usable only with
the internal rte_vhost copy and with the upstream rte_vhost
it simply won't be compiled. This allows us at least to
compile with that upstream rte_vhost, where we can pursue
adding the full integration.

Change-Id: Ic8bc5497c4d77bfef77c57f3d5a1f8681ffb6d1f
Signed-off-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/446082
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>

CONFIG

@@ -94,6 +94,7 @@ CONFIG_RBD=n
 
 # Build vhost library.
 CONFIG_VHOST=y
+CONFIG_VHOST_INTERNAL_LIB=y
 
 # Build vhost initiator (Virtio) driver.
 CONFIG_VIRTIO=y

app/spdk_tgt/Makefile

@@ -43,7 +43,10 @@ SPDK_LIB_LIST = $(ALL_MODULES_LIST)
 
 ifeq ($(OS),Linux)
 ifeq ($(CONFIG_VHOST),y)
-SPDK_LIB_LIST += vhost rte_vhost event_vhost
+SPDK_LIB_LIST += vhost event_vhost
+ifeq ($(CONFIG_VHOST_INTERNAL_LIB),y)
+SPDK_LIB_LIST += rte_vhost
+endif
 endif
 endif
 

app/vhost/Makefile

@@ -40,7 +40,12 @@ APP = vhost
 C_SRCS := vhost.c
 
 SPDK_LIB_LIST = $(ALL_MODULES_LIST)
-SPDK_LIB_LIST += vhost rte_vhost event_vhost
+SPDK_LIB_LIST += vhost event_vhost
+
+ifeq ($(CONFIG_VHOST_INTERNAL_LIB),y)
+SPDK_LIB_LIST += rte_vhost
+endif
+
 SPDK_LIB_LIST += event_bdev event_copy event_net event_scsi event
 SPDK_LIB_LIST += jsonrpc json rpc bdev_rpc bdev scsi copy trace conf
 SPDK_LIB_LIST += thread util log log_rpc trace_rpc app_rpc

configure

@@ -48,6 +48,8 @@ function usage()
 	echo "                           example: /usr/src/fio"
 	echo " vhost                     Required to build vhost target."
 	echo "                           No path required."
+	echo " internal-vhost-lib        Use the internal copy of rte_vhost."
+	echo "                           No path required."
 	echo " virtio                    Required to build vhost initiator (Virtio) bdev module."
 	echo "                           No path required."
 	echo " pmdk                      Required to build persistent memory bdev."
@@ -223,6 +225,12 @@ for i in "$@"; do
 		--without-vhost)
 			CONFIG[VHOST]=n
 			;;
+		--with-internal-vhost-lib)
+			CONFIG[VHOST_INTERNAL_LIB]=y
+			;;
+		--without-internal-vhost-lib)
+			CONFIG[VHOST_INTERNAL_LIB]=n
+			;;
 		--with-virtio)
 			CONFIG[VIRTIO]=y
 			;;

lib/env_dpdk/env.mk

@@ -100,6 +100,15 @@ ifneq (, $(wildcard $(DPDK_ABS_DIR)/lib/librte_kvargs.*))
 DPDK_LIB_LIST += rte_kvargs
 endif
 
+ifneq ($(CONFIG_VHOST_INTERNAL_LIB),y)
+ifneq (, $(wildcard $(DPDK_ABS_DIR)/lib/librte_vhost.*))
+DPDK_LIB_LIST += rte_vhost rte_net rte_hash rte_mbuf
+ifneq ($(DPDK_FRAMEWORK),y)
+DPDK_LIB_LIST += rte_cryptodev
+endif
+endif
+endif
+
 DPDK_LIB = $(DPDK_LIB_LIST:%=$(DPDK_ABS_DIR)/lib/lib%$(DPDK_LIB_EXT))
 
 # SPDK memory registration requires experimental (deprecated) rte_memory API for DPDK 18.05

lib/vhost/Makefile

@@ -35,13 +35,16 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
 CFLAGS += -I.
-CFLAGS += -Irte_vhost
 CFLAGS += $(ENV_CFLAGS)
 
-C_SRCS = vhost.c vhost_rpc.c vhost_scsi.c vhost_blk.c vhost_nvme.c
+C_SRCS = vhost.c vhost_rpc.c vhost_scsi.c vhost_blk.c
+
+ifeq ($(CONFIG_VHOST_INTERNAL_LIB),y)
+C_SRCS += vhost_nvme.c
+DIRS-y += rte_vhost
+CFLAGS := -Irte_vhost $(CFLAGS)
+endif
 
 LIBNAME = vhost
 
-DIRS-y += rte_vhost
-
 include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

lib/vhost/vhost.c

@@ -69,21 +69,26 @@ static int new_connection(int vid);
 static int start_device(int vid);
 static void stop_device(int vid);
 static void destroy_connection(int vid);
+
+#ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB
 static int get_config(int vid, uint8_t *config, uint32_t len);
 static int set_config(int vid, uint8_t *config, uint32_t offset,
 		      uint32_t size, uint32_t flags);
+#endif
 
 const struct vhost_device_ops g_spdk_vhost_ops = {
 	.new_device =  start_device,
 	.destroy_device = stop_device,
-	.get_config = get_config,
-	.set_config = set_config,
 	.new_connection = new_connection,
 	.destroy_connection = destroy_connection,
+#ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB
+	.get_config = get_config,
+	.set_config = set_config,
 	.vhost_nvme_admin_passthrough = spdk_vhost_nvme_admin_passthrough,
 	.vhost_nvme_set_cq_call = spdk_vhost_nvme_set_cq_call,
 	.vhost_nvme_get_cap = spdk_vhost_nvme_get_cap,
 	.vhost_nvme_set_bar_mr = spdk_vhost_nvme_set_bar_mr,
+#endif
 };
 
 static TAILQ_HEAD(, spdk_vhost_dev) g_spdk_vhost_devices = TAILQ_HEAD_INITIALIZER(
@@ -1142,6 +1147,7 @@ out:
 	return rc;
 }
 
+#ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB
 static int
 get_config(int vid, uint8_t *config, uint32_t len)
 {
@@ -1189,6 +1195,7 @@ out:
 	pthread_mutex_unlock(&g_spdk_vhost_mutex);
 	return rc;
 }
+#endif
 
 int
 spdk_vhost_set_socket_path(const char *basename)
@@ -1414,11 +1421,13 @@ spdk_vhost_init(void)
 		return -1;
 	}
 
+#ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB
 	ret = spdk_vhost_nvme_controller_construct();
 	if (ret != 0) {
 		SPDK_ERRLOG("Cannot construct vhost NVMe controllers\n");
 		return -1;
 	}
+#endif
 
 	return 0;
 }

lib/vhost/vhost_internal.h

@@ -41,6 +41,7 @@
 #include "spdk_internal/log.h"
 #include "spdk/event.h"
 #include "spdk/rpc.h"
+#include "spdk/config.h"
 
 #define SPDK_CACHE_LINE_SIZE RTE_CACHE_LINE_SIZE
 
@@ -348,6 +349,8 @@ void spdk_vhost_free_reactor(uint32_t lcore);
 uint32_t spdk_vhost_allocate_reactor(struct spdk_cpuset *cpumask);
 
 int spdk_remove_vhost_controller(struct spdk_vhost_dev *vdev);
+
+#ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB
 int spdk_vhost_nvme_admin_passthrough(int vid, void *cmd, void *cqe, void *buf);
 int spdk_vhost_nvme_set_cq_call(int vid, uint16_t qid, int fd);
 int spdk_vhost_nvme_set_bar_mr(int vid, void *bar_addr, uint64_t bar_size);
@@ -357,5 +360,6 @@ int spdk_vhost_nvme_dev_construct(const char *name, const char *cpumask, uint32_
 int spdk_vhost_nvme_dev_remove(struct spdk_vhost_dev *vdev);
 int spdk_vhost_nvme_dev_add_ns(struct spdk_vhost_dev *vdev,
 			       const char *bdev_name);
+#endif
 
 #endif /* SPDK_VHOST_INTERNAL_H */

lib/vhost/vhost_rpc.c

@@ -544,6 +544,8 @@ invalid:
 SPDK_RPC_REGISTER("set_vhost_controller_coalescing", spdk_rpc_set_vhost_controller_coalescing,
 		  SPDK_RPC_RUNTIME)
 
+#ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB
+
 struct rpc_vhost_nvme_ctrlr {
 	char *ctrlr;
 	uint32_t io_queues;
@@ -668,5 +670,6 @@ invalid:
 }
 SPDK_RPC_REGISTER("add_vhost_nvme_ns", spdk_rpc_add_vhost_nvme_ns, SPDK_RPC_RUNTIME)
 
+#endif /* SPDK_CONFIG_VHOST_INTERNAL_LIB */
 
 SPDK_LOG_REGISTER_COMPONENT("vhost_rpc", SPDK_LOG_VHOST_RPC)

test/unit/lib/vhost/vhost.c/Makefile

@@ -32,8 +32,12 @@
 #
 
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../../..)
+include $(SPDK_ROOT_DIR)/mk/config.mk
 
+ifeq ($(CONFIG_VHOST_INTERNAL_LIB),y)
 CFLAGS += -I$(SPDK_ROOT_DIR)/lib/vhost/rte_vhost
+endif
+
 CFLAGS += $(ENV_CFLAGS)
 TEST_FILE = vhost_ut.c