env: add spdk_env_dpdk_post_init

Based on requirement of initializing SPDK env
from a DPDK application, relative to spdk_env_init,
add spdk_env_dpdk_post_init for calling after rte_env_init.
More details, visit
https://github.com/spdk/spdk/issues/529

Change-Id: I6fda1593e0296ef93b705e31cc76bcd0d248673a
Signed-off-by: Xiaodong Liu <xiaodong.liu@intel.com>
Reviewed-on: https://review.gerrithub.io/c/437225
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## v19.01: (Upcoming Release)
 
+### environment
+
+A new public header file env_dpdk.h has been introduced, and function spdk_env_dpdk_post_init
+is added into it. If user is using DPDK, and already called rte_eal_init, then include
+include/spdk/env_dpdk.h, and call spdk_env_dpdk_post_init() instead of spdk_env_init.
+
 ### util
 
 A new uuid API `spdk_uuid_copy` was added to make a copy of the source uuid.

include/spdk/env_dpdk.h

@@ -0,0 +1,59 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * Encapsulated DPDK specific dependencies
+ */
+
+#ifndef SPDK_ENV_DPDK_H
+#define SPDK_ENV_DPDK_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Initialize the environment library after DPDK env is already initialized.
+ * If DPDK's rte_eal_init is already called, this function must be called
+ * instead of spdk_env_init, prior to using any other functions in SPDK
+ * env library.
+ *
+ * \return 0 on success, or negative errno on failure.
+ */
+int spdk_env_dpdk_post_init(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

lib/env_dpdk/init.c

@@ -36,6 +36,7 @@
 #include "env_internal.h"
 
 #include "spdk/version.h"
+#include "spdk/env_dpdk.h"
 
 #include <rte_config.h>
 #include <rte_eal.h>
@@ -342,7 +343,25 @@ spdk_build_eal_cmdline(const struct spdk_env_opts *opts)
 	return argcount;
 }
 
-int spdk_env_init(const struct spdk_env_opts *opts)
+int
+spdk_env_dpdk_post_init(void)
+{
+	spdk_pci_init();
+
+	if (spdk_mem_map_init() < 0) {
+		fprintf(stderr, "Failed to allocate mem_map\n");
+		return -1;
+	}
+	if (spdk_vtophys_init() < 0) {
+		fprintf(stderr, "Failed to initialize vtophys\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+spdk_env_init(const struct spdk_env_opts *opts)
 {
 	char **dpdk_args = NULL;
 	int i, rc;
@@ -396,16 +415,5 @@ int spdk_env_init(const struct spdk_env_opts *opts)
 		spdk_env_unlink_shared_files();
 	}
 
-	spdk_pci_init();
+	return spdk_env_dpdk_post_init();
-
-	if (spdk_mem_map_init() < 0) {
-		fprintf(stderr, "Failed to allocate mem_map\n");
-		return -1;
-	}
-	if (spdk_vtophys_init() < 0) {
-		fprintf(stderr, "Failed to initialize vtophys\n");
-		return -1;
-	}
-
-	return 0;
 }

test/env/Makefile

@@ -39,7 +39,7 @@ ENV_NAME := $(notdir $(CONFIG_ENV))
 DIRS-y = vtophys
 
 ifeq ($(ENV_NAME),env_dpdk)
-DIRS-y += memory pci
+DIRS-y += env_dpdk_post_init memory pci
 endif
 
 .PHONY: all clean $(DIRS-y)

test/env/env.sh

@@ -20,5 +20,9 @@ timing_enter pci
 $testdir/pci/pci_ut
 timing_exit pci
 
+timing_enter env_dpdk_post_init
+$testdir/env_dpdk_post_init/env_dpdk_post_init
+timing_exit env_dpdk_post_init
+
 report_test_completion "env"
 timing_exit env

test/env/env_dpdk_post_init/.gitignore

@@ -0,0 +1 @@
+env_dpdk_post_init

test/env/env_dpdk_post_init/Makefile

@@ -0,0 +1,40 @@
+#
+#  BSD LICENSE
+#
+#  Copyright (c) Intel Corporation.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+CFLAGS += $(ENV_CFLAGS)
+APP = env_dpdk_post_init
+
+include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk

test/env/env_dpdk_post_init/env_dpdk_post_init.c

@@ -0,0 +1,128 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk/nvme.h"
+#include "spdk/env.h"
+#include "spdk/env_dpdk.h"
+#include <rte_eal.h>
+
+#define MAX_DEVS 64
+
+struct dev {
+	struct spdk_nvme_ctrlr				*ctrlr;
+	struct spdk_nvme_ns				*ns;
+	struct spdk_nvme_qpair				*qpair;
+	char						name[SPDK_NVMF_TRADDR_MAX_LEN + 1];
+};
+
+static struct dev g_nvme_devs[MAX_DEVS];
+static int g_num_devs = 0;
+static int g_failed = 0;
+
+static bool
+probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+	 struct spdk_nvme_ctrlr_opts *opts)
+{
+	printf("Attaching to %s\n", trid->traddr);
+
+	return true;
+}
+
+static void
+attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+	  struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
+{
+	struct dev *dev;
+	uint32_t nsid;
+
+	/* add to dev list */
+	dev = &g_nvme_devs[g_num_devs++];
+	if (g_num_devs >= MAX_DEVS) {
+		return;
+	}
+
+	dev->ctrlr = ctrlr;
+	nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
+	dev->ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
+	if (dev->ns == NULL) {
+		g_failed = 1;
+		return;
+	}
+	dev->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, NULL, 0);
+	if (dev->qpair == NULL) {
+		g_failed = 1;
+		return;
+	}
+
+	snprintf(dev->name, sizeof(dev->name), "%s",
+		 trid->traddr);
+
+	printf("Attached to %s\n", dev->name);
+}
+
+int
+main(int argc, char **argv)
+{
+	int ret;
+	int i;
+
+	printf("Starting DPDK initialization...\n");
+	ret = rte_eal_init(argc, argv);
+	if (ret < 0) {
+		fprintf(stderr, "Failed to initialize DPDK\n");
+		return -1;
+	}
+
+	printf("Starting SPDK post initialization...\n");
+	ret = spdk_env_dpdk_post_init();
+	if (ret < 0) {
+		fprintf(stderr, "Failed to initialize SPDK\n");
+		return -1;
+	}
+
+	printf("SPDK NVMe probe\n");
+	if (spdk_nvme_probe(NULL, NULL, probe_cb, attach_cb, NULL) != 0) {
+		fprintf(stderr, "spdk_nvme_probe() failed\n");
+		return 1;
+	}
+
+	printf("Cleaning up...\n");
+	for (i = 0; i < g_num_devs; i++) {
+		struct dev *dev = &g_nvme_devs[i];
+		spdk_nvme_detach(dev->ctrlr);
+	}
+
+	return g_failed;
+}