From 81551144be0b1b4b35c5966e2c651ee8df87a5b4 Mon Sep 17 00:00:00 2001 From: Xiaodong Liu Date: Thu, 13 Dec 2018 23:54:01 +0800 Subject: [PATCH] env: add spdk_env_dpdk_post_init Based on requirement of initializing SPDK env from a DPDK application, relative to spdk_env_init, add spdk_env_dpdk_post_init for calling after rte_env_init. More details, visit https://github.com/spdk/spdk/issues/529 Change-Id: I6fda1593e0296ef93b705e31cc76bcd0d248673a Signed-off-by: Xiaodong Liu Reviewed-on: https://review.gerrithub.io/c/437225 Tested-by: SPDK CI Jenkins Reviewed-by: Darek Stojaczyk Reviewed-by: Shuhei Matsumoto Reviewed-by: Jim Harris --- CHANGELOG.md | 6 + include/spdk/env_dpdk.h | 59 ++++++++ lib/env_dpdk/init.c | 34 +++-- test/env/Makefile | 2 +- test/env/env.sh | 4 + test/env/env_dpdk_post_init/.gitignore | 1 + test/env/env_dpdk_post_init/Makefile | 40 ++++++ .../env_dpdk_post_init/env_dpdk_post_init.c | 128 ++++++++++++++++++ 8 files changed, 260 insertions(+), 14 deletions(-) create mode 100644 include/spdk/env_dpdk.h create mode 100644 test/env/env_dpdk_post_init/.gitignore create mode 100644 test/env/env_dpdk_post_init/Makefile create mode 100644 test/env/env_dpdk_post_init/env_dpdk_post_init.c diff --git a/CHANGELOG.md b/CHANGELOG.md index ca1561a3e..25016941c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## v19.01: (Upcoming Release) +### environment + +A new public header file env_dpdk.h has been introduced, and function spdk_env_dpdk_post_init +is added into it. If user is using DPDK, and already called rte_eal_init, then include +include/spdk/env_dpdk.h, and call spdk_env_dpdk_post_init() instead of spdk_env_init. + ### util A new uuid API `spdk_uuid_copy` was added to make a copy of the source uuid. diff --git a/include/spdk/env_dpdk.h b/include/spdk/env_dpdk.h new file mode 100644 index 000000000..40e76f65d --- /dev/null +++ b/include/spdk/env_dpdk.h @@ -0,0 +1,59 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * Encapsulated DPDK specific dependencies + */ + +#ifndef SPDK_ENV_DPDK_H +#define SPDK_ENV_DPDK_H + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Initialize the environment library after DPDK env is already initialized. + * If DPDK's rte_eal_init is already called, this function must be called + * instead of spdk_env_init, prior to using any other functions in SPDK + * env library. + * + * \return 0 on success, or negative errno on failure. + */ +int spdk_env_dpdk_post_init(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/lib/env_dpdk/init.c b/lib/env_dpdk/init.c index ddf04ada6..f62136197 100644 --- a/lib/env_dpdk/init.c +++ b/lib/env_dpdk/init.c @@ -36,6 +36,7 @@ #include "env_internal.h" #include "spdk/version.h" +#include "spdk/env_dpdk.h" #include #include @@ -342,7 +343,25 @@ spdk_build_eal_cmdline(const struct spdk_env_opts *opts) return argcount; } -int spdk_env_init(const struct spdk_env_opts *opts) +int +spdk_env_dpdk_post_init(void) +{ + spdk_pci_init(); + + if (spdk_mem_map_init() < 0) { + fprintf(stderr, "Failed to allocate mem_map\n"); + return -1; + } + if (spdk_vtophys_init() < 0) { + fprintf(stderr, "Failed to initialize vtophys\n"); + return -1; + } + + return 0; +} + +int +spdk_env_init(const struct spdk_env_opts *opts) { char **dpdk_args = NULL; int i, rc; @@ -396,16 +415,5 @@ int spdk_env_init(const struct spdk_env_opts *opts) spdk_env_unlink_shared_files(); } - spdk_pci_init(); - - if (spdk_mem_map_init() < 0) { - fprintf(stderr, "Failed to allocate mem_map\n"); - return -1; - } - if (spdk_vtophys_init() < 0) { - fprintf(stderr, "Failed to initialize vtophys\n"); - return -1; - } - - return 0; + return spdk_env_dpdk_post_init(); } diff --git a/test/env/Makefile b/test/env/Makefile index d90696a10..c5bbebdb6 100644 --- a/test/env/Makefile +++ b/test/env/Makefile @@ -39,7 +39,7 @@ ENV_NAME := $(notdir $(CONFIG_ENV)) DIRS-y = vtophys ifeq ($(ENV_NAME),env_dpdk) -DIRS-y += memory pci +DIRS-y += env_dpdk_post_init memory pci endif .PHONY: all clean $(DIRS-y) diff --git a/test/env/env.sh b/test/env/env.sh index 7aa560f2e..973ecdb49 100755 --- a/test/env/env.sh +++ b/test/env/env.sh @@ -20,5 +20,9 @@ timing_enter pci $testdir/pci/pci_ut timing_exit pci +timing_enter env_dpdk_post_init +$testdir/env_dpdk_post_init/env_dpdk_post_init +timing_exit env_dpdk_post_init + report_test_completion "env" timing_exit env diff --git a/test/env/env_dpdk_post_init/.gitignore b/test/env/env_dpdk_post_init/.gitignore new file mode 100644 index 000000000..39bd89884 --- /dev/null +++ b/test/env/env_dpdk_post_init/.gitignore @@ -0,0 +1 @@ +env_dpdk_post_init diff --git a/test/env/env_dpdk_post_init/Makefile b/test/env/env_dpdk_post_init/Makefile new file mode 100644 index 000000000..f4b8804e8 --- /dev/null +++ b/test/env/env_dpdk_post_init/Makefile @@ -0,0 +1,40 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +CFLAGS += $(ENV_CFLAGS) +APP = env_dpdk_post_init + +include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk diff --git a/test/env/env_dpdk_post_init/env_dpdk_post_init.c b/test/env/env_dpdk_post_init/env_dpdk_post_init.c new file mode 100644 index 000000000..fec374177 --- /dev/null +++ b/test/env/env_dpdk_post_init/env_dpdk_post_init.c @@ -0,0 +1,128 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" + +#include "spdk/nvme.h" +#include "spdk/env.h" +#include "spdk/env_dpdk.h" +#include + +#define MAX_DEVS 64 + +struct dev { + struct spdk_nvme_ctrlr *ctrlr; + struct spdk_nvme_ns *ns; + struct spdk_nvme_qpair *qpair; + char name[SPDK_NVMF_TRADDR_MAX_LEN + 1]; +}; + +static struct dev g_nvme_devs[MAX_DEVS]; +static int g_num_devs = 0; +static int g_failed = 0; + +static bool +probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, + struct spdk_nvme_ctrlr_opts *opts) +{ + printf("Attaching to %s\n", trid->traddr); + + return true; +} + +static void +attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, + struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) +{ + struct dev *dev; + uint32_t nsid; + + /* add to dev list */ + dev = &g_nvme_devs[g_num_devs++]; + if (g_num_devs >= MAX_DEVS) { + return; + } + + dev->ctrlr = ctrlr; + nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); + dev->ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); + if (dev->ns == NULL) { + g_failed = 1; + return; + } + dev->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, NULL, 0); + if (dev->qpair == NULL) { + g_failed = 1; + return; + } + + snprintf(dev->name, sizeof(dev->name), "%s", + trid->traddr); + + printf("Attached to %s\n", dev->name); +} + +int +main(int argc, char **argv) +{ + int ret; + int i; + + printf("Starting DPDK initialization...\n"); + ret = rte_eal_init(argc, argv); + if (ret < 0) { + fprintf(stderr, "Failed to initialize DPDK\n"); + return -1; + } + + printf("Starting SPDK post initialization...\n"); + ret = spdk_env_dpdk_post_init(); + if (ret < 0) { + fprintf(stderr, "Failed to initialize SPDK\n"); + return -1; + } + + printf("SPDK NVMe probe\n"); + if (spdk_nvme_probe(NULL, NULL, probe_cb, attach_cb, NULL) != 0) { + fprintf(stderr, "spdk_nvme_probe() failed\n"); + return 1; + } + + printf("Cleaning up...\n"); + for (i = 0; i < g_num_devs; i++) { + struct dev *dev = &g_nvme_devs[i]; + spdk_nvme_detach(dev->ctrlr); + } + + return g_failed; +}