diff --git a/Makefile b/Makefile index 5cef1c0a4..24742e69c 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,9 @@ DIRS-$(CONFIG_ISAL) += isalbuild cc_version cxx_version .libs_only_other .ldflags ldflags install \ uninstall +# Workaround for ninja. See dpdkbuild/Makefile +export MAKE_PID := $(shell echo $$PPID) + ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV)) ifeq ($(CURDIR)/dpdk/build,$(CONFIG_DPDK_DIR)) ifneq ($(SKIP_DPDK_BUILD),1) diff --git a/dpdkbuild/Makefile b/dpdkbuild/Makefile index e17675852..f6429eb44 100644 --- a/dpdkbuild/Makefile +++ b/dpdkbuild/Makefile @@ -36,87 +36,39 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk .PHONY: all clean install uninstall -DPDK_FRAMEWORK = n -DPDK_OPTS = +DPDK_OPTS = -Denable_docs=false DPDK_CFLAGS = -DPDK_OPTS += CONFIG_RTE_BUILD_SHARED_LIB=n -DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_AESNI_MB=n -DPDK_OPTS += CONFIG_RTE_LIBRTE_REORDER=n -DPDK_OPTS += CONFIG_RTE_LIBRTE_ETHER=n -DPDK_OPTS += CONFIG_RTE_LIBRTE_CMDLINE=n -DPDK_OPTS += CONFIG_RTE_LIBRTE_METER=n -DPDK_OPTS += CONFIG_RTE_LIBRTE_HASH=n -DPDK_OPTS += CONFIG_RTE_LIBRTE_VHOST=n -DPDK_OPTS += CONFIG_RTE_EAL_IGB_UIO=n -DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_QAT=n -DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_QAT_SYM=n -DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_ISAL=n -ifeq ($(CONFIG_SHARED),y) -DPDK_OPTS += CONFIG_RTE_BUILD_SHARED_LIB=y -DPDK_LDFLAGS+= -rpath $(SPDK_ROOT_DIR)/dpdk/build/lib +DPDK_KMODS = false +ifeq ($(CONFIG_IGB_UIO_DRIVER),y) +DPDK_KMODS = true endif +ifeq ($(OS),FreeBSD) +DPDK_KMODS = true +endif +DPDK_OPTS += -Denable_kmods=$(DPDK_KMODS) + +# the drivers we use +DPDK_DRIVERS = bus bus/pci bus/vdev mempool/ring ifeq ($(CONFIG_CRYPTO),y) -DPDK_FRAMEWORK = y -DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_AESNI_MB=y -DPDK_OPTS += CONFIG_RTE_LIBRTE_REORDER=y +# crypto/qat is just a stub, the compress/qat pmd is used instead +DPDK_DRIVERS += crypto crypto/aesni_mb crypto/qat compress/qat common/qat DPDK_CFLAGS += -I$(IPSEC_MB_DIR) DPDK_LDFLAGS += -L$(IPSEC_MB_DIR) endif ifeq ($(CONFIG_REDUCE),y) -DPDK_FRAMEWORK = y -DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_ISAL=y +DPDK_DRIVERS += compress compress/isal compress/qat common/qat DPDK_CFLAGS += -I$(ISAL_DIR) -DPDK_LDFLAGS += -L$(ISAL_DIR)/.libs +DPDK_LDFLAGS += -L$(ISAL_DIR)/.libs -lisal endif -ifeq ($(CONFIG_VHOST),y) -DPDK_OPTS += CONFIG_RTE_LIBRTE_ETHER=y -DPDK_OPTS += CONFIG_RTE_LIBRTE_CMDLINE=y -DPDK_OPTS += CONFIG_RTE_LIBRTE_METER=y -DPDK_OPTS += CONFIG_RTE_LIBRTE_HASH=y -DPDK_OPTS += CONFIG_RTE_LIBRTE_VHOST=y -endif - -ifeq ($(CONFIG_IGB_UIO_DRIVER),y) -DPDK_OPTS += CONFIG_RTE_EAL_IGB_UIO=y -endif - -ifeq ($(CONFIG_RAID5),y) -DPDK_OPTS += CONFIG_RTE_LIBRTE_HASH=y -endif - -ifeq ($(DPDK_FRAMEWORK),y) -DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_QAT=y -DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_QAT_SYM=y -endif - -ifeq ($(TARGET_MACHINE),aarch64) -DPDK_CONFIG := arm64-armv8a -else -DPDK_CONFIG := $(TARGET_MACHINE)-native -endif +DPDK_OPTS += -Dmachine=$(TARGET_ARCHITECTURE) ifneq ($(CONFIG_CROSS_PREFIX),) -DPDK_OPTS += CROSS=$(CONFIG_CROSS_PREFIX)- -endif - -ifeq ($(OS),Linux) -DPDK_CONFIG := $(DPDK_CONFIG)-linuxapp -NPROC := $(shell nproc) -else -ifeq ($(OS),FreeBSD) -DPDK_CONFIG := $(DPDK_CONFIG)-bsdapp -NPROC := $(shell sysctl hw.ncpu | awk '{print $$NF}') -endif -endif - -ifeq ($(CC_TYPE),clang) -DPDK_CONFIG := $(DPDK_CONFIG)-clang -else -DPDK_CONFIG := $(DPDK_CONFIG)-gcc +$(error Automatic DPDK cross build is not supported. Please compile DPDK manually \ +with e.g. `meson build --cross-file config/arm/arm64_armv8_linux_gcc`) endif DPDK_CFLAGS += -fPIC @@ -154,15 +106,58 @@ endif # Force-disable scan-build SUB_CC = $(patsubst %ccc-analyzer,$(DEFAULT_CC),$(CC)) -$(SPDK_ROOT_DIR)/dpdk/build: $(SPDK_ROOT_DIR)/mk/cc.mk $(SPDK_ROOT_DIR)/include/spdk/config.h - $(Q)rm -rf $(SPDK_ROOT_DIR)/dpdk/build - $(Q)$(MAKE) -C $(SPDK_ROOT_DIR)/dpdk config T=$(DPDK_CONFIG) $(DPDK_OPTS) +DPDK_ALL_DRIVER_DIRS = $(shell find $(SPDK_ROOT_DIR)/dpdk/drivers -mindepth 1 -type d) +DPDK_ALL_DRIVERS = $(DPDK_ALL_DRIVER_DIRS:$(SPDK_ROOT_DIR)/dpdk/drivers/%=%) +DPDK_DISABLED_DRVERS = $(filter-out $(DPDK_DRIVERS),$(DPDK_ALL_DRIVERS)) -all: $(SPDK_ROOT_DIR)/dpdk/build - $(Q)$(MAKE) -C $(SPDK_ROOT_DIR)/dpdk/build EXTRA_CFLAGS="$(DPDK_CFLAGS)" EXTRA_LDFLAGS="$(DPDK_LDFLAGS)" CC=$(SUB_CC) T="$(DPDK_CONFIG)" $(DPDK_OPTS) +ifeq ($(OS),Linux) +SED_INPLACE_FLAG = "-i" +MESON_PREFIX = $(SPDK_ROOT_DIR)/dpdk/build +else +SED_INPLACE_FLAG = "-i ''" +MESON_PREFIX = "/" +endif + +# Some ninja versions come with a (broken?) jobserver which defaults to use +# only 1 thread for the build. We workaround this by specifying -j to ninja +# with the same value as top-makefile. This is OK as long as DPDK is not built +# in parralel with anything else, which is the case for now. +ifeq ($(MAKE_PID),) +MAKE_PID := $(shell echo $$PPID) +endif + +MAKE_NUMJOBS := $(shell ps T | sed -nE 's/\s*$(MAKE_PID)\s.* (-j|--jobs=)( *[0-9]+).*/\1\2/p') + +all: $(SPDK_ROOT_DIR)/dpdk/build-tmp + $(Q)# DPDK doesn't handle nested make calls, so unset MAKEFLAGS + $(Q)env -u MAKEFLAGS ninja -C $(SPDK_ROOT_DIR)/dpdk/build-tmp $(MAKE_NUMJOBS) + $(Q) \ + # Meson on FreeBSD sometimes appends --prefix value to the default DESTDIR (which is e.g. \ + # /usr/local) instead of replacing it. --prefix needs to be an absolute path, so we set \ + # it to / and then set DESTDIR directly, so libs and headers are copied to "DESTDIR//". \ + # DPDK kernel modules are set to install in $DESTDIR/boot/modules, but we move them \ + # to DESTDIR/kmod to be consistent with the makefile build. + $(Q)if [ "$(OS)" = "FreeBSD" ]; then \ + env -u MAKEFLAGS DESTDIR=$(SPDK_ROOT_DIR)/dpdk/build ninja -C $(SPDK_ROOT_DIR)/dpdk/build-tmp $(MAKE_NUMJOBS) install > /dev/null && \ + mv $(SPDK_ROOT_DIR)/dpdk/build/boot/modules $(SPDK_ROOT_DIR)/dpdk/build/kmod; \ + else \ + env -u MAKEFLAGS ninja -C $(SPDK_ROOT_DIR)/dpdk/build-tmp $(MAKE_NUMJOBS) install > /dev/null; \ + fi + +$(SPDK_ROOT_DIR)/dpdk/build-tmp: $(SPDK_ROOT_DIR)/mk/cc.mk $(SPDK_ROOT_DIR)/include/spdk/config.h + $(Q)rm -rf $(SPDK_ROOT_DIR)/dpdk/build $(SPDK_ROOT_DIR)/dpdk/build-tmp + $(Q)cd "$(SPDK_ROOT_DIR)/dpdk"; CC="$(SUB_CC)" meson --prefix="$(MESON_PREFIX)" --libdir lib -Dc_args="$(DPDK_CFLAGS)" -Dc_link_args="$(DPDK_LDFLAGS)" $(DPDK_OPTS) -Ddisable_drivers="$(shell echo $(DPDK_DISABLED_DRVERS) | sed -E "s/ +/,/g")" build-tmp + $(Q)sed $(SED_INPLACE_FLAG) 's/#define RTE_EAL_PMD_PATH .*/#define RTE_EAL_PMD_PATH ""/g' $(SPDK_ROOT_DIR)/dpdk/build-tmp/rte_build_config.h + $(Q) \ + # TODO Meson build adds libbsd dependency when it's available. This means any app will be \ + # forced to link with -lbsd, but only if it's available on the system. The clean way to \ + # handle this would be to rely on DPDK's pkg-config file which will contain the -lbsd when \ + # required. For now just remove the libbsd dependency. DPDK will fallback to its internal \ + # functions. + $(Q)sed $(SED_INPLACE_FLAG) 's/#define RTE_USE_LIBBSD .*//g' $(SPDK_ROOT_DIR)/dpdk/build-tmp/rte_build_config.h clean: - $(Q)rm -rf $(SPDK_ROOT_DIR)/dpdk/build + $(Q)rm -rf $(SPDK_ROOT_DIR)/dpdk/build $(SPDK_ROOT_DIR)/dpdk/build-tmp install: @: diff --git a/lib/env_dpdk/env.mk b/lib/env_dpdk/env.mk index 9410e3ae9..c2bfb0d19 100644 --- a/lib/env_dpdk/env.mk +++ b/lib/env_dpdk/env.mk @@ -48,10 +48,10 @@ DPDK_INC_DIR := $(DPDK_ABS_DIR)/include/dpdk endif DPDK_INC := -I$(DPDK_INC_DIR) -ifneq (, $(wildcard $(DPDK_ABS_DIR)/lib/librte_eal.a)) -DPDK_LIB_EXT = .a -else +ifeq ($(CONFIG_SHARED),y) DPDK_LIB_EXT = .so +else +DPDK_LIB_EXT = .a endif DPDK_LIB_LIST = rte_eal rte_mempool rte_ring rte_mbuf @@ -94,7 +94,7 @@ endif ifeq ($(CONFIG_REDUCE),y) DPDK_FRAMEWORK=y -DPDK_LIB_LIST += rte_pmd_isal_comp +DPDK_LIB_LIST += rte_pmd_isal endif ifeq ($(DPDK_FRAMEWORK),y) @@ -144,7 +144,8 @@ else ENV_DPDK_FILE = $(call spdk_lib_list_to_static_libs,env_dpdk) endif ENV_LIBS = $(ENV_DPDK_FILE) $(DPDK_LIB) -ENV_LINKER_ARGS = $(call dpdk_env_linker_args,$(DPDK_LIB_LIST)) +ENV_LINKER_ARGS = -Wl,-rpath-link $(DPDK_ABS_DIR)/lib +ENV_LINKER_ARGS += $(call dpdk_env_linker_args,$(DPDK_LIB_LIST)) ifeq ($(CONFIG_IPSEC_MB),y) ENV_LINKER_ARGS += -lIPSec_MB -L$(IPSEC_MB_DIR) @@ -160,6 +161,13 @@ ENV_LINKER_ARGS += -lnuma endif endif +# DPDK built with meson puts those defines elsewhere +ifneq (,$(wildcard $(DPDK_INC_DIR)/rte_build_config.h)) +ifneq (,$(shell grep -e "define RTE_LIBRTE_VHOST_NUMA 1" -e "define RTE_EAL_NUMA_AWARE_HUGEPAGES 1" $(DPDK_INC_DIR)/rte_build_config.h)) +ENV_LINKER_ARGS += -lnuma +endif +endif + ifeq ($(OS),Linux) ENV_LINKER_ARGS += -ldl endif diff --git a/test/external_code/hello_world/Makefile b/test/external_code/hello_world/Makefile index 263992e2d..9f6c9cf30 100644 --- a/test/external_code/hello_world/Makefile +++ b/test/external_code/hello_world/Makefile @@ -31,36 +31,43 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -# Shows how to compile both an external bdev and an external application against the SPDK combined shared object and dpdk shared object. +# Shows how to compile both an external bdev and an external application against the SPDK combined shared object and dpdk shared objects. bdev_shared_combo: - $(CC) $(COMMON_CFLAGS) -L../passthru -o hello_bdev ./hello_bdev.c -lpassthru_external -lspdk -lspdk_env_dpdk -ldpdk + $(CC) $(COMMON_CFLAGS) -L../passthru -Wl,-rpath=$(SPDK_LIB_DIR),--no-as-needed -o hello_bdev ./hello_bdev.c -lpassthru_external \ + -lspdk -lspdk_env_dpdk -lrte_eal -lrte_mempool -lrte_ring -lrte_mbuf -lrte_mempool_ring -lrte_pci -lrte_bus_pci -lrte_kvargs \ + -lrte_vhost -lrte_net -lrte_hash -lrte_cryptodev -Wl,--no-whole-archive -# Shows how to compile both an external bdev and an external application against the SPDK individual shared objects and dpdk shared object. +# Shows how to compile both an external bdev and an external application against the SPDK individual shared objects and dpdk shared objects. bdev_shared_iso: $(CC) $(COMMON_CFLAGS) -L../passthru -Wl,-rpath=$(SPDK_LIB_DIR),--no-as-needed -o hello_bdev ./hello_bdev.c \ - -lpassthru_external -lspdk_event_bdev -lspdk_bdev -lspdk_bdev_malloc -lspdk_log -lspdk_thread -lspdk_util -lspdk_event -lspdk_env_dpdk -ldpdk + -lpassthru_external -lspdk_event_bdev -lspdk_event_accel -lspdk_event_vmd -lspdk_bdev -lspdk_bdev_malloc -lspdk_log -lspdk_thread -lspdk_util -lspdk_event \ + -lspdk_env_dpdk -lrte_eal -lrte_mempool -lrte_ring -lrte_mbuf -lrte_mempool_ring -lrte_pci -lrte_bus_pci -lrte_kvargs \ + -lrte_vhost -lrte_net -lrte_hash -lrte_cryptodev -Wl,--no-whole-archive -lnuma -# Shows how to compile an external application against the SPDK combined shared object and dpdk shared object. +# Shows how to compile an external application against the SPDK combined shared object and dpdk shared objects. alone_shared_combo: - $(CC) $(COMMON_CFLAGS) -o hello_bdev ./hello_bdev.c -lspdk -lspdk_env_dpdk -ldpdk + $(CC) $(COMMON_CFLAGS) -Wl,-rpath=$(SPDK_LIB_DIR),--no-as-needed -o hello_bdev ./hello_bdev.c -lspdk -lspdk_env_dpdk -lrte_eal \ + -lrte_mempool -lrte_ring -lrte_mbuf -lrte_mempool_ring -lrte_pci -lrte_bus_pci -lrte_kvargs -lrte_vhost -lrte_net -lrte_hash -lrte_cryptodev -# Shows how to compile an external application against the SPDK individual shared objects and dpdk shared object. +# Shows how to compile an external application against the SPDK individual shared objects and dpdk shared objects. alone_shared_iso: $(CC) $(COMMON_CFLAGS) -Wl,-rpath=$(SPDK_LIB_DIR),--no-as-needed -o hello_bdev ./hello_bdev.c -lspdk_event_bdev \ - -lspdk_bdev -lspdk_bdev_malloc -lspdk_log -lspdk_thread -lspdk_util -lspdk_event -lspdk_env_dpdk -ldpdk + -lspdk_event_accel -lspdk_event_vmd -lspdk_bdev -lspdk_bdev_malloc -lspdk_log -lspdk_thread -lspdk_util -lspdk_event \ + -lspdk_env_dpdk -lrte_eal -lrte_mempool -lrte_ring -lrte_mbuf -lrte_mempool_ring -lrte_pci -lrte_bus_pci -lrte_kvargs \ + -lrte_vhost -lrte_net -lrte_hash -lrte_cryptodev # Shows how to compile an external application against the SPDK archives. alone_static: - $(CC) $(COMMON_CFLAGS) -o hello_bdev ./hello_bdev.c -Wl,--whole-archive -lspdk_bdev_malloc -lspdk_event_bdev -lspdk_event_accel -lspdk_event_vmd \ + $(CC) $(COMMON_CFLAGS) -o hello_bdev ./hello_bdev.c -Wl,--whole-archive,-Bstatic -lspdk_bdev_malloc -lspdk_event_bdev -lspdk_event_accel -lspdk_event_vmd \ -lspdk_event_sock -lspdk_bdev -lspdk_accel -lspdk_event -lspdk_thread -lspdk_util -lspdk_conf -lspdk_trace -lspdk_log -lspdk_json \ -lspdk_jsonrpc -lspdk_rpc -lspdk_sock -lspdk_notify -lspdk_vmd -lspdk_env_dpdk -lrte_eal -lrte_mempool -lrte_ring \ -lrte_mbuf -lrte_mempool_ring -lrte_pci -lrte_bus_pci -lrte_kvargs -lrte_vhost -lrte_net -lrte_hash -lrte_telemetry \ - -lrte_cryptodev -Wl,--no-whole-archive -lnuma -luuid -lpthread -ldl -lrt + -lrte_cryptodev -Wl,--no-whole-archive,-Bdynamic -lnuma -luuid -lpthread -ldl -lrt # Shows how to compile and external bdev and application sgainst the SPDK archives. bdev_static: - $(CC) $(COMMON_CFLAGS) -L../passthru -o hello_bdev ./hello_bdev.c -Wl,--whole-archive -lpassthru_external -lspdk_bdev_malloc -lspdk_event_bdev \ + $(CC) $(COMMON_CFLAGS) -L../passthru -o hello_bdev ./hello_bdev.c -Wl,--whole-archive,-Bstatic -lpassthru_external -lspdk_bdev_malloc -lspdk_event_bdev \ -lspdk_event_accel -lspdk_event_vmd -lspdk_event_sock -lspdk_bdev -lspdk_accel -lspdk_event -lspdk_thread -lspdk_util -lspdk_conf -lspdk_trace \ -lspdk_log -lspdk_json -lspdk_jsonrpc -lspdk_rpc -lspdk_sock -lspdk_notify -lspdk_vmd -lspdk_env_dpdk -lrte_eal -lrte_mempool \ -lrte_ring -lrte_mbuf -lrte_mempool_ring -lrte_pci -lrte_bus_pci -lrte_kvargs -lrte_vhost -lrte_net -lrte_hash -lrte_telemetry -lrte_cryptodev \ - -Wl,--no-whole-archive -lnuma -luuid -lpthread -ldl -lrt + -Wl,--no-whole-archive,-Bdynamic -lnuma -luuid -lpthread -ldl -lrt