From 2a6ddeb83756d5c2f2c6754cbc9eb8bdb26c1c63 Mon Sep 17 00:00:00 2001 From: Yuriy Umanets Date: Wed, 19 Jan 2022 17:28:13 +0200 Subject: [PATCH] configure: Changes to support MLX5 crypto - Added build system logic for checking mlx5 crypto PMD support. Added required libs in make files. - Changes in mlx5 reduce build system logic since both crypto and reduce use common libs and libmlx5 related checks. - Both mlx5 crypto and reduce require -libverbs. Signed-off-by: Yuriy Umanets Change-Id: Ice1b88fe74bb04bf715ecaac70c4a8d4f3b5d782 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11620 Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Aleksey Marchuk Reviewed-by: Paul Luse Reviewed-by: Shuhei Matsumoto Reviewed-by: Jim Harris --- CONFIG | 3 ++ autobuild.sh | 19 ++++++-- configure | 109 ++++++++++++++++++++++++++++++++------------ dpdkbuild/Makefile | 10 +++- lib/env_dpdk/env.mk | 29 +++++++++--- mk/spdk.modules.mk | 5 +- 6 files changed, 136 insertions(+), 39 deletions(-) diff --git a/CONFIG b/CONFIG index 6237af1b5..29c558b37 100644 --- a/CONFIG +++ b/CONFIG @@ -149,6 +149,9 @@ CONFIG_REDUCE=n # Enable mlx5_pci dpdk compress PMD, enabled automatically if CONFIG_REDUCE=y and libmlx5 exists CONFIG_REDUCE_MLX5=n +# Enable mlx5_pci dpdk crypto PMD, enabled automatically if CONFIG_CRYPTO=y and libmlx5 exists +CONFIG_CRYPTO_MLX5=n + # Requires libiscsi development libraries. CONFIG_ISCSI_INITIATOR=n diff --git a/autobuild.sh b/autobuild.sh index 8b34f3920..8c171c55d 100755 --- a/autobuild.sh +++ b/autobuild.sh @@ -104,6 +104,7 @@ function build_native_dpdk() { # for DPDK issue: https://bugs.dpdk.org/show_bug.cgi?id=576 DPDK_DRIVERS=("bus" "bus/pci" "bus/vdev" "mempool/ring" "net/i40e" "net/i40e/base") + local mlx5_libs_added="n" if [[ "$SPDK_TEST_CRYPTO" -eq 1 ]]; then intel_ipsec_mb_ver=v0.54 intel_ipsec_mb_drv=crypto/aesni_mb @@ -125,6 +126,16 @@ function build_native_dpdk() { DPDK_DRIVERS+=("crypto/qat") DPDK_DRIVERS+=("compress/qat") DPDK_DRIVERS+=("common/qat") + # 22.03.0 is version of DPDK with stable support for mlx5 crypto. + if ge "$dpdk_ver" 22.03.0; then + # SPDK enables CRYPTO_MLX in case supported version of DPDK is detected + # so make sure proper libs are built. + DPDK_DRIVERS+=("bus/auxiliary") + DPDK_DRIVERS+=("common/mlx5") + DPDK_DRIVERS+=("common/mlx5/linux") + DPDK_DRIVERS+=("crypto/mlx5") + mlx5_libs_added="y" + fi dpdk_cflags+=" -I$external_dpdk_base_dir/intel-ipsec-mb/$intel_ipsec_lib" dpdk_ldflags+=" -L$external_dpdk_base_dir/intel-ipsec-mb/$intel_ipsec_lib" export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$external_dpdk_base_dir/intel-ipsec-mb/$intel_ipsec_lib" @@ -147,9 +158,11 @@ function build_native_dpdk() { if ge "$dpdk_ver" 21.02.0; then # SPDK enables REDUCE_MLX in case supported version of DPDK is detected # so make sure proper libs are built. - DPDK_DRIVERS+=("bus/auxiliary") - DPDK_DRIVERS+=("common/mlx5") - DPDK_DRIVERS+=("common/mlx5/linux") + if test $mlx5_libs_added = "n"; then + DPDK_DRIVERS+=("bus/auxiliary") + DPDK_DRIVERS+=("common/mlx5") + DPDK_DRIVERS+=("common/mlx5/linux") + fi DPDK_DRIVERS+=("compress/mlx5") fi export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$isal_dir/build/lib/pkgconfig" diff --git a/configure b/configure index 56963f7a7..fb81e8a46 100755 --- a/configure +++ b/configure @@ -870,6 +870,31 @@ if [[ "${CONFIG[PMDK]}" = "y" ]]; then fi fi +function dpdk_version() { + # Check DPDK version to determine if mlx5_pci driver is supported + local dpdk_ver="none" + if [[ "${CONFIG[DPDK_DIR]}" == "$rootdir/dpdk/build" ]]; then + # DPDK_DIR points at our submodule so ./build may not exist yet. Use + # absolute path to lookup the version. + dpdk_ver=$(< "$rootdir/dpdk/VERSION") + elif [[ -f "${CONFIG[DPDK_DIR]}"/../VERSION ]]; then + dpdk_ver=$(< "${CONFIG[DPDK_DIR]}"/../VERSION) + fi + echo $dpdk_ver +} + +function mlx5_build() { + # Check if libmlx5 exists to enable mlx5_pci compress/crypto PMD + if ! echo -e '#include \n' \ + '#include \n' \ + '#include \n' \ + 'int main(void) { return 0; }\n' \ + | "${BUILD_CMD[@]}" -lmlx5 -libverbs -I${rootdir}/include -c - 2> /dev/null; then + return 1 + fi + return 0 +} + if [[ "${CONFIG[REDUCE]}" = "y" ]]; then if ! echo -e '#include \nint main(void) { return 0; }\n' \ | "${BUILD_CMD[@]}" -lpmem - 2> /dev/null; then @@ -877,41 +902,69 @@ if [[ "${CONFIG[REDUCE]}" = "y" ]]; then echo "Please install then re-run this script." exit 1 fi - + # Try to enable mlx5 compress CONFIG[REDUCE_MLX5]="y" + # Check if libmlx5 exists to enable mlx5_pci compress PMD - if ! echo -e '#include \n' \ - '#include \n' \ - '#include \n' \ - 'int main(void) { return 0; }\n' \ - | "${BUILD_CMD[@]}" -lmlx5 -I${rootdir}/include -c - 2> /dev/null; then + if ! mlx5_build; then echo "libmlx5 is not found, so disabling DPDK mlx5_pci compress PMD" CONFIG[REDUCE_MLX5]="n" - fi - - if [[ "${CONFIG[DPDK_PKG_CONFIG]}" = "y" ]]; then - # Check if librte_compress_mlx5 exists in DPDK package - if [ ! -f "${CONFIG[DPDK_LIB_DIR]}"/librte_compress_mlx5.so ]; then - echo "librte_compress_mlx5 is not found, so disabling DPDK mlx5_pci compress PMD" - CONFIG[REDUCE_MLX5]="n" - fi else - # Check DPDK version to determine if mlx5_pci driver is supported - dpdk_ver="" - if [[ "${CONFIG[DPDK_DIR]}" == "$rootdir/dpdk/build" ]]; then - # DPDK_DIR points at our submodule so ./build may not exist yet. Use - # absolute path to lookup the version. - dpdk_ver=$(< "$rootdir/dpdk/VERSION") - elif [[ -f "${CONFIG[DPDK_DIR]}"/../VERSION ]]; then - dpdk_ver=$(< "${CONFIG[DPDK_DIR]}"/../VERSION) + if [[ "${CONFIG[DPDK_PKG_CONFIG]}" = "y" ]]; then + # Check if librte_compress_mlx5 exists in DPDK package + if [ ! -f "${CONFIG[DPDK_LIB_DIR]}"/librte_compress_mlx5.so ]; then + echo "librte_compress_mlx5 is not found, so disabling DPDK mlx5_pci compress PMD" + CONFIG[REDUCE_MLX5]="n" + fi else - echo "Cannot get DPDK version, so disabling DPDK mlx5_pci compress PMD" - CONFIG[REDUCE_MLX5]="n" + # Check DPDK version to determine if mlx5_pci driver is supported + dpdk_ver=$(dpdk_version) + if [[ $dpdk_ver = "none" ]]; then + echo "Cannot get DPDK version, so disabling DPDK mlx5_pci compress PMD" + CONFIG[REDUCE_MLX5]="n" + elif [[ -n $dpdk_ver ]] && lt "$dpdk_ver" 21.02.0; then + # mlx5_pci for compress is supported by DPDK >- 21.02.0 + echo "DPDK version ${dpdk_ver} doesn't support mlx5_pci compress PMD" + CONFIG[REDUCE_MLX5]="n" + elif [[ -n ${CONFIG[DPDK_LIB_DIR]} ]] && [ ! -f "${CONFIG[DPDK_LIB_DIR]}"/librte_compress_mlx5.so ]; then + # This is only checked when --with-dpdk or --with-dpdk=* is used + echo "librte_compress_mlx5 is not found, so disabling DPDK mlx5_pci compress PMD" + CONFIG[REDUCE_MLX5]="n" + fi fi - # mlx5_pci is supported by DPDK >- 21.02.0 - if [[ -n $dpdk_ver ]] && lt "$dpdk_ver" 21.02.0; then - echo "DPDK version ${dpdk_ver} doesn't support mlx5_pci compress PMD" - CONFIG[REDUCE_MLX5]="n" + fi +fi + +if [[ "${CONFIG[CRYPTO]}" = "y" ]]; then + # Try to enable mlx5 crypto + CONFIG[CRYPTO_MLX5]="y" + + # Check if libmlx5 exists to enable mlx5_pci compress PMD + if ! mlx5_build; then + echo "libmlx5 is not found, so disabling DPDK mlx5_pci crypto PMD" + CONFIG[CRYPTO_MLX5]="n" + else + if [[ "${CONFIG[DPDK_PKG_CONFIG]}" = "y" ]]; then + # Check if librte_crypto_mlx5 exists in DPDK package + if [ ! -f "${CONFIG[DPDK_LIB_DIR]}"/librte_crypto_mlx5.so ]; then + echo "librte_crypto_mlx5 is not found, so disabling DPDK mlx5_pci crypto PMD" + CONFIG[CRYPTO_MLX5]="n" + fi + else + # Check DPDK version to determine if mlx5_pci driver is supported + dpdk_ver=$(dpdk_version) + if [[ $dpdk_ver = "none" ]]; then + echo "Cannot get DPDK version, so disabling DPDK mlx5_pci crypto PMD" + CONFIG[CRYPTO_MLX5]="n" + elif [[ -n $dpdk_ver ]] && lt "$dpdk_ver" 22.03.0; then + # mlx5_pci for crypto is supported by DPDK >- 22.03.0 + echo "DPDK version ${dpdk_ver} doesn't support mlx5_pci crypto PMD" + CONFIG[CRYPTO_MLX5]="n" + elif [[ -n ${CONFIG[DPDK_LIB_DIR]} ]] && [ ! -f "${CONFIG[DPDK_LIB_DIR]}"/librte_crypto_mlx5.so ]; then + # This is only checked when --with-dpdk or --with-dpdk=* is used + echo "librte_crypto_mlx5 is not found, so disabling DPDK mlx5_pci crypto PMD" + CONFIG[CRYPTO_MLX5]="n" + fi fi fi fi diff --git a/dpdkbuild/Makefile b/dpdkbuild/Makefile index 1231fb3a7..9062e3679 100644 --- a/dpdkbuild/Makefile +++ b/dpdkbuild/Makefile @@ -65,10 +65,18 @@ ifeq ($(findstring y,$(CONFIG_CRYPTO)$(CONFIG_REDUCE)),y) DPDK_DRIVERS += crypto/qat compress/qat common/qat endif +# common mlx5 libs +ifeq ($(findstring y,$(CONFIG_CRYPTO_MLX5)$(CONFIG_REDUCE_MLX5)),y) +DPDK_DRIVERS += common/mlx5 bus/auxiliary +endif + ifeq ($(CONFIG_CRYPTO),y) DPDK_DRIVERS += crypto crypto/ipsec_mb # aesni_mb is name of the PMD in DPDK 21.08 and earlier DPDK_DRIVERS += crypto/aesni_mb +ifeq ($(CONFIG_CRYPTO_MLX5),y) +DPDK_DRIVERS += crypto/mlx5 +endif DPDK_CFLAGS += -I$(IPSEC_MB_DIR) DPDK_LDFLAGS += -L$(IPSEC_MB_DIR) endif @@ -76,7 +84,7 @@ endif ifeq ($(CONFIG_REDUCE),y) DPDK_DRIVERS += compress compress/isal ifeq ($(CONFIG_REDUCE_MLX5),y) -DPDK_DRIVERS += common/mlx5 compress/mlx5 bus/auxiliary +DPDK_DRIVERS += compress/mlx5 endif DPDK_CFLAGS += -I$(ISAL_DIR) DPDK_LDFLAGS += -L$(ISAL_DIR)/.libs -lisal diff --git a/lib/env_dpdk/env.mk b/lib/env_dpdk/env.mk index 29c42db19..d3d46af73 100644 --- a/lib/env_dpdk/env.mk +++ b/lib/env_dpdk/env.mk @@ -71,6 +71,15 @@ endif # here we add the feature specific ones and set a flag to add the common # ones after that. DPDK_FRAMEWORK=n + +ifeq ($(findstring y,$(CONFIG_CRYPTO_MLX5)$(CONFIG_REDUCE_MLX5)),y) +DPDK_LIB_LIST += rte_common_mlx5 +# Introduced in DPDK 21.08 +ifneq (, $(wildcard $(DPDK_LIB_DIR)/librte_bus_auxiliary.*)) +DPDK_LIB_LIST += rte_bus_auxiliary +endif +endif + ifeq ($(CONFIG_CRYPTO),y) DPDK_FRAMEWORK=y ifneq (, $(wildcard $(DPDK_LIB_DIR)/librte_crypto_ipsec_mb.*)) @@ -82,6 +91,12 @@ ifneq (, $(wildcard $(DPDK_LIB_DIR)/librte_crypto_aesni_mb.*)) DPDK_LIB_LIST += rte_crypto_aesni_mb endif endif + +ifeq ($(CONFIG_CRYPTO_MLX5),y) +ifneq (, $(wildcard $(DPDK_LIB_DIR)/librte_crypto_mlx5.*)) +DPDK_LIB_LIST += rte_crypto_mlx5 +endif +endif endif ifeq ($(CONFIG_REDUCE),y) @@ -90,11 +105,7 @@ ifneq (, $(wildcard $(DPDK_LIB_DIR)/librte_compress_isal.*)) DPDK_LIB_LIST += rte_compress_isal endif ifeq ($(CONFIG_REDUCE_MLX5),y) -DPDK_LIB_LIST += rte_common_mlx5 rte_compress_mlx5 -# Introduced in DPDK 21.08 -ifneq (, $(wildcard $(DPDK_LIB_DIR)/librte_bus_auxiliary.*)) -DPDK_LIB_LIST += rte_bus_auxiliary -endif +DPDK_LIB_LIST += rte_compress_mlx5 endif endif @@ -153,10 +164,16 @@ ifeq ($(CONFIG_HAVE_LIBBSD),y) DPDK_PRIVATE_LINKER_ARGS += -lbsd endif +ifeq ($(CONFIG_CRYPTO),y) +ifeq ($(CONFIG_CRYPTO_MLX5),y) +DPDK_PRIVATE_LINKER_ARGS += -lmlx5 -libverbs +endif +endif + ifeq ($(CONFIG_REDUCE),y) DPDK_PRIVATE_LINKER_ARGS += -lisal -L$(ISAL_DIR)/.libs ifeq ($(CONFIG_REDUCE_MLX5),y) -DPDK_PRIVATE_LINKER_ARGS += -lmlx5 +DPDK_PRIVATE_LINKER_ARGS += -lmlx5 -libverbs endif endif diff --git a/mk/spdk.modules.mk b/mk/spdk.modules.mk index b944ea62b..744bee0ab 100644 --- a/mk/spdk.modules.mk +++ b/mk/spdk.modules.mk @@ -48,6 +48,9 @@ endif ifeq ($(CONFIG_CRYPTO),y) BLOCKDEV_MODULES_LIST += bdev_crypto +ifeq ($(CONFIG_CRYPTO_MLX5),y) +BLOCKDEV_MODULES_PRIVATE_LIBS += -lmlx5 -libverbs +endif endif ifeq ($(CONFIG_OCF),y) @@ -59,7 +62,7 @@ ifeq ($(CONFIG_REDUCE),y) BLOCKDEV_MODULES_LIST += bdev_compress reduce BLOCKDEV_MODULES_PRIVATE_LIBS += -lpmem ifeq ($(CONFIG_REDUCE_MLX5),y) -BLOCKDEV_MODULES_PRIVATE_LIBS += -lmlx5 +BLOCKDEV_MODULES_PRIVATE_LIBS += -lmlx5 -libverbs endif endif