configure: Changes to support MLX5 crypto

- Added build system logic for checking mlx5 crypto PMD support. Added
  required libs in make files.
- Changes in mlx5 reduce build system logic since both crypto and reduce
  use common libs and libmlx5 related checks.
- Both mlx5 crypto and reduce require -libverbs.

Signed-off-by: Yuriy Umanets <yumanets@nvidia.com>
Change-Id: Ice1b88fe74bb04bf715ecaac70c4a8d4f3b5d782
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11620
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: Paul Luse <paul.e.luse@intel.com>
Reviewed-by: Shuhei Matsumoto <smatsumoto@nvidia.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

CONFIG

@@ -149,6 +149,9 @@ CONFIG_REDUCE=n
 # Enable mlx5_pci dpdk compress PMD, enabled automatically if CONFIG_REDUCE=y and libmlx5 exists
 CONFIG_REDUCE_MLX5=n
 
+# Enable mlx5_pci dpdk crypto PMD, enabled automatically if CONFIG_CRYPTO=y and libmlx5 exists
+CONFIG_CRYPTO_MLX5=n
+
 # Requires libiscsi development libraries.
 CONFIG_ISCSI_INITIATOR=n
 

autobuild.sh

@@ -104,6 +104,7 @@ function build_native_dpdk() {
 	# for DPDK issue: https://bugs.dpdk.org/show_bug.cgi?id=576
 	DPDK_DRIVERS=("bus" "bus/pci" "bus/vdev" "mempool/ring" "net/i40e" "net/i40e/base")
 
+	local mlx5_libs_added="n"
 	if [[ "$SPDK_TEST_CRYPTO" -eq 1 ]]; then
 		intel_ipsec_mb_ver=v0.54
 		intel_ipsec_mb_drv=crypto/aesni_mb
@@ -125,6 +126,16 @@ function build_native_dpdk() {
 		DPDK_DRIVERS+=("crypto/qat")
 		DPDK_DRIVERS+=("compress/qat")
 		DPDK_DRIVERS+=("common/qat")
+		# 22.03.0 is version of DPDK with stable support for mlx5 crypto.
+		if ge "$dpdk_ver" 22.03.0; then
+			# SPDK enables CRYPTO_MLX in case supported version of DPDK is detected
+			# so make sure proper libs are built.
+			DPDK_DRIVERS+=("bus/auxiliary")
+			DPDK_DRIVERS+=("common/mlx5")
+			DPDK_DRIVERS+=("common/mlx5/linux")
+			DPDK_DRIVERS+=("crypto/mlx5")
+			mlx5_libs_added="y"
+		fi
 		dpdk_cflags+=" -I$external_dpdk_base_dir/intel-ipsec-mb/$intel_ipsec_lib"
 		dpdk_ldflags+=" -L$external_dpdk_base_dir/intel-ipsec-mb/$intel_ipsec_lib"
 		export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$external_dpdk_base_dir/intel-ipsec-mb/$intel_ipsec_lib"
@@ -147,9 +158,11 @@ function build_native_dpdk() {
 		if ge "$dpdk_ver" 21.02.0; then
 			# SPDK enables REDUCE_MLX in case supported version of DPDK is detected
 			# so make sure proper libs are built.
-			DPDK_DRIVERS+=("bus/auxiliary")
+			if test $mlx5_libs_added = "n"; then
-			DPDK_DRIVERS+=("common/mlx5")
+				DPDK_DRIVERS+=("bus/auxiliary")
-			DPDK_DRIVERS+=("common/mlx5/linux")
+				DPDK_DRIVERS+=("common/mlx5")
+				DPDK_DRIVERS+=("common/mlx5/linux")
+			fi
 			DPDK_DRIVERS+=("compress/mlx5")
 		fi
 		export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$isal_dir/build/lib/pkgconfig"

configure

@@ -870,6 +870,31 @@ if [[ "${CONFIG[PMDK]}" = "y" ]]; then
 	fi
 fi
 
+function dpdk_version() {
+	# Check DPDK version to determine if mlx5_pci driver is supported
+	local dpdk_ver="none"
+	if [[ "${CONFIG[DPDK_DIR]}" == "$rootdir/dpdk/build" ]]; then
+		# DPDK_DIR points at our submodule so ./build may not exist yet. Use
+		# absolute path to lookup the version.
+		dpdk_ver=$(< "$rootdir/dpdk/VERSION")
+	elif [[ -f "${CONFIG[DPDK_DIR]}"/../VERSION ]]; then
+		dpdk_ver=$(< "${CONFIG[DPDK_DIR]}"/../VERSION)
+	fi
+	echo $dpdk_ver
+}
+
+function mlx5_build() {
+	# Check if libmlx5 exists to enable mlx5_pci compress/crypto PMD
+	if ! echo -e '#include <spdk/stdinc.h>\n' \
+		'#include <infiniband/mlx5dv.h>\n' \
+		'#include <infiniband/verbs.h>\n' \
+		'int main(void) { return 0; }\n' \
+		| "${BUILD_CMD[@]}" -lmlx5 -libverbs -I${rootdir}/include -c - 2> /dev/null; then
+		return 1
+	fi
+	return 0
+}
+
 if [[ "${CONFIG[REDUCE]}" = "y" ]]; then
 	if ! echo -e '#include <libpmem.h>\nint main(void) { return 0; }\n' \
 		| "${BUILD_CMD[@]}" -lpmem - 2> /dev/null; then
@@ -877,41 +902,69 @@ if [[ "${CONFIG[REDUCE]}" = "y" ]]; then
 		echo "Please install then re-run this script."
 		exit 1
 	fi
-
+	# Try to enable mlx5 compress
 	CONFIG[REDUCE_MLX5]="y"
+
 	# Check if libmlx5 exists to enable mlx5_pci compress PMD
-	if ! echo -e '#include <spdk/stdinc.h>\n' \
+	if ! mlx5_build; then
-		'#include <infiniband/mlx5dv.h>\n' \
-		'#include <infiniband/verbs.h>\n' \
-		'int main(void) { return 0; }\n' \
-		| "${BUILD_CMD[@]}" -lmlx5 -I${rootdir}/include -c - 2> /dev/null; then
 		echo "libmlx5 is not found, so disabling DPDK mlx5_pci compress PMD"
 		CONFIG[REDUCE_MLX5]="n"
-	fi
-
-	if [[ "${CONFIG[DPDK_PKG_CONFIG]}" = "y" ]]; then
-		# Check if librte_compress_mlx5 exists in DPDK package
-		if [ ! -f "${CONFIG[DPDK_LIB_DIR]}"/librte_compress_mlx5.so ]; then
-			echo "librte_compress_mlx5 is not found, so disabling DPDK mlx5_pci compress PMD"
-			CONFIG[REDUCE_MLX5]="n"
-		fi
 	else
-		# Check DPDK version to determine if mlx5_pci driver is supported
+		if [[ "${CONFIG[DPDK_PKG_CONFIG]}" = "y" ]]; then
-		dpdk_ver=""
+			# Check if librte_compress_mlx5 exists in DPDK package
-		if [[ "${CONFIG[DPDK_DIR]}" == "$rootdir/dpdk/build" ]]; then
+			if [ ! -f "${CONFIG[DPDK_LIB_DIR]}"/librte_compress_mlx5.so ]; then
-			# DPDK_DIR points at our submodule so ./build may not exist yet. Use
+				echo "librte_compress_mlx5 is not found, so disabling DPDK mlx5_pci compress PMD"
-			# absolute path to lookup the version.
+				CONFIG[REDUCE_MLX5]="n"
-			dpdk_ver=$(< "$rootdir/dpdk/VERSION")
+			fi
-		elif [[ -f "${CONFIG[DPDK_DIR]}"/../VERSION ]]; then
-			dpdk_ver=$(< "${CONFIG[DPDK_DIR]}"/../VERSION)
 		else
-			echo "Cannot get DPDK version, so disabling DPDK mlx5_pci compress PMD"
+			# Check DPDK version to determine if mlx5_pci driver is supported
-			CONFIG[REDUCE_MLX5]="n"
+			dpdk_ver=$(dpdk_version)
+			if [[ $dpdk_ver = "none" ]]; then
+				echo "Cannot get DPDK version, so disabling DPDK mlx5_pci compress PMD"
+				CONFIG[REDUCE_MLX5]="n"
+			elif [[ -n $dpdk_ver ]] && lt "$dpdk_ver" 21.02.0; then
+				# mlx5_pci for compress is supported by DPDK >- 21.02.0
+				echo "DPDK version ${dpdk_ver} doesn't support mlx5_pci compress PMD"
+				CONFIG[REDUCE_MLX5]="n"
+			elif [[ -n ${CONFIG[DPDK_LIB_DIR]} ]] && [ ! -f "${CONFIG[DPDK_LIB_DIR]}"/librte_compress_mlx5.so ]; then
+				# This is only checked when --with-dpdk or --with-dpdk=* is used
+				echo "librte_compress_mlx5 is not found, so disabling DPDK mlx5_pci compress PMD"
+				CONFIG[REDUCE_MLX5]="n"
+			fi
 		fi
-		# mlx5_pci is supported by DPDK >- 21.02.0
+	fi
-		if [[ -n $dpdk_ver ]] && lt "$dpdk_ver" 21.02.0; then
+fi
-			echo "DPDK version ${dpdk_ver} doesn't support mlx5_pci compress PMD"
+
-			CONFIG[REDUCE_MLX5]="n"
+if [[ "${CONFIG[CRYPTO]}" = "y" ]]; then
+	# Try to enable mlx5 crypto
+	CONFIG[CRYPTO_MLX5]="y"
+
+	# Check if libmlx5 exists to enable mlx5_pci compress PMD
+	if ! mlx5_build; then
+		echo "libmlx5 is not found, so disabling DPDK mlx5_pci crypto PMD"
+		CONFIG[CRYPTO_MLX5]="n"
+	else
+		if [[ "${CONFIG[DPDK_PKG_CONFIG]}" = "y" ]]; then
+			# Check if librte_crypto_mlx5 exists in DPDK package
+			if [ ! -f "${CONFIG[DPDK_LIB_DIR]}"/librte_crypto_mlx5.so ]; then
+				echo "librte_crypto_mlx5 is not found, so disabling DPDK mlx5_pci crypto PMD"
+				CONFIG[CRYPTO_MLX5]="n"
+			fi
+		else
+			# Check DPDK version to determine if mlx5_pci driver is supported
+			dpdk_ver=$(dpdk_version)
+			if [[ $dpdk_ver = "none" ]]; then
+				echo "Cannot get DPDK version, so disabling DPDK mlx5_pci crypto PMD"
+				CONFIG[CRYPTO_MLX5]="n"
+			elif [[ -n $dpdk_ver ]] && lt "$dpdk_ver" 22.03.0; then
+				# mlx5_pci for crypto is supported by DPDK >- 22.03.0
+				echo "DPDK version ${dpdk_ver} doesn't support mlx5_pci crypto PMD"
+				CONFIG[CRYPTO_MLX5]="n"
+			elif [[ -n ${CONFIG[DPDK_LIB_DIR]} ]] && [ ! -f "${CONFIG[DPDK_LIB_DIR]}"/librte_crypto_mlx5.so ]; then
+				# This is only checked when --with-dpdk or --with-dpdk=* is used
+				echo "librte_crypto_mlx5 is not found, so disabling DPDK mlx5_pci crypto PMD"
+				CONFIG[CRYPTO_MLX5]="n"
+			fi
 		fi
 	fi
 fi

dpdkbuild/Makefile

@@ -65,10 +65,18 @@ ifeq ($(findstring y,$(CONFIG_CRYPTO)$(CONFIG_REDUCE)),y)
 DPDK_DRIVERS += crypto/qat compress/qat common/qat
 endif
 
+# common mlx5 libs
+ifeq ($(findstring y,$(CONFIG_CRYPTO_MLX5)$(CONFIG_REDUCE_MLX5)),y)
+DPDK_DRIVERS += common/mlx5 bus/auxiliary
+endif
+
 ifeq ($(CONFIG_CRYPTO),y)
 DPDK_DRIVERS += crypto crypto/ipsec_mb
 # aesni_mb is name of the PMD in DPDK 21.08 and earlier
 DPDK_DRIVERS += crypto/aesni_mb
+ifeq ($(CONFIG_CRYPTO_MLX5),y)
+DPDK_DRIVERS += crypto/mlx5
+endif
 DPDK_CFLAGS += -I$(IPSEC_MB_DIR)
 DPDK_LDFLAGS += -L$(IPSEC_MB_DIR)
 endif
@@ -76,7 +84,7 @@ endif
 ifeq ($(CONFIG_REDUCE),y)
 DPDK_DRIVERS += compress compress/isal
 ifeq ($(CONFIG_REDUCE_MLX5),y)
-DPDK_DRIVERS += common/mlx5 compress/mlx5 bus/auxiliary
+DPDK_DRIVERS += compress/mlx5
 endif
 DPDK_CFLAGS += -I$(ISAL_DIR)
 DPDK_LDFLAGS += -L$(ISAL_DIR)/.libs -lisal

lib/env_dpdk/env.mk

@@ -71,6 +71,15 @@ endif
 # here we add the feature specific ones and set a flag to add the common
 # ones after that.
 DPDK_FRAMEWORK=n
+
+ifeq ($(findstring y,$(CONFIG_CRYPTO_MLX5)$(CONFIG_REDUCE_MLX5)),y)
+DPDK_LIB_LIST += rte_common_mlx5
+# Introduced in DPDK 21.08
+ifneq (, $(wildcard $(DPDK_LIB_DIR)/librte_bus_auxiliary.*))
+DPDK_LIB_LIST += rte_bus_auxiliary
+endif
+endif
+
 ifeq ($(CONFIG_CRYPTO),y)
 DPDK_FRAMEWORK=y
 ifneq (, $(wildcard $(DPDK_LIB_DIR)/librte_crypto_ipsec_mb.*))
@@ -82,6 +91,12 @@ ifneq (, $(wildcard $(DPDK_LIB_DIR)/librte_crypto_aesni_mb.*))
 DPDK_LIB_LIST += rte_crypto_aesni_mb
 endif
 endif
+
+ifeq ($(CONFIG_CRYPTO_MLX5),y)
+ifneq (, $(wildcard $(DPDK_LIB_DIR)/librte_crypto_mlx5.*))
+DPDK_LIB_LIST += rte_crypto_mlx5
+endif
+endif
 endif
 
 ifeq ($(CONFIG_REDUCE),y)
@@ -90,11 +105,7 @@ ifneq (, $(wildcard $(DPDK_LIB_DIR)/librte_compress_isal.*))
 DPDK_LIB_LIST += rte_compress_isal
 endif
 ifeq ($(CONFIG_REDUCE_MLX5),y)
-DPDK_LIB_LIST += rte_common_mlx5 rte_compress_mlx5
+DPDK_LIB_LIST += rte_compress_mlx5
-# Introduced in DPDK 21.08
-ifneq (, $(wildcard $(DPDK_LIB_DIR)/librte_bus_auxiliary.*))
-DPDK_LIB_LIST += rte_bus_auxiliary
-endif
 endif
 endif
 
@@ -153,10 +164,16 @@ ifeq ($(CONFIG_HAVE_LIBBSD),y)
 DPDK_PRIVATE_LINKER_ARGS += -lbsd
 endif
 
+ifeq ($(CONFIG_CRYPTO),y)
+ifeq ($(CONFIG_CRYPTO_MLX5),y)
+DPDK_PRIVATE_LINKER_ARGS += -lmlx5 -libverbs
+endif
+endif
+
 ifeq ($(CONFIG_REDUCE),y)
 DPDK_PRIVATE_LINKER_ARGS += -lisal -L$(ISAL_DIR)/.libs
 ifeq ($(CONFIG_REDUCE_MLX5),y)
-DPDK_PRIVATE_LINKER_ARGS += -lmlx5
+DPDK_PRIVATE_LINKER_ARGS += -lmlx5 -libverbs
 endif
 endif
 

mk/spdk.modules.mk

@@ -48,6 +48,9 @@ endif
 
 ifeq ($(CONFIG_CRYPTO),y)
 BLOCKDEV_MODULES_LIST += bdev_crypto
+ifeq ($(CONFIG_CRYPTO_MLX5),y)
+BLOCKDEV_MODULES_PRIVATE_LIBS += -lmlx5 -libverbs
+endif
 endif
 
 ifeq ($(CONFIG_OCF),y)
@@ -59,7 +62,7 @@ ifeq ($(CONFIG_REDUCE),y)
 BLOCKDEV_MODULES_LIST += bdev_compress reduce
 BLOCKDEV_MODULES_PRIVATE_LIBS += -lpmem
 ifeq ($(CONFIG_REDUCE_MLX5),y)
-BLOCKDEV_MODULES_PRIVATE_LIBS += -lmlx5
+BLOCKDEV_MODULES_PRIVATE_LIBS += -lmlx5 -libverbs
 endif
 endif