diff --git a/app/iscsi_tgt/Makefile b/app/iscsi_tgt/Makefile index 0cb3c4d32..24af5bb9a 100644 --- a/app/iscsi_tgt/Makefile +++ b/app/iscsi_tgt/Makefile @@ -52,7 +52,6 @@ ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV)) SPDK_LIB_LIST += env_dpdk_rpc endif - ifeq ($(OS),Linux) SPDK_LIB_LIST += event_nbd nbd endif diff --git a/lib/nvme/Makefile b/lib/nvme/Makefile index ac9439682..3e18b44ae 100644 --- a/lib/nvme/Makefile +++ b/lib/nvme/Makefile @@ -45,24 +45,6 @@ C_SRCS-$(CONFIG_NVME_CUSE) += nvme_cuse.c LIBNAME = nvme LOCAL_SYS_LIBS = -luuid -ifeq ($(CONFIG_RDMA),y) -LOCAL_SYS_LIBS += -libverbs -lrdmacm -#Attach only if FreeBSD and RDMA is specified with configure -ifeq ($(OS),FreeBSD) -# Mellanox - MLX4 HBA Userspace Library -ifneq ("$(wildcard /usr/lib/libmlx4.*)","") -LOCAL_SYS_LIBS += -lmlx4 -endif -# Mellanox - MLX5 HBA Userspace Library -ifneq ("$(wildcard /usr/lib/libmlx5.*)","") -LOCAL_SYS_LIBS += -lmlx5 -endif -# Chelsio HBA Userspace Library -ifneq ("$(wildcard /usr/lib/libcxgb4.*)","") -LOCAL_SYS_LIBS += -lcxgb4 -endif -endif -endif ifeq ($(CONFIG_NVME_CUSE),y) # fuse requires to set _FILE_OFFSET_BITS to 64 bits even for 64 bit machines diff --git a/lib/nvme/nvme_rdma.c b/lib/nvme/nvme_rdma.c index e0cc69e74..59b7e8118 100644 --- a/lib/nvme/nvme_rdma.c +++ b/lib/nvme/nvme_rdma.c @@ -37,10 +37,6 @@ #include "spdk/stdinc.h" -#include -#include -#include - #include "spdk/assert.h" #include "spdk/log.h" #include "spdk/trace.h" @@ -54,6 +50,7 @@ #include "spdk/config.h" #include "nvme_internal.h" +#include "spdk_internal/rdma.h" #define NVME_RDMA_TIME_OUT_IN_MS 2000 #define NVME_RDMA_RW_BUFFER_SIZE 131072 @@ -146,8 +143,8 @@ struct spdk_nvme_recv_wr_list { struct nvme_rdma_qpair { struct spdk_nvme_qpair qpair; + struct spdk_rdma_qp *rdma_qp; struct rdma_cm_id *cm_id; - struct ibv_cq *cq; struct spdk_nvme_rdma_req *rdma_reqs; @@ -503,7 +500,7 @@ static int nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair) { int rc; - struct ibv_qp_init_attr attr; + struct spdk_rdma_qp_init_attr attr = {}; struct ibv_device_attr dev_attr; struct nvme_rdma_ctrlr *rctrlr; @@ -526,8 +523,7 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair) rctrlr->pd = NULL; } - memset(&attr, 0, sizeof(struct ibv_qp_init_attr)); - attr.qp_type = IBV_QPT_RC; + attr.pd = rctrlr->pd; attr.send_cq = rqpair->cq; attr.recv_cq = rqpair->cq; attr.cap.max_send_wr = rqpair->num_entries; /* SEND operations */ @@ -535,10 +531,9 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair) attr.cap.max_send_sge = spdk_min(NVME_RDMA_DEFAULT_TX_SGE, dev_attr.max_sge); attr.cap.max_recv_sge = spdk_min(NVME_RDMA_DEFAULT_RX_SGE, dev_attr.max_sge); - rc = rdma_create_qp(rqpair->cm_id, rctrlr->pd, &attr); + rqpair->rdma_qp = spdk_rdma_qp_create(rqpair->cm_id, &attr); - if (rc) { - SPDK_ERRLOG("rdma_create_qp failed\n"); + if (!rqpair->rdma_qp) { return -1; } @@ -1684,8 +1679,9 @@ nvme_rdma_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme } } - if (rqpair->cm_id->qp) { - rdma_destroy_qp(rqpair->cm_id); + if (rqpair->rdma_qp) { + spdk_rdma_qp_destroy(rqpair->rdma_qp); + rqpair->rdma_qp = NULL; } rdma_destroy_id(rqpair->cm_id); rqpair->cm_id = NULL; diff --git a/mk/nvme.libtest.mk b/mk/nvme.libtest.mk index 861ab6c17..379c87d7f 100644 --- a/mk/nvme.libtest.mk +++ b/mk/nvme.libtest.mk @@ -43,7 +43,7 @@ SPDK_LIB_LIST = $(filter-out sock_vpp,$(SOCK_MODULES_LIST)) SPDK_LIB_LIST += nvme thread util log sock vmd ifeq ($(CONFIG_RDMA),y) -SYS_LIBS += -libverbs -lrdmacm +SPDK_LIB_LIST += rdma endif include $(SPDK_ROOT_DIR)/mk/spdk.app.mk diff --git a/mk/spdk.lib_deps.mk b/mk/spdk.lib_deps.mk index f243afebd..6ad9af526 100644 --- a/mk/spdk.lib_deps.mk +++ b/mk/spdk.lib_deps.mk @@ -52,10 +52,14 @@ DEPDIRS-vmd := log DEPDIRS-conf := log util DEPDIRS-json := log util -DEPDIRS-nvme := log sock util +DEPDIRS-rdma := log util DEPDIRS-reduce := log util DEPDIRS-thread := log util -DEPDIRS-rdma := log util + +DEPDIRS-nvme := log sock util +ifeq ($(CONFIG_RDMA),y) +DEPDIRS-nvme += rdma +endif DEPDIRS-blob := log util thread DEPDIRS-accel := log util thread diff --git a/mk/spdk.nvmecli.mk b/mk/spdk.nvmecli.mk index 66c84612a..0b6f416ef 100644 --- a/mk/spdk.nvmecli.mk +++ b/mk/spdk.nvmecli.mk @@ -47,10 +47,16 @@ ifneq (, $(wildcard $(DPDK_LIB_DIR)/librte_power.*)) DPDK_LIB_LIST += -lrte_power endif +NVMECLI_SPDK_LIBS = -lspdk_log -lspdk_sock -lspdk_nvme -lspdk_env_dpdk -lspdk_util + +ifeq ($(CONFIG_RDMA),y) +NVMECLI_SPDK_LIBS += -lspdk_rdma +endif + override CFLAGS += -I$(SPDK_ROOT_DIR)/include override LDFLAGS += \ -Wl,--whole-archive \ - -L$(SPDK_LIB_DIR) -lspdk_log -lspdk_sock -lspdk_nvme -lspdk_env_dpdk -lspdk_util \ + -L$(SPDK_LIB_DIR) $(NVMECLI_SPDK_LIBS) \ -L$(DPDK_LIB_DIR) $(DPDK_LIB_LIST) \ -Wl,--no-whole-archive \ -ldl -pthread -lrt -lrdmacm -lnuma -libverbs diff --git a/test/app/fuzz/nvme_fuzz/Makefile b/test/app/fuzz/nvme_fuzz/Makefile index aa24a9ec0..b7ad5e172 100644 --- a/test/app/fuzz/nvme_fuzz/Makefile +++ b/test/app/fuzz/nvme_fuzz/Makefile @@ -42,4 +42,8 @@ C_SRCS := nvme_fuzz.c SPDK_LIB_LIST += $(SOCK_MODULES_LIST) SPDK_LIB_LIST += conf event json jsonrpc log nvme rpc sock thread trace util +ifeq ($(CONFIG_RDMA),y) +SPDK_LIB_LIST += rdma +endif + include $(SPDK_ROOT_DIR)/mk/spdk.app.mk diff --git a/test/app/stub/Makefile b/test/app/stub/Makefile index d1acbe4f7..79ca8a912 100644 --- a/test/app/stub/Makefile +++ b/test/app/stub/Makefile @@ -42,4 +42,8 @@ C_SRCS := stub.c SPDK_LIB_LIST = $(SOCK_MODULES_LIST) SPDK_LIB_LIST += event conf nvme log trace rpc jsonrpc json thread util sock notify +ifeq ($(CONFIG_RDMA),y) +SPDK_LIB_LIST += rdma +endif + include $(SPDK_ROOT_DIR)/mk/spdk.app.mk diff --git a/test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c b/test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c index ad61c649b..f2bf22393 100644 --- a/test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c +++ b/test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c @@ -35,6 +35,7 @@ #include "spdk_cunit.h" #include "nvme/nvme_rdma.c" #include "common/lib/nvme/common_stubs.h" +#include "common/lib/test_rdma.c" SPDK_LOG_REGISTER_COMPONENT("nvme", SPDK_LOG_NVME)