ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvme/rdma: Use RDMA provider API to create/destroy qpair

Browse Source 
This patch adds use of RDMA provider API to NVMEoF initiator.
Makefiles have been updated with new RDMA lib dependency

Change-Id: Ieaefeb12ee9681d3db2b618c5cf0c54dc52230af
Signed-off-by: Alexey Marchuk <alexeymar@mellanox.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1657
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Community-CI: Broadcom CI
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Alexey Marchuk 2020-03-14 12:20:44 +03:00 committed by Tomasz Zawadzki
parent ea7a4f3c53
commit b4a9d7d318
9 changed files with 32 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/iscsi_tgt/Makefile
View File

@ -52,7 +52,6 @@ ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
SPDK_LIB_LIST += env_dpdk_rpc
endif
ifeq ($(OS),Linux)
SPDK_LIB_LIST += event_nbd nbd
endif

18
lib/nvme/Makefile
View File

@ -45,24 +45,6 @@ C_SRCS-$(CONFIG_NVME_CUSE) += nvme_cuse.c
LIBNAME = nvme
LOCAL_SYS_LIBS = -luuid
ifeq ($(CONFIG_RDMA),y)
LOCAL_SYS_LIBS += -libverbs -lrdmacm
#Attach only if FreeBSD and RDMA is specified with configure
ifeq ($(OS),FreeBSD)
# Mellanox - MLX4 HBA Userspace Library
ifneq ("$(wildcard /usr/lib/libmlx4.*)","")
LOCAL_SYS_LIBS += -lmlx4
endif
# Mellanox - MLX5 HBA Userspace Library
ifneq ("$(wildcard /usr/lib/libmlx5.*)","")
LOCAL_SYS_LIBS += -lmlx5
endif
# Chelsio HBA Userspace Library
ifneq ("$(wildcard /usr/lib/libcxgb4.*)","")
LOCAL_SYS_LIBS += -lcxgb4
endif
endif
endif
ifeq ($(CONFIG_NVME_CUSE),y)
# fuse requires to set _FILE_OFFSET_BITS to 64 bits even for 64 bit machines

22
lib/nvme/nvme_rdma.c
View File

@ -37,10 +37,6 @@
#include "spdk/stdinc.h"
#include <infiniband/verbs.h>
#include <rdma/rdma_cma.h>
#include <rdma/rdma_verbs.h>
#include "spdk/assert.h"
#include "spdk/log.h"
#include "spdk/trace.h"
@ -54,6 +50,7 @@
#include "spdk/config.h"
#include "nvme_internal.h"
#include "spdk_internal/rdma.h"
#define NVME_RDMA_TIME_OUT_IN_MS 2000
#define NVME_RDMA_RW_BUFFER_SIZE 131072
@ -146,8 +143,8 @@ struct spdk_nvme_recv_wr_list {
struct nvme_rdma_qpair {
struct spdk_nvme_qpair qpair;
struct spdk_rdma_qp *rdma_qp;
struct rdma_cm_id *cm_id;
struct ibv_cq *cq;
struct spdk_nvme_rdma_req *rdma_reqs;
@ -503,7 +500,7 @@ static int
nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair)
{
int rc;
struct ibv_qp_init_attr attr;
struct spdk_rdma_qp_init_attr attr = {};
struct ibv_device_attr dev_attr;
struct nvme_rdma_ctrlr *rctrlr;
@ -526,8 +523,7 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair)
rctrlr->pd = NULL;
}
memset(&attr, 0, sizeof(struct ibv_qp_init_attr));
attr.qp_type = IBV_QPT_RC;
attr.pd = rctrlr->pd;
attr.send_cq = rqpair->cq;
attr.recv_cq = rqpair->cq;
attr.cap.max_send_wr = rqpair->num_entries; /* SEND operations */
@ -535,10 +531,9 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair)
attr.cap.max_send_sge = spdk_min(NVME_RDMA_DEFAULT_TX_SGE, dev_attr.max_sge);
attr.cap.max_recv_sge = spdk_min(NVME_RDMA_DEFAULT_RX_SGE, dev_attr.max_sge);
rc = rdma_create_qp(rqpair->cm_id, rctrlr->pd, &attr);
rqpair->rdma_qp = spdk_rdma_qp_create(rqpair->cm_id, &attr);
if (rc) {
SPDK_ERRLOG("rdma_create_qp failed\n");
if (!rqpair->rdma_qp) {
return -1;
}
@ -1684,8 +1679,9 @@ nvme_rdma_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme
}
}
if (rqpair->cm_id->qp) {
rdma_destroy_qp(rqpair->cm_id);
if (rqpair->rdma_qp) {
spdk_rdma_qp_destroy(rqpair->rdma_qp);
rqpair->rdma_qp = NULL;
}
rdma_destroy_id(rqpair->cm_id);
rqpair->cm_id = NULL;

2
mk/nvme.libtest.mk
View File

@ -43,7 +43,7 @@ SPDK_LIB_LIST = $(filter-out sock_vpp,$(SOCK_MODULES_LIST))
SPDK_LIB_LIST += nvme thread util log sock vmd
ifeq ($(CONFIG_RDMA),y)
SYS_LIBS += -libverbs -lrdmacm
SPDK_LIB_LIST += rdma
endif
include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

8
mk/spdk.lib_deps.mk
View File

@ -52,10 +52,14 @@ DEPDIRS-vmd := log
DEPDIRS-conf := log util
DEPDIRS-json := log util
DEPDIRS-nvme := log sock util
DEPDIRS-rdma := log util
DEPDIRS-reduce := log util
DEPDIRS-thread := log util
DEPDIRS-rdma := log util
DEPDIRS-nvme := log sock util
ifeq ($(CONFIG_RDMA),y)
DEPDIRS-nvme += rdma
endif
DEPDIRS-blob := log util thread
DEPDIRS-accel := log util thread

8
mk/spdk.nvmecli.mk
View File

@ -47,10 +47,16 @@ ifneq (, $(wildcard $(DPDK_LIB_DIR)/librte_power.*))
DPDK_LIB_LIST += -lrte_power
endif
NVMECLI_SPDK_LIBS = -lspdk_log -lspdk_sock -lspdk_nvme -lspdk_env_dpdk -lspdk_util
ifeq ($(CONFIG_RDMA),y)
NVMECLI_SPDK_LIBS += -lspdk_rdma
endif
override CFLAGS += -I$(SPDK_ROOT_DIR)/include
override LDFLAGS += \
-Wl,--whole-archive \
-L$(SPDK_LIB_DIR) -lspdk_log -lspdk_sock -lspdk_nvme -lspdk_env_dpdk -lspdk_util \
-L$(SPDK_LIB_DIR) $(NVMECLI_SPDK_LIBS) \
-L$(DPDK_LIB_DIR) $(DPDK_LIB_LIST) \
-Wl,--no-whole-archive \
-ldl -pthread -lrt -lrdmacm -lnuma -libverbs

4
test/app/fuzz/nvme_fuzz/Makefile
View File

@ -42,4 +42,8 @@ C_SRCS := nvme_fuzz.c
SPDK_LIB_LIST += $(SOCK_MODULES_LIST)
SPDK_LIB_LIST += conf event json jsonrpc log nvme rpc sock thread trace util
ifeq ($(CONFIG_RDMA),y)
SPDK_LIB_LIST += rdma
endif
include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

4
test/app/stub/Makefile
View File

@ -42,4 +42,8 @@ C_SRCS := stub.c
SPDK_LIB_LIST = $(SOCK_MODULES_LIST)
SPDK_LIB_LIST += event conf nvme log trace rpc jsonrpc json thread util sock notify
ifeq ($(CONFIG_RDMA),y)
SPDK_LIB_LIST += rdma
endif
include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

1
test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c
View File

@ -35,6 +35,7 @@
#include "spdk_cunit.h"
#include "nvme/nvme_rdma.c"
#include "common/lib/nvme/common_stubs.h"
#include "common/lib/test_rdma.c"
SPDK_LOG_REGISTER_COMPONENT("nvme", SPDK_LOG_NVME)