ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvmf/rdma: Use RDMA provider API to create/destroy qpair

Browse Source 
This patch updates NVMF target to use RDMA provider API
to create and destroy qpairs.
Makefiles have been updated with new dependency on RDMA lib

Change-Id: Iae35aea601380f8d1a6453a7fd6115f781e126f5
Signed-off-by: Alexey Marchuk <alexeymar@mellanox.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1656
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Community-CI: Broadcom CI
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Seth Howell <seth.howell@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
This commit is contained in:
Alexey Marchuk 2020-03-14 11:32:18 +03:00 committed by Tomasz Zawadzki
parent 7e14f72443
commit ea7a4f3c53
7 changed files with 76 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
lib/nvmf/Makefile
View File

@ -44,24 +44,6 @@ C_SRCS = ctrlr.c ctrlr_discovery.c ctrlr_bdev.c \
C_SRCS-$(CONFIG_RDMA) += rdma.c
LIBNAME = nvmf
LOCAL_SYS_LIBS = -luuid
ifeq ($(CONFIG_RDMA),y)
LOCAL_SYS_LIBS += -libverbs -lrdmacm
#Attach only if FreeBSD and RDMA is specified with configure
ifeq ($(OS),FreeBSD)
# Mellanox - MLX4 HBA Userspace Library
ifneq ("$(wildcard /usr/lib/libmlx4.*)","")
LOCAL_SYS_LIBS += -lmlx4
endif
# Mellanox - MLX5 HBA Userspace Library
ifneq ("$(wildcard /usr/lib/libmlx5.*)","")
LOCAL_SYS_LIBS += -lmlx5
endif
# Chelsio HBA Userspace Library
ifneq ("$(wildcard /usr/lib/libcxgb4.*)","")
LOCAL_SYS_LIBS += -lcxgb4
endif
endif
endif
ifeq ($(CONFIG_FC),y)
C_SRCS += fc.c fc_ls.c

54
lib/nvmf/rdma.c
View File

@ -33,10 +33,6 @@
#include "spdk/stdinc.h"
#include <infiniband/verbs.h>
#include <rdma/rdma_cma.h>
#include <rdma/rdma_verbs.h>
#include "spdk/config.h"
#include "spdk/thread.h"
#include "spdk/likely.h"
@ -47,6 +43,7 @@
#include "spdk_internal/assert.h"
#include "spdk_internal/log.h"
#include "spdk_internal/rdma.h"
struct spdk_nvme_rdma_hooks g_nvmf_hooks = {};
const struct spdk_nvmf_transport_ops spdk_nvmf_transport_rdma;
@ -343,6 +340,7 @@ struct spdk_nvmf_rdma_qpair {
struct spdk_nvmf_rdma_device *device;
struct spdk_nvmf_rdma_poller *poller;
struct spdk_rdma_qp *rdma_qp;
struct rdma_cm_id *cm_id;
struct ibv_srq *srq;
struct rdma_cm_id *listen_id;
@ -896,8 +894,9 @@ nvmf_rdma_qpair_destroy(struct spdk_nvmf_rdma_qpair *rqpair)
}
if (rqpair->cm_id) {
if (rqpair->cm_id->qp != NULL) {
rdma_destroy_qp(rqpair->cm_id);
if (rqpair->rdma_qp != NULL) {
spdk_rdma_qp_destroy(rqpair->rdma_qp);
rqpair->rdma_qp = NULL;
}
rdma_destroy_id(rqpair->cm_id);
@ -955,48 +954,45 @@ static int
nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair)
{
struct spdk_nvmf_rdma_qpair *rqpair;
int rc;
struct spdk_nvmf_rdma_transport *rtransport;
struct spdk_nvmf_transport *transport;
struct spdk_nvmf_rdma_resource_opts opts;
struct spdk_nvmf_rdma_device *device;
struct ibv_qp_init_attr ibv_init_attr;
struct spdk_rdma_qp_init_attr qp_init_attr = {};
rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair);
device = rqpair->device;
memset(&ibv_init_attr, 0, sizeof(struct ibv_qp_init_attr));
ibv_init_attr.qp_context = rqpair;
ibv_init_attr.qp_type = IBV_QPT_RC;
ibv_init_attr.send_cq = rqpair->poller->cq;
ibv_init_attr.recv_cq = rqpair->poller->cq;
qp_init_attr.qp_context = rqpair;
qp_init_attr.pd = device->pd;
qp_init_attr.send_cq = rqpair->poller->cq;
qp_init_attr.recv_cq = rqpair->poller->cq;
if (rqpair->srq) {
ibv_init_attr.srq = rqpair->srq;
qp_init_attr.srq = rqpair->srq;
} else {
ibv_init_attr.cap.max_recv_wr = rqpair->max_queue_depth;
qp_init_attr.cap.max_recv_wr = rqpair->max_queue_depth;
}
ibv_init_attr.cap.max_send_wr = (uint32_t)rqpair->max_queue_depth *
2; /* SEND, READ or WRITE operations */
ibv_init_attr.cap.max_send_sge = spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_TX_SGE);
ibv_init_attr.cap.max_recv_sge = spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_RX_SGE);
/* SEND, READ, and WRITE operations */
qp_init_attr.cap.max_send_wr = (uint32_t)rqpair->max_queue_depth * 2;
qp_init_attr.cap.max_send_sge = spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_TX_SGE);
qp_init_attr.cap.max_recv_sge = spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_RX_SGE);
if (rqpair->srq == NULL && nvmf_rdma_resize_cq(rqpair, device) < 0) {
SPDK_ERRLOG("Failed to resize the completion queue. Cannot initialize qpair.\n");
goto error;
}
rc = rdma_create_qp(rqpair->cm_id, device->pd, &ibv_init_attr);
if (rc) {
SPDK_ERRLOG("rdma_create_qp failed: errno %d: %s\n", errno, spdk_strerror(errno));
rqpair->rdma_qp = spdk_rdma_qp_create(rqpair->cm_id, &qp_init_attr);
if (!rqpair->rdma_qp) {
goto error;
}
rqpair->max_send_depth = spdk_min((uint32_t)(rqpair->max_queue_depth * 2),
ibv_init_attr.cap.max_send_wr);
rqpair->max_send_sge = spdk_min(NVMF_DEFAULT_TX_SGE, ibv_init_attr.cap.max_send_sge);
rqpair->max_recv_sge = spdk_min(NVMF_DEFAULT_RX_SGE, ibv_init_attr.cap.max_recv_sge);
qp_init_attr.cap.max_send_wr);
rqpair->max_send_sge = spdk_min(NVMF_DEFAULT_TX_SGE, qp_init_attr.cap.max_send_sge);
rqpair->max_recv_sge = spdk_min(NVMF_DEFAULT_RX_SGE, qp_init_attr.cap.max_recv_sge);
spdk_trace_record(TRACE_RDMA_QP_CREATE, 0, 0, (uintptr_t)rqpair->cm_id, 0);
SPDK_DEBUGLOG(SPDK_LOG_RDMA, "New RDMA Connection: %p\n", qpair);
@ -1181,6 +1177,14 @@ nvmf_rdma_event_accept(struct rdma_cm_id *id, struct spdk_nvmf_rdma_qpair *rqpai
ctrlr_event_data.rnr_retry_count = 0x7;
}
/* When qpair is created without use of rdma cm API, an additional
* information must be provided to initiator in the connection response:
* whether qpair is using SRQ and its qp_num
* Fields below are ignored by rdma cm if qpair has been
* created using rdma cm API. */
ctrlr_event_data.srq = rqpair->srq ? 1 : 0;
ctrlr_event_data.qp_num = rqpair->rdma_qp->qp->qp_num;
rc = rdma_accept(id, &ctrlr_event_data);
if (rc) {
SPDK_ERRLOG("Error %d on rdma_accept\n", errno);

3
mk/spdk.lib_deps.mk
View File

@ -77,6 +77,9 @@ DEPDIRS-event := log util conf thread $(JSON_LIBS) trace
DEPDIRS-ftl := log util thread trace bdev
DEPDIRS-nbd := log util thread $(JSON_LIBS) bdev
DEPDIRS-nvmf := log sock util nvme thread $(JSON_LIBS) trace bdev
ifeq ($(CONFIG_RDMA),y)
DEPDIRS-nvmf += rdma
endif
DEPDIRS-scsi := log util thread $(JSON_LIBS) trace bdev
DEPDIRS-iscsi := log sock util conf thread $(JSON_LIBS) trace event scsi

1
mk/spdk.modules.mk
View File

@ -52,6 +52,7 @@ endif
ifeq ($(CONFIG_RDMA),y)
SYS_LIBS += -libverbs -lrdmacm
BLOCKDEV_MODULES_LIST += rdma
endif
ifeq ($(OS),Linux)

41
test/common/lib/test_rdma.c Normal file
View File

@ -0,0 +1,41 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation. All rights reserved.
* Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "spdk/stdinc.h"
#include "spdk_internal/rdma.h"
#include "spdk_internal/mock.h"
DEFINE_STUB(spdk_rdma_qp_create, struct spdk_rdma_qp *, (struct rdma_cm_id *cm_id,
struct spdk_rdma_qp_init_attr *qp_attr), NULL);
DEFINE_STUB_V(spdk_rdma_qp_destroy, (struct spdk_rdma_qp *spdk_rdma_qp));

1
test/common/skipped_build_files.txt
View File

@ -42,6 +42,7 @@ test/blobfs/fuse/fuse
test/common/lib/test_env
test/common/lib/test_sock
test/common/lib/ut_multithread
test/common/lib/test_rdma
test/unit/lib/blob/bs_dev_common
test/unit/lib/blob/bs_scheduler
test/unit/lib/ftl/common/utils

1
test/unit/lib/nvmf/rdma.c/rdma_ut.c
View File

@ -34,6 +34,7 @@
#include "spdk/stdinc.h"
#include "spdk_cunit.h"
#include "common/lib/test_env.c"
#include "common/lib/test_rdma.c"
#include "nvmf/rdma.c"
#include "nvmf/transport.c"