From 268aacb24a697d2b510bbfcf7612624410148d18 Mon Sep 17 00:00:00 2001 From: Alexey Marchuk Date: Tue, 16 Jun 2020 11:31:24 +0300 Subject: [PATCH] rdma: Add new API spdk_rdma_qp_accept This API is a wrapper for rdma_accept which allows to remove spdk_rdma_qp_init_attr::initiator_side. Change-Id: Iba2be5e74e537c498fb11c939c922b2bbda95309 Signed-off-by: Alexey Marchuk Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2908 Tested-by: SPDK CI Jenkins Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Reviewed-by: Ben Walker Reviewed-by: Tomasz Zawadzki --- include/spdk_internal/rdma.h | 13 ++++++++++--- lib/nvme/nvme_rdma.c | 1 - lib/nvmf/rdma.c | 5 ++--- lib/rdma/rdma_mlx5_dv.c | 33 +++++++++++++++++++++------------ lib/rdma/rdma_verbs.c | 9 +++++++++ lib/rdma/spdk_rdma.map | 1 + test/common/lib/test_rdma.c | 2 ++ 7 files changed, 45 insertions(+), 19 deletions(-) diff --git a/include/spdk_internal/rdma.h b/include/spdk_internal/rdma.h index fb34e5069..4a6d5104b 100644 --- a/include/spdk_internal/rdma.h +++ b/include/spdk_internal/rdma.h @@ -45,7 +45,6 @@ struct spdk_rdma_qp_init_attr { struct ibv_srq *srq; struct ibv_qp_cap cap; struct ibv_pd *pd; - bool initiator_side; }; struct spdk_rdma_send_wr_list { @@ -68,11 +67,19 @@ struct spdk_rdma_qp { struct spdk_rdma_qp *spdk_rdma_qp_create(struct rdma_cm_id *cm_id, struct spdk_rdma_qp_init_attr *qp_attr); +/** + * Accept a connection request. Called by the passive side (NVMEoF target) + * \param spdk_rdma_qp Pointer to a qpair + * \param conn_param Optional information needed to establish the connection + * \return 0 on success, errno on failure + */ +int spdk_rdma_qp_accept(struct spdk_rdma_qp *spdk_rdma_qp, struct rdma_conn_param *conn_param); + /** * Complete the connection process, must be called by the active * side (NVMEoF initiator) upon receipt RDMA_CM_EVENT_CONNECT_RESPONSE - * @param spdk_rdma_qp pointer to a qpair - * @return 0 on success, errno on failure + * \param spdk_rdma_qp Pointer to a qpair + * \return 0 on success, errno on failure */ int spdk_rdma_qp_complete_connect(struct spdk_rdma_qp *spdk_rdma_qp); diff --git a/lib/nvme/nvme_rdma.c b/lib/nvme/nvme_rdma.c index 9831294a1..5c9895712 100644 --- a/lib/nvme/nvme_rdma.c +++ b/lib/nvme/nvme_rdma.c @@ -619,7 +619,6 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair) attr.cap.max_recv_wr = rqpair->num_entries; /* RECV operations */ attr.cap.max_send_sge = spdk_min(NVME_RDMA_DEFAULT_TX_SGE, dev_attr.max_sge); attr.cap.max_recv_sge = spdk_min(NVME_RDMA_DEFAULT_RX_SGE, dev_attr.max_sge); - attr.initiator_side = true; rqpair->rdma_qp = spdk_rdma_qp_create(rqpair->cm_id, &attr); diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index 8023b4b1c..fc7427e2d 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -975,7 +975,6 @@ nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair) qp_init_attr.cap.max_send_wr = (uint32_t)rqpair->max_queue_depth * 2; qp_init_attr.cap.max_send_sge = spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_TX_SGE); qp_init_attr.cap.max_recv_sge = spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_RX_SGE); - qp_init_attr.initiator_side = false; if (rqpair->srq == NULL && nvmf_rdma_resize_cq(rqpair, device) < 0) { SPDK_ERRLOG("Failed to resize the completion queue. Cannot initialize qpair.\n"); @@ -1168,9 +1167,9 @@ nvmf_rdma_event_accept(struct rdma_cm_id *id, struct spdk_nvmf_rdma_qpair *rqpai ctrlr_event_data.srq = rqpair->srq ? 1 : 0; ctrlr_event_data.qp_num = rqpair->rdma_qp->qp->qp_num; - rc = rdma_accept(id, &ctrlr_event_data); + rc = spdk_rdma_qp_accept(rqpair->rdma_qp, &ctrlr_event_data); if (rc) { - SPDK_ERRLOG("Error %d on rdma_accept\n", errno); + SPDK_ERRLOG("Error %d on spdk_rdma_qp_accept\n", errno); } else { SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Sent back the accept\n"); } diff --git a/lib/rdma/rdma_mlx5_dv.c b/lib/rdma/rdma_mlx5_dv.c index 2002354c2..bae3afdda 100644 --- a/lib/rdma/rdma_mlx5_dv.c +++ b/lib/rdma/rdma_mlx5_dv.c @@ -44,7 +44,6 @@ struct spdk_rdma_mlx5_dv_qp { struct spdk_rdma_qp common; struct ibv_qp_ex *qpex; - bool initiator_side; }; static int @@ -132,25 +131,38 @@ spdk_rdma_qp_create(struct rdma_cm_id *cm_id, struct spdk_rdma_qp_init_attr *qp_ mlx5_qp->common.qp = qp; mlx5_qp->common.cm_id = cm_id; mlx5_qp->qpex = ibv_qp_to_qp_ex(qp); - mlx5_qp->initiator_side = qp_attr->initiator_side; if (!mlx5_qp->qpex) { spdk_rdma_qp_destroy(&mlx5_qp->common); return NULL; } - /* NVMEoF target must move qpair to RTS state */ - if (!mlx5_qp->initiator_side && rdma_mlx5_dv_init_qpair(mlx5_qp) != 0) { - SPDK_ERRLOG("Failed to initialize qpair\n"); - spdk_rdma_qp_destroy(&mlx5_qp->common); - return NULL; - } - qp_attr->cap = dv_qp_attr.cap; return &mlx5_qp->common; } +int +spdk_rdma_qp_accept(struct spdk_rdma_qp *spdk_rdma_qp, struct rdma_conn_param *conn_param) +{ + struct spdk_rdma_mlx5_dv_qp *mlx5_qp; + + assert(spdk_rdma_qp != NULL); + assert(spdk_rdma_qp->cm_id != NULL); + + mlx5_qp = SPDK_CONTAINEROF(spdk_rdma_qp, struct spdk_rdma_mlx5_dv_qp, common); + + /* NVMEoF target must move qpair to RTS state */ + if (rdma_mlx5_dv_init_qpair(mlx5_qp) != 0) { + SPDK_ERRLOG("Failed to initialize qpair\n"); + /* Set errno to be compliant with rdma_accept behaviour */ + errno = ECONNABORTED; + return -1; + } + + return rdma_accept(spdk_rdma_qp->cm_id, conn_param); +} + int spdk_rdma_qp_complete_connect(struct spdk_rdma_qp *spdk_rdma_qp) { @@ -160,9 +172,6 @@ spdk_rdma_qp_complete_connect(struct spdk_rdma_qp *spdk_rdma_qp) assert(spdk_rdma_qp); mlx5_qp = SPDK_CONTAINEROF(spdk_rdma_qp, struct spdk_rdma_mlx5_dv_qp, common); - if (!mlx5_qp->initiator_side) { - return 0; - } rc = rdma_mlx5_dv_init_qpair(mlx5_qp); if (rc) { diff --git a/lib/rdma/rdma_verbs.c b/lib/rdma/rdma_verbs.c index e2896ea60..66be5bf60 100644 --- a/lib/rdma/rdma_verbs.c +++ b/lib/rdma/rdma_verbs.c @@ -74,6 +74,15 @@ spdk_rdma_qp_create(struct rdma_cm_id *cm_id, struct spdk_rdma_qp_init_attr *qp_ return spdk_rdma_qp; } +int +spdk_rdma_qp_accept(struct spdk_rdma_qp *spdk_rdma_qp, struct rdma_conn_param *conn_param) +{ + assert(spdk_rdma_qp != NULL); + assert(spdk_rdma_qp->cm_id != NULL); + + return rdma_accept(spdk_rdma_qp->cm_id, conn_param); +} + int spdk_rdma_qp_complete_connect(struct spdk_rdma_qp *spdk_rdma_qp) { diff --git a/lib/rdma/spdk_rdma.map b/lib/rdma/spdk_rdma.map index f20e47a7a..9268a2191 100644 --- a/lib/rdma/spdk_rdma.map +++ b/lib/rdma/spdk_rdma.map @@ -3,6 +3,7 @@ # Public functions spdk_rdma_qp_create; + spdk_rdma_qp_accept; spdk_rdma_qp_complete_connect; spdk_rdma_qp_destroy; spdk_rdma_qp_disconnect; diff --git a/test/common/lib/test_rdma.c b/test/common/lib/test_rdma.c index f25570c82..109862fe6 100644 --- a/test/common/lib/test_rdma.c +++ b/test/common/lib/test_rdma.c @@ -38,6 +38,8 @@ DEFINE_STUB(spdk_rdma_qp_create, struct spdk_rdma_qp *, (struct rdma_cm_id *cm_id, struct spdk_rdma_qp_init_attr *qp_attr), NULL); +DEFINE_STUB(spdk_rdma_qp_accept, int, (struct spdk_rdma_qp *spdk_rdma_qp, + struct rdma_conn_param *conn_param), 0); DEFINE_STUB(spdk_rdma_qp_complete_connect, int, (struct spdk_rdma_qp *spdk_rdma_qp), 0); DEFINE_STUB_V(spdk_rdma_qp_destroy, (struct spdk_rdma_qp *spdk_rdma_qp)); DEFINE_STUB(spdk_rdma_qp_disconnect, int, (struct spdk_rdma_qp *spdk_rdma_qp), 0);