rdma: Add new API spdk_rdma_qp_accept

This API is a wrapper for rdma_accept which allows
to remove spdk_rdma_qp_init_attr::initiator_side.

Change-Id: Iba2be5e74e537c498fb11c939c922b2bbda95309
Signed-off-by: Alexey Marchuk <alexeymar@mellanox.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2908
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI
Community-CI: Mellanox Build Bot
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>

include/spdk_internal/rdma.h

@@ -45,7 +45,6 @@ struct spdk_rdma_qp_init_attr {
 	struct ibv_srq	       *srq;
 	struct ibv_qp_cap	cap;
 	struct ibv_pd	       *pd;
-	bool			initiator_side;
 };
 
 struct spdk_rdma_send_wr_list {
@@ -68,11 +67,19 @@ struct spdk_rdma_qp {
 struct spdk_rdma_qp *spdk_rdma_qp_create(struct rdma_cm_id *cm_id,
 		struct spdk_rdma_qp_init_attr *qp_attr);
 
+/**
+ * Accept a connection request. Called by the passive side (NVMEoF target)
+ * \param spdk_rdma_qp Pointer to a qpair
+ * \param conn_param Optional information needed to establish the connection
+ * \return 0 on success, errno on failure
+ */
+int spdk_rdma_qp_accept(struct spdk_rdma_qp *spdk_rdma_qp, struct rdma_conn_param *conn_param);
+
 /**
  * Complete the connection process, must be called by the active
  * side (NVMEoF initiator) upon receipt RDMA_CM_EVENT_CONNECT_RESPONSE
- * @param spdk_rdma_qp pointer to a qpair
+ * \param spdk_rdma_qp Pointer to a qpair
- * @return 0 on success, errno on failure
+ * \return 0 on success, errno on failure
  */
 int spdk_rdma_qp_complete_connect(struct spdk_rdma_qp *spdk_rdma_qp);
 

lib/nvme/nvme_rdma.c

@@ -619,7 +619,6 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair)
 	attr.cap.max_recv_wr	= rqpair->num_entries; /* RECV operations */
 	attr.cap.max_send_sge	= spdk_min(NVME_RDMA_DEFAULT_TX_SGE, dev_attr.max_sge);
 	attr.cap.max_recv_sge	= spdk_min(NVME_RDMA_DEFAULT_RX_SGE, dev_attr.max_sge);
-	attr.initiator_side	= true;
 
 	rqpair->rdma_qp = spdk_rdma_qp_create(rqpair->cm_id, &attr);
 

lib/nvmf/rdma.c

@@ -975,7 +975,6 @@ nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair)
 	qp_init_attr.cap.max_send_wr	= (uint32_t)rqpair->max_queue_depth * 2;
 	qp_init_attr.cap.max_send_sge	= spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_TX_SGE);
 	qp_init_attr.cap.max_recv_sge	= spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_RX_SGE);
-	qp_init_attr.initiator_side	= false;
 
 	if (rqpair->srq == NULL && nvmf_rdma_resize_cq(rqpair, device) < 0) {
 		SPDK_ERRLOG("Failed to resize the completion queue. Cannot initialize qpair.\n");
@@ -1168,9 +1167,9 @@ nvmf_rdma_event_accept(struct rdma_cm_id *id, struct spdk_nvmf_rdma_qpair *rqpai
 	ctrlr_event_data.srq = rqpair->srq ? 1 : 0;
 	ctrlr_event_data.qp_num = rqpair->rdma_qp->qp->qp_num;
 
-	rc = rdma_accept(id, &ctrlr_event_data);
+	rc = spdk_rdma_qp_accept(rqpair->rdma_qp, &ctrlr_event_data);
 	if (rc) {
-		SPDK_ERRLOG("Error %d on rdma_accept\n", errno);
+		SPDK_ERRLOG("Error %d on spdk_rdma_qp_accept\n", errno);
 	} else {
 		SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Sent back the accept\n");
 	}

lib/rdma/rdma_mlx5_dv.c

@@ -44,7 +44,6 @@
 struct spdk_rdma_mlx5_dv_qp {
 	struct spdk_rdma_qp common;
 	struct ibv_qp_ex *qpex;
-	bool initiator_side;
 };
 
 static int
@@ -132,25 +131,38 @@ spdk_rdma_qp_create(struct rdma_cm_id *cm_id, struct spdk_rdma_qp_init_attr *qp_
 	mlx5_qp->common.qp = qp;
 	mlx5_qp->common.cm_id = cm_id;
 	mlx5_qp->qpex = ibv_qp_to_qp_ex(qp);
-	mlx5_qp->initiator_side = qp_attr->initiator_side;
 
 	if (!mlx5_qp->qpex) {
 		spdk_rdma_qp_destroy(&mlx5_qp->common);
 		return NULL;
 	}
 
-	/* NVMEoF target must move qpair to RTS state */
-	if (!mlx5_qp->initiator_side && rdma_mlx5_dv_init_qpair(mlx5_qp) != 0) {
-		SPDK_ERRLOG("Failed to initialize qpair\n");
-		spdk_rdma_qp_destroy(&mlx5_qp->common);
-		return NULL;
-	}
-
 	qp_attr->cap = dv_qp_attr.cap;
 
 	return &mlx5_qp->common;
 }
 
+int
+spdk_rdma_qp_accept(struct spdk_rdma_qp *spdk_rdma_qp, struct rdma_conn_param *conn_param)
+{
+	struct spdk_rdma_mlx5_dv_qp *mlx5_qp;
+
+	assert(spdk_rdma_qp != NULL);
+	assert(spdk_rdma_qp->cm_id != NULL);
+
+	mlx5_qp = SPDK_CONTAINEROF(spdk_rdma_qp, struct spdk_rdma_mlx5_dv_qp, common);
+
+	/* NVMEoF target must move qpair to RTS state */
+	if (rdma_mlx5_dv_init_qpair(mlx5_qp) != 0) {
+		SPDK_ERRLOG("Failed to initialize qpair\n");
+		/* Set errno to be compliant with rdma_accept behaviour */
+		errno = ECONNABORTED;
+		return -1;
+	}
+
+	return rdma_accept(spdk_rdma_qp->cm_id, conn_param);
+}
+
 int
 spdk_rdma_qp_complete_connect(struct spdk_rdma_qp *spdk_rdma_qp)
 {
@@ -160,9 +172,6 @@ spdk_rdma_qp_complete_connect(struct spdk_rdma_qp *spdk_rdma_qp)
 	assert(spdk_rdma_qp);
 
 	mlx5_qp = SPDK_CONTAINEROF(spdk_rdma_qp, struct spdk_rdma_mlx5_dv_qp, common);
-	if (!mlx5_qp->initiator_side) {
-		return 0;
-	}
 
 	rc = rdma_mlx5_dv_init_qpair(mlx5_qp);
 	if (rc) {

lib/rdma/rdma_verbs.c

@@ -74,6 +74,15 @@ spdk_rdma_qp_create(struct rdma_cm_id *cm_id, struct spdk_rdma_qp_init_attr *qp_
 	return spdk_rdma_qp;
 }
 
+int
+spdk_rdma_qp_accept(struct spdk_rdma_qp *spdk_rdma_qp, struct rdma_conn_param *conn_param)
+{
+	assert(spdk_rdma_qp != NULL);
+	assert(spdk_rdma_qp->cm_id != NULL);
+
+	return rdma_accept(spdk_rdma_qp->cm_id, conn_param);
+}
+
 int
 spdk_rdma_qp_complete_connect(struct spdk_rdma_qp *spdk_rdma_qp)
 {

lib/rdma/spdk_rdma.map

@@ -3,6 +3,7 @@
 
 	# Public functions
 	spdk_rdma_qp_create;
+	spdk_rdma_qp_accept;
 	spdk_rdma_qp_complete_connect;
 	spdk_rdma_qp_destroy;
 	spdk_rdma_qp_disconnect;

test/common/lib/test_rdma.c

@@ -38,6 +38,8 @@
 
 DEFINE_STUB(spdk_rdma_qp_create, struct spdk_rdma_qp *, (struct rdma_cm_id *cm_id,
 		struct spdk_rdma_qp_init_attr *qp_attr), NULL);
+DEFINE_STUB(spdk_rdma_qp_accept, int, (struct spdk_rdma_qp *spdk_rdma_qp,
+				       struct rdma_conn_param *conn_param), 0);
 DEFINE_STUB(spdk_rdma_qp_complete_connect, int, (struct spdk_rdma_qp *spdk_rdma_qp), 0);
 DEFINE_STUB_V(spdk_rdma_qp_destroy, (struct spdk_rdma_qp *spdk_rdma_qp));
 DEFINE_STUB(spdk_rdma_qp_disconnect, int, (struct spdk_rdma_qp *spdk_rdma_qp), 0);