From eb2ec1b07b58c3a68d94c9e134e1e093d535955f Mon Sep 17 00:00:00 2001
From: Ben Walker <benjamin.walker@intel.com>
Date: Tue, 20 Dec 2016 12:18:30 -0700
Subject: [PATCH] nvme/rdma: Reap multiple completions per poll

This is more CPU efficient than only grabbing one
completion per call to ibv_poll_cq.

Change-Id: I0c70d33639f0f345482d9e7c810f9c6723937058
Signed-off-by: Ben Walker <benjamin.walker@intel.com>
---
 lib/nvme/nvme_rdma.c | 112 ++++++++++++++++++++-----------------------
 1 file changed, 52 insertions(+), 60 deletions(-)

diff --git a/lib/nvme/nvme_rdma.c b/lib/nvme/nvme_rdma.c
index e51a3c361..03497c23a 100644
--- a/lib/nvme/nvme_rdma.c
+++ b/lib/nvme/nvme_rdma.c
@@ -1374,93 +1374,85 @@ nvme_rdma_qpair_fail(struct spdk_nvme_qpair *qpair)
 	return 0;
 }
 
+#define MAX_COMPLETIONS_PER_POLL 128
+
 int
 nvme_rdma_qpair_process_completions(struct spdk_nvme_qpair *qpair,
 				    uint32_t max_completions)
 {
-	struct nvme_rdma_qpair *rqpair;
-	struct ibv_wc wc;
-	uint32_t size;
-	int rc;
-	uint32_t io_completed = 0;
+	struct nvme_rdma_qpair 	*rqpair = nvme_rdma_qpair(qpair);
+	struct ibv_wc 		wc[MAX_COMPLETIONS_PER_POLL];
+	int			i, rc, batch_size;
+	uint32_t 		reaped;
 
-	rqpair = nvme_rdma_qpair(qpair);
-	size = rqpair->num_entries - 1U;
-	if (!max_completions || max_completions > size) {
-		max_completions = size;
+	if (max_completions == 0) {
+		max_completions = rqpair->num_entries;
+	} else {
+		max_completions = nvme_min(max_completions, rqpair->num_entries);
 	}
 
-	/* poll the send_cq */
-	while (true) {
-		rc = ibv_poll_cq(rqpair->cm_id->send_cq, 1, &wc);
-		if (rc == 0) {
-			break;
-		}
-
+	/* Consume all send completions */
+	reaped = 0;
+	do {
+		batch_size = nvme_min((max_completions - reaped),
+				      MAX_COMPLETIONS_PER_POLL);
+		rc = ibv_poll_cq(rqpair->cm_id->send_cq, batch_size, wc);
 		if (rc < 0) {
-			SPDK_ERRLOG("Poll CQ error!(%d): %s\n",
+			SPDK_ERRLOG("Error polling CQ! (%d): %s\n",
 				    errno, strerror(errno));
 			return -1;
-		}
-
-		if (wc.status) {
-			SPDK_ERRLOG("CQ completion error status %d, exiting handler\n",
-				    wc.status);
-			break;
-		}
-
-		if (wc.opcode == IBV_WC_SEND) {
-			SPDK_TRACELOG(SPDK_TRACE_DEBUG, "CQ send completion\n");
-		} else {
-			SPDK_ERRLOG("Poll cq opcode type unknown!!!!! completion\n");
-			return -1;
-		}
-	}
-
-	/* poll the recv_cq */
-	while (true) {
-		rc = ibv_poll_cq(rqpair->cm_id->recv_cq, 1, &wc);
-		if (rc == 0) {
+		} else if (rc == 0) {
+			/* Ran out of completions */
 			break;
 		}
+		reaped += rc;
+	} while (reaped < max_completions);
 
+	/* Poll for recv completions */
+	reaped = 0;
+	do {
+		batch_size = nvme_min((max_completions - reaped),
+				      MAX_COMPLETIONS_PER_POLL);
+		rc = ibv_poll_cq(rqpair->cm_id->recv_cq, batch_size, wc);
 		if (rc < 0) {
-			SPDK_ERRLOG("Poll CQ error!(%d): %s\n",
+			SPDK_ERRLOG("Error polling CQ! (%d): %s\n",
 				    errno, strerror(errno));
 			return -1;
-		}
-
-		if (wc.status) {
-			SPDK_ERRLOG("CQ Completion Error For Response %lu: %d (%s)\n",
-				    wc.wr_id, wc.status, ibv_wc_status_str(wc.status));
+		} else if (rc == 0) {
+			/* Ran out of completions */
 			break;
 		}
 
-		if (wc.opcode == IBV_WC_RECV) {
-			SPDK_TRACELOG(SPDK_TRACE_DEBUG, "CQ recv completion\n");
-			if (wc.byte_len < sizeof(struct spdk_nvme_cpl)) {
-				SPDK_ERRLOG("recv length %u less than expected response size\n", wc.byte_len);
+		reaped += rc;
+		for (i = 0; i < rc; i++) {
+			if (wc[i].status) {
+				SPDK_ERRLOG("CQ error on Queue Pair %p, Response Index %lu (%d): %s\n",
+					    qpair, wc[i].wr_id, wc[i].status, ibv_wc_status_str(wc[i].status));
 				return -1;
 			}
 
-			rc = nvme_rdma_recv(rqpair, wc.wr_id);
-			if (rc) {
-				SPDK_ERRLOG("nvme_rdma_recv processing failure\n");
+			switch (wc[i].opcode) {
+			case IBV_WC_RECV:
+				SPDK_TRACELOG(SPDK_TRACE_DEBUG, "CQ recv completion\n");
+				if (wc[i].byte_len < sizeof(struct spdk_nvme_cpl)) {
+					SPDK_ERRLOG("recv length %u less than expected response size\n", wc[i].byte_len);
+					return -1;
+				}
 
+				if (nvme_rdma_recv(rqpair, wc[i].wr_id)) {
+					SPDK_ERRLOG("nvme_rdma_recv processing failure\n");
+					return -1;
+				}
+				break;
+
+			default:
+				SPDK_ERRLOG("Received an unexpected opcode on the CQ: %d\n", wc[i].opcode);
 				return -1;
 			}
-			io_completed++;
-		} else {
-			SPDK_ERRLOG("Poll cq opcode type unknown!!!!! completion\n");
-			return -1;
 		}
+	} while (reaped < max_completions);
 
-		if (io_completed == max_completions) {
-			break;
-		}
-	}
-
-	return io_completed;
+	return reaped;
 }
 
 uint32_t