ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvmf/rdma: Combine RDMA WRITE and SEND into single post

Browse Source 
When data needs to be transferred from the controller
to the host, do a single ibv_post_send containing
both the data and the completion.

Change-Id: I072c545b31593e0e324c97ed700b42c6a4c358e1
Signed-off-by: Ben Walker <benjamin.walker@intel.com>
This commit is contained in:
Ben Walker 2017-03-15 14:30:15 -07:00 committed by Daniel Verkamp
parent 6696fce7ae
commit 1f38243914
1 changed files with 15 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
lib/nvmf/rdma.c
View File

@ -437,6 +437,7 @@ request_transfer_in(struct spdk_nvmf_request *req)
spdk_trace_record(TRACE_RDMA_READ_START, 0, 0, (uintptr_t)req, 0); spdk_trace_record(TRACE_RDMA_READ_START, 0, 0, (uintptr_t)req, 0);
rdma_req->data.wr.opcode = IBV_WR_RDMA_READ; rdma_req->data.wr.opcode = IBV_WR_RDMA_READ;
rdma_req->data.wr.next = NULL;
rc = ibv_post_send(rdma_conn->cm_id->qp, &rdma_req->data.wr, &bad_wr); rc = ibv_post_send(rdma_conn->cm_id->qp, &rdma_req->data.wr, &bad_wr);
if (rc) { if (rc) {
SPDK_ERRLOG("Unable to transfer data from host to target\n"); SPDK_ERRLOG("Unable to transfer data from host to target\n");
@ -455,7 +456,7 @@ request_transfer_out(struct spdk_nvmf_request *req)
struct spdk_nvmf_rdma_conn *rdma_conn = get_rdma_conn(conn); struct spdk_nvmf_rdma_conn *rdma_conn = get_rdma_conn(conn);
struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
struct ibv_recv_wr *bad_recv_wr = NULL; struct ibv_recv_wr *bad_recv_wr = NULL;
struct ibv_send_wr *bad_send_wr = NULL; struct ibv_send_wr *send_wr, *bad_send_wr = NULL;
/* Advance our sq_head pointer */ /* Advance our sq_head pointer */
if (conn->sq_head == conn->sq_head_max) { if (conn->sq_head == conn->sq_head_max) {
@ -480,25 +481,29 @@ request_transfer_out(struct spdk_nvmf_request *req)
} }
rdma_req->recv = NULL; rdma_req->recv = NULL;
/* Build the response which consists of an optional
* RDMA WRITE to transfer data, plus an RDMA SEND
* containing the response.
*/
send_wr = &rdma_req->rsp.wr;
if (rsp->status.sc == SPDK_NVME_SC_SUCCESS && if (rsp->status.sc == SPDK_NVME_SC_SUCCESS &&
req->xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST) { req->xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST) {
/* Send the write */
rdma_conn->cur_rdma_rw_depth++;
SPDK_TRACELOG(SPDK_TRACE_RDMA, "RDMA WRITE POSTED. Request: %p Connection: %p\n", req, conn); SPDK_TRACELOG(SPDK_TRACE_RDMA, "RDMA WRITE POSTED. Request: %p Connection: %p\n", req, conn);
spdk_trace_record(TRACE_RDMA_WRITE_START, 0, 0, (uintptr_t)req, 0); spdk_trace_record(TRACE_RDMA_WRITE_START, 0, 0, (uintptr_t)req, 0);
rdma_conn->cur_rdma_rw_depth++;
rdma_req->data.wr.opcode = IBV_WR_RDMA_WRITE; rdma_req->data.wr.opcode = IBV_WR_RDMA_WRITE;
rc = ibv_post_send(rdma_conn->cm_id->qp, &rdma_req->data.wr, &bad_send_wr);
if (rc) { rdma_req->data.wr.next = send_wr;
SPDK_ERRLOG("Unable to transfer data from target to host\n"); send_wr = &rdma_req->data.wr;
return -1;
}
} }
/* Send the completion */
SPDK_TRACELOG(SPDK_TRACE_RDMA, "RDMA SEND POSTED. Request: %p Connection: %p\n", req, conn); SPDK_TRACELOG(SPDK_TRACE_RDMA, "RDMA SEND POSTED. Request: %p Connection: %p\n", req, conn);
spdk_trace_record(TRACE_NVMF_IO_COMPLETE, 0, 0, (uintptr_t)req, 0); spdk_trace_record(TRACE_NVMF_IO_COMPLETE, 0, 0, (uintptr_t)req, 0);
rc = ibv_post_send(rdma_conn->cm_id->qp, &rdma_req->rsp.wr, &bad_send_wr);
/* Send the completion */
rc = ibv_post_send(rdma_conn->cm_id->qp, send_wr, &bad_send_wr);
if (rc) { if (rc) {
SPDK_ERRLOG("Unable to send response capsule\n"); SPDK_ERRLOG("Unable to send response capsule\n");
} }