ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvme/rdma: separate req/rsp allocation from registration

Browse Source 
This will simplify some upcoming changes to reconnect
a qpair.  In these cases we only need to re-register
the memory - we shouldn't have to allocate it again.

Signed-off-by: Jim Harris <james.r.harris@intel.com>
Change-Id: Id8adff313f191fbf11d7502127a2b961f2ca2f6e

Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/453478
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Jim Harris 2019-05-06 14:35:21 -07:00 committed by Changpeng Liu
parent ac2b06c809
commit 6949c71dca
1 changed files with 57 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
lib/nvme/nvme_rdma.c
View File

@ -326,13 +326,17 @@ nvme_rdma_post_recv(struct nvme_rdma_qpair *rqpair, uint16_t rsp_idx)
} }
static void static void
nvme_rdma_free_rsps(struct nvme_rdma_qpair *rqpair) nvme_rdma_unregister_rsps(struct nvme_rdma_qpair *rqpair)
{ {
if (rqpair->rsp_mr && rdma_dereg_mr(rqpair->rsp_mr)) { if (rqpair->rsp_mr && rdma_dereg_mr(rqpair->rsp_mr)) {
SPDK_ERRLOG("Unable to de-register rsp_mr\n"); SPDK_ERRLOG("Unable to de-register rsp_mr\n");
} }
rqpair->rsp_mr = NULL; rqpair->rsp_mr = NULL;
}
static void
nvme_rdma_free_rsps(struct nvme_rdma_qpair *rqpair)
{
free(rqpair->rsps); free(rqpair->rsps);
rqpair->rsps = NULL; rqpair->rsps = NULL;
free(rqpair->rsp_sgls); free(rqpair->rsp_sgls);
@ -344,9 +348,6 @@ nvme_rdma_free_rsps(struct nvme_rdma_qpair *rqpair)
static int static int
nvme_rdma_alloc_rsps(struct nvme_rdma_qpair *rqpair) nvme_rdma_alloc_rsps(struct nvme_rdma_qpair *rqpair)
{ {
uint16_t i;
rqpair->rsp_mr = NULL;
rqpair->rsps = NULL; rqpair->rsps = NULL;
rqpair->rsp_recv_wrs = NULL; rqpair->rsp_recv_wrs = NULL;
@ -369,6 +370,17 @@ nvme_rdma_alloc_rsps(struct nvme_rdma_qpair *rqpair)
goto fail; goto fail;
} }
return 0;
fail:
nvme_rdma_free_rsps(rqpair);
return -ENOMEM;
}
static int
nvme_rdma_register_rsps(struct nvme_rdma_qpair *rqpair)
{
int i;
rqpair->rsp_mr = rdma_reg_msgs(rqpair->cm_id, rqpair->rsps, rqpair->rsp_mr = rdma_reg_msgs(rqpair->cm_id, rqpair->rsps,
rqpair->num_entries * sizeof(*rqpair->rsps)); rqpair->num_entries * sizeof(*rqpair->rsps));
if (rqpair->rsp_mr == NULL) { if (rqpair->rsp_mr == NULL) {
@ -397,10 +409,19 @@ nvme_rdma_alloc_rsps(struct nvme_rdma_qpair *rqpair)
return 0; return 0;
fail: fail:
nvme_rdma_free_rsps(rqpair); nvme_rdma_unregister_rsps(rqpair);
return -ENOMEM; return -ENOMEM;
} }
static void
nvme_rdma_unregister_reqs(struct nvme_rdma_qpair *rqpair)
{
if (rqpair->cmd_mr && rdma_dereg_mr(rqpair->cmd_mr)) {
SPDK_ERRLOG("Unable to de-register cmd_mr\n");
}
rqpair->cmd_mr = NULL;
}
static void static void
nvme_rdma_free_reqs(struct nvme_rdma_qpair *rqpair) nvme_rdma_free_reqs(struct nvme_rdma_qpair *rqpair)
{ {
@ -408,11 +429,6 @@ nvme_rdma_free_reqs(struct nvme_rdma_qpair *rqpair)
return; return;
} }
if (rqpair->cmd_mr && rdma_dereg_mr(rqpair->cmd_mr)) {
SPDK_ERRLOG("Unable to de-register cmd_mr\n");
}
rqpair->cmd_mr = NULL;
free(rqpair->cmds); free(rqpair->cmds);
rqpair->cmds = NULL; rqpair->cmds = NULL;
@ -423,8 +439,6 @@ nvme_rdma_free_reqs(struct nvme_rdma_qpair *rqpair)
static int static int
nvme_rdma_alloc_reqs(struct nvme_rdma_qpair *rqpair) nvme_rdma_alloc_reqs(struct nvme_rdma_qpair *rqpair)
{ {
int i;
rqpair->rdma_reqs = calloc(rqpair->num_entries, sizeof(struct spdk_nvme_rdma_req)); rqpair->rdma_reqs = calloc(rqpair->num_entries, sizeof(struct spdk_nvme_rdma_req));
if (rqpair->rdma_reqs == NULL) { if (rqpair->rdma_reqs == NULL) {
SPDK_ERRLOG("Failed to allocate rdma_reqs\n"); SPDK_ERRLOG("Failed to allocate rdma_reqs\n");
@ -437,6 +451,17 @@ nvme_rdma_alloc_reqs(struct nvme_rdma_qpair *rqpair)
goto fail; goto fail;
} }
return 0;
fail:
nvme_rdma_free_reqs(rqpair);
return -ENOMEM;
}
static int
nvme_rdma_register_reqs(struct nvme_rdma_qpair *rqpair)
{
int i;
rqpair->cmd_mr = rdma_reg_msgs(rqpair->cm_id, rqpair->cmds, rqpair->cmd_mr = rdma_reg_msgs(rqpair->cm_id, rqpair->cmds,
rqpair->num_entries * sizeof(*rqpair->cmds)); rqpair->num_entries * sizeof(*rqpair->cmds));
if (!rqpair->cmd_mr) { if (!rqpair->cmd_mr) {
@ -475,7 +500,7 @@ nvme_rdma_alloc_reqs(struct nvme_rdma_qpair *rqpair)
return 0; return 0;
fail: fail:
nvme_rdma_free_reqs(rqpair); nvme_rdma_unregister_reqs(rqpair);
return -ENOMEM; return -ENOMEM;
} }
@ -849,11 +874,19 @@ nvme_rdma_qpair_connect(struct nvme_rdma_qpair *rqpair)
rc = nvme_rdma_alloc_reqs(rqpair); rc = nvme_rdma_alloc_reqs(rqpair);
SPDK_DEBUGLOG(SPDK_LOG_NVME, "rc =%d\n", rc); SPDK_DEBUGLOG(SPDK_LOG_NVME, "rc =%d\n", rc);
if (rc) { if (rc) {
SPDK_ERRLOG("Unable to allocate rqpair RDMA requests\n"); SPDK_ERRLOG("Unable to allocate rqpair RDMA requests\n");
return -1; return -1;
} }
SPDK_DEBUGLOG(SPDK_LOG_NVME, "RDMA requests allocated\n"); SPDK_DEBUGLOG(SPDK_LOG_NVME, "RDMA requests allocated\n");
rc = nvme_rdma_register_reqs(rqpair);
SPDK_DEBUGLOG(SPDK_LOG_NVME, "rc =%d\n", rc);
if (rc) {
SPDK_ERRLOG("Unable to register rqpair RDMA requests\n");
return -1;
}
SPDK_DEBUGLOG(SPDK_LOG_NVME, "RDMA requests registered\n");
rc = nvme_rdma_alloc_rsps(rqpair); rc = nvme_rdma_alloc_rsps(rqpair);
SPDK_DEBUGLOG(SPDK_LOG_NVME, "rc =%d\n", rc); SPDK_DEBUGLOG(SPDK_LOG_NVME, "rc =%d\n", rc);
if (rc < 0) { if (rc < 0) {
@ -862,6 +895,14 @@ nvme_rdma_qpair_connect(struct nvme_rdma_qpair *rqpair)
} }
SPDK_DEBUGLOG(SPDK_LOG_NVME, "RDMA responses allocated\n"); SPDK_DEBUGLOG(SPDK_LOG_NVME, "RDMA responses allocated\n");
rc = nvme_rdma_register_rsps(rqpair);
SPDK_DEBUGLOG(SPDK_LOG_NVME, "rc =%d\n", rc);
if (rc < 0) {
SPDK_ERRLOG("Unable to register rqpair RDMA responses\n");
return -1;
}
SPDK_DEBUGLOG(SPDK_LOG_NVME, "RDMA responses registered\n");
rc = nvme_rdma_register_mem(rqpair); rc = nvme_rdma_register_mem(rqpair);
if (rc < 0) { if (rc < 0) {
SPDK_ERRLOG("Unable to register memory for RDMA\n"); SPDK_ERRLOG("Unable to register memory for RDMA\n");
@ -1299,7 +1340,9 @@ nvme_rdma_qpair_destroy(struct spdk_nvme_qpair *qpair)
rqpair = nvme_rdma_qpair(qpair); rqpair = nvme_rdma_qpair(qpair);
nvme_rdma_unregister_mem(rqpair); nvme_rdma_unregister_mem(rqpair);
nvme_rdma_unregister_reqs(rqpair);
nvme_rdma_free_reqs(rqpair); nvme_rdma_free_reqs(rqpair);
nvme_rdma_unregister_rsps(rqpair);
nvme_rdma_free_rsps(rqpair); nvme_rdma_free_rsps(rqpair);
if (rqpair->cm_id) { if (rqpair->cm_id) {