ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvme_rdma: Start qpair after resolving address and route when poll group is used

Browse Source 
Signed-off-by: Shuhei Matsumoto <smatsumoto@nvidia.com>
Change-Id: I0b0f314c98368247582f2dfcaf69f78e24d715f9
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11366
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
This commit is contained in:
Shuhei Matsumoto 2022-01-31 22:53:03 +09:00 committed by Tomasz Zawadzki
parent 531c1b0f04
commit 0a61427ecc
2 changed files with 57 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
lib/nvme/nvme_rdma.c
View File

@ -712,6 +712,60 @@ nvme_rdma_process_event(struct nvme_rdma_qpair *rqpair,
return rc; return rc;
} }
static int
nvme_rdma_resize_cq(struct nvme_rdma_qpair *rqpair, struct nvme_rdma_poller *poller)
{
int current_num_wc, required_num_wc;
required_num_wc = poller->required_num_wc + WC_PER_QPAIR(rqpair->num_entries);
current_num_wc = poller->current_num_wc;
if (current_num_wc < required_num_wc) {
current_num_wc = spdk_max(current_num_wc * 2, required_num_wc);
}
if (poller->current_num_wc != current_num_wc) {
SPDK_DEBUGLOG(nvme, "Resize RDMA CQ from %d to %d\n", poller->current_num_wc,
current_num_wc);
if (ibv_resize_cq(poller->cq, current_num_wc)) {
SPDK_ERRLOG("RDMA CQ resize failed: errno %d: %s\n", errno, spdk_strerror(errno));
return -1;
}
poller->current_num_wc = current_num_wc;
}
poller->required_num_wc = required_num_wc;
return 0;
}
static int
nvme_rdma_poll_group_set_cq(struct spdk_nvme_qpair *qpair)
{
struct nvme_rdma_qpair *rqpair = nvme_rdma_qpair(qpair);
struct nvme_rdma_poll_group *group = nvme_rdma_poll_group(qpair->poll_group);
struct nvme_rdma_poller *poller;
assert(rqpair->cq == NULL);
STAILQ_FOREACH(poller, &group->pollers, link) {
if (poller->device == rqpair->cm_id->verbs) {
if (nvme_rdma_resize_cq(rqpair, poller)) {
return -EPROTO;
}
rqpair->cq = poller->cq;
rqpair->poller = poller;
break;
}
}
if (rqpair->cq == NULL) {
SPDK_ERRLOG("Unable to find a cq for qpair %p on poll group %p\n", qpair, qpair->poll_group);
return -EINVAL;
}
return 0;
}
static int static int
nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair) nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair)
{ {
@ -728,7 +782,7 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair)
if (rqpair->qpair.poll_group) { if (rqpair->qpair.poll_group) {
assert(!rqpair->cq); assert(!rqpair->cq);
rc = nvme_poll_group_connect_qpair(&rqpair->qpair); rc = nvme_rdma_poll_group_set_cq(&rqpair->qpair);
if (rc) { if (rc) {
SPDK_ERRLOG("Unable to activate the rdmaqpair.\n"); SPDK_ERRLOG("Unable to activate the rdmaqpair.\n");
return -1; return -1;
@ -2701,57 +2755,10 @@ nvme_rdma_poll_group_get_qpair_by_id(struct nvme_rdma_poll_group *group, uint32_
return NULL; return NULL;
} }
static int
nvme_rdma_resize_cq(struct nvme_rdma_qpair *rqpair, struct nvme_rdma_poller *poller)
{
int current_num_wc, required_num_wc;
required_num_wc = poller->required_num_wc + WC_PER_QPAIR(rqpair->num_entries);
current_num_wc = poller->current_num_wc;
if (current_num_wc < required_num_wc) {
current_num_wc = spdk_max(current_num_wc * 2, required_num_wc);
}
if (poller->current_num_wc != current_num_wc) {
SPDK_DEBUGLOG(nvme, "Resize RDMA CQ from %d to %d\n", poller->current_num_wc,
current_num_wc);
if (ibv_resize_cq(poller->cq, current_num_wc)) {
SPDK_ERRLOG("RDMA CQ resize failed: errno %d: %s\n", errno, spdk_strerror(errno));
return -1;
}
poller->current_num_wc = current_num_wc;
}
poller->required_num_wc = required_num_wc;
return 0;
}
static int static int
nvme_rdma_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair) nvme_rdma_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)
{ {
struct nvme_rdma_qpair *rqpair = nvme_rdma_qpair(qpair);
struct nvme_rdma_poll_group *group = nvme_rdma_poll_group(qpair->poll_group);
struct nvme_rdma_poller *poller;
assert(rqpair->cq == NULL);
STAILQ_FOREACH(poller, &group->pollers, link) {
if (poller->device == rqpair->cm_id->verbs) {
if (nvme_rdma_resize_cq(rqpair, poller)) {
return -EPROTO;
}
rqpair->cq = poller->cq;
rqpair->poller = poller;
break;
}
}
if (rqpair->cq == NULL) {
SPDK_ERRLOG("Unable to find a cq for qpair %p on poll group %p\n", qpair, qpair->poll_group);
return -EINVAL;
}
return 0; return 0;
} }

4
test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c
View File

@ -1052,7 +1052,7 @@ test_nvme_rdma_poll_group_connect_disconnect_qpair(void)
rc = nvme_rdma_poller_create(&group, contexts); rc = nvme_rdma_poller_create(&group, contexts);
SPDK_CU_ASSERT_FATAL(rc == 0); SPDK_CU_ASSERT_FATAL(rc == 0);
rc = nvme_rdma_poll_group_connect_qpair(&rqpair->qpair); rc = nvme_rdma_poll_group_set_cq(&rqpair->qpair);
CU_ASSERT(rc == 0); CU_ASSERT(rc == 0);
CU_ASSERT(rqpair->cq == (void *)0xFEEDBEEF); CU_ASSERT(rqpair->cq == (void *)0xFEEDBEEF);
CU_ASSERT(rqpair->poller != NULL); CU_ASSERT(rqpair->poller != NULL);
@ -1086,7 +1086,7 @@ test_nvme_rdma_poll_group_connect_disconnect_qpair(void)
rqpair->qpair.state = NVME_QPAIR_DESTROYING; rqpair->qpair.state = NVME_QPAIR_DESTROYING;
cm_id.verbs = (void *)0xDEADBEEF; cm_id.verbs = (void *)0xDEADBEEF;
rc = nvme_rdma_poll_group_connect_qpair(&rqpair->qpair); rc = nvme_rdma_poll_group_set_cq(&rqpair->qpair);
CU_ASSERT(rc == -EINVAL); CU_ASSERT(rc == -EINVAL);
CU_ASSERT(rqpair->cq == NULL); CU_ASSERT(rqpair->cq == NULL);