ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvme/rdma: add cq resizing.

Browse Source 
Signed-off-by: Seth Howell <seth.howell@intel.com>
Change-Id: I6350d76b8c1e778c18e693b2dfbb10dd36b3e3d8
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1927
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
This commit is contained in:
Seth Howell 2020-04-18 14:26:02 -07:00 committed by Tomasz Zawadzki
parent 67b0dcfe29
commit 1039254319
1 changed files with 35 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
lib/nvme/nvme_rdma.c
View File

@ -95,6 +95,8 @@
*/
#define NVME_RDMA_DESTROYED_QPAIR_EXPIRATION_CYCLES 50
#define WC_PER_QPAIR(queue_depth) (queue_depth * 2)
enum nvme_rdma_wr_type {
RDMA_WR_TYPE_RECV,
RDMA_WR_TYPE_SEND,
@ -152,6 +154,8 @@ struct nvme_rdma_destroyed_qpair {
struct nvme_rdma_poller {
struct ibv_context *device;
struct ibv_cq *cq;
int required_num_wc;
int current_num_wc;
STAILQ_ENTRY(nvme_rdma_poller) link;
};
@ -2427,6 +2431,8 @@ nvme_rdma_poller_create(struct nvme_rdma_poll_group *group, struct ibv_context *
STAILQ_INSERT_HEAD(&group->pollers, poller, link);
group->num_pollers++;
poller->current_num_wc = DEFAULT_NVME_RDMA_CQ_SIZE;
poller->required_num_wc = 0;
return 0;
}
@ -2512,6 +2518,32 @@ nvme_rdma_poll_group_get_qpair_by_id(struct nvme_rdma_poll_group *group, uint32_
return NULL;
}
static int
nvme_rdma_resize_cq(struct nvme_rdma_qpair *rqpair, struct nvme_rdma_poller *poller)
{
int current_num_wc, required_num_wc;
required_num_wc = poller->required_num_wc + WC_PER_QPAIR(rqpair->num_entries);
current_num_wc = poller->current_num_wc;
if (current_num_wc < required_num_wc) {
current_num_wc = spdk_max(current_num_wc * 2, required_num_wc);
}
if (poller->current_num_wc != current_num_wc) {
SPDK_DEBUGLOG(SPDK_LOG_NVME, "Resize RDMA CQ from %d to %d\n", poller->current_num_wc,
current_num_wc);
if (ibv_resize_cq(poller->cq, current_num_wc)) {
SPDK_ERRLOG("RDMA CQ resize failed: errno %d: %s\n", errno, spdk_strerror(errno));
return -1;
}
poller->current_num_wc = current_num_wc;
}
poller->required_num_wc = required_num_wc;
return 0;
}
static int
nvme_rdma_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)
{
@ -2523,6 +2555,9 @@ nvme_rdma_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)
STAILQ_FOREACH(poller, &group->pollers, link) {
if (poller->device == rqpair->cm_id->verbs) {
if (nvme_rdma_resize_cq(rqpair, poller)) {
return -EPROTO;
}
rqpair->cq = poller->cq;
break;
}