ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvme_rdma: properly configure and store max_sges

Browse Source 
The max_send_sge and max_recv_sge values can be set to any value from
0...dev_attr->max_sge. WHen we actually set the attributes, we will
receive a qpair with values for max_sge greater than or equal to what we
initially set. We need to store the maximum number of SGEs for later use
when constructing work requests.
Previously we have not relied on these values since we assumed that we
would always be able to have more sges than we asked for initially. This
may change as we try to allocate more SGEs to handle splitting buffers
across memory regions.

Change-Id: Ibbeae1908b86baa3a96d9c6cd2051401aaa2197b
Signed-off-by: Seth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/433307
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
This commit is contained in:
Seth Howell 2018-11-14 08:34:17 -07:00 committed by Jim Harris
parent 1180bf8343
commit e688d1ccf1
1 changed files with 23 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
lib/nvme/nvme_rdma.c
View File

@ -92,14 +92,16 @@ struct nvme_rdma_ctrlr {
struct nvme_rdma_qpair {
struct spdk_nvme_qpair qpair;
struct rdma_event_channel *cm_channel;
struct rdma_cm_id *cm_id;
struct ibv_cq *cq;
struct spdk_nvme_rdma_req *rdma_reqs;
uint32_t max_send_sge;
uint32_t max_recv_sge;
uint16_t num_entries;
/* Parallel arrays of response buffers + response SGLs of size num_entries */
@ -124,6 +126,9 @@ struct nvme_rdma_qpair {
TAILQ_HEAD(, spdk_nvme_rdma_req) free_reqs;
TAILQ_HEAD(, spdk_nvme_rdma_req) outstanding_reqs;
/* Placed at the end of the struct since it is not used frequently */
struct rdma_event_channel *cm_channel;
};
struct spdk_nvme_rdma_req {
@ -245,8 +250,15 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair)
{
int rc;
struct ibv_qp_init_attr attr;
struct ibv_device_attr dev_attr;
struct nvme_rdma_ctrlr *rctrlr;
rc = ibv_query_device(rqpair->cm_id->verbs, &dev_attr);
if (rc != 0) {
SPDK_ERRLOG("Failed to query RDMA device attributes.\n");
return -1;
}
rqpair->cq = ibv_create_cq(rqpair->cm_id->verbs, rqpair->num_entries * 2, rqpair, NULL, 0);
if (!rqpair->cq) {
SPDK_ERRLOG("Unable to create completion queue: errno %d: %s\n", errno, spdk_strerror(errno));
@ -266,14 +278,20 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair)
attr.recv_cq = rqpair->cq;
attr.cap.max_send_wr = rqpair->num_entries; /* SEND operations */
attr.cap.max_recv_wr = rqpair->num_entries; /* RECV operations */
attr.cap.max_send_sge = NVME_RDMA_DEFAULT_TX_SGE;
attr.cap.max_recv_sge = NVME_RDMA_DEFAULT_RX_SGE;
attr.cap.max_send_sge = spdk_min(NVME_RDMA_DEFAULT_TX_SGE, dev_attr.max_sge);
attr.cap.max_recv_sge = spdk_min(NVME_RDMA_DEFAULT_RX_SGE, dev_attr.max_sge);
rc = rdma_create_qp(rqpair->cm_id, rctrlr->pd, &attr);
if (rc) {
SPDK_ERRLOG("rdma_create_qp failed\n");
return -1;
}
/* ibv_create_qp will change the values in attr.cap. Make sure we store the proper value. */
rqpair->max_send_sge = spdk_min(NVME_RDMA_DEFAULT_TX_SGE, attr.cap.max_send_sge);
rqpair->max_recv_sge = spdk_min(NVME_RDMA_DEFAULT_RX_SGE, attr.cap.max_recv_sge);
rctrlr->pd = rqpair->cm_id->qp->pd;
rqpair->cm_id->context = &rqpair->qpair;