diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index 89e086e28..f851bcbc4 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -1115,6 +1115,26 @@ spdk_nvmf_rdma_check_contiguous_entries(uint64_t addr_1, uint64_t addr_2) return addr_1 == addr_2; } +static void +spdk_nvmf_rdma_request_free_buffers(struct spdk_nvmf_rdma_request *rdma_req, + struct spdk_nvmf_transport_poll_group *group, struct spdk_nvmf_transport *transport) +{ + for (uint32_t i = 0; i < rdma_req->req.iovcnt; i++) { + if (group->buf_cache_count < group->buf_cache_size) { + STAILQ_INSERT_HEAD(&group->buf_cache, + (struct spdk_nvmf_transport_pg_cache_buf *)rdma_req->data.buffers[i], link); + group->buf_cache_count++; + } else { + spdk_mempool_put(transport->data_buf_pool, rdma_req->data.buffers[i]); + } + rdma_req->req.iov[i].iov_base = NULL; + rdma_req->data.buffers[i] = NULL; + rdma_req->req.iov[i].iov_len = 0; + + } + rdma_req->data_from_pool = false; +} + typedef enum spdk_nvme_data_transfer spdk_nvme_data_transfer_t; static spdk_nvme_data_transfer_t @@ -1182,18 +1202,29 @@ spdk_nvmf_rdma_request_fill_iovs(struct spdk_nvmf_rdma_transport *rtransport, struct spdk_nvmf_rdma_device *device, struct spdk_nvmf_rdma_request *rdma_req) { - void *buf = NULL; - uint32_t length = rdma_req->req.length; - uint64_t translation_len; - uint32_t i = 0; - int rc = 0; + struct spdk_nvmf_rdma_qpair *rqpair; + struct spdk_nvmf_rdma_poll_group *rgroup; + void *buf = NULL; + uint32_t length = rdma_req->req.length; + uint64_t translation_len; + uint32_t i = 0; + int rc = 0; + rqpair = SPDK_CONTAINEROF(rdma_req->req.qpair, struct spdk_nvmf_rdma_qpair, qpair); + rgroup = rqpair->poller->group; rdma_req->req.iovcnt = 0; while (length) { - buf = spdk_mempool_get(rtransport->transport.data_buf_pool); - if (!buf) { - rc = -ENOMEM; - goto err_exit; + if (!(STAILQ_EMPTY(&rgroup->group.buf_cache))) { + rgroup->group.buf_cache_count--; + buf = STAILQ_FIRST(&rgroup->group.buf_cache); + STAILQ_REMOVE_HEAD(&rgroup->group.buf_cache, link); + assert(buf != NULL); + } else { + buf = spdk_mempool_get(rtransport->transport.data_buf_pool); + if (!buf) { + rc = -ENOMEM; + goto err_exit; + } } rdma_req->req.iov[i].iov_base = (void *)((uintptr_t)(buf + NVMF_DATA_BUFFER_MASK) & @@ -1221,12 +1252,9 @@ spdk_nvmf_rdma_request_fill_iovs(struct spdk_nvmf_rdma_transport *rtransport, return rc; err_exit: + spdk_nvmf_rdma_request_free_buffers(rdma_req, &rgroup->group, &rtransport->transport); while (i) { i--; - spdk_mempool_put(rtransport->transport.data_buf_pool, rdma_req->data.buffers[i]); - rdma_req->req.iov[i].iov_base = NULL; - rdma_req->req.iov[i].iov_len = 0; - rdma_req->data.wr.sg_list[i].addr = 0; rdma_req->data.wr.sg_list[i].length = 0; rdma_req->data.wr.sg_list[i].lkey = 0; @@ -1331,14 +1359,14 @@ static void nvmf_rdma_request_free(struct spdk_nvmf_rdma_request *rdma_req, struct spdk_nvmf_rdma_transport *rtransport) { + struct spdk_nvmf_rdma_qpair *rqpair; + struct spdk_nvmf_rdma_poll_group *rgroup; + if (rdma_req->data_from_pool) { - /* Put the buffer/s back in the pool */ - for (uint32_t i = 0; i < rdma_req->req.iovcnt; i++) { - spdk_mempool_put(rtransport->transport.data_buf_pool, rdma_req->data.buffers[i]); - rdma_req->req.iov[i].iov_base = NULL; - rdma_req->data.buffers[i] = NULL; - } - rdma_req->data_from_pool = false; + rqpair = SPDK_CONTAINEROF(rdma_req->req.qpair, struct spdk_nvmf_rdma_qpair, qpair); + rgroup = rqpair->poller->group; + + spdk_nvmf_rdma_request_free_buffers(rdma_req, &rgroup->group, &rtransport->transport); } rdma_req->req.length = 0; rdma_req->req.iovcnt = 0; diff --git a/test/unit/lib/nvmf/rdma.c/rdma_ut.c b/test/unit/lib/nvmf/rdma.c/rdma_ut.c index d98feea68..90f97eb65 100644 --- a/test/unit/lib/nvmf/rdma.c/rdma_ut.c +++ b/test/unit/lib/nvmf/rdma.c/rdma_ut.c @@ -114,16 +114,27 @@ test_spdk_nvmf_rdma_request_parse_sgl(void) struct spdk_nvmf_rdma_device device; struct spdk_nvmf_rdma_request rdma_req; struct spdk_nvmf_rdma_recv recv; + struct spdk_nvmf_rdma_poll_group group; + struct spdk_nvmf_rdma_qpair rqpair; + struct spdk_nvmf_rdma_poller poller; union nvmf_c2h_msg cpl; union nvmf_h2c_msg cmd; struct spdk_nvme_sgl_descriptor *sgl; + struct spdk_nvmf_transport_pg_cache_buf bufs[4]; int rc, i; + STAILQ_INIT(&group.group.buf_cache); + group.group.buf_cache_size = 0; + group.group.buf_cache_count = 0; + poller.group = &group; + rqpair.poller = &poller; + sgl = &cmd.nvme_cmd.dptr.sgl1; rdma_req.recv = &recv; rdma_req.req.cmd = &cmd; rdma_req.req.rsp = &cpl; rdma_req.data.wr.sg_list = rdma_req.data.sgl; + rdma_req.req.qpair = &rqpair.qpair; rtransport.transport.opts = g_rdma_ut_transport_opts; @@ -227,6 +238,93 @@ test_spdk_nvmf_rdma_request_parse_sgl(void) rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); CU_ASSERT(rc == -1); + /* Test 3: use PG buffer cache */ + sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK; + sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS; + sgl->address = 0xFFFF; + rdma_req.recv->buf = (void *)0xDDDD; + g_rdma_mr.lkey = 0xABCD; + sgl->keyed.key = 0xEEEE; + + for (i = 0; i < 4; i++) { + STAILQ_INSERT_TAIL(&group.group.buf_cache, &bufs[i], link); + } + + /* part 1: use the four buffers from the pg cache */ + + group.group.buf_cache_size = 4; + group.group.buf_cache_count = 4; + MOCK_SET(spdk_mempool_get, (void *)0x2000); + reset_nvmf_rdma_request(&rdma_req); + sgl->keyed.length = rtransport.transport.opts.io_unit_size * 4; + rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); + + SPDK_CU_ASSERT_FATAL(rc == 0); + CU_ASSERT(rdma_req.data_from_pool == true); + CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); + CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&bufs[0] + NVMF_DATA_BUFFER_MASK) & + ~NVMF_DATA_BUFFER_MASK)); + CU_ASSERT(rdma_req.data.wr.num_sge == 4); + CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); + CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); + CU_ASSERT(group.group.buf_cache_count == 0); + CU_ASSERT(STAILQ_EMPTY(&group.group.buf_cache)); + for (i = 0; i < 4; i++) { + CU_ASSERT((uint64_t)rdma_req.data.buffers[i] == (uint64_t)&bufs[i]); + CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (((uint64_t)&bufs[i] + NVMF_DATA_BUFFER_MASK) & + ~NVMF_DATA_BUFFER_MASK)); + CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); + } + /* part 2: now that we have used the buffers from the cache, try again. We should get mempool buffers. */ + + reset_nvmf_rdma_request(&rdma_req); + rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); + + SPDK_CU_ASSERT_FATAL(rc == 0); + CU_ASSERT(rdma_req.data_from_pool == true); + CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); + CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000); + CU_ASSERT(rdma_req.data.wr.num_sge == 4); + CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); + CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); + CU_ASSERT(group.group.buf_cache_count == 0); + CU_ASSERT(STAILQ_EMPTY(&group.group.buf_cache)); + for (i = 0; i < 4; i++) { + CU_ASSERT((uint64_t)rdma_req.data.buffers[i] == 0x2000); + CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); + CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); + CU_ASSERT(group.group.buf_cache_count == 0); + } + + /* part 3: half and half */ + group.group.buf_cache_count = 2; + + for (i = 0; i < 2; i++) { + STAILQ_INSERT_TAIL(&group.group.buf_cache, &bufs[i], link); + } + reset_nvmf_rdma_request(&rdma_req); + rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req); + + SPDK_CU_ASSERT_FATAL(rc == 0); + CU_ASSERT(rdma_req.data_from_pool == true); + CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4); + CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&bufs[0] + NVMF_DATA_BUFFER_MASK) & + ~NVMF_DATA_BUFFER_MASK)); + CU_ASSERT(rdma_req.data.wr.num_sge == 4); + CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE); + CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF); + CU_ASSERT(group.group.buf_cache_count == 0); + for (i = 0; i < 2; i++) { + CU_ASSERT((uint64_t)rdma_req.data.buffers[i] == (uint64_t)&bufs[i]); + CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (((uint64_t)&bufs[i] + NVMF_DATA_BUFFER_MASK) & + ~NVMF_DATA_BUFFER_MASK)); + CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); + } + for (i = 2; i < 4; i++) { + CU_ASSERT((uint64_t)rdma_req.data.buffers[i] == 0x2000); + CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000); + CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size); + } } int main(int argc, char **argv)