diff --git a/lib/nvmf/fc.c b/lib/nvmf/fc.c index 12090fbc7..a90a5a23a 100644 --- a/lib/nvmf/fc.c +++ b/lib/nvmf/fc.c @@ -1135,10 +1135,10 @@ nvmf_fc_req_in_bdev(struct spdk_nvmf_fc_request *fc_req) static inline bool nvmf_fc_req_in_pending(struct spdk_nvmf_fc_request *fc_req) { - struct spdk_nvmf_fc_request *tmp = NULL; + struct spdk_nvmf_request *tmp = NULL; - STAILQ_FOREACH(tmp, &fc_req->hwqp->fgroup->pending_data_buf_queue, pending_link) { - if (tmp == fc_req) { + STAILQ_FOREACH(tmp, &fc_req->hwqp->fgroup->group.pending_buf_queue, buf_link) { + if (tmp == &fc_req->req) { return true; } } @@ -1258,8 +1258,8 @@ spdk_nvmf_fc_request_abort(struct spdk_nvmf_fc_request *fc_req, bool send_abts, SPDK_DEBUGLOG(SPDK_LOG_NVMF_FC, "Abort req when getting buffers.\n"); } else if (nvmf_fc_req_in_pending(fc_req)) { /* Remove from pending */ - STAILQ_REMOVE(&fc_req->hwqp->fgroup->pending_data_buf_queue, fc_req, - spdk_nvmf_fc_request, pending_link); + STAILQ_REMOVE(&fc_req->hwqp->fgroup->group.pending_buf_queue, &fc_req->req, + spdk_nvmf_request, buf_link); goto complete; } else { /* Should never happen */ @@ -1454,7 +1454,7 @@ nvmf_fc_hwqp_handle_request(struct spdk_nvmf_fc_hwqp *hwqp, struct spdk_nvmf_fc_ nvmf_fc_record_req_trace_point(fc_req, SPDK_NVMF_FC_REQ_INIT); if (nvmf_fc_request_execute(fc_req)) { - STAILQ_INSERT_TAIL(&hwqp->fgroup->pending_data_buf_queue, fc_req, pending_link); + STAILQ_INSERT_TAIL(&hwqp->fgroup->group.pending_buf_queue, &fc_req->req, buf_link); } return 0; @@ -1626,13 +1626,15 @@ spdk_nvmf_fc_hwqp_process_frame(struct spdk_nvmf_fc_hwqp *hwqp, void spdk_nvmf_fc_hwqp_process_pending_reqs(struct spdk_nvmf_fc_hwqp *hwqp) { - struct spdk_nvmf_fc_request *fc_req = NULL, *tmp; + struct spdk_nvmf_request *req = NULL, *tmp; + struct spdk_nvmf_fc_request *fc_req; int budget = 64; - STAILQ_FOREACH_SAFE(fc_req, &hwqp->fgroup->pending_data_buf_queue, pending_link, tmp) { + STAILQ_FOREACH_SAFE(req, &hwqp->fgroup->group.pending_buf_queue, buf_link, tmp) { + fc_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_fc_request, req); if (!nvmf_fc_request_execute(fc_req)) { /* Succesfuly posted, Delete from pending. */ - STAILQ_REMOVE_HEAD(&hwqp->fgroup->pending_data_buf_queue, pending_link); + STAILQ_REMOVE_HEAD(&hwqp->fgroup->group.pending_buf_queue, buf_link); } if (budget) { @@ -1999,7 +2001,6 @@ nvmf_fc_poll_group_create(struct spdk_nvmf_transport *transport) } TAILQ_INIT(&fgroup->hwqp_list); - STAILQ_INIT(&fgroup->pending_data_buf_queue); pthread_mutex_lock(&ftransport->lock); TAILQ_INSERT_TAIL(&g_nvmf_fgroups, fgroup, link); diff --git a/lib/nvmf/nvmf_fc.h b/lib/nvmf/nvmf_fc.h index a0f8d9d87..d92499a01 100644 --- a/lib/nvmf/nvmf_fc.h +++ b/lib/nvmf/nvmf_fc.h @@ -264,8 +264,6 @@ struct spdk_nvmf_fc_poll_group { struct spdk_nvmf_tgt *nvmf_tgt; uint32_t hwqp_count; /* number of hwqp's assigned to this pg */ TAILQ_HEAD(, spdk_nvmf_fc_hwqp) hwqp_list; - /* requests that are waiting to obtain xchg/buffer */ - STAILQ_HEAD(, spdk_nvmf_fc_request) pending_data_buf_queue; TAILQ_ENTRY(spdk_nvmf_fc_poll_group) link; }; diff --git a/lib/nvmf/nvmf_internal.h b/lib/nvmf/nvmf_internal.h index f8b6ec678..99b2f2d76 100644 --- a/lib/nvmf/nvmf_internal.h +++ b/lib/nvmf/nvmf_internal.h @@ -111,6 +111,8 @@ struct spdk_nvmf_transport_pg_cache_buf { struct spdk_nvmf_transport_poll_group { struct spdk_nvmf_transport *transport; + /* Requests that are waiting to obtain a data buffer */ + STAILQ_HEAD(, spdk_nvmf_request) pending_buf_queue; STAILQ_HEAD(, spdk_nvmf_transport_pg_cache_buf) buf_cache; uint32_t buf_cache_count; uint32_t buf_cache_size; @@ -216,6 +218,7 @@ struct spdk_nvmf_request { bool data_from_pool; struct spdk_bdev_io_wait_entry bdev_io_wait; + STAILQ_ENTRY(spdk_nvmf_request) buf_link; TAILQ_ENTRY(spdk_nvmf_request) link; }; diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index 1da3dd9c5..74d6cbedb 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -452,9 +452,6 @@ struct spdk_nvmf_rdma_poll_group_stat { struct spdk_nvmf_rdma_poll_group { struct spdk_nvmf_transport_poll_group group; - /* Requests that are waiting to obtain a data buffer */ - STAILQ_HEAD(, spdk_nvmf_rdma_request) pending_data_buf_queue; - TAILQ_HEAD(, spdk_nvmf_rdma_poller) pollers; struct spdk_nvmf_rdma_poll_group_stat stat; @@ -1826,7 +1823,7 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport, * to release resources. */ if (rqpair->ibv_state == IBV_QPS_ERR || rqpair->qpair.state != SPDK_NVMF_QPAIR_ACTIVE) { if (rdma_req->state == RDMA_REQUEST_STATE_NEED_BUFFER) { - STAILQ_REMOVE(&rgroup->pending_data_buf_queue, rdma_req, spdk_nvmf_rdma_request, state_link); + STAILQ_REMOVE(&rgroup->group.pending_buf_queue, &rdma_req->req, spdk_nvmf_request, buf_link); } else if (rdma_req->state == RDMA_REQUEST_STATE_DATA_TRANSFER_TO_CONTROLLER_PENDING) { STAILQ_REMOVE(&rqpair->pending_rdma_read_queue, rdma_req, spdk_nvmf_rdma_request, state_link); } else if (rdma_req->state == RDMA_REQUEST_STATE_DATA_TRANSFER_TO_HOST_PENDING) { @@ -1870,7 +1867,7 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport, } rdma_req->state = RDMA_REQUEST_STATE_NEED_BUFFER; - STAILQ_INSERT_TAIL(&rgroup->pending_data_buf_queue, rdma_req, state_link); + STAILQ_INSERT_TAIL(&rgroup->group.pending_buf_queue, &rdma_req->req, buf_link); break; case RDMA_REQUEST_STATE_NEED_BUFFER: spdk_trace_record(TRACE_RDMA_REQUEST_STATE_NEED_BUFFER, 0, 0, @@ -1878,7 +1875,7 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport, assert(rdma_req->req.xfer != SPDK_NVME_DATA_NONE); - if (rdma_req != STAILQ_FIRST(&rgroup->pending_data_buf_queue)) { + if (&rdma_req->req != STAILQ_FIRST(&rgroup->group.pending_buf_queue)) { /* This request needs to wait in line to obtain a buffer */ break; } @@ -1886,7 +1883,7 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport, /* Try to get a data buffer */ rc = spdk_nvmf_rdma_request_parse_sgl(rtransport, device, rdma_req); if (rc < 0) { - STAILQ_REMOVE_HEAD(&rgroup->pending_data_buf_queue, state_link); + STAILQ_REMOVE_HEAD(&rgroup->group.pending_buf_queue, buf_link); rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; rdma_req->state = RDMA_REQUEST_STATE_READY_TO_COMPLETE; break; @@ -1898,7 +1895,7 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport, break; } - STAILQ_REMOVE_HEAD(&rgroup->pending_data_buf_queue, state_link); + STAILQ_REMOVE_HEAD(&rgroup->group.pending_buf_queue, buf_link); /* If data is transferring from host to controller and the data didn't * arrive using in capsule data, we need to do a transfer from the host. @@ -2521,6 +2518,7 @@ static void spdk_nvmf_rdma_qpair_process_pending(struct spdk_nvmf_rdma_transport *rtransport, struct spdk_nvmf_rdma_qpair *rqpair, bool drain) { + struct spdk_nvmf_request *req, *tmp; struct spdk_nvmf_rdma_request *rdma_req, *req_tmp; struct spdk_nvmf_rdma_resources *resources; @@ -2539,8 +2537,8 @@ spdk_nvmf_rdma_qpair_process_pending(struct spdk_nvmf_rdma_transport *rtransport } /* The second highest priority is I/O waiting on memory buffers. */ - STAILQ_FOREACH_SAFE(rdma_req, &rqpair->poller->group->pending_data_buf_queue, state_link, - req_tmp) { + STAILQ_FOREACH_SAFE(req, &rqpair->poller->group->group.pending_buf_queue, buf_link, tmp) { + rdma_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_rdma_request, req); if (spdk_nvmf_rdma_request_process(rtransport, rdma_req) == false && drain == false) { break; } @@ -2937,7 +2935,6 @@ spdk_nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport) } TAILQ_INIT(&rgroup->pollers); - STAILQ_INIT(&rgroup->pending_data_buf_queue); pthread_mutex_lock(&rtransport->lock); TAILQ_FOREACH(device, &rtransport->devices, link) { @@ -3047,10 +3044,6 @@ spdk_nvmf_rdma_poll_group_destroy(struct spdk_nvmf_transport_poll_group *group) free(poller); } - if (!STAILQ_EMPTY(&rgroup->pending_data_buf_queue)) { - SPDK_ERRLOG("Pending I/O list wasn't empty on poll group destruction\n"); - } - free(rgroup); } diff --git a/lib/nvmf/tcp.c b/lib/nvmf/tcp.c index 6b0eeb658..8afb81aa7 100644 --- a/lib/nvmf/tcp.c +++ b/lib/nvmf/tcp.c @@ -273,9 +273,6 @@ struct spdk_nvmf_tcp_poll_group { struct spdk_nvmf_transport_poll_group group; struct spdk_sock_group *sock_group; - /* Requests that are waiting to obtain a data buffer */ - STAILQ_HEAD(, spdk_nvmf_tcp_req) pending_data_buf_queue; - TAILQ_HEAD(, spdk_nvmf_tcp_qpair) qpairs; }; @@ -442,7 +439,8 @@ spdk_nvmf_tcp_cleanup_all_states(struct spdk_nvmf_tcp_qpair *tqpair) /* Wipe the requests waiting for buffer from the global list */ TAILQ_FOREACH_SAFE(tcp_req, &tqpair->state_queue[TCP_REQUEST_STATE_NEED_BUFFER], state_link, req_tmp) { - STAILQ_REMOVE(&tqpair->group->pending_data_buf_queue, tcp_req, spdk_nvmf_tcp_req, link); + STAILQ_REMOVE(&tqpair->group->group.pending_buf_queue, &tcp_req->req, + spdk_nvmf_request, buf_link); } spdk_nvmf_tcp_drain_state_queue(tqpair, TCP_REQUEST_STATE_NEED_BUFFER); @@ -1211,7 +1209,6 @@ spdk_nvmf_tcp_poll_group_create(struct spdk_nvmf_transport *transport) } TAILQ_INIT(&tgroup->qpairs); - STAILQ_INIT(&tgroup->pending_data_buf_queue); return &tgroup->group; @@ -1244,10 +1241,6 @@ spdk_nvmf_tcp_poll_group_destroy(struct spdk_nvmf_transport_poll_group *group) tgroup = SPDK_CONTAINEROF(group, struct spdk_nvmf_tcp_poll_group, group); spdk_sock_group_close(&tgroup->sock_group); - if (!STAILQ_EMPTY(&tgroup->pending_data_buf_queue)) { - SPDK_ERRLOG("Pending I/O list wasn't empty on poll group destruction\n"); - } - free(tgroup); } @@ -2590,14 +2583,14 @@ spdk_nvmf_tcp_req_process(struct spdk_nvmf_tcp_transport *ttransport, } spdk_nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_NEED_BUFFER); - STAILQ_INSERT_TAIL(&tqpair->group->pending_data_buf_queue, tcp_req, link); + STAILQ_INSERT_TAIL(&group->pending_buf_queue, &tcp_req->req, buf_link); break; case TCP_REQUEST_STATE_NEED_BUFFER: spdk_trace_record(TRACE_TCP_REQUEST_STATE_NEED_BUFFER, 0, 0, (uintptr_t)tcp_req, 0); assert(tcp_req->req.xfer != SPDK_NVME_DATA_NONE); - if (tcp_req != STAILQ_FIRST(&tqpair->group->pending_data_buf_queue)) { + if (&tcp_req->req != STAILQ_FIRST(&group->pending_buf_queue)) { SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "Not the first element to wait for the buf for tcp_req(%p) on tqpair=%p\n", tcp_req, tqpair); @@ -2608,7 +2601,7 @@ spdk_nvmf_tcp_req_process(struct spdk_nvmf_tcp_transport *ttransport, /* Try to get a data buffer */ rc = spdk_nvmf_tcp_req_parse_sgl(ttransport, tcp_req); if (rc < 0) { - STAILQ_REMOVE_HEAD(&tqpair->group->pending_data_buf_queue, link); + STAILQ_REMOVE_HEAD(&group->pending_buf_queue, buf_link); rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; /* Reset the tqpair receving pdu state */ spdk_nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR); @@ -2623,7 +2616,7 @@ spdk_nvmf_tcp_req_process(struct spdk_nvmf_tcp_transport *ttransport, break; } - STAILQ_REMOVE_HEAD(&tqpair->group->pending_data_buf_queue, link); + STAILQ_REMOVE_HEAD(&group->pending_buf_queue, buf_link); /* If data is transferring from host to controller, we need to do a transfer from the host. */ if (tcp_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER) { @@ -2825,7 +2818,8 @@ spdk_nvmf_tcp_poll_group_poll(struct spdk_nvmf_transport_poll_group *group) { struct spdk_nvmf_tcp_poll_group *tgroup; int rc; - struct spdk_nvmf_tcp_req *tcp_req, *req_tmp; + struct spdk_nvmf_request *req, *req_tmp; + struct spdk_nvmf_tcp_req *tcp_req; struct spdk_nvmf_tcp_transport *ttransport = SPDK_CONTAINEROF(group->transport, struct spdk_nvmf_tcp_transport, transport); @@ -2835,7 +2829,8 @@ spdk_nvmf_tcp_poll_group_poll(struct spdk_nvmf_transport_poll_group *group) return 0; } - STAILQ_FOREACH_SAFE(tcp_req, &tgroup->pending_data_buf_queue, link, req_tmp) { + STAILQ_FOREACH_SAFE(req, &group->pending_buf_queue, buf_link, req_tmp) { + tcp_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_tcp_req, req); if (spdk_nvmf_tcp_req_process(ttransport, tcp_req) == false) { break; } diff --git a/lib/nvmf/transport.c b/lib/nvmf/transport.c index ab1c3c596..d7856502c 100644 --- a/lib/nvmf/transport.c +++ b/lib/nvmf/transport.c @@ -196,6 +196,7 @@ spdk_nvmf_transport_poll_group_create(struct spdk_nvmf_transport *transport) } group->transport = transport; + STAILQ_INIT(&group->pending_buf_queue); STAILQ_INIT(&group->buf_cache); if (transport->opts.buf_cache_size) { @@ -230,6 +231,10 @@ spdk_nvmf_transport_poll_group_destroy(struct spdk_nvmf_transport_poll_group *gr { struct spdk_nvmf_transport_pg_cache_buf *buf, *tmp; + if (!STAILQ_EMPTY(&group->pending_buf_queue)) { + SPDK_ERRLOG("Pending I/O list wasn't empty on poll group destruction\n"); + } + STAILQ_FOREACH_SAFE(buf, &group->buf_cache, link, tmp) { STAILQ_REMOVE(&group->buf_cache, buf, spdk_nvmf_transport_pg_cache_buf, link); spdk_mempool_put(group->transport->data_buf_pool, buf); diff --git a/test/unit/lib/nvmf/rdma.c/rdma_ut.c b/test/unit/lib/nvmf/rdma.c/rdma_ut.c index 4e960992a..968c8cf1b 100644 --- a/test/unit/lib/nvmf/rdma.c/rdma_ut.c +++ b/test/unit/lib/nvmf/rdma.c/rdma_ut.c @@ -586,7 +586,7 @@ test_spdk_nvmf_rdma_request_process(void) bool progress; STAILQ_INIT(&group.group.buf_cache); - STAILQ_INIT(&group.pending_data_buf_queue); + STAILQ_INIT(&group.group.pending_buf_queue); group.group.buf_cache_size = 0; group.group.buf_cache_count = 0; port.device = &device;