From bf102998937f4e8bd707be296cd4cac96d466b2f Mon Sep 17 00:00:00 2001 From: Ziye Yang Date: Sat, 16 Jan 2021 17:50:17 +0800 Subject: [PATCH] Revert "nvmf/tcp: Remove the await_req list." This reverts commit 0bcaf050d74138f1191a3c18814f0991dd182db2. Reason: After this patch, the last nvme command cannot be executed on the qpair with the following two conditions together: (1) The qpair is in waiting for available req state. (2) There is no incoming data again (i.e.. no read in event). Fixes issue: 1746 Signed-off-by: Ziye Yang Reported by: John K Kariuki Change-Id: Ie16a89e19baeaa784002990ae034ae59d90a51f7 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5951 Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Aleksey Marchuk Reviewed-by: Shuhei Matsumoto Reviewed-by: John Kariuki Reviewed-by: Jim Harris --- lib/nvmf/tcp.c | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/nvmf/tcp.c b/lib/nvmf/tcp.c index a4d45f1ed..1a422cdfa 100644 --- a/lib/nvmf/tcp.c +++ b/lib/nvmf/tcp.c @@ -271,6 +271,7 @@ struct spdk_nvmf_tcp_poll_group { struct spdk_sock_group *sock_group; TAILQ_HEAD(, spdk_nvmf_tcp_qpair) qpairs; + TAILQ_HEAD(, spdk_nvmf_tcp_qpair) await_req; struct spdk_nvmf_tcp_control_msg_list *control_msg_list; }; @@ -1132,6 +1133,7 @@ nvmf_tcp_poll_group_create(struct spdk_nvmf_transport *transport) } TAILQ_INIT(&tgroup->qpairs); + TAILQ_INIT(&tgroup->await_req); ttransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_tcp_transport, transport); @@ -1192,6 +1194,12 @@ nvmf_tcp_qpair_set_recv_state(struct spdk_nvmf_tcp_qpair *tqpair, return; } + if (tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_REQ) { + /* When leaving the await req state, move the qpair to the main list */ + TAILQ_REMOVE(&tqpair->group->await_req, tqpair, link); + TAILQ_INSERT_TAIL(&tqpair->group->qpairs, tqpair, link); + } + SPDK_DEBUGLOG(nvmf_tcp, "tqpair(%p) recv state=%d\n", tqpair, state); tqpair->recv_state = state; @@ -1199,7 +1207,10 @@ nvmf_tcp_qpair_set_recv_state(struct spdk_nvmf_tcp_qpair *tqpair, case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_CH: case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH: case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD: + break; case NVME_TCP_PDU_RECV_STATE_AWAIT_REQ: + TAILQ_REMOVE(&tqpair->group->qpairs, tqpair, link); + TAILQ_INSERT_TAIL(&tqpair->group->await_req, tqpair, link); break; case NVME_TCP_PDU_RECV_STATE_ERROR: case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY: @@ -2576,7 +2587,11 @@ nvmf_tcp_poll_group_remove(struct spdk_nvmf_transport_poll_group *group, assert(tqpair->group == tgroup); SPDK_DEBUGLOG(nvmf_tcp, "remove tqpair=%p from the tgroup=%p\n", tqpair, tgroup); - TAILQ_REMOVE(&tgroup->qpairs, tqpair, link); + if (tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_REQ) { + TAILQ_REMOVE(&tgroup->await_req, tqpair, link); + } else { + TAILQ_REMOVE(&tgroup->qpairs, tqpair, link); + } rc = spdk_sock_group_remove_sock(tgroup->sock_group, tqpair->sock); if (rc != 0) { @@ -2626,12 +2641,13 @@ nvmf_tcp_poll_group_poll(struct spdk_nvmf_transport_poll_group *group) int rc; struct spdk_nvmf_request *req, *req_tmp; struct spdk_nvmf_tcp_req *tcp_req; + struct spdk_nvmf_tcp_qpair *tqpair, *tqpair_tmp; struct spdk_nvmf_tcp_transport *ttransport = SPDK_CONTAINEROF(group->transport, struct spdk_nvmf_tcp_transport, transport); tgroup = SPDK_CONTAINEROF(group, struct spdk_nvmf_tcp_poll_group, group); - if (spdk_unlikely(TAILQ_EMPTY(&tgroup->qpairs))) { + if (spdk_unlikely(TAILQ_EMPTY(&tgroup->qpairs) && TAILQ_EMPTY(&tgroup->await_req))) { return 0; } @@ -2647,6 +2663,10 @@ nvmf_tcp_poll_group_poll(struct spdk_nvmf_transport_poll_group *group) SPDK_ERRLOG("Failed to poll sock_group=%p\n", tgroup->sock_group); } + TAILQ_FOREACH_SAFE(tqpair, &tgroup->await_req, link, tqpair_tmp) { + nvmf_tcp_sock_process(tqpair); + } + return rc; }