From ea0aaf5e859160a04cf7be775b137a97dd239ce0 Mon Sep 17 00:00:00 2001 From: Ben Walker Date: Thu, 22 Jul 2021 12:08:38 -0700 Subject: [PATCH] nvme: Transports now set qpair state to NVME_QPAIR_CONNECTED inside .ctrlr_connect_qpair Previously this was assumed to be a synchronous process so the generic layer transport code updated the state after .ctrlr_connect_qpair returned. In preparation for making this support asynchronous mode, shift that responsibility down into the individual transports. While none of the transports actually do this asynchronously, insert a busy wait in nvme_transport_ctrlr_connect_qpair to wait for the qpair to exit from the CONNECTING state. None of the upper layer code can actually correct handle a transport doing this asynchronously, so the busy wait will cover that. Signed-off-by: Ben Walker Change-Id: I3c1a5c115264ffcb87e549765d891d796e0c81fe Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/8909 Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Monica Kenguva Reviewed-by: Changpeng Liu Reviewed-by: Shuhei Matsumoto Reviewed-by: Jim Harris --- lib/nvme/nvme_pcie_common.c | 14 ++++++++++---- lib/nvme/nvme_rdma.c | 4 ++++ lib/nvme/nvme_tcp.c | 2 ++ lib/nvme/nvme_transport.c | 4 +++- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/lib/nvme/nvme_pcie_common.c b/lib/nvme/nvme_pcie_common.c index 4ba7622e6..98b79254e 100644 --- a/lib/nvme/nvme_pcie_common.c +++ b/lib/nvme/nvme_pcie_common.c @@ -542,11 +542,17 @@ _nvme_pcie_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme int nvme_pcie_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair) { - if (nvme_qpair_is_admin_queue(qpair)) { - return 0; - } else { - return _nvme_pcie_ctrlr_create_io_qpair(ctrlr, qpair, qpair->id); + int rc = 0; + + if (!nvme_qpair_is_admin_queue(qpair)) { + rc = _nvme_pcie_ctrlr_create_io_qpair(ctrlr, qpair, qpair->id); } + + if (rc == 0) { + nvme_qpair_set_state(qpair, NVME_QPAIR_CONNECTED); + } + + return rc; } void diff --git a/lib/nvme/nvme_rdma.c b/lib/nvme/nvme_rdma.c index 9d38e9869..4d1e8c4d0 100644 --- a/lib/nvme/nvme_rdma.c +++ b/lib/nvme/nvme_rdma.c @@ -1245,6 +1245,10 @@ nvme_rdma_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qp } while (rc == -EAGAIN && retry_count < NVME_RDMA_STALE_CONN_RETRY_MAX); } + if (rc == 0) { + nvme_qpair_set_state(qpair, NVME_QPAIR_CONNECTED); + } + return rc; } diff --git a/lib/nvme/nvme_tcp.c b/lib/nvme/nvme_tcp.c index fa762d0fb..1c674850b 100644 --- a/lib/nvme/nvme_tcp.c +++ b/lib/nvme/nvme_tcp.c @@ -1941,6 +1941,8 @@ nvme_tcp_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpa return rc; } + nvme_qpair_set_state(qpair, NVME_QPAIR_CONNECTED); + return 0; } diff --git a/lib/nvme/nvme_transport.c b/lib/nvme/nvme_transport.c index d59185cdd..6345d00b6 100644 --- a/lib/nvme/nvme_transport.c +++ b/lib/nvme/nvme_transport.c @@ -369,7 +369,9 @@ nvme_transport_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nv goto err; } - nvme_qpair_set_state(qpair, NVME_QPAIR_CONNECTED); + /* Busy wait until the qpair exits the connecting state */ + while (nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTING) { } + if (qpair->poll_group) { rc = nvme_poll_group_connect_qpair(qpair); if (rc) {