From cdf61c2f22f26b8914cccf7d942268bbe6b5e470 Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Thu, 1 Sep 2022 06:46:37 +0900 Subject: [PATCH] nvme: Polls only the qpair if ctrlr is not fabrics when connecting synchronously For non-fabric controllers, the corresponding I/O qpairs are simply re-enabled at controller reset. This had a issue when I/O qpairs span multiple threads and poll group is used. spdk_nvme_ctrlr_reconnect_poll_async() calls nvme_transport_ctrlr_connect_qpair() with qpair->async being false. Then nvme_transport_ctrlr_connect_qpair() calls spdk_nvme_poll_group_process_completions() until the qpair is connected. spdk_nvme_poll_group_process_completions() may poll other qpairs. This may cause I/O to complete on a wrong thread. For PCIe controller, spdk_nvme_poll_group_process_completions() calls spdk_nvme_qpair_process_completions() simply for each qpair. Hence change nvme_transport_ctrlr_connect_qpair() to call spdk_nvme_qpair_process_completions() if the controller is non-fabrics. Signed-off-by: Shuhei Matsumoto Change-Id: Ieb270c2fb154124021ef6d25577b817d05e5ca9e Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/14295 Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Michael Haeuptle Reviewed-by: Jim Harris Reviewed-by: Dong Yi Reviewed-by: Aleksey Marchuk --- lib/nvme/nvme_transport.c | 2 +- test/unit/lib/nvme/nvme_transport.c/nvme_transport_ut.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/nvme/nvme_transport.c b/lib/nvme/nvme_transport.c index 3e060e590..4ecbc68d7 100644 --- a/lib/nvme/nvme_transport.c +++ b/lib/nvme/nvme_transport.c @@ -484,7 +484,7 @@ nvme_transport_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nv if (!qpair->async) { /* Busy wait until the qpair exits the connecting state */ while (nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTING) { - if (qpair->poll_group) { + if (qpair->poll_group && spdk_nvme_ctrlr_is_fabrics(ctrlr)) { rc = spdk_nvme_poll_group_process_completions( qpair->poll_group->group, 0, nvme_transport_connect_qpair_fail); diff --git a/test/unit/lib/nvme/nvme_transport.c/nvme_transport_ut.c b/test/unit/lib/nvme/nvme_transport.c/nvme_transport_ut.c index 825e6d360..982f927de 100644 --- a/test/unit/lib/nvme/nvme_transport.c/nvme_transport_ut.c +++ b/test/unit/lib/nvme/nvme_transport.c/nvme_transport_ut.c @@ -24,6 +24,7 @@ DEFINE_STUB(spdk_nvme_poll_group_process_completions, int64_t, (struct spdk_nvme spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb), 0); DEFINE_STUB(nvme_ctrlr_get_current_process, struct spdk_nvme_ctrlr_process *, (struct spdk_nvme_ctrlr *ctrlr), NULL); +DEFINE_STUB(spdk_nvme_ctrlr_is_fabrics, bool, (struct spdk_nvme_ctrlr *ctrlr), false); static void ut_construct_transport(struct spdk_nvme_transport *transport, const char name[])