From 4b47b6e31c253e3ed67642ce1434b1e8422a5b7a Mon Sep 17 00:00:00 2001 From: Changpeng Liu Date: Tue, 26 Oct 2021 20:49:31 +0800 Subject: [PATCH] nvmf/vfio-user: avoid recursive lock when disconnecting qpair When destroying controller, we will disconnect each connected qpair, and in the spdk_nvmf_qpair_disconnect() call, qpair_fini() will also try to hold the same lock, so existing vfio-user implementation assume that qpair_fini() will not be called in the same context. Patch https://review.spdk.io/gerrit/c/spdk/spdk/+/8963 remind me that vfio-user has this issue. While here, we add one more thread poll to avoid such issue. Change-Id: I83b82ddcce3eb54c724291223e794dcb53a08059 Signed-off-by: Changpeng Liu Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9998 Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins Reviewed-by: Aleksey Marchuk Reviewed-by: Ben Walker --- lib/nvmf/vfio_user.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/nvmf/vfio_user.c b/lib/nvmf/vfio_user.c index 6d790172c..a04e10666 100644 --- a/lib/nvmf/vfio_user.c +++ b/lib/nvmf/vfio_user.c @@ -2303,12 +2303,23 @@ vfio_user_qpair_disconnect_cb(void *ctx) if (TAILQ_EMPTY(&ctrlr->connected_qps)) { endpoint->ctrlr = NULL; free_ctrlr(ctrlr, false); - pthread_mutex_unlock(&endpoint->lock); - return; } pthread_mutex_unlock(&endpoint->lock); } +static void +_vfio_user_qpair_disconnect(void *ctx) +{ + struct nvmf_vfio_user_qpair *vu_qpair = ctx; + struct nvmf_vfio_user_ctrlr *vu_ctrlr; + struct nvmf_vfio_user_endpoint *endpoint; + + vu_ctrlr = vu_qpair->ctrlr; + endpoint = vu_ctrlr->endpoint; + + spdk_nvmf_qpair_disconnect(&vu_qpair->qpair, vfio_user_qpair_disconnect_cb, endpoint); +} + static int vfio_user_destroy_ctrlr(struct nvmf_vfio_user_ctrlr *ctrlr) { @@ -2329,7 +2340,8 @@ vfio_user_destroy_ctrlr(struct nvmf_vfio_user_ctrlr *ctrlr) } TAILQ_FOREACH(qpair, &ctrlr->connected_qps, tailq) { - spdk_nvmf_qpair_disconnect(&qpair->qpair, vfio_user_qpair_disconnect_cb, endpoint); + /* add another round thread poll to avoid recursive endpoint lock */ + spdk_thread_send_msg(ctrlr->thread, _vfio_user_qpair_disconnect, qpair); } pthread_mutex_unlock(&endpoint->lock);