From 5fd77e32a953c832d8fcfc340e5f8c9d37680c16 Mon Sep 17 00:00:00 2001 From: Changpeng Liu Date: Wed, 1 Sep 2021 18:21:39 +0800 Subject: [PATCH] nvmf/vfio-user: unmap queue pairs before spdk_mem_unregister() Ideally, SPDK should make sure no pending I/Os in this queue pair are using the removed memory region. Currently we just stop the submission path and leave a TODO comment here until we have an asynchronous way to do this. Also use the `<=` for the boundary check. Change-Id: I63a2189022978811dc21f92f2599f28a5191ecd7 Signed-off-by: Changpeng Liu Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9352 Community-CI: Mellanox Build Bot Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Jim Harris --- lib/nvmf/vfio_user.c | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/lib/nvmf/vfio_user.c b/lib/nvmf/vfio_user.c index 1f154dc7e..9218a743a 100644 --- a/lib/nvmf/vfio_user.c +++ b/lib/nvmf/vfio_user.c @@ -1518,6 +1518,22 @@ memory_region_remove_cb(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info) (uintptr_t)info->mapping.iov_base, (uintptr_t)info->mapping.iov_base + info->mapping.iov_len); + map_start = info->mapping.iov_base; + map_end = info->mapping.iov_base + info->mapping.iov_len; + + pthread_mutex_lock(&endpoint->lock); + TAILQ_FOREACH(qpair, &ctrlr->connected_qps, tailq) { + if ((qpair->cq.addr >= map_start && qpair->cq.addr <= map_end) || + (qpair->sq.addr >= map_start && qpair->sq.addr <= map_end)) { + /* TODO: Ideally we should disconnect this queue pair + * before returning to caller. + */ + unmap_qp(qpair); + qpair->state = VFIO_USER_QPAIR_INACTIVE; + } + } + pthread_mutex_unlock(&endpoint->lock); + if (info->prot == (PROT_WRITE | PROT_READ)) { ret = spdk_mem_unregister(info->mapping.iov_base, info->mapping.iov_len); if (ret) { @@ -1528,19 +1544,6 @@ memory_region_remove_cb(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info) } } - map_start = info->mapping.iov_base; - map_end = info->mapping.iov_base + info->mapping.iov_len; - - pthread_mutex_lock(&endpoint->lock); - TAILQ_FOREACH(qpair, &ctrlr->connected_qps, tailq) { - if ((qpair->cq.addr >= map_start && qpair->cq.addr < map_end) || - (qpair->sq.addr >= map_start && qpair->sq.addr < map_end)) { - unmap_qp(qpair); - qpair->state = VFIO_USER_QPAIR_INACTIVE; - } - } - pthread_mutex_unlock(&endpoint->lock); - return 0; }