diff --git a/lib/vhost/rte_vhost_user.c b/lib/vhost/rte_vhost_user.c
index d465153f6..05d935cfa 100644
--- a/lib/vhost/rte_vhost_user.c
+++ b/lib/vhost/rte_vhost_user.c
@@ -1756,6 +1756,10 @@ vhost_user_dev_register(struct spdk_vhost_dev *vdev, const char *name, struct sp
 int
 vhost_user_dev_unregister(struct spdk_vhost_dev *vdev)
 {
+	if (vdev->pending_async_op_num) {
+		return -EBUSY;
+	}
+
 	if (!TAILQ_EMPTY(&vdev->vsessions)) {
 		SPDK_ERRLOG("Controller %s has still valid connection.\n", vdev->name);
 		return -EBUSY;
diff --git a/lib/vhost/vhost.c b/lib/vhost/vhost.c
index eea486699..09176dc3c 100644
--- a/lib/vhost/vhost.c
+++ b/lib/vhost/vhost.c
@@ -196,10 +196,6 @@ vhost_dump_info_json(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w)
 int
 spdk_vhost_dev_remove(struct spdk_vhost_dev *vdev)
 {
-	if (vdev->pending_async_op_num) {
-		return -EBUSY;
-	}
-
 	return vdev->backend->remove_device(vdev);
 }
 
diff --git a/lib/vhost/vhost_scsi.c b/lib/vhost/vhost_scsi.c
index 6ff07e130..b9e47c672 100644
--- a/lib/vhost/vhost_scsi.c
+++ b/lib/vhost/vhost_scsi.c
@@ -895,6 +895,10 @@ vhost_scsi_dev_remove(struct spdk_vhost_dev *vdev)
 	struct spdk_vhost_scsi_dev *svdev = to_scsi_dev(vdev);
 	int rc, i;
 
+	if (vdev->pending_async_op_num) {
+		return -EBUSY;
+	}
+
 	assert(svdev != NULL);
 	for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; ++i) {
 		if (svdev->scsi_dev_state[i].dev) {