diff --git a/lib/vhost/rte_vhost/vhost_user.c b/lib/vhost/rte_vhost/vhost_user.c
index 9aef88c9b..0ce4798d0 100644
--- a/lib/vhost/rte_vhost/vhost_user.c
+++ b/lib/vhost/rte_vhost/vhost_user.c
@@ -364,21 +364,20 @@ vhost_user_set_vring_addr(struct virtio_net *dev, VhostUserMsg *msg)
 	struct vhost_virtqueue *vq;
 	uint64_t len;
 
-	if (dev->has_new_mem_table) {
-		vhost_setup_mem_table(dev);
-		dev->has_new_mem_table = 0;
-	}
-
-
-	if (dev->mem == NULL)
-		return -1;
-
 	/* Remove from the data plane. */
 	if (dev->flags & VIRTIO_DEV_RUNNING) {
 		dev->flags &= ~VIRTIO_DEV_RUNNING;
 		dev->notify_ops->destroy_device(dev->vid);
 	}
 
+	if (dev->has_new_mem_table) {
+		vhost_setup_mem_table(dev);
+		dev->has_new_mem_table = 0;
+	}
+
+	if (dev->mem == NULL)
+		return -1;
+
 	/* addr->index refers to the queue index. The txq 1, rxq is 0. */
 	vq = dev->virtqueue[msg->payload.addr.index];