nvmf/vfio-user: re-check doorbells upon resuming

If we're in interrupt mode and live migrating a guest, there is a window
where the I/O queues haven't been set up but the device is in running
state, during which the guest might write to a doorbell. This doorbell
write will go unnoticed. This patch ensures that we re-check the
doorbells after an I/O queue has been set up.

Fixes #2410

Signed-off-by: Thanos Makatos <thanos.makatos@nutanix.com>
Signed-off-by: John Levon <john.levon@nutanix.com>
Change-Id: I161d2a0e7ab3065022b2bccbe17f019640cceeba
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11809
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

lib/nvmf/vfio_user.c

@@ -349,6 +349,8 @@ struct nvmf_vfio_user_ctrlr {
 	TAILQ_ENTRY(nvmf_vfio_user_ctrlr)	link;
 
 	volatile uint32_t			*doorbells;
+
+	bool					self_kick_requested;
 };
 
 struct nvmf_vfio_user_endpoint {
@@ -497,6 +499,38 @@ vfio_user_migr_data_len(void)
 	return SPDK_ALIGN_CEIL(sizeof(struct vfio_user_nvme_migr_state), PAGE_SIZE);
 }
 
+static int
+vfio_user_handle_intr(void *ctx);
+
+/*
+ * Wrap vfio_user_handle_intr() such that it can be used with
+ * spdk_thread_send_msg().
+ * Pollers have type int (*)(void *) while message functions should have type
+ * void (*)(void *), so simply discard the returned value.
+ */
+static void
+vfio_user_handle_intr_wrapper(void *ctx)
+{
+	vfio_user_handle_intr(ctx);
+}
+
+static inline int
+self_kick(struct nvmf_vfio_user_ctrlr *ctrlr)
+{
+	assert(ctrlr != NULL);
+	assert(ctrlr->thread != NULL);
+
+	if (ctrlr->self_kick_requested) {
+		return 0;
+	}
+
+	ctrlr->self_kick_requested = true;
+
+	return spdk_thread_send_msg(ctrlr->thread,
+				    vfio_user_handle_intr_wrapper,
+				    ctrlr);
+}
+
 static int
 nvme_cmd_map_prps(void *prv, struct spdk_nvme_cmd *cmd, struct iovec *iovs,
 		  uint32_t max_iovcnt, uint32_t len, size_t mps,
@@ -3550,6 +3584,13 @@ nvmf_vfio_user_poll_group_create(struct spdk_nvmf_transport *transport,
 	return &vu_group->group;
 }
 
+static bool
+in_interrupt_mode(struct nvmf_vfio_user_transport *vu_transport)
+{
+	return spdk_interrupt_mode_is_enabled() &&
+	       vu_transport->intr_mode_supported;
+}
+
 static struct spdk_nvmf_transport_poll_group *
 nvmf_vfio_user_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair)
 {
@@ -3586,8 +3627,7 @@ nvmf_vfio_user_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair)
 		 * on the same poll group, to avoid complications in
 		 * vfio_user_handle_intr().
 		 */
-		if (spdk_interrupt_mode_is_enabled() &&
+		if (in_interrupt_mode(vu_transport)) {
-		    vu_transport->intr_mode_supported) {
 			result = sq->ctrlr->sqs[0]->group;
 			goto out;
 		}
@@ -3746,6 +3786,8 @@ vfio_user_handle_intr(void *ctx)
 	assert(ctrlr->sqs[0] != NULL);
 	assert(ctrlr->sqs[0]->group != NULL);
 
+	ctrlr->self_kick_requested = false;
+
 	vfio_user_poll_vfu_ctx(ctrlr);
 
 	/*
@@ -3797,8 +3839,7 @@ handle_queue_connect_rsp(struct nvmf_vfio_user_req *req, void *cb_arg)
 
 		cq->thread = spdk_get_thread();
 
-		if (spdk_interrupt_mode_is_enabled() &&
+		if (in_interrupt_mode(endpoint->transport)) {
-		    endpoint->transport->intr_mode_supported) {
 			vu_ctrlr->intr_fd = vfu_get_poll_fd(vu_ctrlr->endpoint->vfu_ctx);
 			assert(vu_ctrlr->intr_fd != -1);
 
@@ -3840,6 +3881,14 @@ handle_queue_connect_rsp(struct nvmf_vfio_user_req *req, void *cb_arg)
 						sq->create_io_sq_cmd.cid, SPDK_NVME_SC_SUCCESS, SPDK_NVME_SCT_GENERIC);
 			}
 			sq->post_create_io_sq_completion = false;
+		} else if (in_interrupt_mode(endpoint->transport)) {
+			/*
+			 * FIXME self_kick() ends up polling all queues on the
+			 * controller thread, and this will be wrong if we ever
+			 * support interrupt mode with I/O queues in a
+			 * different poll group than the controller's.
+			 */
+			self_kick(vu_ctrlr);
 		}
 		sq->sq_state = VFIO_USER_SQ_ACTIVE;
 	}