vhost-blk: record inflight descs of packed ring

This patch is for packed ring live recovery.
First step is to track the packed ring descs.

This feature base on this QEMU patch:
https://patchwork.kernel.org/patch/11766697/

Change-Id: Id0ccb6fd12b3623cc367424f496b4ffc5323f390
Signed-off-by: Jin Yu <jin.yu@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/4126
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>

lib/vhost/vhost.c

@@ -546,7 +546,7 @@ void
 vhost_vq_packed_ring_enqueue(struct spdk_vhost_session *vsession,
 			     struct spdk_vhost_virtqueue *virtqueue,
 			     uint16_t num_descs, uint16_t buffer_id,
-			     uint32_t length)
+			     uint32_t length, uint16_t inflight_head)
 {
 	struct vring_packed_desc *desc = &virtqueue->vring.desc_packed[virtqueue->last_used_idx];
 	bool used, avail;
@@ -587,6 +587,8 @@ vhost_vq_packed_ring_enqueue(struct spdk_vhost_session *vsession,
 	 * written to the descriptor.
 	 */
 	spdk_smp_wmb();
+
+	rte_vhost_set_last_inflight_io_packed(vsession->vid, virtqueue->vring_idx, inflight_head);
 	/* To mark a desc as used, the device sets the F_USED bit in flags to match
 	 * the internal Device ring wrap counter. It also sets the F_AVAIL bit to
 	 * match the same value.
@@ -596,6 +598,7 @@ vhost_vq_packed_ring_enqueue(struct spdk_vhost_session *vsession,
 	} else {
 		desc->flags &= ~VRING_DESC_F_AVAIL_USED;
 	}
+	rte_vhost_clr_inflight_desc_packed(vsession->vid, virtqueue->vring_idx, inflight_head);
 
 	vhost_log_used_vring_elem(vsession, virtqueue, virtqueue->last_used_idx);
 	virtqueue->last_used_idx += num_descs;

lib/vhost/vhost_blk.c

@@ -72,6 +72,7 @@ struct spdk_vhost_blk_task {
 	uint16_t req_idx;
 	uint16_t num_descs;
 	uint16_t buffer_id;
+	uint16_t inflight_head;
 
 	/* for io wait */
 	struct spdk_bdev_io_wait_entry bdev_io_wait;
@@ -141,7 +142,8 @@ blk_task_enqueue(struct spdk_vhost_blk_task *task)
 	if (task->vq->packed.packed_ring) {
 		vhost_vq_packed_ring_enqueue(&task->bvsession->vsession, task->vq,
 					     task->num_descs,
-					     task->buffer_id, task->used_len);
+					     task->buffer_id, task->used_len,
+					     task->inflight_head);
 	} else {
 		vhost_vq_used_ring_enqueue(&task->bvsession->vsession, task->vq,
 					   task->req_idx, task->used_len);
@@ -578,6 +580,10 @@ process_blk_task(struct spdk_vhost_virtqueue *vq, uint16_t req_idx)
 		task->req_idx = req_idx;
 		task->num_descs = num_descs;
 		task->buffer_id = task_idx;
+
+		rte_vhost_set_inflight_desc_packed(task->bvsession->vsession.vid, vq->vring_idx,
+						   req_idx, (req_idx + num_descs - 1) % vq->vring.size,
+						   &task->inflight_head);
 	}
 
 	task->bvsession->vsession.task_cnt++;
@@ -752,7 +758,8 @@ no_bdev_process_packed_vq(struct spdk_vhost_blk_session *bvsession, struct spdk_
 		SPDK_ERRLOG("%s: request with idx '%"PRIu16"' is already pending.\n",
 			    vsession->name, req_idx);
 		vhost_vq_packed_ring_enqueue(vsession, vq, num_descs,
-					     task->buffer_id, task->used_len);
+					     task->buffer_id, task->used_len,
+					     task->inflight_head);
 		return;
 	}
 
@@ -769,7 +776,8 @@ no_bdev_process_packed_vq(struct spdk_vhost_blk_session *bvsession, struct spdk_
 
 	task->used = false;
 	vhost_vq_packed_ring_enqueue(vsession, vq, num_descs,
-				     task->buffer_id, task->used_len);
+				     task->buffer_id, task->used_len,
+				     task->inflight_head);
 }
 
 static int

lib/vhost/vhost_internal.h

@@ -324,11 +324,12 @@ void vhost_vq_used_ring_enqueue(struct spdk_vhost_session *vsession,
  * \buffer_id descriptor buffer ID.
  * \length device write length. Specify the length of the buffer that has been initialized
  * (written to) by the device
+ * \inflight_head the head idx of this IO inflight desc chain.
  */
 void vhost_vq_packed_ring_enqueue(struct spdk_vhost_session *vsession,
 				  struct spdk_vhost_virtqueue *virtqueue,
 				  uint16_t num_descs, uint16_t buffer_id,
-				  uint32_t length);
+				  uint32_t length, uint16_t inflight_head);
 
 /**
  * Get subsequent descriptor from given table.

test/unit/lib/vhost/vhost.c/vhost_ut.c

@@ -75,6 +75,10 @@ DEFINE_STUB(rte_vhost_set_last_inflight_io_split, int,
 	    (int vid, uint16_t vring_idx, uint16_t idx), 0);
 DEFINE_STUB(rte_vhost_clr_inflight_desc_split, int,
 	    (int vid, uint16_t vring_idx, uint16_t last_used_idx, uint16_t idx), 0);
+DEFINE_STUB(rte_vhost_set_last_inflight_io_packed, int,
+	    (int vid, uint16_t vring_idx, uint16_t head), 0);
+DEFINE_STUB(rte_vhost_clr_inflight_desc_packed, int,
+	    (int vid, uint16_t vring_idx, uint16_t head), 0);
 DEFINE_STUB_V(rte_vhost_log_write, (int vid, uint64_t addr, uint64_t len));
 DEFINE_STUB_V(vhost_session_mem_register, (struct rte_vhost_memory *mem));
 DEFINE_STUB_V(vhost_session_mem_unregister, (struct rte_vhost_memory *mem));
@@ -473,9 +477,9 @@ vq_packed_ring_test(void)
 	}
 
 	/* Host complete them out of order: 1, 0, 2. */
-	vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 1, 1);
+	vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 1, 1, 0);
-	vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 0, 1);
+	vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 0, 1, 0);
-	vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 2, 1);
+	vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 2, 1, 0);
 
 	/* Host has got all the available request but only complete three requests */
 	CU_ASSERT(vq.last_avail_idx == 0);
@@ -511,10 +515,10 @@ vq_packed_ring_test(void)
 	CU_ASSERT(vq.packed.avail_phase == 0);
 
 	/* Host complete all the requests */
-	vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 1, 1);
+	vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 1, 1, 0);
-	vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 0, 1);
+	vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 0, 1, 0);
-	vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 3, 1);
+	vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 3, 1, 0);
-	vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 2, 1);
+	vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 2, 1, 0);
 
 	CU_ASSERT(vq.last_used_idx == vq.last_avail_idx);
 	CU_ASSERT(vq.packed.used_phase == vq.packed.avail_phase);