bdev: add bdev channel a link

Add a link to the bdev channel for linking all the bdev
IOs that were submitted to this channel so that we can
monitor each IO's consuming-time.

Change-Id: I1e425b2059f20fd7b158eb3d6b023ce8629e7a30
Signed-off-by: JinYu <jin.yu@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/469227
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>

include/spdk/bdev_module.h

@@ -609,6 +609,9 @@ struct spdk_bdev_io {
 		/** Entry to the list need_buf of struct spdk_bdev. */
 		STAILQ_ENTRY(spdk_bdev_io) buf_link;
 
+		/** Entry to the list io_submitted of struct spdk_bdev_channel */
+		TAILQ_ENTRY(spdk_bdev_io) ch_link;
+
 		/** Enables queuing parent I/O when no bdev_ios available for split children. */
 		struct spdk_bdev_io_wait_entry waitq_entry;
 	} internal;

lib/bdev/bdev.c

@@ -252,12 +252,16 @@ struct spdk_bdev_channel {
 	struct spdk_bdev_io_stat stat;
 
 	/*
-	 * Count of I/O submitted through this channel and waiting for completion.
+	 * Count of I/O submitted to the underlying dev module through this channel
-	 * Incremented before submit_request() is called on an spdk_bdev_io.
+	 * and waiting for completion.
 	 */
 	uint64_t		io_outstanding;
 
-	bdev_io_tailq_t		queued_resets;
+	/*
+	 * List of spdk_bdev_io directly associated with a call to the public bdev API.
+	 * It does not include any spdk_bdev_io that are generated via splitting.
+	 */
+	bdev_io_tailq_t		io_submitted;
 
 	uint32_t		flags;
 
@@ -270,6 +274,7 @@ struct spdk_bdev_channel {
 	struct spdk_bdev_io_stat prev_stat;
 #endif
 
+	bdev_io_tailq_t		queued_resets;
 };
 
 struct spdk_bdev_desc {
@@ -1746,6 +1751,7 @@ _bdev_io_split(void *_bdev_io)
 			} else {
 				bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
 				if (bdev_io->u.bdev.split_outstanding == 0) {
+					TAILQ_REMOVE(&bdev_io->internal.ch->io_submitted, bdev_io, internal.ch_link);
 					bdev_io->internal.cb(bdev_io, false, bdev_io->internal.caller_ctx);
 				}
 			}
@@ -1774,6 +1780,8 @@ bdev_io_split_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
 	 * Parent I/O finishes when all blocks are consumed.
 	 */
 	if (parent_io->u.bdev.split_remaining_num_blocks == 0) {
+		assert(parent_io->internal.cb != bdev_io_split_done);
+		TAILQ_REMOVE(&parent_io->internal.ch->io_submitted, parent_io, internal.ch_link);
 		parent_io->internal.cb(parent_io, parent_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS,
 				       parent_io->internal.caller_ctx);
 		return;
@@ -1866,6 +1874,16 @@ bdev_io_submit(struct spdk_bdev_io *bdev_io)
 	assert(thread != NULL);
 	assert(bdev_io->internal.status == SPDK_BDEV_IO_STATUS_PENDING);
 
+	/* Add the bdev_io to io_submitted only if it is the original
+	 * submission from the bdev user.  When a bdev_io is split,
+	 * it comes back through this code path, so we need to make sure
+	 * we don't try to add it a second time.
+	 */
+	if (bdev_io->internal.cb != bdev_io_split_done) {
+		TAILQ_INSERT_TAIL(&bdev_io->internal.ch->io_submitted, bdev_io,
+				  internal.ch_link);
+	}
+
 	if (bdev->split_on_optimal_io_boundary && bdev_io_should_split(bdev_io)) {
 		bdev_io_split(NULL, bdev_io);
 		return;
@@ -2030,6 +2048,7 @@ bdev_channel_destroy_resource(struct spdk_bdev_channel *ch)
 
 	shared_resource = ch->shared_resource;
 
+	assert(TAILQ_EMPTY(&ch->io_submitted));
 	assert(ch->io_outstanding == 0);
 	assert(shared_resource->ref > 0);
 	shared_resource->ref--;
@@ -2155,6 +2174,8 @@ bdev_channel_create(void *io_device, void *ctx_buf)
 	ch->flags = 0;
 	ch->shared_resource = shared_resource;
 
+	TAILQ_INIT(&ch->io_submitted);
+
 #ifdef SPDK_CONFIG_VTUNE
 	{
 		char *name;
@@ -3482,6 +3503,9 @@ spdk_bdev_reset(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
 	TAILQ_INSERT_TAIL(&channel->queued_resets, bdev_io, internal.link);
 	pthread_mutex_unlock(&bdev->internal.mutex);
 
+	TAILQ_INSERT_TAIL(&bdev_io->internal.ch->io_submitted, bdev_io,
+			  internal.ch_link);
+
 	bdev_channel_start_reset(channel);
 
 	return 0;
@@ -3718,6 +3742,7 @@ static inline void
 bdev_io_complete(void *ctx)
 {
 	struct spdk_bdev_io *bdev_io = ctx;
+	struct spdk_bdev_channel *bdev_ch = bdev_io->internal.ch;
 	uint64_t tsc, tsc_diff;
 
 	if (spdk_unlikely(bdev_io->internal.in_submit_request || bdev_io->internal.io_submit_ch)) {
@@ -3742,6 +3767,13 @@ bdev_io_complete(void *ctx)
 	tsc = spdk_get_ticks();
 	tsc_diff = tsc - bdev_io->internal.submit_tsc;
 	spdk_trace_record_tsc(tsc, TRACE_BDEV_IO_DONE, 0, 0, (uintptr_t)bdev_io, 0);
+	/* When a bdev_io is split, the children bdev_io are not added
+	 * to the io_submitted list.  So don't try to remove them in that
+	 * case.
+	 */
+	if (bdev_io->internal.cb != bdev_io_split_done) {
+		TAILQ_REMOVE(&bdev_ch->io_submitted, bdev_io, internal.ch_link);
+	}
 
 	if (bdev_io->internal.ch->histogram) {
 		spdk_histogram_data_tally(bdev_io->internal.ch->histogram, tsc_diff);