From 360ccc4e1e3ee51169ac90ef7f00f34b4c661fbd Mon Sep 17 00:00:00 2001 From: Ni Xun Date: Tue, 25 Sep 2018 11:26:06 +0800 Subject: [PATCH] bdev_split: add support for bdev_io_wait when no memory to submit according to commit: bdev: add spdk_bdev_queue_io_wait This patch will make io_wait to support bdev_split Change-Id: Ie5c57db3fbd9e777f69daf65b080f2659d85032a Signed-off-by: Ni Xun Signed-off-by: Li Lin Signed-off-by: Zhang Yu Reviewed-on: https://review.gerrithub.io/426649 Reviewed-by: Shuhei Matsumoto Reviewed-by: Ben Walker Chandler-Test-Pool: SPDK Automated Test System Tested-by: SPDK CI Jenkins --- include/spdk/bdev_module.h | 3 +- lib/bdev/error/vbdev_error.c | 7 +++- lib/bdev/gpt/vbdev_gpt.c | 7 +++- lib/bdev/part.c | 10 +++--- lib/bdev/split/vbdev_split.c | 64 +++++++++++++++++++++++++++++++++++- 5 files changed, 81 insertions(+), 10 deletions(-) diff --git a/include/spdk/bdev_module.h b/include/spdk/bdev_module.h index 39e571b53..0cfcbbbc3 100644 --- a/include/spdk/bdev_module.h +++ b/include/spdk/bdev_module.h @@ -904,8 +904,9 @@ int spdk_bdev_part_construct(struct spdk_bdev_part *part, struct spdk_bdev_part_ * * \param ch The I/O channel associated with the spdk_bdev_part. * \param bdev_io The I/O to be submitted to the underlying bdev. + * \return 0 on success or non-zero if submit request failed. */ -void spdk_bdev_part_submit_request(struct spdk_bdev_part_channel *ch, struct spdk_bdev_io *bdev_io); +int spdk_bdev_part_submit_request(struct spdk_bdev_part_channel *ch, struct spdk_bdev_io *bdev_io); /** * Return a pointer to this part's spdk_bdev. diff --git a/lib/bdev/error/vbdev_error.c b/lib/bdev/error/vbdev_error.c index 8f4807248..4bab9426e 100644 --- a/lib/bdev/error/vbdev_error.c +++ b/lib/bdev/error/vbdev_error.c @@ -191,7 +191,12 @@ vbdev_error_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bde error_type = vbdev_error_get_error_type(error_disk, bdev_io->type); if (error_type == 0) { - spdk_bdev_part_submit_request(&ch->part_ch, bdev_io); + int rc = spdk_bdev_part_submit_request(&ch->part_ch, bdev_io); + + if (rc) { + SPDK_ERRLOG("bdev_error: submit request failed, rc=%d\n", rc); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + } return; } else if (error_type == VBDEV_IO_FAILURE) { error_disk->error_vector[bdev_io->type].error_num--; diff --git a/lib/bdev/gpt/vbdev_gpt.c b/lib/bdev/gpt/vbdev_gpt.c index 8a0b4e419..4c6063778 100644 --- a/lib/bdev/gpt/vbdev_gpt.c +++ b/lib/bdev/gpt/vbdev_gpt.c @@ -160,8 +160,13 @@ static void vbdev_gpt_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) { struct gpt_channel *ch = spdk_io_channel_get_ctx(_ch); + int rc; - spdk_bdev_part_submit_request(&ch->part_ch, bdev_io); + rc = spdk_bdev_part_submit_request(&ch->part_ch, bdev_io); + if (rc) { + SPDK_ERRLOG("gpt: error on bdev_io submission, rc=%d.\n", rc); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + } } static void diff --git a/lib/bdev/part.c b/lib/bdev/part.c index f10a70555..0cb4759b7 100644 --- a/lib/bdev/part.c +++ b/lib/bdev/part.c @@ -192,7 +192,7 @@ spdk_bdev_part_complete_io(struct spdk_bdev_io *bdev_io, bool success, void *cb_ spdk_bdev_free_io(bdev_io); } -void +int spdk_bdev_part_submit_request(struct spdk_bdev_part_channel *ch, struct spdk_bdev_io *bdev_io) { struct spdk_bdev_part *part = ch->part; @@ -238,14 +238,12 @@ spdk_bdev_part_submit_request(struct spdk_bdev_part_channel *ch, struct spdk_bde break; default: SPDK_ERRLOG("split: unknown I/O type %d\n", bdev_io->type); - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); - return; + return SPDK_BDEV_IO_STATUS_FAILED; } - if (rc != 0) { - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); - } + return rc; } + static int spdk_bdev_part_channel_create_cb(void *io_device, void *ctx_buf) { diff --git a/lib/bdev/split/vbdev_split.c b/lib/bdev/split/vbdev_split.c index e04da5cc4..97f119841 100644 --- a/lib/bdev/split/vbdev_split.c +++ b/lib/bdev/split/vbdev_split.c @@ -67,17 +67,30 @@ struct vbdev_split_channel { struct spdk_bdev_part_channel part_ch; }; +struct vbdev_split_bdev_io { + struct spdk_io_channel *ch; + struct spdk_bdev_io *bdev_io; + + /* for bdev_io_wait */ + struct spdk_bdev_io_wait_entry bdev_io_wait; +}; + static void vbdev_split_del_config(struct spdk_vbdev_split_config *cfg); static int vbdev_split_init(void); static void vbdev_split_fini(void); static void vbdev_split_examine(struct spdk_bdev *bdev); static int vbdev_split_config_json(struct spdk_json_write_ctx *w); +static int vbdev_split_get_ctx_size(void); + +static void +vbdev_split_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io); static struct spdk_bdev_module split_if = { .name = "split", .module_init = vbdev_split_init, .module_fini = vbdev_split_fini, + .get_ctx_size = vbdev_split_get_ctx_size, .examine_config = vbdev_split_examine, .config_json = vbdev_split_config_json, }; @@ -109,12 +122,50 @@ vbdev_split_base_bdev_hotremove_cb(void *_base_bdev) spdk_bdev_part_base_hotremove(_base_bdev, &g_split_disks); } +static void +vbdev_split_resubmit_io(void *arg) +{ + struct vbdev_split_bdev_io *split_io = (struct vbdev_split_bdev_io *)arg; + + vbdev_split_submit_request(split_io->ch, split_io->bdev_io); +} + +static void +vbdev_split_queue_io(struct vbdev_split_bdev_io *split_io) +{ + int rc; + + split_io->bdev_io_wait.bdev = split_io->bdev_io->bdev; + split_io->bdev_io_wait.cb_fn = vbdev_split_resubmit_io; + split_io->bdev_io_wait.cb_arg = split_io; + + rc = spdk_bdev_queue_io_wait(split_io->bdev_io->bdev, + split_io->ch, &split_io->bdev_io_wait); + if (rc != 0) { + SPDK_ERRLOG("Queue io failed in vbdev_split_queue_io, rc=%d\n", rc); + spdk_bdev_io_complete(split_io->bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + } +} + static void vbdev_split_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io) { struct vbdev_split_channel *ch = spdk_io_channel_get_ctx(_ch); + struct vbdev_split_bdev_io *io_ctx = (struct vbdev_split_bdev_io *)bdev_io->driver_ctx; + int rc; - spdk_bdev_part_submit_request(&ch->part_ch, bdev_io); + rc = spdk_bdev_part_submit_request(&ch->part_ch, bdev_io); + if (rc) { + if (rc == -ENOMEM) { + SPDK_DEBUGLOG(SPDK_LOG_VBDEV_SPLIT, "split: no memory, queue io.\n"); + io_ctx->ch = _ch; + io_ctx->bdev_io = bdev_io; + vbdev_split_queue_io(io_ctx); + } else { + SPDK_ERRLOG("split: error on io submission, rc=%d.\n", rc); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + } + } } static int @@ -500,4 +551,15 @@ spdk_vbdev_split_get_part_base(struct spdk_bdev *bdev) return cfg->split_base; } +/* + * During init we'll be asked how much memory we'd like passed to us + * in bev_io structures as context. Here's where we specify how + * much context we want per IO. + */ +static int +vbdev_split_get_ctx_size(void) +{ + return sizeof(struct vbdev_split_bdev_io); +} + SPDK_LOG_REGISTER_COMPONENT("vbdev_split", SPDK_LOG_VBDEV_SPLIT)