From f2a9507af639ab367b95ad40d6e93a3399586296 Mon Sep 17 00:00:00 2001 From: Jim Harris Date: Thu, 23 Aug 2018 14:18:42 -0700 Subject: [PATCH] bdev/raid: enable multiple iovs The bdevio tests will now work on RAID volumes, so enable RAID testing there too. It is probably safe now to just build RAID by default, and not require --with-raid - but let's do some more testing with vhost and NVMe-oF against it first before we do so. Signed-off-by: Jim Harris Change-Id: I15697f035cb688574a7ecb8be24d0c84fc622d83 Reviewed-on: https://review.gerrithub.io/423408 Reviewed-by: Kunal Sablok Reviewed-by: Ben Walker Reviewed-by: Shuhei Matsumoto Tested-by: SPDK CI Jenkins Chandler-Test-Pool: SPDK Automated Test System --- lib/bdev/raid/bdev_raid.c | 27 +++++++------------ test/bdev/bdev.conf.in | 7 +++++ test/bdev/blockdev.sh | 5 ---- test/bdev/raid.conf | 6 ----- test/unit/lib/bdev/bdev_raid.c/bdev_raid_ut.c | 18 ++++++------- 5 files changed, 25 insertions(+), 38 deletions(-) delete mode 100644 test/bdev/raid.conf diff --git a/lib/bdev/raid/bdev_raid.c b/lib/bdev/raid/bdev_raid.c index 7cae5370b..ac66192ab 100644 --- a/lib/bdev/raid/bdev_raid.c +++ b/lib/bdev/raid/bdev_raid.c @@ -297,7 +297,6 @@ raid_bdev_submit_children(struct spdk_bdev_io *bdev_io, uint64_t start_strip) struct raid_bdev_io *raid_io = (struct raid_bdev_io *)bdev_io->driver_ctx; struct raid_bdev_io_channel *raid_ch = spdk_io_channel_get_ctx(raid_io->ch); struct raid_bdev *raid_bdev = (struct raid_bdev *)bdev_io->bdev->ctxt; - uint8_t *buf = bdev_io->u.bdev.iovs->iov_base; uint64_t pd_strip; uint32_t offset_in_strip; uint64_t pd_lba; @@ -321,15 +320,17 @@ raid_bdev_submit_children(struct spdk_bdev_io *bdev_io, uint64_t start_strip) * function and function callback context */ if (bdev_io->type == SPDK_BDEV_IO_TYPE_READ) { - ret = spdk_bdev_read_blocks(raid_bdev->base_bdev_info[pd_idx].desc, - raid_ch->base_channel[pd_idx], - buf, pd_lba, pd_blocks, raid_bdev_io_completion, - bdev_io); - } else if (bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE) { - ret = spdk_bdev_write_blocks(raid_bdev->base_bdev_info[pd_idx].desc, + ret = spdk_bdev_readv_blocks(raid_bdev->base_bdev_info[pd_idx].desc, raid_ch->base_channel[pd_idx], - buf, pd_lba, pd_blocks, raid_bdev_io_completion, + bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, + pd_lba, pd_blocks, raid_bdev_io_completion, bdev_io); + } else if (bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE) { + ret = spdk_bdev_writev_blocks(raid_bdev->base_bdev_info[pd_idx].desc, + raid_ch->base_channel[pd_idx], + bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, + pd_lba, pd_blocks, raid_bdev_io_completion, + bdev_io); } else { SPDK_ERRLOG("Recvd not supported io type %u\n", bdev_io->type); assert(0); @@ -446,15 +447,6 @@ _raid_bdev_submit_rw_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bd uint64_t end_strip = 0; int ret; - if (bdev_io->u.bdev.iovcnt != 1) { - SPDK_ERRLOG("iov vector count is not 1\n"); - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); - return; - } - - /* - * IO parameters used during io split and io completion - */ raid_bdev = (struct raid_bdev *)bdev_io->bdev->ctxt; raid_io = (struct raid_bdev_io *)bdev_io->driver_ctx; raid_io->ch = ch; @@ -465,6 +457,7 @@ _raid_bdev_submit_rw_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bd assert(false); SPDK_ERRLOG("I/O spans strip boundary!\n"); spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + return; } ret = raid_bdev_submit_children(bdev_io, start_strip); if (ret != 0) { diff --git a/test/bdev/bdev.conf.in b/test/bdev/bdev.conf.in index f4cff0297..22cfcca4d 100644 --- a/test/bdev/bdev.conf.in +++ b/test/bdev/bdev.conf.in @@ -29,3 +29,10 @@ # # Assign 20000 IOPS for the Malloc0 block device Limit_IOPS Malloc0 20000 + +[RAID0] + Name raid0 + StripSize 64 + NumDevices 2 + RaidLevel 0 + Devices Malloc4 Malloc5 diff --git a/test/bdev/blockdev.sh b/test/bdev/blockdev.sh index 001ed1b2d..712aa8c82 100755 --- a/test/bdev/blockdev.sh +++ b/test/bdev/blockdev.sh @@ -84,11 +84,6 @@ timing_enter bounds $testdir/bdevio/bdevio -c $testdir/bdev.conf timing_exit bounds -# RAID module doesn't support multi-iov yet, so bdevio test -# would fail. So wait to append the RAID configuration until -# after bdevio has run. -cat $testdir/raid.conf >> $testdir/bdev.conf - timing_enter nbd_gpt if grep -q Nvme0 $testdir/bdev.conf; then part_dev_by_gpt $testdir/bdev.conf Nvme0n1 $rootdir diff --git a/test/bdev/raid.conf b/test/bdev/raid.conf deleted file mode 100644 index 7b9e0e8b9..000000000 --- a/test/bdev/raid.conf +++ /dev/null @@ -1,6 +0,0 @@ -[RAID0] - Name raid0 - StripSize 64 - NumDevices 2 - RaidLevel 0 - Devices Malloc4 Malloc5 diff --git a/test/unit/lib/bdev/bdev_raid.c/bdev_raid_ut.c b/test/unit/lib/bdev/bdev_raid.c/bdev_raid_ut.c index 1d988dc57..7994ec5e2 100644 --- a/test/unit/lib/bdev/bdev_raid.c/bdev_raid_ut.c +++ b/test/unit/lib/bdev/bdev_raid.c/bdev_raid_ut.c @@ -46,7 +46,6 @@ struct io_output { struct spdk_bdev_desc *desc; struct spdk_io_channel *ch; - void *buf; uint64_t offset_blocks; uint64_t num_blocks; spdk_bdev_io_completion_cb cb; @@ -204,9 +203,10 @@ spdk_bdev_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status sta /* It will cache the split IOs for verification */ int -spdk_bdev_write_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - void *buf, uint64_t offset_blocks, uint64_t num_blocks, - spdk_bdev_io_completion_cb cb, void *cb_arg) +spdk_bdev_writev_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + struct iovec *iov, int iovcnt, + uint64_t offset_blocks, uint64_t num_blocks, + spdk_bdev_io_completion_cb cb, void *cb_arg) { struct io_output *p = &g_io_output[g_io_output_index]; struct spdk_bdev_io *child_io; @@ -223,7 +223,6 @@ spdk_bdev_write_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, if (g_bdev_io_submit_status == 0) { p->desc = desc; p->ch = ch; - p->buf = buf; p->offset_blocks = offset_blocks; p->num_blocks = num_blocks; p->cb = cb; @@ -404,9 +403,10 @@ spdk_bdev_free_io(struct spdk_bdev_io *bdev_io) /* It will cache split IOs for verification */ int -spdk_bdev_read_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - void *buf, uint64_t offset_blocks, uint64_t num_blocks, - spdk_bdev_io_completion_cb cb, void *cb_arg) +spdk_bdev_readv_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + struct iovec *iov, int iovcnt, + uint64_t offset_blocks, uint64_t num_blocks, + spdk_bdev_io_completion_cb cb, void *cb_arg) { struct io_output *p = &g_io_output[g_io_output_index]; struct spdk_bdev_io *child_io; @@ -419,7 +419,6 @@ spdk_bdev_read_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, if (g_bdev_io_submit_status == 0) { p->desc = desc; p->ch = ch; - p->buf = buf; p->offset_blocks = offset_blocks; p->num_blocks = num_blocks; p->cb = cb; @@ -756,7 +755,6 @@ verify_io(struct spdk_bdev_io *bdev_io, uint8_t num_base_drives, CU_ASSERT(pd_blocks == g_io_output[index].num_blocks); CU_ASSERT(ch_ctx->base_channel[pd_idx] == g_io_output[index].ch); CU_ASSERT(raid_bdev->base_bdev_info[pd_idx].desc == g_io_output[index].desc); - CU_ASSERT(buf == g_io_output[index].buf); CU_ASSERT(bdev_io->type == g_io_output[index].iotype); buf += (pd_blocks << spdk_u32log2(g_block_len)); }