From a30b59fa336fa563c4d6ecb2a997c3133a9c0a7a Mon Sep 17 00:00:00 2001 From: Jim Harris Date: Tue, 20 Sep 2016 17:47:25 -0700 Subject: [PATCH] aio: create blockdev_aio_io_channel abstraction This breaks out the resources needed to perform aio-based I/O into a separate data structure, as a steps towards some future patches that will enable per-thread resources to enable parallel I/O without synchronization. Signed-off-by: Jim Harris Change-Id: I84b95713133f9c411863ff0aeef8f886a08e0857 --- lib/bdev/aio/blockdev_aio.c | 60 ++++++++++++++++++++++++------------- lib/bdev/aio/blockdev_aio.h | 13 ++++---- 2 files changed, 47 insertions(+), 26 deletions(-) diff --git a/lib/bdev/aio/blockdev_aio.c b/lib/bdev/aio/blockdev_aio.c index 3e815ea00..a5165606c 100644 --- a/lib/bdev/aio/blockdev_aio.c +++ b/lib/bdev/aio/blockdev_aio.c @@ -81,7 +81,7 @@ blockdev_aio_close(struct file_disk *disk) { int rc; - io_destroy(disk->io_ctx); + io_destroy(disk->ch.io_ctx); if (disk->fd == -1) { return 0; @@ -117,7 +117,7 @@ blockdev_aio_read(struct file_disk *fdisk, struct blockdev_aio_task *aio_task, SPDK_TRACELOG(SPDK_TRACE_AIO, "read from %p of size %lu to off: %#lx\n", buf, nbytes, offset); - rc = io_submit(fdisk->io_ctx, 1, &iocb); + rc = io_submit(fdisk->ch.io_ctx, 1, &iocb); if (rc < 0) { SPDK_ERRLOG("%s: io_submit returned %d\n", __func__, rc); return -1; @@ -145,7 +145,7 @@ blockdev_aio_writev(struct file_disk *fdisk, struct blockdev_aio_task *aio_task, SPDK_TRACELOG(SPDK_TRACE_AIO, "write %d iovs size %lu from off: %#lx\n", iovcnt, len, offset); - rc = io_submit(fdisk->io_ctx, 1, &iocb); + rc = io_submit(fdisk->ch.io_ctx, 1, &iocb); if (rc < 0) { SPDK_ERRLOG("%s: io_submit returned %d\n", __func__, rc); return -1; @@ -181,28 +181,47 @@ blockdev_aio_destruct(struct spdk_bdev *bdev) } static int -blockdev_aio_check_io(struct spdk_bdev *bdev) +blockdev_aio_initialize_io_channel(struct blockdev_aio_io_channel *ch) { + ch->queue_depth = 128; + + if (io_setup(ch->queue_depth, &ch->io_ctx) < 0) { + SPDK_ERRLOG("async I/O context setup failure\n"); + return -1; + } + + ch->events = calloc(sizeof(struct io_event), ch->queue_depth); + if (!ch->events) { + io_destroy(ch->io_ctx); + return -1; + } + + return 0; +} + +static void +blockdev_aio_poll(void *arg) +{ + struct blockdev_aio_io_channel *ch = arg; int nr, i; enum spdk_bdev_io_status status; struct blockdev_aio_task *aio_task; - struct file_disk *fdisk = (struct file_disk *)bdev; struct timespec timeout; timeout.tv_sec = 0; timeout.tv_nsec = 0; - nr = io_getevents(fdisk->io_ctx, 1, fdisk->queue_depth, - fdisk->events, &timeout); + nr = io_getevents(ch->io_ctx, 1, ch->queue_depth, + ch->events, &timeout); if (nr < 0) { SPDK_ERRLOG("%s: io_getevents returned %d\n", __func__, nr); - return -1; + return; } for (i = 0; i < nr; i++) { - aio_task = fdisk->events[i].data; - if (fdisk->events[i].res != aio_task->len) { + aio_task = ch->events[i].data; + if (ch->events[i].res != aio_task->len) { status = SPDK_BDEV_IO_STATUS_FAILED; } else { status = SPDK_BDEV_IO_STATUS_SUCCESS; @@ -210,7 +229,14 @@ blockdev_aio_check_io(struct spdk_bdev *bdev) spdk_bdev_io_complete(spdk_bdev_io_from_ctx(aio_task), status); } +} +static int +blockdev_aio_check_io(struct spdk_bdev *bdev) +{ + struct file_disk *fdisk = (struct file_disk *)bdev; + + blockdev_aio_poll(&fdisk->ch); return 0; } @@ -299,8 +325,8 @@ static void aio_free_disk(struct file_disk *fdisk) { if (fdisk == NULL) return; - if (fdisk->events != NULL) - free(fdisk->events); + if (fdisk->ch.events != NULL) + free(fdisk->ch.events); free(fdisk); } @@ -322,7 +348,6 @@ create_aio_disk(char *fname) } fdisk->size = spdk_fd_get_size(fdisk->fd); - fdisk->queue_depth = 128; // TODO: where do we get the queue depth from. TAILQ_INIT(&fdisk->sync_completion_list); snprintf(fdisk->disk.name, SPDK_BDEV_MAX_NAME_LENGTH, "AIO%d", @@ -336,14 +361,7 @@ create_aio_disk(char *fname) fdisk->disk.ctxt = fdisk; fdisk->disk.fn_table = &aio_fn_table; - if (io_setup(fdisk->queue_depth, &fdisk->io_ctx) < 0) { - SPDK_ERRLOG("async I/O context setup failure\n"); - goto error_return; - } - - fdisk->events = calloc(sizeof(struct io_event), fdisk->queue_depth); - if (!fdisk->events) { - SPDK_ERRLOG("unable to allocate async events\n"); + if (blockdev_aio_initialize_io_channel(&fdisk->ch) != 0) { goto error_return; } diff --git a/lib/bdev/aio/blockdev_aio.h b/lib/bdev/aio/blockdev_aio.h index db2fe412c..851fc22f8 100644 --- a/lib/bdev/aio/blockdev_aio.h +++ b/lib/bdev/aio/blockdev_aio.h @@ -48,23 +48,26 @@ struct blockdev_aio_task { TAILQ_ENTRY(blockdev_aio_task) link; }; +struct blockdev_aio_io_channel { + io_context_t io_ctx; + long queue_depth; + struct io_event *events; +}; + struct file_disk { struct spdk_bdev disk; /* this must be first element */ char *file; int fd; char disk_name[SPDK_BDEV_MAX_NAME_LENGTH]; - long queue_depth; uint64_t size; + struct blockdev_aio_io_channel ch; + /** * For storing I/O that were completed synchronously, and will be * completed during next check_io call. */ TAILQ_HEAD(, blockdev_aio_task) sync_completion_list; - - /* for libaio */ - io_context_t io_ctx; - struct io_event *events; }; struct file_disk *create_aio_disk(char *fname);