diff --git a/examples/interrupt_tgt/Makefile b/examples/interrupt_tgt/Makefile index b2166fa7a..960a1ba24 100644 --- a/examples/interrupt_tgt/Makefile +++ b/examples/interrupt_tgt/Makefile @@ -55,6 +55,9 @@ SPDK_LIB_LIST += bdev_malloc bdev_passthru bdev_error bdev_gpt bdev_split bdev_r SPDK_LIB_LIST += bdev_lvol lvol blob_bdev blob # blobfs libraries SPDK_LIB_LIST += blobfs blobfs_bdev +# vhost blk related libraries. +SPDK_LIB_LIST += vhost event_vhost +SPDK_LIB_LIST += scsi event_scsi # vhost-scsi is not supported, just because vhost lib requires scsi related libs ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV)) SPDK_LIB_LIST += env_dpdk_rpc diff --git a/examples/interrupt_tgt/interrupt_tgt.c b/examples/interrupt_tgt/interrupt_tgt.c index 3c4676ad5..7b1a7a99f 100644 --- a/examples/interrupt_tgt/interrupt_tgt.c +++ b/examples/interrupt_tgt/interrupt_tgt.c @@ -34,17 +34,22 @@ #include "spdk/stdinc.h" #include "spdk/conf.h" #include "spdk/event.h" +#include "spdk/vhost.h" static void interrupt_tgt_usage(void) { printf(" -E Set interrupt mode\n"); + printf(" -S directory where to create vhost sockets (default: pwd)\n"); } static int interrupt_tgt_parse_arg(int ch, char *arg) { switch (ch) { + case 'S': + spdk_vhost_set_socket_path(arg); + break; case 'E': spdk_interrupt_mode_enable(); break; @@ -68,7 +73,7 @@ main(int argc, char *argv[]) spdk_app_opts_init(&opts); opts.name = "interrupt_tgt"; - if ((rc = spdk_app_parse_args(argc, argv, &opts, "E", NULL, + if ((rc = spdk_app_parse_args(argc, argv, &opts, "S:E", NULL, interrupt_tgt_parse_arg, interrupt_tgt_usage)) != SPDK_APP_PARSE_ARGS_SUCCESS) { exit(rc); diff --git a/lib/vhost/vhost.c b/lib/vhost/vhost.c index 4a04cabdb..227d5489b 100644 --- a/lib/vhost/vhost.c +++ b/lib/vhost/vhost.c @@ -199,6 +199,38 @@ vhost_vq_avail_ring_get(struct spdk_vhost_virtqueue *virtqueue, uint16_t *reqs, } count = spdk_min(count, reqs_len); + if (virtqueue->vsession && virtqueue->vsession->interrupt_mode) { + /* if completed IO number is larger than SPDK_AIO_QUEUE_DEPTH, + * io_getevent should be called again to ensure all completed IO are processed. + */ + int rc; + uint64_t num_events; + + rc = read(vring->kickfd, &num_events, sizeof(num_events)); + if (rc < 0) { + SPDK_ERRLOG("failed to acknowledge kickfd: %s.\n", spdk_strerror(errno)); + return -errno; + } + + if ((uint16_t)(avail_idx - last_idx) != num_events) { + SPDK_DEBUGLOG(vhost_ring, + "virtqueue gets %d reqs, but kickfd shows %lu reqs\n", + avail_idx - last_idx, num_events); + } + + if (num_events > count) { + SPDK_DEBUGLOG(vhost_ring, + "virtqueue kickfd shows %lu reqs, take %d, send notice for other reqs\n", + num_events, reqs_len); + num_events -= count; + rc = write(vring->kickfd, &num_events, sizeof(num_events)); + if (rc < 0) { + SPDK_ERRLOG("failed to kick vring: %s.\n", spdk_strerror(errno)); + return -errno; + } + } + } + virtqueue->last_avail_idx += count; for (i = 0; i < count; i++) { reqs[i] = vring->avail->ring[(last_idx + i) & size_mask]; @@ -500,6 +532,14 @@ vhost_vq_used_ring_enqueue(struct spdk_vhost_session *vsession, rte_vhost_clr_inflight_desc_split(vsession->vid, vq_idx, virtqueue->last_used_idx, id); virtqueue->used_req_cnt++; + + if (vsession->interrupt_mode) { + if (virtqueue->vring.desc == NULL || vhost_vq_event_is_suppressed(virtqueue)) { + return; + } + + vhost_vq_used_signal(vsession, virtqueue); + } } void @@ -1194,6 +1234,10 @@ vhost_start_device_cb(int vid) goto out; } + if (spdk_interrupt_mode_is_enabled()) { + vsession->interrupt_mode = true; + } + vdev = vsession->vdev; if (vsession->started) { /* already started, nothing to do */ @@ -1242,11 +1286,15 @@ vhost_start_device_cb(int vid) q->packed.used_phase = q->last_used_idx >> 15; q->last_used_idx = q->last_used_idx & 0x7FFF; - /* Disable I/O submission notifications, we'll be polling. */ - q->vring.device_event->flags = VRING_PACKED_EVENT_FLAG_DISABLE; + if (!vsession->interrupt_mode) { + /* Disable I/O submission notifications, we'll be polling. */ + q->vring.device_event->flags = VRING_PACKED_EVENT_FLAG_DISABLE; + } } else { - /* Disable I/O submission notifications, we'll be polling. */ - q->vring.used->flags = VRING_USED_F_NO_NOTIFY; + if (!vsession->interrupt_mode) { + /* Disable I/O submission notifications, we'll be polling. */ + q->vring.used->flags = VRING_USED_F_NO_NOTIFY; + } } q->packed.packed_ring = packed_ring; diff --git a/lib/vhost/vhost_blk.c b/lib/vhost/vhost_blk.c index 641533bd2..6867d579a 100644 --- a/lib/vhost/vhost_blk.c +++ b/lib/vhost/vhost_blk.c @@ -690,6 +690,14 @@ _vdev_vq_worker(struct spdk_vhost_virtqueue *vq) } +static int +vdev_vq_worker(void *arg) +{ + struct spdk_vhost_virtqueue *vq = arg; + + return _vdev_vq_worker(vq); +} + static int vdev_worker(void *arg) { @@ -788,6 +796,14 @@ _no_bdev_vdev_vq_worker(struct spdk_vhost_virtqueue *vq) return SPDK_POLLER_BUSY; } +static int +no_bdev_vdev_vq_worker(void *arg) +{ + struct spdk_vhost_virtqueue *vq = arg; + + return _no_bdev_vdev_vq_worker(vq); +} + static int no_bdev_vdev_worker(void *arg) { @@ -802,6 +818,55 @@ no_bdev_vdev_worker(void *arg) return SPDK_POLLER_BUSY; } +static void +vhost_blk_session_unregister_interrupts(struct spdk_vhost_blk_session *bvsession) +{ + struct spdk_vhost_session *vsession = &bvsession->vsession; + struct spdk_vhost_virtqueue *vq; + int i; + + SPDK_DEBUGLOG(vhost_blk, "unregister virtqueues interrupt\n"); + for (i = 0; i < vsession->max_queues; i++) { + vq = &vsession->virtqueue[i]; + if (vq->intr == NULL) { + break; + } + + SPDK_DEBUGLOG(vhost_blk, "unregister vq[%d]'s kickfd is %d\n", + i, vq->vring.kickfd); + spdk_interrupt_unregister(&vq->intr); + } +} + +static int +vhost_blk_session_register_interrupts(struct spdk_vhost_blk_session *bvsession, + spdk_interrupt_fn fn) +{ + struct spdk_vhost_session *vsession = &bvsession->vsession; + struct spdk_vhost_virtqueue *vq = NULL; + int i; + + SPDK_DEBUGLOG(vhost_blk, "Register virtqueues interrupt\n"); + for (i = 0; i < vsession->max_queues; i++) { + vq = &vsession->virtqueue[i]; + SPDK_DEBUGLOG(vhost_blk, "Register vq[%d]'s kickfd is %d\n", + i, vq->vring.kickfd); + + vq->intr = SPDK_INTERRUPT_REGISTER(vq->vring.kickfd, fn, vq); + if (vq->intr == NULL) { + SPDK_ERRLOG("Fail to register req notifier handler.\n"); + goto err; + } + } + + return 0; + +err: + vhost_blk_session_unregister_interrupts(bvsession); + + return -1; +} + static struct spdk_vhost_blk_dev * to_blk_dev(struct spdk_vhost_dev *vdev) { @@ -863,6 +928,7 @@ vhost_session_bdev_remove_cb(struct spdk_vhost_dev *vdev, void *ctx) { struct spdk_vhost_blk_session *bvsession; + int rc; bvsession = (struct spdk_vhost_blk_session *)vsession; if (bvsession->requestq_poller) { @@ -870,6 +936,16 @@ vhost_session_bdev_remove_cb(struct spdk_vhost_dev *vdev, bvsession->requestq_poller = SPDK_POLLER_REGISTER(no_bdev_vdev_worker, bvsession, 0); } + if (vsession->virtqueue[0].intr) { + vhost_blk_session_unregister_interrupts(bvsession); + rc = vhost_blk_session_register_interrupts(bvsession, no_bdev_vdev_vq_worker); + if (rc) { + SPDK_ERRLOG("%s: Interrupt register failed\n", vsession->name); + return -1; + } + + } + return 0; } @@ -1013,10 +1089,22 @@ vhost_blk_start_cb(struct spdk_vhost_dev *vdev, } } - bvsession->requestq_poller = SPDK_POLLER_REGISTER(bvdev->bdev ? vdev_worker : no_bdev_vdev_worker, - bvsession, 0); - SPDK_INFOLOG(vhost, "%s: started poller on lcore %d\n", - vsession->name, spdk_env_get_current_core()); + if (spdk_interrupt_mode_is_enabled()) { + rc = vhost_blk_session_register_interrupts(bvsession, + bvdev->bdev ? vdev_vq_worker : no_bdev_vdev_vq_worker); + if (rc) { + SPDK_ERRLOG("%s: Interrupt register failed\n", vsession->name); + goto out; + } + SPDK_INFOLOG(vhost, "%s: started interrupt source on lcore %d\n", + vsession->name, spdk_env_get_current_core()); + } else { + bvsession->requestq_poller = SPDK_POLLER_REGISTER(bvdev->bdev ? vdev_worker : no_bdev_vdev_worker, + bvsession, 0); + SPDK_INFOLOG(vhost, "%s: started poller on lcore %d\n", + vsession->name, spdk_env_get_current_core()); + } + out: vhost_session_start_done(vsession, rc); return rc; @@ -1072,6 +1160,11 @@ vhost_blk_stop_cb(struct spdk_vhost_dev *vdev, struct spdk_vhost_blk_session *bvsession = to_blk_session(vsession); spdk_poller_unregister(&bvsession->requestq_poller); + + if (vsession->virtqueue[0].intr) { + vhost_blk_session_unregister_interrupts(bvsession); + } + bvsession->stop_poller = SPDK_POLLER_REGISTER(destroy_session_poller_cb, bvsession, 1000); return 0; diff --git a/lib/vhost/vhost_internal.h b/lib/vhost/vhost_internal.h index f38360679..319735d5c 100644 --- a/lib/vhost/vhost_internal.h +++ b/lib/vhost/vhost_internal.h @@ -124,6 +124,8 @@ struct spdk_vhost_virtqueue { uint32_t vring_idx; struct spdk_vhost_session *vsession; + + struct spdk_interrupt *intr; } __attribute((aligned(SPDK_CACHE_LINE_SIZE))); struct spdk_vhost_session { @@ -141,6 +143,7 @@ struct spdk_vhost_session { bool started; bool needs_restart; bool forced_polling; + bool interrupt_mode; struct rte_vhost_memory *mem; diff --git a/test/unit/lib/vhost/vhost.c/vhost_ut.c b/test/unit/lib/vhost/vhost.c/vhost_ut.c index b7e746a8b..b4bd5c22f 100644 --- a/test/unit/lib/vhost/vhost.c/vhost_ut.c +++ b/test/unit/lib/vhost/vhost.c/vhost_ut.c @@ -319,7 +319,7 @@ remove_controller_test(void) static void vq_avail_ring_get_test(void) { - struct spdk_vhost_virtqueue vq; + struct spdk_vhost_virtqueue vq = {}; uint16_t avail_mem[34]; uint16_t reqs[32]; uint16_t reqs_len, ret, i;