diff --git a/CHANGELOG.md b/CHANGELOG.md index 97a02067a..800172b13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,9 @@ existing listener. Users should now explicitly add listeners for the discovery s Host can still connect to the discovery subsystem as before, but a warning message will be emitted if no listener was configured for the transport ID of the incoming connection. +Added adaptive interrupt feature for vfio-user transport. New parameter `disable_adaptive_irq` +is added to the RPC `nvmf_create_transport`. + ### thread Added `spdk_thread_exec_msg()` API. diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index d43d9a513..96055a7d1 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -6268,6 +6268,7 @@ abort_timeout_sec | Optional | number | Abort execution timeout value no_wr_batching | Optional | boolean | Disable work requests batching (RDMA only) control_msg_num | Optional | number | The number of control messages per poll group (TCP only) disable_mappable_bar0 | Optional | boolean | disable client mmap() of BAR0 (VFIO-USER only) +disable_adaptive_irq | Optional | boolean | Disable adaptive interrupt feature (VFIO-USER only) zcopy | Optional | boolean | Use zero-copy operations if the underlying bdev supports them #### Example diff --git a/lib/nvmf/vfio_user.c b/lib/nvmf/vfio_user.c index ce11d7e3d..8115b13c5 100644 --- a/lib/nvmf/vfio_user.c +++ b/lib/nvmf/vfio_user.c @@ -311,6 +311,9 @@ struct nvmf_vfio_user_cq { uint16_t iv; bool ien; + + uint32_t last_head; + uint32_t last_trigger_irq_tail; }; struct nvmf_vfio_user_poll_group { @@ -382,6 +385,7 @@ struct nvmf_vfio_user_endpoint { struct nvmf_vfio_user_transport_opts { bool disable_mappable_bar0; + bool disable_adaptive_irq; }; struct nvmf_vfio_user_transport { @@ -857,6 +861,11 @@ static const struct spdk_json_object_decoder vfio_user_transport_opts_decoder[] offsetof(struct nvmf_vfio_user_transport, transport_opts.disable_mappable_bar0), spdk_json_decode_bool, true }, + { + "disable_adaptive_irq", + offsetof(struct nvmf_vfio_user_transport, transport_opts.disable_adaptive_irq), + spdk_json_decode_bool, true + }, }; static struct spdk_nvmf_transport * @@ -902,6 +911,8 @@ nvmf_vfio_user_create(struct spdk_nvmf_transport_opts *opts) SPDK_DEBUGLOG(nvmf_vfio, "vfio_user transport: disable_mappable_bar0=%d\n", vu_transport->transport_opts.disable_mappable_bar0); + SPDK_DEBUGLOG(nvmf_vfio, "vfio_user transport: disable_adaptive_irq=%d\n", + vu_transport->transport_opts.disable_adaptive_irq); /* * To support interrupt mode, the transport must be configured with @@ -1105,6 +1116,14 @@ static int handle_cmd_req(struct nvmf_vfio_user_ctrlr *ctrlr, struct spdk_nvme_cmd *cmd, struct nvmf_vfio_user_sq *sq); +static inline int +adaptive_irq_enabled(struct nvmf_vfio_user_ctrlr *ctrlr, struct nvmf_vfio_user_cq *cq) +{ + return (!spdk_interrupt_mode_is_enabled() && cq->qid != 0 && + !ctrlr->transport->transport_opts.disable_adaptive_irq); + +} + /* * Posts a CQE in the completion queue. * @@ -1179,7 +1198,8 @@ post_completion(struct nvmf_vfio_user_ctrlr *ctrlr, struct nvmf_vfio_user_cq *cq * might be triggering interrupts from vfio-user thread context so * check for race conditions. */ - if (ctrlr_interrupt_enabled(ctrlr) && cq->ien) { + if (!adaptive_irq_enabled(ctrlr, cq) && + cq->ien && ctrlr_interrupt_enabled(ctrlr)) { err = vfu_irq_trigger(ctrlr->endpoint->vfu_ctx, cq->iv); if (err != 0) { SPDK_ERRLOG("%s: failed to trigger interrupt: %m\n", @@ -4301,12 +4321,38 @@ static int nvmf_vfio_user_sq_poll(struct nvmf_vfio_user_sq *sq) { struct nvmf_vfio_user_ctrlr *ctrlr; + struct nvmf_vfio_user_cq *cq; uint32_t new_tail; int count = 0; + uint32_t cq_head; + uint32_t cq_tail; + int err; assert(sq != NULL); ctrlr = sq->ctrlr; + cq = ctrlr->cqs[sq->cqid]; + + if (cq->ien && ctrlr_interrupt_enabled(ctrlr) && + adaptive_irq_enabled(ctrlr, cq)) { + cq_tail = *cq_tailp(cq); + + if (cq_tail != cq->last_trigger_irq_tail) { + spdk_ivdt_dcache(cq_dbl_headp(ctrlr, cq)); + cq_head = *cq_dbl_headp(ctrlr, cq); + + if (cq_head != cq_tail && cq_head == cq->last_head) { + err = vfu_irq_trigger(ctrlr->endpoint->vfu_ctx, cq->iv); + if (err != 0) { + SPDK_ERRLOG("%s: failed to trigger interrupt: %m\n", + ctrlr_id(ctrlr)); + } else { + cq->last_trigger_irq_tail = cq_tail; + } + } + cq->last_head = cq_head; + } + } /* On aarch64 platforms, doorbells update from guest VM may not be seen * on SPDK target side. This is because there is memory type mismatch diff --git a/scripts/rpc.py b/scripts/rpc.py index c9e0215e3..6ce5d0e73 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -2088,6 +2088,8 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse Relevant only for TCP transport""", type=int) p.add_argument('-M', '--disable-mappable-bar0', action='store_true', help="""Disable mmap() of BAR0. Relevant only for VFIO-USER transport""") + p.add_argument('-I', '--disable-adaptive-irq', action='store_true', help="""Disable adaptive interrupt feature. + Relevant only for VFIO-USER transport""") p.add_argument('--acceptor-poll-rate', help='Polling interval of the acceptor for incoming connections (usec)', type=int) p.set_defaults(func=nvmf_create_transport) diff --git a/scripts/rpc/nvmf.py b/scripts/rpc/nvmf.py index f899c2acf..0811221e6 100644 --- a/scripts/rpc/nvmf.py +++ b/scripts/rpc/nvmf.py @@ -120,6 +120,7 @@ def nvmf_create_transport(client, **params): no_wr_batching: Boolean flag to disable work requests batching - RDMA specific (optional) control_msg_num: The number of control messages per poll group - TCP specific (optional) disable_mappable_bar0: disable client mmap() of BAR0 - VFIO-USER specific (optional) + disable_adaptive_irq: Disable adaptive interrupt feature - VFIO-USER specific (optional) acceptor_poll_rate: Acceptor poll period in microseconds (optional) Returns: True or False