From e6464f32fa1e8634d335c57c11ed35ef3777b13a Mon Sep 17 00:00:00 2001 From: Changpeng Liu Date: Wed, 30 Jun 2021 16:46:30 +0800 Subject: [PATCH] nvmf: abort AERs when doing controller reset and shutdown The vfio-user target emulated NVMe device is treated as PCIe NVMe SSD in the Guest VM, so when doing controller reset or shutdown, we should abort the AERs which in the NVMf library. Users may switch kernel NVMe driver to SPDK NVMe driver in the VM, without this fix, we will got "AERL exceeded" response very frequently, because the AERs submitted by previous driver will never be aborted in runtime. Change-Id: I0222ed509629ccb0e98217414dd9043857105686 Signed-off-by: Changpeng Liu Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/8558 Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Jim Harris --- lib/nvmf/ctrlr.c | 4 ++++ lib/nvmf/vfio_user.c | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/lib/nvmf/ctrlr.c b/lib/nvmf/ctrlr.c index dd4bfffe3..98e2a8b84 100644 --- a/lib/nvmf/ctrlr.c +++ b/lib/nvmf/ctrlr.c @@ -3229,6 +3229,10 @@ nvmf_ctrlr_abort_aer(struct spdk_nvmf_ctrlr *ctrlr) struct spdk_nvmf_request *req; int i; + if (!ctrlr->nr_aer_reqs) { + return; + } + for (i = 0; i < ctrlr->nr_aer_reqs; i++) { req = ctrlr->aer_req[i]; diff --git a/lib/nvmf/vfio_user.c b/lib/nvmf/vfio_user.c index a27ef54da..9f9e7e0cf 100644 --- a/lib/nvmf/vfio_user.c +++ b/lib/nvmf/vfio_user.c @@ -1032,6 +1032,27 @@ handle_cmd_rsp(struct nvmf_vfio_user_req *req, void *cb_arg) req->req.rsp->nvme_cpl.status.sct); } +static int +handle_admin_aer_rsp(struct nvmf_vfio_user_req *req, void *cb_arg) +{ + struct nvmf_vfio_user_qpair *qpair = cb_arg; + + assert(qpair != NULL); + assert(req != NULL); + + vfu_unmap_sg(qpair->ctrlr->endpoint->vfu_ctx, req->sg, req->iov, req->iovcnt); + + if (qpair->state != VFIO_USER_QPAIR_ACTIVE) { + return 0; + } + + return post_completion(qpair->ctrlr, &req->req.cmd->nvme_cmd, + &qpair->ctrlr->qp[req->req.qpair->qid]->cq, + req->req.rsp->nvme_cpl.cdw0, + req->req.rsp->nvme_cpl.status.sc, + req->req.rsp->nvme_cpl.status.sct); +} + static int consume_cmd(struct nvmf_vfio_user_ctrlr *ctrlr, struct nvmf_vfio_user_qpair *qpair, struct spdk_nvme_cmd *cmd) @@ -1284,6 +1305,8 @@ nvmf_vfio_user_prop_req_rsp(struct nvmf_vfio_user_req *req, void *cb_arg) ctrlr_id(qpair->ctrlr)); unmap_admin_queue(qpair->ctrlr); qpair->state = VFIO_USER_QPAIR_INACTIVE; + /* For PCIe controller reset, we will drop all AER responses */ + nvmf_ctrlr_abort_aer(req->req.qpair->ctrlr); } } } @@ -2350,6 +2373,9 @@ handle_cmd_req(struct nvmf_vfio_user_ctrlr *ctrlr, struct spdk_nvme_cmd *cmd, req->cmd->nvme_cmd = *cmd; if (nvmf_qpair_is_admin_queue(req->qpair)) { err = map_admin_cmd_req(ctrlr, req); + if (cmd->opc == SPDK_NVME_OPC_ASYNC_EVENT_REQUEST) { + vu_req->cb_fn = handle_admin_aer_rsp; + } } else { err = map_io_cmd_req(ctrlr, req); }