From f0f6413d136204dcc4b1681712a3b82f36989f72 Mon Sep 17 00:00:00 2001 From: Changpeng Liu Date: Fri, 10 Sep 2021 19:14:06 +0800 Subject: [PATCH] nvmf/vfio-user: post CREATE IO SQ response to VM based on flag Previously we didn't post the response for CREATE IO SQ command until the queue pair is connected finally, but for coming live migration support, we will connect IO queue pairs in the destination VM, and this function will also be called for this case, so here we add a flag to indicate the CREATE IO SQ case. Change-Id: Iab4c64a7ebb72bcffbfff712dc729c40eead7c7d Signed-off-by: Changpeng Liu Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9464 Tested-by: SPDK CI Jenkins Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Reviewed-by: Jim Harris Reviewed-by: Ben Walker --- lib/nvmf/vfio_user.c | 49 +++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/lib/nvmf/vfio_user.c b/lib/nvmf/vfio_user.c index 394757cad..aac545fff 100644 --- a/lib/nvmf/vfio_user.c +++ b/lib/nvmf/vfio_user.c @@ -267,7 +267,15 @@ struct nvmf_vfio_user_sq { struct nvme_q sq; enum nvmf_vfio_user_sq_state sq_state; - /* Copy of Create IO SQ command */ + /* handle_queue_connect_rsp() can be used both for CREATE IO SQ response + * and SQ re-connect response in the destination VM, for the prior case, + * we will post a NVMe completion to VM, we will not set this flag when + * re-connecting SQs in the destination VM. + */ + bool post_create_io_sq_completion; + /* Copy of Create IO SQ command, this field is used together with + * `post_create_io_sq_completion` flag. + */ struct spdk_nvme_cmd create_io_sq_cmd; TAILQ_HEAD(, nvmf_vfio_user_req) reqs; @@ -1463,6 +1471,7 @@ handle_create_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, * then eventually completed via handle_queue_connect_rsp(). */ vu_sq->create_io_sq_cmd = *cmd; + vu_sq->post_create_io_sq_completion = true; spdk_nvmf_tgt_new_qpair(ctrlr->transport->transport.tgt, &vu_sq->qpair); return 0; @@ -2927,27 +2936,29 @@ handle_queue_connect_rsp(struct nvmf_vfio_user_req *req, void *cb_arg) * ADMIN I/O CREATE SUBMISSION QUEUE command which has not yet * been completed. Complete it now. */ + if (vu_sq->post_create_io_sq_completion) { + assert(vu_cq->thread != NULL); + if (vu_cq->thread != spdk_get_thread()) { + struct vfio_user_post_cpl_ctx *cpl_ctx; - assert(vu_cq->thread != NULL); - if (vu_cq->thread != spdk_get_thread()) { - struct vfio_user_post_cpl_ctx *cpl_ctx; + cpl_ctx = calloc(1, sizeof(*cpl_ctx)); + if (!cpl_ctx) { + return -ENOMEM; + } + cpl_ctx->ctrlr = vu_ctrlr; + cpl_ctx->cq = vu_cq; + cpl_ctx->cpl.sqid = 0; + cpl_ctx->cpl.cdw0 = 0; + cpl_ctx->cpl.cid = vu_sq->create_io_sq_cmd.cid; + cpl_ctx->cpl.status.sc = SPDK_NVME_SC_SUCCESS; + cpl_ctx->cpl.status.sct = SPDK_NVME_SCT_GENERIC; - cpl_ctx = calloc(1, sizeof(*cpl_ctx)); - if (!cpl_ctx) { - return -ENOMEM; + spdk_thread_send_msg(vu_cq->thread, _post_completion_msg, cpl_ctx); + } else { + post_completion(vu_ctrlr, vu_cq, 0, 0, + vu_sq->create_io_sq_cmd.cid, SPDK_NVME_SC_SUCCESS, SPDK_NVME_SCT_GENERIC); } - cpl_ctx->ctrlr = vu_ctrlr; - cpl_ctx->cq = vu_cq; - cpl_ctx->cpl.sqid = 0; - cpl_ctx->cpl.cdw0 = 0; - cpl_ctx->cpl.cid = vu_sq->create_io_sq_cmd.cid; - cpl_ctx->cpl.status.sc = SPDK_NVME_SC_SUCCESS; - cpl_ctx->cpl.status.sct = SPDK_NVME_SCT_GENERIC; - - spdk_thread_send_msg(vu_cq->thread, _post_completion_msg, cpl_ctx); - } else { - post_completion(vu_ctrlr, vu_cq, 0, 0, - vu_sq->create_io_sq_cmd.cid, SPDK_NVME_SC_SUCCESS, SPDK_NVME_SCT_GENERIC); + vu_sq->post_create_io_sq_completion = false; } }