From 72800826ec9007443b260d16532994d146f0c309 Mon Sep 17 00:00:00 2001 From: Ben Walker Date: Tue, 22 May 2018 10:34:56 -0700 Subject: [PATCH] nvmf: Quiesce I/O before closing spdk_nvmf_qpairs Change-Id: Iff58943f3f9db39b6fce1437a0cfe7380991daa6 Signed-off-by: Ben Walker Reviewed-on: https://review.gerrithub.io/412078 Tested-by: SPDK Automated Test System Reviewed-by: Changpeng Liu Reviewed-by: Jim Harris Reviewed-by: Shuhei Matsumoto --- lib/nvmf/ctrlr.c | 27 ++++++++++++++++++++------- lib/nvmf/nvmf_internal.h | 4 ++++ lib/nvmf/request.c | 15 ++++++++++++++- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/lib/nvmf/ctrlr.c b/lib/nvmf/ctrlr.c index ffe736945..3a906c9aa 100644 --- a/lib/nvmf/ctrlr.c +++ b/lib/nvmf/ctrlr.c @@ -455,27 +455,40 @@ _spdk_nvmf_ctrlr_remove_qpair(void *ctx) } } +static void +_spdk_nvmf_qpair_quiesced(void *cb_arg, int status) +{ + struct spdk_nvmf_qpair *qpair = cb_arg; + + /* Send a message to the controller thread to remove the qpair from its internal + * list. */ + spdk_thread_send_msg(qpair->ctrlr->admin_qpair->group->thread, _spdk_nvmf_ctrlr_remove_qpair, + qpair); +} + static void _spdk_nvmf_qpair_deactivate(void *ctx) { struct spdk_nvmf_qpair *qpair = ctx; - struct spdk_nvmf_ctrlr *ctrlr; assert(qpair->state == SPDK_NVMF_QPAIR_ACTIVE); qpair->state = SPDK_NVMF_QPAIR_DEACTIVATING; - ctrlr = qpair->ctrlr; - - if (ctrlr == NULL) { + if (qpair->ctrlr == NULL) { /* This qpair was never added to a controller. Skip a step * and destroy it immediately. */ _spdk_nvmf_qpair_destroy(qpair); return; } - /* Send a message to the controller thread to remove the qpair from its internal - * list. */ - spdk_thread_send_msg(ctrlr->admin_qpair->group->thread, _spdk_nvmf_ctrlr_remove_qpair, qpair); + /* Check for outstanding I/O */ + if (!TAILQ_EMPTY(&qpair->outstanding)) { + qpair->state_cb = _spdk_nvmf_qpair_quiesced; + qpair->state_cb_arg = qpair; + return; + } + + _spdk_nvmf_qpair_quiesced(qpair, 0); } void diff --git a/lib/nvmf/nvmf_internal.h b/lib/nvmf/nvmf_internal.h index 8958449fd..e33003486 100644 --- a/lib/nvmf/nvmf_internal.h +++ b/lib/nvmf/nvmf_internal.h @@ -62,6 +62,8 @@ enum spdk_nvmf_qpair_state { SPDK_NVMF_QPAIR_DEACTIVATING, }; +typedef void (*spdk_nvmf_state_change_done)(void *cb_arg, int status); + struct spdk_nvmf_tgt { struct spdk_nvmf_tgt_opts opts; @@ -155,6 +157,8 @@ struct spdk_nvmf_ns { struct spdk_nvmf_qpair { enum spdk_nvmf_qpair_state state; + spdk_nvmf_state_change_done state_cb; + void *state_cb_arg; struct spdk_nvmf_transport *transport; struct spdk_nvmf_ctrlr *ctrlr; diff --git a/lib/nvmf/request.c b/lib/nvmf/request.c index ff4434de9..3e0b2fd7a 100644 --- a/lib/nvmf/request.c +++ b/lib/nvmf/request.c @@ -49,21 +49,34 @@ int spdk_nvmf_request_complete(struct spdk_nvmf_request *req) { struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; + struct spdk_nvmf_qpair *qpair; rsp->sqid = 0; rsp->status.p = 0; rsp->cid = req->cmd->nvme_cmd.cid; + qpair = req->qpair; + SPDK_DEBUGLOG(SPDK_LOG_NVMF, "cpl: cid=%u cdw0=0x%08x rsvd1=%u status=0x%04x\n", rsp->cid, rsp->cdw0, rsp->rsvd1, *(uint16_t *)&rsp->status); - TAILQ_REMOVE(&req->qpair->outstanding, req, link); + TAILQ_REMOVE(&qpair->outstanding, req, link); if (spdk_nvmf_transport_req_complete(req)) { SPDK_ERRLOG("Transport request completion error!\n"); } + if (qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING) { + assert(qpair->state_cb != NULL); + + if (TAILQ_EMPTY(&qpair->outstanding)) { + qpair->state_cb(qpair->state_cb_arg, 0); + } + } else { + assert(qpair->state == SPDK_NVMF_QPAIR_ACTIVE); + } + return 0; }