From bad2c8e86cc9de65297431f7b6aa715850e6c67a Mon Sep 17 00:00:00 2001 From: Changpeng Liu Date: Mon, 24 Feb 2020 01:02:27 -0500 Subject: [PATCH] nvme: detach the controller in STUB and flush the admin active requests at last In the autotest, when calling kill_stub() function, there is error log like this: "Device 0000:83:00.0 is still attached at shutdown!", so it's better to detach the controller when exit the stub process. But after call spdk_nvme_detach() in the stub process, there is another issue: 1. NVMe stub running as the primary process, and it will send 4 AERs. 2. Using NVMe reset tool as the secondary process. When doing NVMe reset from the secondary process, it will abort all the outstanding requests, so for the 4 AERs from the primary process, the 4 requests will be added to the active_proc->active_reqs list. When calling spdk_nvme_detach() to detach a controller, there is a assertion in the nvme_ctrlr_free_processes() at last to check the active requests list of this active process data structure. We can add a check before destructing the controller to poll the completion queue, so that the active requests list can be flushed. Change-Id: I0c473e935333a28d16f4c9fb443341fc47c5c24f Signed-off-by: Changpeng Liu Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/977 Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Shuhei Matsumoto --- lib/nvme/nvme_ctrlr.c | 1 + test/app/stub/stub.c | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/lib/nvme/nvme_ctrlr.c b/lib/nvme/nvme_ctrlr.c index c8c7670a0..d4c2b9956 100644 --- a/lib/nvme/nvme_ctrlr.c +++ b/lib/nvme/nvme_ctrlr.c @@ -2639,6 +2639,7 @@ nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr) SPDK_DEBUGLOG(SPDK_LOG_NVME, "Prepare to destruct SSD: %s\n", ctrlr->trid.traddr); + spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); nvme_transport_admin_qpair_abort_aers(ctrlr->adminq); TAILQ_FOREACH_SAFE(qpair, &ctrlr->active_io_qpairs, tailq, tmp) { diff --git a/test/app/stub/stub.c b/test/app/stub/stub.c index 2b525f40d..0481e2625 100644 --- a/test/app/stub/stub.c +++ b/test/app/stub/stub.c @@ -41,6 +41,27 @@ static char g_path[256]; static struct spdk_poller *g_poller; +struct ctrlr_entry { + struct spdk_nvme_ctrlr *ctrlr; + struct ctrlr_entry *next; +}; + +static struct ctrlr_entry *g_controllers = NULL; + +static void +cleanup(void) +{ + struct ctrlr_entry *ctrlr_entry = g_controllers; + + while (ctrlr_entry) { + struct ctrlr_entry *next = ctrlr_entry->next; + + spdk_nvme_detach(ctrlr_entry->ctrlr); + free(ctrlr_entry); + ctrlr_entry = next; + } +} + static void usage(char *executable_name) { @@ -70,6 +91,17 @@ static void attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) { + struct ctrlr_entry *entry; + + entry = malloc(sizeof(struct ctrlr_entry)); + if (entry == NULL) { + fprintf(stderr, "Malloc error\n"); + exit(1); + } + + entry->ctrlr = ctrlr; + entry->next = g_controllers; + g_controllers = entry; } static int @@ -163,6 +195,8 @@ main(int argc, char **argv) opts.shutdown_cb = stub_shutdown; ch = spdk_app_start(&opts, stub_start, (void *)(intptr_t)opts.shm_id); + + cleanup(); spdk_app_fini(); return ch;