From 305088e594a447da8f5acdd00b08e4a423fba066 Mon Sep 17 00:00:00 2001 From: Konstantin Vyshetsky Date: Tue, 22 Aug 2017 14:22:20 -0700 Subject: [PATCH] nvme: Obtain number of queues controller allocated via "Get Features/Number of Queues" Certain vendors do not report correct number of queues allocated in "Set Features/Number of Queues" completion CDW0 per spec. As a work around, issue "Get Features/Number of Queues" and rely on the value provided there. Change-Id: Ib9cc4dcf1bdb732413becc751883a7311c6f672f Signed-off-by: Konstantin Vyshetsky Reviewed-on: https://review.gerrithub.io/375234 Tested-by: SPDK Automated Test System Reviewed-by: Jim Harris Reviewed-by: Daniel Verkamp Reviewed-by: Ben Walker --- lib/nvme/nvme_ctrlr.c | 29 ++++++++++++++++--- lib/nvme/nvme_ctrlr_cmd.c | 8 +++++ lib/nvme/nvme_internal.h | 2 ++ .../lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c | 8 +++++ 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/lib/nvme/nvme_ctrlr.c b/lib/nvme/nvme_ctrlr.c index 8efaa66b4..79d2c888e 100644 --- a/lib/nvme/nvme_ctrlr.c +++ b/lib/nvme/nvme_ctrlr.c @@ -674,9 +674,8 @@ static int nvme_ctrlr_set_num_qpairs(struct spdk_nvme_ctrlr *ctrlr) { struct nvme_completion_poll_status status; - int cq_allocated, sq_allocated; - int rc; - uint32_t i; + uint32_t cq_allocated, sq_allocated, min_allocated, i; + int rc; status.done = false; @@ -703,6 +702,21 @@ nvme_ctrlr_set_num_qpairs(struct spdk_nvme_ctrlr *ctrlr) return -ENXIO; } + /* Obtain the number of queues allocated using Get Features. */ + status.done = false; + rc = nvme_ctrlr_cmd_get_num_queues(ctrlr, nvme_completion_poll_cb, &status); + if (rc != 0) { + return rc; + } + + while (status.done == false) { + spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); + } + if (spdk_nvme_cpl_is_error(&status.cpl)) { + SPDK_ERRLOG("nvme_set_num_queues failed!\n"); + return -ENXIO; + } + /* * Data in cdw0 is 0-based. * Lower 16-bits indicate number of submission queues allocated. @@ -711,7 +725,14 @@ nvme_ctrlr_set_num_qpairs(struct spdk_nvme_ctrlr *ctrlr) sq_allocated = (status.cpl.cdw0 & 0xFFFF) + 1; cq_allocated = (status.cpl.cdw0 >> 16) + 1; - ctrlr->opts.num_io_queues = spdk_min(sq_allocated, cq_allocated); + /* + * For 1:1 queue mapping, set number of allocated queues to be minimum of + * submission and completion queues. + */ + min_allocated = spdk_min(sq_allocated, cq_allocated); + + /* Set number of queues to be minimum of requested and actually allocated. */ + ctrlr->opts.num_io_queues = spdk_min(min_allocated, ctrlr->opts.num_io_queues); ctrlr->free_io_qids = spdk_bit_array_create(ctrlr->opts.num_io_queues + 1); if (ctrlr->free_io_qids == NULL) { diff --git a/lib/nvme/nvme_ctrlr_cmd.c b/lib/nvme/nvme_ctrlr_cmd.c index 2709d92b5..3726f0689 100644 --- a/lib/nvme/nvme_ctrlr_cmd.c +++ b/lib/nvme/nvme_ctrlr_cmd.c @@ -330,6 +330,14 @@ nvme_ctrlr_cmd_set_num_queues(struct spdk_nvme_ctrlr *ctrlr, NULL, 0, cb_fn, cb_arg); } +int +nvme_ctrlr_cmd_get_num_queues(struct spdk_nvme_ctrlr *ctrlr, + spdk_nvme_cmd_cb cb_fn, void *cb_arg) +{ + return spdk_nvme_ctrlr_cmd_get_feature(ctrlr, SPDK_NVME_FEAT_NUMBER_OF_QUEUES, 0, NULL, 0, + cb_fn, cb_arg); +} + int nvme_ctrlr_cmd_set_async_event_config(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_critical_warning_state state, spdk_nvme_cmd_cb cb_fn, diff --git a/lib/nvme/nvme_internal.h b/lib/nvme/nvme_internal.h index cdf0a751c..c4980d0bc 100644 --- a/lib/nvme/nvme_internal.h +++ b/lib/nvme/nvme_internal.h @@ -518,6 +518,8 @@ int nvme_ctrlr_cmd_identify_namespace(struct spdk_nvme_ctrlr *ctrlr, int nvme_ctrlr_cmd_set_num_queues(struct spdk_nvme_ctrlr *ctrlr, uint32_t num_queues, spdk_nvme_cmd_cb cb_fn, void *cb_arg); +int nvme_ctrlr_cmd_get_num_queues(struct spdk_nvme_ctrlr *ctrlr, + spdk_nvme_cmd_cb cb_fn, void *cb_arg); int nvme_ctrlr_cmd_set_async_event_config(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_critical_warning_state state, spdk_nvme_cmd_cb cb_fn, void *cb_arg); diff --git a/test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c b/test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c index 3068110ca..5b415c5fa 100644 --- a/test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c +++ b/test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c @@ -274,6 +274,14 @@ nvme_ctrlr_cmd_set_num_queues(struct spdk_nvme_ctrlr *ctrlr, return 0; } +int +nvme_ctrlr_cmd_get_num_queues(struct spdk_nvme_ctrlr *ctrlr, + spdk_nvme_cmd_cb cb_fn, void *cb_arg) +{ + fake_cpl_success(cb_fn, cb_arg); + return 0; +} + int nvme_ctrlr_cmd_attach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, struct spdk_nvme_ctrlr_list *payload, spdk_nvme_cmd_cb cb_fn, void *cb_arg)