ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvme: extend ctrlr opts with admin queue size

Browse Source 
Align rdma and tcp to respect opts. Reduce default number of entries
for admin queue so it becomes memory optimization.

Linux driver by default creates admin queue with 32 depth, there is no
good reason to enlarge that queue by default within SPDK NVMe driver.

Signed-off-by: Jacek Kalwas <jacek.kalwas@intel.com>
Change-Id: I97ceea8f350c52313021a63190fb0980f604c48e
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1110
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
This commit is contained in:
Jacek Kalwas 2020-03-03 16:27:30 +01:00 committed by Tomasz Zawadzki
parent ea863bb0b3
commit daa8f941e4
6 changed files with 30 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
include/spdk/nvme.h
View File

@ -234,6 +234,11 @@ struct spdk_nvme_ctrlr_opts {
* 4.096 * 2^(transport_ack_timeout) usec.
*/
uint8_t transport_ack_timeout;
/**
* The queue depth of NVMe Admin queue.
*/
uint16_t admin_queue_size;
};
/**

16
lib/nvme/nvme_ctrlr.c
View File

@ -178,6 +178,10 @@ spdk_nvme_ctrlr_get_default_ctrlr_opts(struct spdk_nvme_ctrlr_opts *opts, size_t
if (FIELD_OK(transport_ack_timeout)) {
opts->transport_ack_timeout = SPDK_NVME_DEFAULT_TRANSPORT_ACK_TIMEOUT;
}
if (FIELD_OK(admin_queue_size)) {
opts->admin_queue_size = DEFAULT_ADMIN_QUEUE_SIZE;
}
#undef FIELD_OK
}
@ -2576,6 +2580,18 @@ nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr)
nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE);
}
if (ctrlr->opts.admin_queue_size > SPDK_NVME_ADMIN_QUEUE_MAX_ENTRIES) {
SPDK_ERRLOG("admin_queue_size %u exceeds max defined by NVMe spec, use max value\n",
ctrlr->opts.admin_queue_size);
ctrlr->opts.admin_queue_size = SPDK_NVME_ADMIN_QUEUE_MAX_ENTRIES;
}
if (ctrlr->opts.admin_queue_size < SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES) {
SPDK_ERRLOG("admin_queue_size %u is less than minimum defined by NVMe spec, use min value\n",
ctrlr->opts.admin_queue_size);
ctrlr->opts.admin_queue_size = SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES;
}
ctrlr->flags = 0;
ctrlr->free_io_qids = NULL;
ctrlr->is_resetting = false;

2
lib/nvme/nvme_internal.h
View File

@ -151,10 +151,10 @@ extern pid_t g_spdk_nvme_pid;
* try to configure, if available.
*/
#define DEFAULT_MAX_IO_QUEUES (1024)
#define DEFAULT_ADMIN_QUEUE_SIZE (32)
#define DEFAULT_IO_QUEUE_SIZE (256)
#define DEFAULT_IO_QUEUE_SIZE_FOR_QUIRK (1024) /* Matches Linux kernel driver */
#define DEFAULT_ADMIN_QUEUE_REQUESTS (32)
#define DEFAULT_IO_QUEUE_REQUESTS (512)
#define SPDK_NVME_DEFAULT_RETRY_COUNT (4)

10
lib/nvme/nvme_pcie.c
View File

@ -50,8 +50,6 @@
#define NVME_MIN_COMPLETIONS (1)
#define NVME_MAX_COMPLETIONS (128)
#define NVME_ADMIN_ENTRIES (128)
/*
* NVME_MAX_SGL_DESCRIPTORS defines the maximum number of descriptors in one SGL
* segment.
@ -673,7 +671,7 @@ nvme_pcie_ctrlr_free_bars(struct nvme_pcie_ctrlr *pctrlr)
}
static int
nvme_pcie_ctrlr_construct_admin_qpair(struct spdk_nvme_ctrlr *ctrlr)
nvme_pcie_ctrlr_construct_admin_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t num_entries)
{
struct nvme_pcie_qpair *pqpair;
int rc;
@ -683,7 +681,7 @@ nvme_pcie_ctrlr_construct_admin_qpair(struct spdk_nvme_ctrlr *ctrlr)
return -ENOMEM;
}
pqpair->num_entries = NVME_ADMIN_ENTRIES;
pqpair->num_entries = num_entries;
pqpair->flags.delay_cmd_submit = 0;
ctrlr->adminq = &pqpair->qpair;
@ -692,7 +690,7 @@ nvme_pcie_ctrlr_construct_admin_qpair(struct spdk_nvme_ctrlr *ctrlr)
0, /* qpair ID */
ctrlr,
SPDK_NVME_QPRIO_URGENT,
NVME_ADMIN_ENTRIES);
num_entries);
if (rc != 0) {
return rc;
}
@ -856,7 +854,7 @@ static struct spdk_nvme_ctrlr *nvme_pcie_ctrlr_construct(const struct spdk_nvme_
pci_id = spdk_pci_device_get_id(pci_dev);
pctrlr->ctrlr.quirks = nvme_get_quirks(&pci_id);
rc = nvme_pcie_ctrlr_construct_admin_qpair(&pctrlr->ctrlr);
rc = nvme_pcie_ctrlr_construct_admin_qpair(&pctrlr->ctrlr, pctrlr->ctrlr.opts.admin_queue_size);
if (rc != 0) {
nvme_ctrlr_destruct(&pctrlr->ctrlr);
return NULL;

4
lib/nvme/nvme_rdma.c
View File

@ -1816,8 +1816,8 @@ static struct spdk_nvme_ctrlr *nvme_rdma_ctrlr_construct(const struct spdk_nvme_
}
rctrlr->ctrlr.adminq = nvme_rdma_ctrlr_create_qpair(&rctrlr->ctrlr, 0,
SPDK_NVMF_MIN_ADMIN_QUEUE_ENTRIES, 0, SPDK_NVMF_MIN_ADMIN_QUEUE_ENTRIES,
false);
rctrlr->ctrlr.opts.admin_queue_size, 0,
rctrlr->ctrlr.opts.admin_queue_size, false);
if (!rctrlr->ctrlr.adminq) {
SPDK_ERRLOG("failed to create admin qpair\n");
nvme_rdma_ctrlr_destruct(&rctrlr->ctrlr);

3
lib/nvme/nvme_tcp.c
View File

@ -1620,7 +1620,8 @@ static struct spdk_nvme_ctrlr *nvme_tcp_ctrlr_construct(const struct spdk_nvme_t
}
tctrlr->ctrlr.adminq = nvme_tcp_ctrlr_create_qpair(&tctrlr->ctrlr, 0,
SPDK_NVMF_MIN_ADMIN_QUEUE_ENTRIES, 0, SPDK_NVMF_MIN_ADMIN_QUEUE_ENTRIES);
tctrlr->ctrlr.opts.admin_queue_size, 0,
tctrlr->ctrlr.opts.admin_queue_size);
if (!tctrlr->ctrlr.adminq) {
SPDK_ERRLOG("failed to create admin qpair\n");
nvme_tcp_ctrlr_destruct(&tctrlr->ctrlr);