ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

env/pci: move the vfio init delay to nvme/pci

Browse Source 
This is an NVMe-specific issue and I/OA or VirtIO devices don't
need it. Additionally, the delay is now asynchronous, meaning
that potentially multiple NVMe controllers can wait all at once.

The drawback of this change is that we're needlessly waiting
even when using uio_pci_generic. However, since the delay does
not block anymore, its impact is significantly minimized.

Change-Id: I5d16a7fd7cb66c785acb687f14690e95f6188b9e
Signed-off-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-on: https://review.gerrithub.io/429414
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
This commit is contained in:
Darek Stojaczyk 2018-10-22 14:10:41 +02:00 committed by Jim Harris
parent a084a906ce
commit 951bb3a458
3 changed files with 29 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
lib/env_dpdk/pci.c
View File

@ -59,14 +59,6 @@ spdk_pci_device_init(struct rte_pci_driver *driver,
return 1;
}
if (device->kdrv == RTE_KDRV_VFIO && spdk_process_is_primary()) {
/*
* TODO: This is a workaround for an issue where the device is not ready after VFIO reset.
* Figure out what is actually going wrong and remove this sleep.
*/
usleep(500 * 1000);
}
rc = ctx->cb_fn(ctx->cb_arg, (struct spdk_pci_device *)device);
if (rc != 0) {
return rc;

25
lib/nvme/nvme_ctrlr.c
View File

@ -608,6 +608,8 @@ static const char *
nvme_ctrlr_state_string(enum nvme_ctrlr_state state)
{
switch (state) {
case NVME_CTRLR_STATE_INIT_DELAY:
return "delay init";
case NVME_CTRLR_STATE_INIT:
return "init";
case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1:
@ -1869,6 +1871,22 @@ nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr)
* Check if the current initialization step is done or has timed out.
*/
switch (ctrlr->state) {
case NVME_CTRLR_STATE_INIT_DELAY:
nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, ready_timeout_in_ms);
/*
* Controller may need some delay before it's enabled.
*
* This is a workaround for an issue where the PCIe-attached NVMe controller
* is not ready after VFIO reset. We delay the initialization rather than the
* enabling itself, because this is required only for the very first enabling
* - directly after a VFIO reset.
*
* TODO: Figure out what is actually going wrong.
*/
SPDK_DEBUGLOG(SPDK_LOG_NVME, "Adding 500 ms delay before initializing the controller\n");
ctrlr->sleep_timeout_tsc = spdk_get_ticks() + (500 * spdk_get_ticks_hz() / 1000);
break;
case NVME_CTRLR_STATE_INIT:
/* Begin the hardware initialization by making sure the controller is disabled. */
if (cc.bits.en) {
@ -2113,7 +2131,12 @@ nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr)
{
int rc;
nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE);
if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT_DELAY, NVME_TIMEOUT_INFINITE);
} else {
nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE);
}
ctrlr->flags = 0;
ctrlr->free_io_qids = NULL;
ctrlr->is_resetting = false;

5
lib/nvme/nvme_internal.h
View File

@ -388,6 +388,11 @@ struct spdk_nvme_ns {
* State of struct spdk_nvme_ctrlr (in particular, during initialization).
*/
enum nvme_ctrlr_state {
/**
* Wait before initializing the controller.
*/
NVME_CTRLR_STATE_INIT_DELAY,
/**
* Controller has not been initialized yet.
*/