ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvme: use spdk_likely/unlikely macros in I/O path

Browse Source 
Also avoid an spdk_get_ticks() call in the default
case where a timeout_cb_fn is not defined.

On my Intel(R) Xeon(R) E5-2699 v3 system with an
Intel(R) P3700 SSD, these modifications reduce software
overhead per I/O by 3-5% (as measured by the SPDK
overhead tool).

Signed-off-by: Jim Harris <james.r.harris@intel.com>
Change-Id: I5bb5a87747b15d9e27655fabcd2bc1a40b0b990e
This commit is contained in:
Jim Harris 2017-04-10 15:33:10 -07:00 committed by Ben Walker
parent 0911aa83fb
commit fd5d6901f2
1 changed files with 14 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
lib/nvme/nvme_pcie.c
View File

@ -40,6 +40,7 @@
#include <sys/syscall.h> #include <sys/syscall.h>
#include <sys/types.h> #include <sys/types.h>
#include "spdk/likely.h"
#include "nvme_internal.h" #include "nvme_internal.h"
#include "nvme_uevent.h" #include "nvme_uevent.h"
@ -1021,8 +1022,10 @@ nvme_pcie_qpair_submit_tracker(struct spdk_nvme_qpair *qpair, struct nvme_tracke
struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair); struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
struct nvme_pcie_ctrlr *pctrlr = nvme_pcie_ctrlr(qpair->ctrlr); struct nvme_pcie_ctrlr *pctrlr = nvme_pcie_ctrlr(qpair->ctrlr);
tr->submit_tick = spdk_get_ticks();
tr->timed_out = 0; tr->timed_out = 0;
if (spdk_unlikely(qpair->ctrlr->timeout_cb_fn != NULL)) {
tr->submit_tick = spdk_get_ticks();
}
req = tr->req; req = tr->req;
pqpair->tr[tr->cid].active = true; pqpair->tr[tr->cid].active = true;
@ -1865,7 +1868,7 @@ nvme_pcie_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_
uint32_t num_completions = 0; uint32_t num_completions = 0;
struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr; struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
if (!nvme_pcie_qpair_check_enabled(qpair)) { if (spdk_unlikely(!nvme_pcie_qpair_check_enabled(qpair))) {
/* /*
* qpair is not enabled, likely because a controller reset is * qpair is not enabled, likely because a controller reset is
* is in progress. Ignore the interrupt - any I/O that was * is in progress. Ignore the interrupt - any I/O that was
@ -1875,7 +1878,7 @@ nvme_pcie_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_
return 0; return 0;
} }
if (nvme_qpair_is_admin_queue(qpair)) { if (spdk_unlikely(nvme_qpair_is_admin_queue(qpair))) {
nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
} }
@ -1905,7 +1908,7 @@ nvme_pcie_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_
assert(0); assert(0);
} }
if (++pqpair->cq_head == pqpair->num_entries) { if (spdk_unlikely(++pqpair->cq_head == pqpair->num_entries)) {
pqpair->cq_head = 0; pqpair->cq_head = 0;
pqpair->phase = !pqpair->phase; pqpair->phase = !pqpair->phase;
} }
@ -1921,17 +1924,16 @@ nvme_pcie_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_
g_thread_mmio_ctrlr = NULL; g_thread_mmio_ctrlr = NULL;
} }
if (qpair->ctrlr->state == NVME_CTRLR_STATE_READY) { if (spdk_unlikely(qpair->ctrlr->timeout_cb_fn != NULL) &&
if (qpair->ctrlr->timeout_cb_fn) { qpair->ctrlr->state == NVME_CTRLR_STATE_READY) {
/* /*
* User registered for timeout callback * User registered for timeout callback
*/ */
nvme_pcie_qpair_check_timeout(qpair); nvme_pcie_qpair_check_timeout(qpair);
} }
}
/* Before returning, complete any pending admin request. */ /* Before returning, complete any pending admin request. */
if (nvme_qpair_is_admin_queue(qpair)) { if (spdk_unlikely(nvme_qpair_is_admin_queue(qpair))) {
nvme_pcie_qpair_complete_pending_admin_request(qpair); nvme_pcie_qpair_complete_pending_admin_request(qpair);
nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);