ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvmf/vfio-user: add some unlikely on the hot path

Browse Source 
Signed-off-by: Thanos Makatos <thanos.makatos@nutanix.com>
Change-Id: Ib7977f34fc2fc312f0a502405dcd1b5266a22d3f
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15430
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: John Levon <levon@movementarian.org>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
This commit is contained in:
Thanos Makatos 2022-11-14 09:08:28 -05:00 committed by Tomasz Zawadzki
parent 6b71006dfe
commit 4475295e15
1 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
lib/nvmf/vfio_user.c
View File

@ -2433,7 +2433,7 @@ consume_cmd(struct nvmf_vfio_user_ctrlr *ctrlr, struct nvmf_vfio_user_sq *sq,
struct spdk_nvme_cmd *cmd) struct spdk_nvme_cmd *cmd)
{ {
assert(sq != NULL); assert(sq != NULL);
if (nvmf_qpair_is_admin_queue(&sq->qpair)) { if (spdk_unlikely(nvmf_qpair_is_admin_queue(&sq->qpair))) {
return consume_admin_cmd(ctrlr, cmd); return consume_admin_cmd(ctrlr, cmd);
} }
@ -2473,7 +2473,7 @@ handle_sq_tdbl_write(struct nvmf_vfio_user_ctrlr *ctrlr, const uint32_t new_tail
sq_head_advance(sq); sq_head_advance(sq);
err = consume_cmd(ctrlr, sq, cmd); err = consume_cmd(ctrlr, sq, cmd);
if (err != 0) { if (spdk_unlikely(err != 0)) {
return err; return err;
} }
} }
@ -2797,14 +2797,14 @@ handle_dbl_access(struct nvmf_vfio_user_ctrlr *ctrlr, uint32_t *buf,
assert(ctrlr != NULL); assert(ctrlr != NULL);
assert(buf != NULL); assert(buf != NULL);
if (!is_write) { if (spdk_unlikely(!is_write)) {
SPDK_WARNLOG("%s: host tried to read BAR0 doorbell %#lx\n", SPDK_WARNLOG("%s: host tried to read BAR0 doorbell %#lx\n",
ctrlr_id(ctrlr), pos); ctrlr_id(ctrlr), pos);
errno = EPERM; errno = EPERM;
return -1; return -1;
} }
if (count != sizeof(uint32_t)) { if (spdk_unlikely(count != sizeof(uint32_t))) {
SPDK_ERRLOG("%s: bad doorbell buffer size %ld\n", SPDK_ERRLOG("%s: bad doorbell buffer size %ld\n",
ctrlr_id(ctrlr), count); ctrlr_id(ctrlr), count);
errno = EINVAL; errno = EINVAL;
@ -2814,7 +2814,7 @@ handle_dbl_access(struct nvmf_vfio_user_ctrlr *ctrlr, uint32_t *buf,
pos -= NVME_DOORBELLS_OFFSET; pos -= NVME_DOORBELLS_OFFSET;
/* pos must be dword aligned */ /* pos must be dword aligned */
if ((pos & 0x3) != 0) { if (spdk_unlikely((pos & 0x3) != 0)) {
SPDK_ERRLOG("%s: bad doorbell offset %#lx\n", ctrlr_id(ctrlr), pos); SPDK_ERRLOG("%s: bad doorbell offset %#lx\n", ctrlr_id(ctrlr), pos);
errno = EINVAL; errno = EINVAL;
return -1; return -1;
@ -2823,7 +2823,7 @@ handle_dbl_access(struct nvmf_vfio_user_ctrlr *ctrlr, uint32_t *buf,
/* convert byte offset to array index */ /* convert byte offset to array index */
pos >>= 2; pos >>= 2;
if (pos >= NVMF_VFIO_USER_MAX_QPAIRS_PER_CTRLR * 2) { if (spdk_unlikely(pos >= NVMF_VFIO_USER_MAX_QPAIRS_PER_CTRLR * 2)) {
SPDK_ERRLOG("%s: bad doorbell index %#lx\n", ctrlr_id(ctrlr), pos); SPDK_ERRLOG("%s: bad doorbell index %#lx\n", ctrlr_id(ctrlr), pos);
errno = EINVAL; errno = EINVAL;
return -1; return -1;
@ -2900,7 +2900,7 @@ access_bar0_fn(vfu_ctx_t *vfu_ctx, char *buf, size_t count, loff_t pos,
int ret; int ret;
ctrlr = endpoint->ctrlr; ctrlr = endpoint->ctrlr;
if (endpoint->need_async_destroy || !ctrlr) { if (spdk_unlikely(endpoint->need_async_destroy || !ctrlr)) {
errno = EIO; errno = EIO;
return -1; return -1;
} }
@ -3670,7 +3670,7 @@ vfio_user_migr_ctrlr_restore(struct nvmf_vfio_user_ctrlr *vu_ctrlr)
cmd.opc = SPDK_NVME_OPC_ASYNC_EVENT_REQUEST; cmd.opc = SPDK_NVME_OPC_ASYNC_EVENT_REQUEST;
cmd.cid = migr_state.nvmf_data.aer_cids[i]; cmd.cid = migr_state.nvmf_data.aer_cids[i];
rc = handle_cmd_req(vu_ctrlr, &cmd, vu_ctrlr->sqs[0]); rc = handle_cmd_req(vu_ctrlr, &cmd, vu_ctrlr->sqs[0]);
if (rc) { if (spdk_unlikely(rc)) {
break; break;
} }
} }
@ -5553,7 +5553,7 @@ nvmf_vfio_user_sq_poll(struct nvmf_vfio_user_sq *sq)
spdk_rmb(); spdk_rmb();
count = handle_sq_tdbl_write(ctrlr, new_tail, sq); count = handle_sq_tdbl_write(ctrlr, new_tail, sq);
if (count < 0) { if (spdk_unlikely(count < 0)) {
fail_ctrlr(ctrlr); fail_ctrlr(ctrlr);
} }
@ -5589,7 +5589,7 @@ nvmf_vfio_user_poll_group_poll(struct spdk_nvmf_transport_poll_group *group)
ret = nvmf_vfio_user_sq_poll(sq); ret = nvmf_vfio_user_sq_poll(sq);
if (ret < 0) { if (spdk_unlikely(ret < 0)) {
return ret; return ret;
} }