diff --git a/include/spdk/nvme.h b/include/spdk/nvme.h index 95b23096e..a93090a24 100644 --- a/include/spdk/nvme.h +++ b/include/spdk/nvme.h @@ -1649,8 +1649,6 @@ int spdk_nvme_ctrlr_io_cmd_raw_no_payload_build(struct spdk_nvme_ctrlr *ctrlr, * \return 0 if successfully submitted, negated errnos on the following error conditions: * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ctrlr_cmd_io_raw(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair, @@ -1685,8 +1683,6 @@ int spdk_nvme_ctrlr_cmd_io_raw(struct spdk_nvme_ctrlr *ctrlr, * \return 0 if successfully submitted, negated errnos on the following error conditions: * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ctrlr_cmd_io_raw_with_md(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair, @@ -2844,8 +2840,6 @@ typedef int (*spdk_nvme_req_next_sge_cb)(void *cb_arg, void **address, uint32_t * -EINVAL: The request is malformed. * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ns_cmd_write(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, void *payload, uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, @@ -2873,8 +2867,6 @@ int spdk_nvme_ns_cmd_write(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpai * -EINVAL: The request is malformed. * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ns_cmd_writev(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, uint64_t lba, uint32_t lba_count, @@ -2908,8 +2900,6 @@ int spdk_nvme_ns_cmd_writev(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpa * -EINVAL: The request is malformed. * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ns_cmd_writev_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, uint64_t lba, uint32_t lba_count, @@ -2976,8 +2966,6 @@ int spdk_nvme_ns_cmd_writev_ext(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair * -EINVAL: The request is malformed. * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ns_cmd_write_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, void *payload, void *metadata, @@ -3005,8 +2993,6 @@ int spdk_nvme_ns_cmd_write_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpa * -EINVAL: The request is malformed. * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ns_cmd_write_zeroes(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, uint64_t lba, uint32_t lba_count, @@ -3031,8 +3017,6 @@ int spdk_nvme_ns_cmd_write_zeroes(struct spdk_nvme_ns *ns, struct spdk_nvme_qpai * -EINVAL: The request is malformed. * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ns_cmd_write_uncorrectable(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, uint64_t lba, uint32_t lba_count, @@ -3058,8 +3042,6 @@ int spdk_nvme_ns_cmd_write_uncorrectable(struct spdk_nvme_ns *ns, struct spdk_nv * -EINVAL: The request is malformed. * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ns_cmd_read(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, void *payload, uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, @@ -3087,8 +3069,6 @@ int spdk_nvme_ns_cmd_read(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair * -EINVAL: The request is malformed. * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ns_cmd_readv(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, uint64_t lba, uint32_t lba_count, @@ -3121,8 +3101,6 @@ int spdk_nvme_ns_cmd_readv(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpai * -EINVAL: The request is malformed. * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ns_cmd_readv_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, uint64_t lba, uint32_t lba_count, @@ -3186,8 +3164,6 @@ int spdk_nvme_ns_cmd_readv_ext(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair * * -EINVAL: The request is malformed. * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ns_cmd_read_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, void *payload, void *metadata, @@ -3300,8 +3276,6 @@ int spdk_nvme_ns_cmd_flush(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpai * \return 0 if successfully submitted, negated errnos on the following error conditions: * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ns_cmd_reservation_register(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, @@ -3330,8 +3304,6 @@ int spdk_nvme_ns_cmd_reservation_register(struct spdk_nvme_ns *ns, * \return 0 if successfully submitted, negated errnos on the following error conditions: * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ns_cmd_reservation_release(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, @@ -3360,8 +3332,6 @@ int spdk_nvme_ns_cmd_reservation_release(struct spdk_nvme_ns *ns, * \return 0 if successfully submitted, negated errnos on the following error conditions: * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ns_cmd_reservation_acquire(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, @@ -3388,8 +3358,6 @@ int spdk_nvme_ns_cmd_reservation_acquire(struct spdk_nvme_ns *ns, * \return 0 if successfully submitted, negated errnos on the following error conditions: * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ns_cmd_reservation_report(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, @@ -3416,8 +3384,6 @@ int spdk_nvme_ns_cmd_reservation_report(struct spdk_nvme_ns *ns, * -EINVAL: The request is malformed. * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ns_cmd_compare(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, void *payload, uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, @@ -3445,8 +3411,6 @@ int spdk_nvme_ns_cmd_compare(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qp * -EINVAL: The request is malformed. * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ns_cmd_comparev(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, uint64_t lba, uint32_t lba_count, @@ -3480,8 +3444,6 @@ int spdk_nvme_ns_cmd_comparev(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *q * -EINVAL: The request is malformed. * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ns_cmd_comparev_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, @@ -3515,8 +3477,6 @@ spdk_nvme_ns_cmd_comparev_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpai * -EINVAL: The request is malformed. * -ENOMEM: The request cannot be allocated. * -ENXIO: The qpair is failed at the transport level. - * -EFAULT: Invalid address was specified as part of payload. cb_fn is also called - * with error status including dnr=1 in this case. */ int spdk_nvme_ns_cmd_compare_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, void *payload, void *metadata, diff --git a/lib/nvme/nvme_pcie_common.c b/lib/nvme/nvme_pcie_common.c index b738a5c45..2fde567f7 100644 --- a/lib/nvme/nvme_pcie_common.c +++ b/lib/nvme/nvme_pcie_common.c @@ -43,6 +43,9 @@ __thread struct nvme_pcie_ctrlr *g_thread_mmio_ctrlr = NULL; +static void +nvme_pcie_fail_request_bad_vtophys(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr); + static inline uint64_t nvme_pcie_vtophys(struct spdk_nvme_ctrlr *ctrlr, const void *buf, uint64_t *size) { @@ -935,6 +938,18 @@ nvme_pcie_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_ nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); } + if (spdk_unlikely(pqpair->flags.has_pending_vtophys_failures)) { + struct nvme_tracker *tr, *tmp; + + TAILQ_FOREACH_SAFE(tr, &pqpair->outstanding_tr, tq_list, tmp) { + if (tr->bad_vtophys) { + tr->bad_vtophys = 0; + nvme_pcie_fail_request_bad_vtophys(qpair, tr); + } + } + pqpair->flags.has_pending_vtophys_failures = 0; + } + return num_completions; } @@ -1090,10 +1105,19 @@ free: static void nvme_pcie_fail_request_bad_vtophys(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr) { + if (!qpair->in_completion_context) { + struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair); + + tr->bad_vtophys = 1; + pqpair->flags.has_pending_vtophys_failures = 1; + return; + } + /* * Bad vtophys translation, so abort this request and return * immediately. */ + SPDK_ERRLOG("vtophys or other payload buffer related error\n"); nvme_pcie_qpair_manual_complete_tracker(qpair, tr, SPDK_NVME_SCT_GENERIC, SPDK_NVME_SC_INVALID_FIELD, 1 /* do not retry */, true); @@ -1582,13 +1606,22 @@ nvme_pcie_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_reques if (sgl_supported && !(ctrlr->flags & SPDK_NVME_CTRLR_SGL_REQUIRES_DWORD_ALIGNMENT)) { dword_aligned = false; } + + /* If we fail to build the request or the metadata, do not return the -EFAULT back up + * the stack. This ensures that we always fail these types of requests via a + * completion callback, and never in the context of the submission. + */ rc = g_nvme_pcie_build_req_table[payload_type][sgl_supported](qpair, req, tr, dword_aligned); if (rc < 0) { + assert(rc == -EFAULT); + rc = 0; goto exit; } rc = nvme_pcie_qpair_build_metadata(qpair, tr, sgl_supported, dword_aligned); if (rc < 0) { + assert(rc == -EFAULT); + rc = 0; goto exit; } } diff --git a/lib/nvme/nvme_pcie_internal.h b/lib/nvme/nvme_pcie_internal.h index 8c5cb9c2a..c3fd27ab9 100644 --- a/lib/nvme/nvme_pcie_internal.h +++ b/lib/nvme/nvme_pcie_internal.h @@ -110,7 +110,8 @@ struct nvme_tracker { struct nvme_request *req; uint16_t cid; - uint16_t rsvd0; + uint16_t bad_vtophys : 1; + uint16_t rsvd0 : 15; uint32_t rsvd1; spdk_nvme_cmd_cb cb_fn; @@ -184,6 +185,7 @@ struct nvme_pcie_qpair { uint8_t phase : 1; uint8_t delay_cmd_submit : 1; uint8_t has_shadow_doorbell : 1; + uint8_t has_pending_vtophys_failures : 1; } flags; /*