diff --git a/lib/nvme/nvme_ctrlr.c b/lib/nvme/nvme_ctrlr.c index b55478344..fce7e9162 100644 --- a/lib/nvme/nvme_ctrlr.c +++ b/lib/nvme/nvme_ctrlr.c @@ -842,6 +842,11 @@ nvme_ctrlr_start(struct spdk_nvme_ctrlr *ctrlr) nvme_ctrlr_set_supported_log_pages(ctrlr); nvme_ctrlr_set_supported_features(ctrlr); + + if (ctrlr->cdata.sgls.supported) { + ctrlr->flags |= SPDK_NVME_CTRLR_SGL_SUPPORTED; + } + return 0; } @@ -908,6 +913,7 @@ nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr, void *devhandle) ctrlr->is_resetting = false; ctrlr->is_failed = false; + ctrlr->flags = 0; nvme_mutex_init_recursive(&ctrlr->ctrlr_lock); diff --git a/lib/nvme/nvme_internal.h b/lib/nvme/nvme_internal.h index c1016fb73..1e09d7f19 100644 --- a/lib/nvme/nvme_internal.h +++ b/lib/nvme/nvme_internal.h @@ -98,6 +98,12 @@ #define NVME_MIN_IO_TRACKERS (4) #define NVME_MAX_IO_TRACKERS (1024) +/* + * NVME_MAX_SGL_DESCRIPTORS defines the maximum number of descriptors in one SGL + * segment. + */ +#define NVME_MAX_SGL_DESCRIPTORS (256) + /* * NVME_MAX_IO_ENTRIES is not defined, since it is specified in CC.MQES * for each controller. @@ -128,6 +134,13 @@ enum nvme_payload_type { NVME_PAYLOAD_TYPE_SGL, }; +/* + * Controller support flags. + */ +enum spdk_nvme_ctrlr_flags { + SPDK_NVME_CTRLR_SGL_SUPPORTED = 0x1, /**< The SGL is supported */ +}; + /** * Descriptor for a request data payload. * @@ -233,8 +246,11 @@ struct nvme_tracker { struct nvme_request *req; uint16_t cid; - uint64_t prp_bus_addr; - uint64_t prp[NVME_MAX_PRP_LIST_ENTRIES]; + uint64_t prp_sgl_bus_addr; + union { + uint64_t prp[NVME_MAX_PRP_LIST_ENTRIES]; + struct spdk_nvme_sgl_descriptor sgl[NVME_MAX_SGL_DESCRIPTORS]; + } u; }; struct spdk_nvme_qpair { @@ -342,6 +358,9 @@ struct spdk_nvme_ctrlr { bool is_failed; + /** Controller support flags */ + uint64_t flags; + /* Cold data (not accessed in normal I/O path) is after this point. */ enum nvme_ctrlr_state state; diff --git a/lib/nvme/nvme_qpair.c b/lib/nvme/nvme_qpair.c index cfff01e32..fb1887d54 100644 --- a/lib/nvme/nvme_qpair.c +++ b/lib/nvme/nvme_qpair.c @@ -289,7 +289,7 @@ nvme_completion_is_retry(const struct spdk_nvme_cpl *cpl) static void nvme_qpair_construct_tracker(struct nvme_tracker *tr, uint16_t cid, uint64_t phys_addr) { - tr->prp_bus_addr = phys_addr + offsetof(struct nvme_tracker, prp); + tr->prp_sgl_bus_addr = phys_addr + offsetof(struct nvme_tracker, u.prp); tr->cid = cid; } @@ -702,7 +702,7 @@ _nvme_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_requ tr->req->cmd.dptr.prp.prp2 = nvme_vtophys(seg_addr); } else if (nseg > 2) { cur_nseg = 1; - tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_bus_addr; + tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_sgl_bus_addr; while (cur_nseg < nseg) { seg_addr = payload + cur_nseg * PAGE_SIZE - unaligned; phys_addr = nvme_vtophys(seg_addr); @@ -710,7 +710,7 @@ _nvme_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_requ _nvme_fail_request_bad_vtophys(qpair, tr); return -1; } - tr->prp[cur_nseg - 1] = phys_addr; + tr->u.prp[cur_nseg - 1] = phys_addr; cur_nseg++; } } @@ -718,9 +718,79 @@ _nvme_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_requ return 0; } +/** + * Build SGL list describing scattered payload buffer. + */ static int -_nvme_qpair_build_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req, - struct nvme_tracker *tr) +_nvme_qpair_build_hw_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req, + struct nvme_tracker *tr) +{ + int rc; + uint64_t phys_addr; + uint32_t remaining_transfer_len, length; + struct spdk_nvme_sgl_descriptor *sgl; + uint32_t nseg = 0; + + /* + * Build scattered payloads. + */ + nvme_assert(req->payload.type == NVME_PAYLOAD_TYPE_SGL, ("sgl payload type required\n")); + nvme_assert(req->payload.u.sgl.reset_sgl_fn != NULL, ("sgl reset callback required\n")); + req->payload.u.sgl.reset_sgl_fn(req->payload.u.sgl.cb_arg, req->payload_offset); + + sgl = (struct spdk_nvme_sgl_descriptor *)tr->u.sgl; + req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_SGL; + req->cmd.dptr.sgl1.type_specific = 0; + + remaining_transfer_len = req->payload_size; + + while (remaining_transfer_len > 0) { + nvme_assert(req->payload.u.sgl.next_sge_fn != NULL, ("sgl callback required\n")); + rc = req->payload.u.sgl.next_sge_fn(req->payload.u.sgl.cb_arg, &phys_addr, &length); + if (rc) { + _nvme_fail_request_bad_vtophys(qpair, tr); + return -1; + } + + remaining_transfer_len -= length; + + sgl->type = SPDK_NVME_SGL_TYPE_DATA_BLOCK; + sgl->length = length; + sgl->address = phys_addr; + sgl->type_specific = 0; + + sgl++; + nseg++; + + if (nseg >= NVME_MAX_SGL_DESCRIPTORS) { + _nvme_fail_request_bad_vtophys(qpair, tr); + return -1; + } + } + + if (nseg == 1) { + req->cmd.dptr.sgl1.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK; + req->cmd.dptr.sgl1.address = phys_addr; + req->cmd.dptr.sgl1.length = length; + } else if (nseg > 1) { + /* For now we can only support 1 SGL segment in NVMe controller */ + req->cmd.dptr.sgl1.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT; + req->cmd.dptr.sgl1.address = tr->prp_sgl_bus_addr; + req->cmd.dptr.sgl1.length = nseg * sizeof(struct spdk_nvme_sgl_descriptor); + } else { + _nvme_fail_request_bad_vtophys(qpair, tr); + return -1; + } + + return 0; +} + +/** + * Build PRP list describing scattered payload buffer. + */ +static int +_nvme_qpair_build_prps_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req, + struct nvme_tracker *tr) { int rc; uint64_t phys_addr; @@ -779,13 +849,13 @@ _nvme_qpair_build_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request else cur_nseg = 0; - tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_bus_addr; + tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_sgl_bus_addr; while (cur_nseg < nseg) { if (prp2) { - tr->prp[0] = prp2; - tr->prp[last_nseg + 1] = phys_addr + cur_nseg * PAGE_SIZE - unaligned; + tr->u.prp[0] = prp2; + tr->u.prp[last_nseg + 1] = phys_addr + cur_nseg * PAGE_SIZE - unaligned; } else - tr->prp[last_nseg] = phys_addr + cur_nseg * PAGE_SIZE - unaligned; + tr->u.prp[last_nseg] = phys_addr + cur_nseg * PAGE_SIZE - unaligned; last_nseg++; cur_nseg++; @@ -810,6 +880,7 @@ nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *re int rc; struct nvme_tracker *tr; struct nvme_request *child_req; + struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr; nvme_qpair_check_enabled(qpair); @@ -860,7 +931,10 @@ nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *re return; } } else if (req->payload.type == NVME_PAYLOAD_TYPE_SGL) { - rc = _nvme_qpair_build_sgl_request(qpair, req, tr); + if (ctrlr->flags & SPDK_NVME_CTRLR_SGL_SUPPORTED) + rc = _nvme_qpair_build_hw_sgl_request(qpair, req, tr); + else + rc = _nvme_qpair_build_prps_sgl_request(qpair, req, tr); if (rc < 0) { return; } diff --git a/test/lib/nvme/sgl/nvme_sgl.c b/test/lib/nvme/sgl/nvme_sgl.c index 36669a747..d8d6bdd51 100644 --- a/test/lib/nvme/sgl/nvme_sgl.c +++ b/test/lib/nvme/sgl/nvme_sgl.c @@ -129,6 +129,22 @@ io_complete(void *ctx, const struct spdk_nvme_cpl *cpl) io_complete_flag = 1; } +static uint32_t build_io_request_0(struct io_request *req) +{ + int i; + uint32_t len = 0; + + req->nseg = 1; + + req->iovs[0].iov_base = rte_zmalloc(NULL, 0x800, 4); + req->iovs[0].iov_len = 0x800; + + for (i = 0; i < req->nseg; i++) + len += req->iovs[i].iov_len; + + return len; +} + static uint32_t build_io_request_1(struct io_request *req) { int i, found = 0; @@ -456,7 +472,8 @@ int main(int argc, char **argv) } foreach_dev(iter) { - if (writev_readv_tests(iter, build_io_request_1) + if (writev_readv_tests(iter, build_io_request_0) + || writev_readv_tests(iter, build_io_request_1) || writev_readv_tests(iter, build_io_request_2) || writev_readv_tests(iter, build_io_request_3) || writev_readv_tests(iter, build_io_request_4) diff --git a/test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c b/test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c index c80fd35a5..ab3cd11c6 100644 --- a/test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c +++ b/test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c @@ -389,13 +389,14 @@ test_sgl_req(void) CU_ASSERT(req->cmd.dptr.prp.prp1 == 0); CU_ASSERT(qpair.sq_tail == 1); sgl_tr = LIST_FIRST(&qpair.outstanding_tr); - CU_ASSERT(sgl_tr != NULL); - for (i = 0; i < 32; i++) { - CU_ASSERT(sgl_tr->prp[i] == (PAGE_SIZE * (i + 1))); - } + if (sgl_tr != NULL) { + for (i = 0; i < 32; i++) { + CU_ASSERT(sgl_tr->u.prp[i] == (PAGE_SIZE * (i + 1))); + } - LIST_REMOVE(sgl_tr, list); - free(sgl_tr); + LIST_REMOVE(sgl_tr, list); + free(sgl_tr); + } cleanup_submit_request_test(&qpair); nvme_free_request(req); }