ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvme: add qpair operations to transport

Browse Source 
Change-Id: Id354ba13515d54bb54a8293569ee83ea72111183
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>
This commit is contained in:
Daniel Verkamp 2016-10-13 15:23:55 -07:00
parent 706bace35a
commit 03aead3903
6 changed files with 827 additions and 702 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
lib/nvme/nvme_ctrlr.c
View File

@ -335,7 +335,6 @@ nvme_ctrlr_construct_admin_qpair(struct spdk_nvme_ctrlr *ctrlr)
return nvme_qpair_construct(&ctrlr->adminq,
0, /* qpair ID */
NVME_ADMIN_ENTRIES,
NVME_ADMIN_TRACKERS,
ctrlr);
}
@ -344,7 +343,7 @@ nvme_ctrlr_construct_io_qpairs(struct spdk_nvme_ctrlr *ctrlr)
{
struct spdk_nvme_qpair *qpair;
union spdk_nvme_cap_register cap;
uint32_t i, num_entries, num_trackers;
uint32_t i, num_entries;
int rc;
uint64_t phys_addr = 0;
@ -370,13 +369,6 @@ nvme_ctrlr_construct_io_qpairs(struct spdk_nvme_ctrlr *ctrlr)
*/
num_entries = nvme_min(NVME_IO_ENTRIES, cap.bits.mqes + 1);
/*
* No need to have more trackers than entries in the submit queue.
* Note also that for a queue size of N, we can only have (N-1)
* commands outstanding, hence the "-1" here.
*/
num_trackers = nvme_min(NVME_IO_TRACKERS, (num_entries - 1));
ctrlr->ioq = spdk_zmalloc(ctrlr->opts.num_io_queues * sizeof(struct spdk_nvme_qpair),
64, &phys_addr);
@ -393,7 +385,6 @@ nvme_ctrlr_construct_io_qpairs(struct spdk_nvme_ctrlr *ctrlr)
rc = nvme_qpair_construct(qpair,
i + 1, /* qpair ID */
num_entries,
num_trackers,
ctrlr);
if (rc)
return -1;
@ -969,7 +960,7 @@ nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr)
int
nvme_ctrlr_start(struct spdk_nvme_ctrlr *ctrlr)
{
nvme_qpair_reset(&ctrlr->adminq);
ctrlr->transport->qpair_reset(&ctrlr->adminq);
nvme_qpair_enable(&ctrlr->adminq);

30
lib/nvme/nvme_internal.h
View File

@ -259,6 +259,18 @@ struct spdk_nvme_transport {
int (*ctrlr_create_io_qpair)(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair);
int (*ctrlr_delete_io_qpair)(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair);
int (*qpair_construct)(struct spdk_nvme_qpair *qpair);
void (*qpair_destroy)(struct spdk_nvme_qpair *qpair);
void (*qpair_enable)(struct spdk_nvme_qpair *qpair);
void (*qpair_disable)(struct spdk_nvme_qpair *qpair);
void (*qpair_reset)(struct spdk_nvme_qpair *qpair);
void (*qpair_fail)(struct spdk_nvme_qpair *qpair);
int (*qpair_submit_request)(struct spdk_nvme_qpair *qpair, struct nvme_request *req);
int32_t (*qpair_process_completions)(struct spdk_nvme_qpair *qpair, uint32_t max_completions);
};
struct nvme_completion_poll_status {
@ -521,6 +533,18 @@ nvme_align32pow2(uint32_t x)
return 1u << (1 + nvme_u32log2(x - 1));
}
static inline bool
nvme_qpair_is_admin_queue(struct spdk_nvme_qpair *qpair)
{
return qpair->id == 0;
}
static inline bool
nvme_qpair_is_io_queue(struct spdk_nvme_qpair *qpair)
{
return qpair->id != 0;
}
/* Admin functions */
int nvme_ctrlr_cmd_identify_controller(struct spdk_nvme_ctrlr *ctrlr,
void *payload,
@ -575,14 +599,12 @@ int nvme_ctrlr_alloc_cmb(struct spdk_nvme_ctrlr *ctrlr, uint64_t length, uint64_
uint64_t *offset);
int nvme_qpair_construct(struct spdk_nvme_qpair *qpair, uint16_t id,
uint16_t num_entries,
uint16_t num_trackers,
struct spdk_nvme_ctrlr *ctrlr);
void nvme_qpair_destroy(struct spdk_nvme_qpair *qpair);
void nvme_qpair_enable(struct spdk_nvme_qpair *qpair);
void nvme_qpair_disable(struct spdk_nvme_qpair *qpair);
int nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair,
struct nvme_request *req);
void nvme_qpair_reset(struct spdk_nvme_qpair *qpair);
void nvme_qpair_fail(struct spdk_nvme_qpair *qpair);
int nvme_ns_construct(struct spdk_nvme_ns *ns, uint16_t id,
@ -604,4 +626,8 @@ void spdk_nvme_ctrlr_opts_set_defaults(struct spdk_nvme_ctrlr_opts *opts);
int nvme_mutex_init_shared(pthread_mutex_t *mtx);
bool nvme_completion_is_retry(const struct spdk_nvme_cpl *cpl);
void nvme_qpair_print_command(struct spdk_nvme_qpair *qpair, struct spdk_nvme_cmd *cmd);
void nvme_qpair_print_completion(struct spdk_nvme_qpair *qpair, struct spdk_nvme_cpl *cpl);
#endif /* __NVME_INTERNAL_H__ */

716
lib/nvme/nvme_pcie.c
View File

@ -96,6 +96,353 @@ nvme_pcie_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64
return 0;
}
static void
nvme_qpair_construct_tracker(struct nvme_tracker *tr, uint16_t cid, uint64_t phys_addr)
{
tr->prp_sgl_bus_addr = phys_addr + offsetof(struct nvme_tracker, u.prp);
tr->cid = cid;
tr->active = false;
}
static void
nvme_pcie_qpair_reset(struct spdk_nvme_qpair *qpair)
{
qpair->sq_tail = qpair->cq_head = 0;
/*
* First time through the completion queue, HW will set phase
* bit on completions to 1. So set this to 1 here, indicating
* we're looking for a 1 to know which entries have completed.
* we'll toggle the bit each time when the completion queue
* rolls over.
*/
qpair->phase = 1;
memset(qpair->cmd, 0,
qpair->num_entries * sizeof(struct spdk_nvme_cmd));
memset(qpair->cpl, 0,
qpair->num_entries * sizeof(struct spdk_nvme_cpl));
}
static int
nvme_pcie_qpair_construct(struct spdk_nvme_qpair *qpair)
{
struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
struct nvme_tracker *tr;
uint16_t i;
volatile uint32_t *doorbell_base;
uint64_t phys_addr = 0;
uint64_t offset;
uint16_t num_trackers;
if (qpair->id == 0) {
num_trackers = NVME_ADMIN_TRACKERS;
} else {
/*
* No need to have more trackers than entries in the submit queue.
* Note also that for a queue size of N, we can only have (N-1)
* commands outstanding, hence the "-1" here.
*/
num_trackers = nvme_min(NVME_IO_TRACKERS, qpair->num_entries - 1);
}
assert(num_trackers != 0);
qpair->sq_in_cmb = false;
/* cmd and cpl rings must be aligned on 4KB boundaries. */
if (ctrlr->opts.use_cmb_sqs) {
if (nvme_ctrlr_alloc_cmb(ctrlr, qpair->num_entries * sizeof(struct spdk_nvme_cmd),
0x1000, &offset) == 0) {
qpair->cmd = ctrlr->cmb_bar_virt_addr + offset;
qpair->cmd_bus_addr = ctrlr->cmb_bar_phys_addr + offset;
qpair->sq_in_cmb = true;
}
}
if (qpair->sq_in_cmb == false) {
qpair->cmd = spdk_zmalloc(qpair->num_entries * sizeof(struct spdk_nvme_cmd),
0x1000,
&qpair->cmd_bus_addr);
if (qpair->cmd == NULL) {
SPDK_ERRLOG("alloc qpair_cmd failed\n");
return -ENOMEM;
}
}
qpair->cpl = spdk_zmalloc(qpair->num_entries * sizeof(struct spdk_nvme_cpl),
0x1000,
&qpair->cpl_bus_addr);
if (qpair->cpl == NULL) {
SPDK_ERRLOG("alloc qpair_cpl failed\n");
return -ENOMEM;
}
doorbell_base = &ctrlr->regs->doorbell[0].sq_tdbl;
qpair->sq_tdbl = doorbell_base + (2 * qpair->id + 0) * ctrlr->doorbell_stride_u32;
qpair->cq_hdbl = doorbell_base + (2 * qpair->id + 1) * ctrlr->doorbell_stride_u32;
/*
* Reserve space for all of the trackers in a single allocation.
* struct nvme_tracker must be padded so that its size is already a power of 2.
* This ensures the PRP list embedded in the nvme_tracker object will not span a
* 4KB boundary, while allowing access to trackers in tr[] via normal array indexing.
*/
qpair->tr = spdk_zmalloc(num_trackers * sizeof(*tr), sizeof(*tr), &phys_addr);
if (qpair->tr == NULL) {
SPDK_ERRLOG("nvme_tr failed\n");
return -ENOMEM;
}
for (i = 0; i < num_trackers; i++) {
tr = &qpair->tr[i];
nvme_qpair_construct_tracker(tr, i, phys_addr);
LIST_INSERT_HEAD(&qpair->free_tr, tr, list);
phys_addr += sizeof(struct nvme_tracker);
}
nvme_pcie_qpair_reset(qpair);
return 0;
}
static inline void
nvme_pcie_copy_command(struct spdk_nvme_cmd *dst, const struct spdk_nvme_cmd *src)
{
/* dst and src are known to be non-overlapping and 64-byte aligned. */
#if defined(__AVX__)
__m256i *d256 = (__m256i *)dst;
const __m256i *s256 = (const __m256i *)src;
_mm256_store_si256(&d256[0], _mm256_load_si256(&s256[0]));
_mm256_store_si256(&d256[1], _mm256_load_si256(&s256[1]));
#elif defined(__SSE2__)
__m128i *d128 = (__m128i *)dst;
const __m128i *s128 = (const __m128i *)src;
_mm_store_si128(&d128[0], _mm_load_si128(&s128[0]));
_mm_store_si128(&d128[1], _mm_load_si128(&s128[1]));
_mm_store_si128(&d128[2], _mm_load_si128(&s128[2]));
_mm_store_si128(&d128[3], _mm_load_si128(&s128[3]));
#else
*dst = *src;
#endif
}
static void
nvme_pcie_qpair_submit_tracker(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr)
{
struct nvme_request *req;
req = tr->req;
qpair->tr[tr->cid].active = true;
/* Copy the command from the tracker to the submission queue. */
nvme_pcie_copy_command(&qpair->cmd[qpair->sq_tail], &req->cmd);
if (++qpair->sq_tail == qpair->num_entries) {
qpair->sq_tail = 0;
}
spdk_wmb();
spdk_mmio_write_4(qpair->sq_tdbl, qpair->sq_tail);
}
static void
nvme_pcie_qpair_complete_tracker(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr,
struct spdk_nvme_cpl *cpl, bool print_on_error)
{
struct nvme_request *req;
bool retry, error, was_active;
req = tr->req;
assert(req != NULL);
error = spdk_nvme_cpl_is_error(cpl);
retry = error && nvme_completion_is_retry(cpl) &&
req->retries < spdk_nvme_retry_count;
if (error && print_on_error) {
nvme_qpair_print_command(qpair, &req->cmd);
nvme_qpair_print_completion(qpair, cpl);
}
was_active = qpair->tr[cpl->cid].active;
qpair->tr[cpl->cid].active = false;
assert(cpl->cid == req->cmd.cid);
if (retry) {
req->retries++;
nvme_pcie_qpair_submit_tracker(qpair, tr);
} else {
if (was_active && req->cb_fn) {
req->cb_fn(req->cb_arg, cpl);
}
nvme_free_request(req);
tr->req = NULL;
LIST_REMOVE(tr, list);
LIST_INSERT_HEAD(&qpair->free_tr, tr, list);
/*
* If the controller is in the middle of resetting, don't
* try to submit queued requests here - let the reset logic
* handle that instead.
*/
if (!STAILQ_EMPTY(&qpair->queued_req) &&
!qpair->ctrlr->is_resetting) {
req = STAILQ_FIRST(&qpair->queued_req);
STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
nvme_qpair_submit_request(qpair, req);
}
}
}
static void
nvme_pcie_qpair_manual_complete_tracker(struct spdk_nvme_qpair *qpair,
struct nvme_tracker *tr, uint32_t sct, uint32_t sc, uint32_t dnr,
bool print_on_error)
{
struct spdk_nvme_cpl cpl;
memset(&cpl, 0, sizeof(cpl));
cpl.sqid = qpair->id;
cpl.cid = tr->cid;
cpl.status.sct = sct;
cpl.status.sc = sc;
cpl.status.dnr = dnr;
nvme_pcie_qpair_complete_tracker(qpair, tr, &cpl, print_on_error);
}
static void
nvme_pcie_admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair)
{
struct nvme_tracker *tr;
tr = LIST_FIRST(&qpair->outstanding_tr);
while (tr != NULL) {
assert(tr->req != NULL);
if (tr->req->cmd.opc == SPDK_NVME_OPC_ASYNC_EVENT_REQUEST) {
nvme_pcie_qpair_manual_complete_tracker(qpair, tr,
SPDK_NVME_SCT_GENERIC, SPDK_NVME_SC_ABORTED_SQ_DELETION, 0,
false);
tr = LIST_FIRST(&qpair->outstanding_tr);
} else {
tr = LIST_NEXT(tr, list);
}
}
}
static void
nvme_pcie_admin_qpair_destroy(struct spdk_nvme_qpair *qpair)
{
nvme_pcie_admin_qpair_abort_aers(qpair);
}
static void
nvme_pcie_qpair_destroy(struct spdk_nvme_qpair *qpair)
{
if (nvme_qpair_is_admin_queue(qpair)) {
nvme_pcie_admin_qpair_destroy(qpair);
}
if (qpair->cmd && !qpair->sq_in_cmb) {
spdk_free(qpair->cmd);
qpair->cmd = NULL;
}
if (qpair->cpl) {
spdk_free(qpair->cpl);
qpair->cpl = NULL;
}
if (qpair->tr) {
spdk_free(qpair->tr);
qpair->tr = NULL;
}
}
static void
nvme_pcie_admin_qpair_enable(struct spdk_nvme_qpair *qpair)
{
struct nvme_tracker *tr, *temp;
/*
* Manually abort each outstanding admin command. Do not retry
* admin commands found here, since they will be left over from
* a controller reset and its likely the context in which the
* command was issued no longer applies.
*/
LIST_FOREACH_SAFE(tr, &qpair->outstanding_tr, list, temp) {
SPDK_ERRLOG("aborting outstanding admin command\n");
nvme_pcie_qpair_manual_complete_tracker(qpair, tr, SPDK_NVME_SCT_GENERIC,
SPDK_NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, true);
}
}
static void
nvme_pcie_io_qpair_enable(struct spdk_nvme_qpair *qpair)
{
struct nvme_tracker *tr, *temp;
/* Manually abort each outstanding I/O. */
LIST_FOREACH_SAFE(tr, &qpair->outstanding_tr, list, temp) {
SPDK_ERRLOG("aborting outstanding i/o\n");
nvme_pcie_qpair_manual_complete_tracker(qpair, tr, SPDK_NVME_SCT_GENERIC,
SPDK_NVME_SC_ABORTED_BY_REQUEST, 0, true);
}
}
static void
nvme_pcie_qpair_enable(struct spdk_nvme_qpair *qpair)
{
if (nvme_qpair_is_io_queue(qpair)) {
nvme_pcie_io_qpair_enable(qpair);
} else {
nvme_pcie_admin_qpair_enable(qpair);
}
}
static void
nvme_pcie_admin_qpair_disable(struct spdk_nvme_qpair *qpair)
{
nvme_pcie_admin_qpair_abort_aers(qpair);
}
static void
nvme_pcie_io_qpair_disable(struct spdk_nvme_qpair *qpair)
{
}
static void
nvme_pcie_qpair_disable(struct spdk_nvme_qpair *qpair)
{
qpair->is_enabled = false;
if (nvme_qpair_is_io_queue(qpair)) {
nvme_pcie_io_qpair_disable(qpair);
} else {
nvme_pcie_admin_qpair_disable(qpair);
}
}
static void
nvme_pcie_qpair_fail(struct spdk_nvme_qpair *qpair)
{
struct nvme_tracker *tr;
/* Manually abort each outstanding I/O. */
while (!LIST_EMPTY(&qpair->outstanding_tr)) {
tr = LIST_FIRST(&qpair->outstanding_tr);
/*
* Do not remove the tracker. The abort_tracker path will
* do that for us.
*/
SPDK_ERRLOG("failing outstanding i/o\n");
nvme_pcie_qpair_manual_complete_tracker(qpair, tr, SPDK_NVME_SCT_GENERIC,
SPDK_NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, true);
}
}
static int
nvme_pcie_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
{
@ -142,7 +489,7 @@ nvme_pcie_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_
return -1;
}
nvme_qpair_reset(qpair);
nvme_pcie_qpair_reset(qpair);
return 0;
}
@ -185,6 +532,361 @@ nvme_pcie_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_
return 0;
}
static void
nvme_pcie_fail_request_bad_vtophys(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr)
{
/*
* Bad vtophys translation, so abort this request and return
* immediately.
*/
nvme_pcie_qpair_manual_complete_tracker(qpair, tr, SPDK_NVME_SCT_GENERIC,
SPDK_NVME_SC_INVALID_FIELD,
1 /* do not retry */, true);
}
/**
* Build PRP list describing physically contiguous payload buffer.
*/
static int
nvme_pcie_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
struct nvme_tracker *tr)
{
uint64_t phys_addr;
void *seg_addr;
uint32_t nseg, cur_nseg, modulo, unaligned;
void *md_payload;
void *payload = req->payload.u.contig + req->payload_offset;
phys_addr = spdk_vtophys(payload);
if (phys_addr == SPDK_VTOPHYS_ERROR) {
nvme_pcie_fail_request_bad_vtophys(qpair, tr);
return -1;
}
nseg = req->payload_size >> nvme_u32log2(PAGE_SIZE);
modulo = req->payload_size & (PAGE_SIZE - 1);
unaligned = phys_addr & (PAGE_SIZE - 1);
if (modulo || unaligned) {
nseg += 1 + ((modulo + unaligned - 1) >> nvme_u32log2(PAGE_SIZE));
}
if (req->payload.md) {
md_payload = req->payload.md + req->md_offset;
tr->req->cmd.mptr = spdk_vtophys(md_payload);
if (tr->req->cmd.mptr == SPDK_VTOPHYS_ERROR) {
nvme_pcie_fail_request_bad_vtophys(qpair, tr);
return -1;
}
}
tr->req->cmd.psdt = SPDK_NVME_PSDT_PRP;
tr->req->cmd.dptr.prp.prp1 = phys_addr;
if (nseg == 2) {
seg_addr = payload + PAGE_SIZE - unaligned;
tr->req->cmd.dptr.prp.prp2 = spdk_vtophys(seg_addr);
} else if (nseg > 2) {
cur_nseg = 1;
tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_sgl_bus_addr;
while (cur_nseg < nseg) {
seg_addr = payload + cur_nseg * PAGE_SIZE - unaligned;
phys_addr = spdk_vtophys(seg_addr);
if (phys_addr == SPDK_VTOPHYS_ERROR) {
nvme_pcie_fail_request_bad_vtophys(qpair, tr);
return -1;
}
tr->u.prp[cur_nseg - 1] = phys_addr;
cur_nseg++;
}
}
return 0;
}
/**
* Build SGL list describing scattered payload buffer.
*/
static int
nvme_pcie_qpair_build_hw_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
struct nvme_tracker *tr)
{
int rc;
uint64_t phys_addr;
uint32_t remaining_transfer_len, length;
struct spdk_nvme_sgl_descriptor *sgl;
uint32_t nseg = 0;
/*
* Build scattered payloads.
*/
assert(req->payload_size != 0);
assert(req->payload.type == NVME_PAYLOAD_TYPE_SGL);
assert(req->payload.u.sgl.reset_sgl_fn != NULL);
assert(req->payload.u.sgl.next_sge_fn != NULL);
req->payload.u.sgl.reset_sgl_fn(req->payload.u.sgl.cb_arg, req->payload_offset);
sgl = tr->u.sgl;
req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_SGL;
req->cmd.dptr.sgl1.unkeyed.subtype = 0;
remaining_transfer_len = req->payload_size;
while (remaining_transfer_len > 0) {
if (nseg >= NVME_MAX_SGL_DESCRIPTORS) {
nvme_pcie_fail_request_bad_vtophys(qpair, tr);
return -1;
}
rc = req->payload.u.sgl.next_sge_fn(req->payload.u.sgl.cb_arg, &phys_addr, &length);
if (rc) {
nvme_pcie_fail_request_bad_vtophys(qpair, tr);
return -1;
}
length = nvme_min(remaining_transfer_len, length);
remaining_transfer_len -= length;
sgl->unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
sgl->unkeyed.length = length;
sgl->address = phys_addr;
sgl->unkeyed.subtype = 0;
sgl++;
nseg++;
}
if (nseg == 1) {
/*
* The whole transfer can be described by a single SGL descriptor.
* Use the special case described by the spec where SGL1's type is Data Block.
* This means the SGL in the tracker is not used at all, so copy the first (and only)
* SGL element into SGL1.
*/
req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
req->cmd.dptr.sgl1.address = tr->u.sgl[0].address;
req->cmd.dptr.sgl1.unkeyed.length = tr->u.sgl[0].unkeyed.length;
} else {
/* For now we can only support 1 SGL segment in NVMe controller */
req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT;
req->cmd.dptr.sgl1.address = tr->prp_sgl_bus_addr;
req->cmd.dptr.sgl1.unkeyed.length = nseg * sizeof(struct spdk_nvme_sgl_descriptor);
}
return 0;
}
/**
* Build PRP list describing scattered payload buffer.
*/
static int
nvme_pcie_qpair_build_prps_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
struct nvme_tracker *tr)
{
int rc;
uint64_t phys_addr;
uint32_t data_transferred, remaining_transfer_len, length;
uint32_t nseg, cur_nseg, total_nseg, last_nseg, modulo, unaligned;
uint32_t sge_count = 0;
uint64_t prp2 = 0;
/*
* Build scattered payloads.
*/
assert(req->payload.type == NVME_PAYLOAD_TYPE_SGL);
assert(req->payload.u.sgl.reset_sgl_fn != NULL);
req->payload.u.sgl.reset_sgl_fn(req->payload.u.sgl.cb_arg, req->payload_offset);
remaining_transfer_len = req->payload_size;
total_nseg = 0;
last_nseg = 0;
while (remaining_transfer_len > 0) {
assert(req->payload.u.sgl.next_sge_fn != NULL);
rc = req->payload.u.sgl.next_sge_fn(req->payload.u.sgl.cb_arg, &phys_addr, &length);
if (rc) {
nvme_pcie_fail_request_bad_vtophys(qpair, tr);
return -1;
}
/* Confirm that this sge is prp compatible. */
if (phys_addr & 0x3 ||
(length < remaining_transfer_len && ((phys_addr + length) & (PAGE_SIZE - 1)))) {
nvme_pcie_fail_request_bad_vtophys(qpair, tr);
return -1;
}
data_transferred = nvme_min(remaining_transfer_len, length);
nseg = data_transferred >> nvme_u32log2(PAGE_SIZE);
modulo = data_transferred & (PAGE_SIZE - 1);
unaligned = phys_addr & (PAGE_SIZE - 1);
if (modulo || unaligned) {
nseg += 1 + ((modulo + unaligned - 1) >> nvme_u32log2(PAGE_SIZE));
}
if (total_nseg == 0) {
req->cmd.psdt = SPDK_NVME_PSDT_PRP;
req->cmd.dptr.prp.prp1 = phys_addr;
phys_addr -= unaligned;
}
total_nseg += nseg;
sge_count++;
remaining_transfer_len -= data_transferred;
if (total_nseg == 2) {
if (sge_count == 1)
tr->req->cmd.dptr.prp.prp2 = phys_addr + PAGE_SIZE;
else if (sge_count == 2)
tr->req->cmd.dptr.prp.prp2 = phys_addr;
/* save prp2 value */
prp2 = tr->req->cmd.dptr.prp.prp2;
} else if (total_nseg > 2) {
if (sge_count == 1)
cur_nseg = 1;
else
cur_nseg = 0;
tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_sgl_bus_addr;
while (cur_nseg < nseg) {
if (prp2) {
tr->u.prp[0] = prp2;
tr->u.prp[last_nseg + 1] = phys_addr + cur_nseg * PAGE_SIZE;
} else
tr->u.prp[last_nseg] = phys_addr + cur_nseg * PAGE_SIZE;
last_nseg++;
cur_nseg++;
}
}
}
return 0;
}
static inline bool
nvme_pcie_qpair_check_enabled(struct spdk_nvme_qpair *qpair)
{
if (!qpair->is_enabled &&
!qpair->ctrlr->is_resetting) {
nvme_qpair_enable(qpair);
}
return qpair->is_enabled;
}
static int
nvme_pcie_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
{
struct nvme_tracker *tr;
int rc;
struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
nvme_pcie_qpair_check_enabled(qpair);
tr = LIST_FIRST(&qpair->free_tr);
if (tr == NULL || !qpair->is_enabled) {
/*
* No tracker is available, or the qpair is disabled due to
* an in-progress controller-level reset.
*
* Put the request on the qpair's request queue to be
* processed when a tracker frees up via a command
* completion or when the controller reset is
* completed.
*/
STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq);
return 0;
}
LIST_REMOVE(tr, list); /* remove tr from free_tr */
LIST_INSERT_HEAD(&qpair->outstanding_tr, tr, list);
tr->req = req;
req->cmd.cid = tr->cid;
if (req->payload_size == 0) {
/* Null payload - leave PRP fields zeroed */
rc = 0;
} else if (req->payload.type == NVME_PAYLOAD_TYPE_CONTIG) {
rc = nvme_pcie_qpair_build_contig_request(qpair, req, tr);
} else if (req->payload.type == NVME_PAYLOAD_TYPE_SGL) {
if (ctrlr->flags & SPDK_NVME_CTRLR_SGL_SUPPORTED) {
rc = nvme_pcie_qpair_build_hw_sgl_request(qpair, req, tr);
} else {
rc = nvme_pcie_qpair_build_prps_sgl_request(qpair, req, tr);
}
} else {
assert(0);
nvme_pcie_fail_request_bad_vtophys(qpair, tr);
rc = -EINVAL;
}
if (rc < 0) {
return rc;
}
nvme_pcie_qpair_submit_tracker(qpair, tr);
return 0;
}
static int32_t
nvme_pcie_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions)
{
struct nvme_tracker *tr;
struct spdk_nvme_cpl *cpl;
uint32_t num_completions = 0;
if (!nvme_pcie_qpair_check_enabled(qpair)) {
/*
* qpair is not enabled, likely because a controller reset is
* is in progress. Ignore the interrupt - any I/O that was
* associated with this interrupt will get retried when the
* reset is complete.
*/
return 0;
}
if (max_completions == 0 || (max_completions > (qpair->num_entries - 1U))) {
/*
* max_completions == 0 means unlimited, but complete at most one
* queue depth batch of I/O at a time so that the completion
* queue doorbells don't wrap around.
*/
max_completions = qpair->num_entries - 1;
}
while (1) {
cpl = &qpair->cpl[qpair->cq_head];
if (cpl->status.p != qpair->phase)
break;
tr = &qpair->tr[cpl->cid];
if (tr->active) {
nvme_pcie_qpair_complete_tracker(qpair, tr, cpl, true);
} else {
SPDK_ERRLOG("cpl does not map to outstanding cmd\n");
nvme_qpair_print_completion(qpair, cpl);
assert(0);
}
if (++qpair->cq_head == qpair->num_entries) {
qpair->cq_head = 0;
qpair->phase = !qpair->phase;
}
if (++num_completions == max_completions) {
break;
}
}
if (num_completions > 0) {
spdk_mmio_write_4(qpair->cq_hdbl, qpair->cq_head);
}
return num_completions;
}
const struct spdk_nvme_transport spdk_nvme_transport_pcie = {
.ctrlr_get_pci_id = nvme_pcie_ctrlr_get_pci_id,
@ -196,4 +898,16 @@ const struct spdk_nvme_transport spdk_nvme_transport_pcie = {
.ctrlr_create_io_qpair = nvme_pcie_ctrlr_create_io_qpair,
.ctrlr_delete_io_qpair = nvme_pcie_ctrlr_delete_io_qpair,
.qpair_construct = nvme_pcie_qpair_construct,
.qpair_destroy = nvme_pcie_qpair_destroy,
.qpair_enable = nvme_pcie_qpair_enable,
.qpair_disable = nvme_pcie_qpair_disable,
.qpair_reset = nvme_pcie_qpair_reset,
.qpair_fail = nvme_pcie_qpair_fail,
.qpair_submit_request = nvme_pcie_qpair_submit_request,
.qpair_process_completions = nvme_pcie_qpair_process_completions,
};

676
lib/nvme/nvme_qpair.c
View File

@ -33,16 +33,6 @@
#include "nvme_internal.h"
static inline bool nvme_qpair_is_admin_queue(struct spdk_nvme_qpair *qpair)
{
return qpair->id == 0;
}
static inline bool nvme_qpair_is_io_queue(struct spdk_nvme_qpair *qpair)
{
return qpair->id != 0;
}
struct nvme_string {
uint16_t value;
const char *str;
@ -143,7 +133,7 @@ nvme_io_qpair_print_command(struct spdk_nvme_qpair *qpair,
}
}
static void
void
nvme_qpair_print_command(struct spdk_nvme_qpair *qpair, struct spdk_nvme_cmd *cmd)
{
assert(qpair != NULL);
@ -256,7 +246,7 @@ get_status_string(uint16_t sct, uint16_t sc)
return nvme_get_string(entry, sc);
}
static void
void
nvme_qpair_print_completion(struct spdk_nvme_qpair *qpair,
struct spdk_nvme_cpl *cpl)
{
@ -266,7 +256,7 @@ nvme_qpair_print_completion(struct spdk_nvme_qpair *qpair,
cpl->sqhd, cpl->status.p, cpl->status.m, cpl->status.dnr);
}
static bool
bool
nvme_completion_is_retry(const struct spdk_nvme_cpl *cpl)
{
/*
@ -310,125 +300,6 @@ nvme_completion_is_retry(const struct spdk_nvme_cpl *cpl)
}
}
static void
nvme_qpair_construct_tracker(struct nvme_tracker *tr, uint16_t cid, uint64_t phys_addr)
{
tr->prp_sgl_bus_addr = phys_addr + offsetof(struct nvme_tracker, u.prp);
tr->cid = cid;
tr->active = false;
}
static inline void
nvme_copy_command(struct spdk_nvme_cmd *dst, const struct spdk_nvme_cmd *src)
{
/* dst and src are known to be non-overlapping and 64-byte aligned. */
#if defined(__AVX__)
__m256i *d256 = (__m256i *)dst;
const __m256i *s256 = (const __m256i *)src;
_mm256_store_si256(&d256[0], _mm256_load_si256(&s256[0]));
_mm256_store_si256(&d256[1], _mm256_load_si256(&s256[1]));
#elif defined(__SSE2__)
__m128i *d128 = (__m128i *)dst;
const __m128i *s128 = (const __m128i *)src;
_mm_store_si128(&d128[0], _mm_load_si128(&s128[0]));
_mm_store_si128(&d128[1], _mm_load_si128(&s128[1]));
_mm_store_si128(&d128[2], _mm_load_si128(&s128[2]));
_mm_store_si128(&d128[3], _mm_load_si128(&s128[3]));
#else
*dst = *src;
#endif
}
static void
nvme_qpair_submit_tracker(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr)
{
struct nvme_request *req;
req = tr->req;
qpair->tr[tr->cid].active = true;
/* Copy the command from the tracker to the submission queue. */
nvme_copy_command(&qpair->cmd[qpair->sq_tail], &req->cmd);
if (++qpair->sq_tail == qpair->num_entries) {
qpair->sq_tail = 0;
}
spdk_wmb();
spdk_mmio_write_4(qpair->sq_tdbl, qpair->sq_tail);
}
static void
nvme_qpair_complete_tracker(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr,
struct spdk_nvme_cpl *cpl, bool print_on_error)
{
struct nvme_request *req;
bool retry, error, was_active;
req = tr->req;
assert(req != NULL);
error = spdk_nvme_cpl_is_error(cpl);
retry = error && nvme_completion_is_retry(cpl) &&
req->retries < spdk_nvme_retry_count;
if (error && print_on_error) {
nvme_qpair_print_command(qpair, &req->cmd);
nvme_qpair_print_completion(qpair, cpl);
}
was_active = qpair->tr[cpl->cid].active;
qpair->tr[cpl->cid].active = false;
assert(cpl->cid == req->cmd.cid);
if (retry) {
req->retries++;
nvme_qpair_submit_tracker(qpair, tr);
} else {
if (was_active && req->cb_fn) {
req->cb_fn(req->cb_arg, cpl);
}
nvme_free_request(req);
tr->req = NULL;
LIST_REMOVE(tr, list);
LIST_INSERT_HEAD(&qpair->free_tr, tr, list);
/*
* If the controller is in the middle of resetting, don't
* try to submit queued requests here - let the reset logic
* handle that instead.
*/
if (!STAILQ_EMPTY(&qpair->queued_req) &&
!qpair->ctrlr->is_resetting) {
req = STAILQ_FIRST(&qpair->queued_req);
STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
nvme_qpair_submit_request(qpair, req);
}
}
}
static void
nvme_qpair_manual_complete_tracker(struct spdk_nvme_qpair *qpair,
struct nvme_tracker *tr, uint32_t sct, uint32_t sc, uint32_t dnr,
bool print_on_error)
{
struct spdk_nvme_cpl cpl;
memset(&cpl, 0, sizeof(cpl));
cpl.sqid = qpair->id;
cpl.cid = tr->cid;
cpl.status.sct = sct;
cpl.status.sc = sc;
cpl.status.dnr = dnr;
nvme_qpair_complete_tracker(qpair, tr, &cpl, print_on_error);
}
static void
nvme_qpair_manual_complete_request(struct spdk_nvme_qpair *qpair,
struct nvme_request *req, uint32_t sct, uint32_t sc,
@ -456,440 +327,49 @@ nvme_qpair_manual_complete_request(struct spdk_nvme_qpair *qpair,
nvme_free_request(req);
}
static inline bool
nvme_qpair_check_enabled(struct spdk_nvme_qpair *qpair)
{
if (!qpair->is_enabled &&
!qpair->ctrlr->is_resetting) {
nvme_qpair_enable(qpair);
}
return qpair->is_enabled;
}
int32_t
spdk_nvme_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions)
{
struct nvme_tracker *tr;
struct spdk_nvme_cpl *cpl;
uint32_t num_completions = 0;
if (!nvme_qpair_check_enabled(qpair)) {
/*
* qpair is not enabled, likely because a controller reset is
* is in progress. Ignore the interrupt - any I/O that was
* associated with this interrupt will get retried when the
* reset is complete.
*/
return 0;
}
if (max_completions == 0 || (max_completions > (qpair->num_entries - 1U))) {
/*
* max_completions == 0 means unlimited, but complete at most one
* queue depth batch of I/O at a time so that the completion
* queue doorbells don't wrap around.
*/
max_completions = qpair->num_entries - 1;
}
while (1) {
cpl = &qpair->cpl[qpair->cq_head];
if (cpl->status.p != qpair->phase)
break;
tr = &qpair->tr[cpl->cid];
if (tr->active) {
nvme_qpair_complete_tracker(qpair, tr, cpl, true);
} else {
SPDK_ERRLOG("cpl does not map to outstanding cmd\n");
nvme_qpair_print_completion(qpair, cpl);
assert(0);
}
if (++qpair->cq_head == qpair->num_entries) {
qpair->cq_head = 0;
qpair->phase = !qpair->phase;
}
if (++num_completions == max_completions) {
break;
}
}
if (num_completions > 0) {
spdk_mmio_write_4(qpair->cq_hdbl, qpair->cq_head);
}
return num_completions;
return qpair->transport->qpair_process_completions(qpair, max_completions);
}
int
nvme_qpair_construct(struct spdk_nvme_qpair *qpair, uint16_t id,
uint16_t num_entries, uint16_t num_trackers,
uint16_t num_entries,
struct spdk_nvme_ctrlr *ctrlr)
{
struct nvme_tracker *tr;
uint16_t i;
volatile uint32_t *doorbell_base;
uint64_t phys_addr = 0;
uint64_t offset;
assert(num_entries != 0);
assert(num_trackers != 0);
qpair->id = id;
qpair->num_entries = num_entries;
qpair->qprio = 0;
qpair->sq_in_cmb = false;
qpair->ctrlr = ctrlr;
qpair->transport = ctrlr->transport;
/* cmd and cpl rings must be aligned on 4KB boundaries. */
if (ctrlr->opts.use_cmb_sqs) {
if (nvme_ctrlr_alloc_cmb(ctrlr, qpair->num_entries * sizeof(struct spdk_nvme_cmd),
0x1000, &offset) == 0) {
qpair->cmd = ctrlr->cmb_bar_virt_addr + offset;
qpair->cmd_bus_addr = ctrlr->cmb_bar_phys_addr + offset;
qpair->sq_in_cmb = true;
}
}
if (qpair->sq_in_cmb == false) {
qpair->cmd = spdk_zmalloc(qpair->num_entries * sizeof(struct spdk_nvme_cmd),
0x1000,
&qpair->cmd_bus_addr);
if (qpair->cmd == NULL) {
SPDK_ERRLOG("alloc qpair_cmd failed\n");
goto fail;
}
}
qpair->cpl = spdk_zmalloc(qpair->num_entries * sizeof(struct spdk_nvme_cpl),
0x1000,
&qpair->cpl_bus_addr);
if (qpair->cpl == NULL) {
SPDK_ERRLOG("alloc qpair_cpl failed\n");
goto fail;
}
doorbell_base = &ctrlr->regs->doorbell[0].sq_tdbl;
qpair->sq_tdbl = doorbell_base + (2 * id + 0) * ctrlr->doorbell_stride_u32;
qpair->cq_hdbl = doorbell_base + (2 * id + 1) * ctrlr->doorbell_stride_u32;
LIST_INIT(&qpair->free_tr);
LIST_INIT(&qpair->outstanding_tr);
STAILQ_INIT(&qpair->queued_req);
/*
* Reserve space for all of the trackers in a single allocation.
* struct nvme_tracker must be padded so that its size is already a power of 2.
* This ensures the PRP list embedded in the nvme_tracker object will not span a
* 4KB boundary, while allowing access to trackers in tr[] via normal array indexing.
*/
qpair->tr = spdk_zmalloc(num_trackers * sizeof(*tr), sizeof(*tr), &phys_addr);
if (qpair->tr == NULL) {
SPDK_ERRLOG("nvme_tr failed\n");
goto fail;
if (qpair->transport->qpair_construct(qpair)) {
SPDK_TRACELOG(SPDK_TRACE_NVME, "qpair_construct() failed\n");
nvme_qpair_destroy(qpair);
return -1;
}
for (i = 0; i < num_trackers; i++) {
tr = &qpair->tr[i];
nvme_qpair_construct_tracker(tr, i, phys_addr);
LIST_INSERT_HEAD(&qpair->free_tr, tr, list);
phys_addr += sizeof(struct nvme_tracker);
}
nvme_qpair_reset(qpair);
return 0;
fail:
nvme_qpair_destroy(qpair);
return -1;
}
static void
nvme_admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair)
{
struct nvme_tracker *tr;
tr = LIST_FIRST(&qpair->outstanding_tr);
while (tr != NULL) {
assert(tr->req != NULL);
if (tr->req->cmd.opc == SPDK_NVME_OPC_ASYNC_EVENT_REQUEST) {
nvme_qpair_manual_complete_tracker(qpair, tr,
SPDK_NVME_SCT_GENERIC, SPDK_NVME_SC_ABORTED_SQ_DELETION, 0,
false);
tr = LIST_FIRST(&qpair->outstanding_tr);
} else {
tr = LIST_NEXT(tr, list);
}
}
}
static void
_nvme_admin_qpair_destroy(struct spdk_nvme_qpair *qpair)
{
nvme_admin_qpair_abort_aers(qpair);
}
void
nvme_qpair_destroy(struct spdk_nvme_qpair *qpair)
{
if (nvme_qpair_is_admin_queue(qpair)) {
_nvme_admin_qpair_destroy(qpair);
}
if (qpair->cmd && !qpair->sq_in_cmb) {
spdk_free(qpair->cmd);
qpair->cmd = NULL;
}
if (qpair->cpl) {
spdk_free(qpair->cpl);
qpair->cpl = NULL;
}
if (qpair->tr) {
spdk_free(qpair->tr);
qpair->tr = NULL;
}
}
static void
_nvme_fail_request_bad_vtophys(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr)
{
/*
* Bad vtophys translation, so abort this request and return
* immediately.
*/
nvme_qpair_manual_complete_tracker(qpair, tr, SPDK_NVME_SCT_GENERIC,
SPDK_NVME_SC_INVALID_FIELD,
1 /* do not retry */, true);
}
/**
* Build PRP list describing physically contiguous payload buffer.
*/
static int
_nvme_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
struct nvme_tracker *tr)
{
uint64_t phys_addr;
void *seg_addr;
uint32_t nseg, cur_nseg, modulo, unaligned;
void *md_payload;
void *payload = req->payload.u.contig + req->payload_offset;
phys_addr = spdk_vtophys(payload);
if (phys_addr == SPDK_VTOPHYS_ERROR) {
_nvme_fail_request_bad_vtophys(qpair, tr);
return -1;
}
nseg = req->payload_size >> nvme_u32log2(PAGE_SIZE);
modulo = req->payload_size & (PAGE_SIZE - 1);
unaligned = phys_addr & (PAGE_SIZE - 1);
if (modulo || unaligned) {
nseg += 1 + ((modulo + unaligned - 1) >> nvme_u32log2(PAGE_SIZE));
}
if (req->payload.md) {
md_payload = req->payload.md + req->md_offset;
tr->req->cmd.mptr = spdk_vtophys(md_payload);
if (tr->req->cmd.mptr == SPDK_VTOPHYS_ERROR) {
_nvme_fail_request_bad_vtophys(qpair, tr);
return -1;
}
}
tr->req->cmd.psdt = SPDK_NVME_PSDT_PRP;
tr->req->cmd.dptr.prp.prp1 = phys_addr;
if (nseg == 2) {
seg_addr = payload + PAGE_SIZE - unaligned;
tr->req->cmd.dptr.prp.prp2 = spdk_vtophys(seg_addr);
} else if (nseg > 2) {
cur_nseg = 1;
tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_sgl_bus_addr;
while (cur_nseg < nseg) {
seg_addr = payload + cur_nseg * PAGE_SIZE - unaligned;
phys_addr = spdk_vtophys(seg_addr);
if (phys_addr == SPDK_VTOPHYS_ERROR) {
_nvme_fail_request_bad_vtophys(qpair, tr);
return -1;
}
tr->u.prp[cur_nseg - 1] = phys_addr;
cur_nseg++;
}
}
return 0;
}
/**
* Build SGL list describing scattered payload buffer.
*/
static int
_nvme_qpair_build_hw_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
struct nvme_tracker *tr)
{
int rc;
uint64_t phys_addr;
uint32_t remaining_transfer_len, length;
struct spdk_nvme_sgl_descriptor *sgl;
uint32_t nseg = 0;
/*
* Build scattered payloads.
*/
assert(req->payload_size != 0);
assert(req->payload.type == NVME_PAYLOAD_TYPE_SGL);
assert(req->payload.u.sgl.reset_sgl_fn != NULL);
assert(req->payload.u.sgl.next_sge_fn != NULL);
req->payload.u.sgl.reset_sgl_fn(req->payload.u.sgl.cb_arg, req->payload_offset);
sgl = tr->u.sgl;
req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_SGL;
req->cmd.dptr.sgl1.unkeyed.subtype = 0;
remaining_transfer_len = req->payload_size;
while (remaining_transfer_len > 0) {
if (nseg >= NVME_MAX_SGL_DESCRIPTORS) {
_nvme_fail_request_bad_vtophys(qpair, tr);
return -1;
}
rc = req->payload.u.sgl.next_sge_fn(req->payload.u.sgl.cb_arg, &phys_addr, &length);
if (rc) {
_nvme_fail_request_bad_vtophys(qpair, tr);
return -1;
}
length = nvme_min(remaining_transfer_len, length);
remaining_transfer_len -= length;
sgl->unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
sgl->unkeyed.length = length;
sgl->address = phys_addr;
sgl->unkeyed.subtype = 0;
sgl++;
nseg++;
}
if (nseg == 1) {
/*
* The whole transfer can be described by a single SGL descriptor.
* Use the special case described by the spec where SGL1's type is Data Block.
* This means the SGL in the tracker is not used at all, so copy the first (and only)
* SGL element into SGL1.
*/
req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
req->cmd.dptr.sgl1.address = tr->u.sgl[0].address;
req->cmd.dptr.sgl1.unkeyed.length = tr->u.sgl[0].unkeyed.length;
} else {
/* For now we can only support 1 SGL segment in NVMe controller */
req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT;
req->cmd.dptr.sgl1.address = tr->prp_sgl_bus_addr;
req->cmd.dptr.sgl1.unkeyed.length = nseg * sizeof(struct spdk_nvme_sgl_descriptor);
}
return 0;
}
/**
* Build PRP list describing scattered payload buffer.
*/
static int
_nvme_qpair_build_prps_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
struct nvme_tracker *tr)
{
int rc;
uint64_t phys_addr;
uint32_t data_transferred, remaining_transfer_len, length;
uint32_t nseg, cur_nseg, total_nseg, last_nseg, modulo, unaligned;
uint32_t sge_count = 0;
uint64_t prp2 = 0;
/*
* Build scattered payloads.
*/
assert(req->payload.type == NVME_PAYLOAD_TYPE_SGL);
assert(req->payload.u.sgl.reset_sgl_fn != NULL);
req->payload.u.sgl.reset_sgl_fn(req->payload.u.sgl.cb_arg, req->payload_offset);
remaining_transfer_len = req->payload_size;
total_nseg = 0;
last_nseg = 0;
while (remaining_transfer_len > 0) {
assert(req->payload.u.sgl.next_sge_fn != NULL);
rc = req->payload.u.sgl.next_sge_fn(req->payload.u.sgl.cb_arg, &phys_addr, &length);
if (rc) {
_nvme_fail_request_bad_vtophys(qpair, tr);
return -1;
}
/* Confirm that this sge is prp compatible. */
if (phys_addr & 0x3 ||
(length < remaining_transfer_len && ((phys_addr + length) & (PAGE_SIZE - 1)))) {
_nvme_fail_request_bad_vtophys(qpair, tr);
return -1;
}
data_transferred = nvme_min(remaining_transfer_len, length);
nseg = data_transferred >> nvme_u32log2(PAGE_SIZE);
modulo = data_transferred & (PAGE_SIZE - 1);
unaligned = phys_addr & (PAGE_SIZE - 1);
if (modulo || unaligned) {
nseg += 1 + ((modulo + unaligned - 1) >> nvme_u32log2(PAGE_SIZE));
}
if (total_nseg == 0) {
req->cmd.psdt = SPDK_NVME_PSDT_PRP;
req->cmd.dptr.prp.prp1 = phys_addr;
phys_addr -= unaligned;
}
total_nseg += nseg;
sge_count++;
remaining_transfer_len -= data_transferred;
if (total_nseg == 2) {
if (sge_count == 1)
tr->req->cmd.dptr.prp.prp2 = phys_addr + PAGE_SIZE;
else if (sge_count == 2)
tr->req->cmd.dptr.prp.prp2 = phys_addr;
/* save prp2 value */
prp2 = tr->req->cmd.dptr.prp.prp2;
} else if (total_nseg > 2) {
if (sge_count == 1)
cur_nseg = 1;
else
cur_nseg = 0;
tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_sgl_bus_addr;
while (cur_nseg < nseg) {
if (prp2) {
tr->u.prp[0] = prp2;
tr->u.prp[last_nseg + 1] = phys_addr + cur_nseg * PAGE_SIZE;
} else
tr->u.prp[last_nseg] = phys_addr + cur_nseg * PAGE_SIZE;
last_nseg++;
cur_nseg++;
}
}
}
return 0;
qpair->transport->qpair_destroy(qpair);
}
int
nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
{
int rc = 0;
struct nvme_tracker *tr;
struct nvme_request *child_req, *tmp;
struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
bool child_req_failed = false;
@ -899,8 +379,6 @@ nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *re
return -ENXIO;
}
nvme_qpair_check_enabled(qpair);
if (req->num_children) {
/*
* This is a split (parent) request. Submit all of the children but not the parent
@ -920,102 +398,14 @@ nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *re
return rc;
}
tr = LIST_FIRST(&qpair->free_tr);
if (tr == NULL || !qpair->is_enabled) {
/*
* No tracker is available, or the qpair is disabled due to
* an in-progress controller-level reset.
*
* Put the request on the qpair's request queue to be
* processed when a tracker frees up via a command
* completion or when the controller reset is
* completed.
*/
STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq);
return 0;
}
LIST_REMOVE(tr, list); /* remove tr from free_tr */
LIST_INSERT_HEAD(&qpair->outstanding_tr, tr, list);
tr->req = req;
req->cmd.cid = tr->cid;
if (req->payload_size == 0) {
/* Null payload - leave PRP fields zeroed */
} else if (req->payload.type == NVME_PAYLOAD_TYPE_CONTIG) {
rc = _nvme_qpair_build_contig_request(qpair, req, tr);
if (rc < 0) {
return rc;
}
} else if (req->payload.type == NVME_PAYLOAD_TYPE_SGL) {
if (ctrlr->flags & SPDK_NVME_CTRLR_SGL_SUPPORTED)
rc = _nvme_qpair_build_hw_sgl_request(qpair, req, tr);
else
rc = _nvme_qpair_build_prps_sgl_request(qpair, req, tr);
if (rc < 0) {
return rc;
}
} else {
assert(0);
_nvme_fail_request_bad_vtophys(qpair, tr);
return -EINVAL;
}
nvme_qpair_submit_tracker(qpair, tr);
return 0;
}
void
nvme_qpair_reset(struct spdk_nvme_qpair *qpair)
{
qpair->sq_tail = qpair->cq_head = 0;
/*
* First time through the completion queue, HW will set phase
* bit on completions to 1. So set this to 1 here, indicating
* we're looking for a 1 to know which entries have completed.
* we'll toggle the bit each time when the completion queue
* rolls over.
*/
qpair->phase = 1;
memset(qpair->cmd, 0,
qpair->num_entries * sizeof(struct spdk_nvme_cmd));
memset(qpair->cpl, 0,
qpair->num_entries * sizeof(struct spdk_nvme_cpl));
}
static void
_nvme_admin_qpair_enable(struct spdk_nvme_qpair *qpair)
{
struct nvme_tracker *tr;
struct nvme_tracker *tr_temp;
/*
* Manually abort each outstanding admin command. Do not retry
* admin commands found here, since they will be left over from
* a controller reset and its likely the context in which the
* command was issued no longer applies.
*/
LIST_FOREACH_SAFE(tr, &qpair->outstanding_tr, list, tr_temp) {
SPDK_ERRLOG("aborting outstanding admin command\n");
nvme_qpair_manual_complete_tracker(qpair, tr, SPDK_NVME_SCT_GENERIC,
SPDK_NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, true);
}
qpair->is_enabled = true;
return qpair->transport->qpair_submit_request(qpair, req);
}
static void
_nvme_io_qpair_enable(struct spdk_nvme_qpair *qpair)
{
struct nvme_tracker *tr;
struct nvme_tracker *temp;
struct nvme_request *req;
qpair->is_enabled = true;
/* Manually abort each queued I/O. */
while (!STAILQ_EMPTY(&qpair->queued_req)) {
req = STAILQ_FIRST(&qpair->queued_req);
@ -1024,52 +414,30 @@ _nvme_io_qpair_enable(struct spdk_nvme_qpair *qpair)
nvme_qpair_manual_complete_request(qpair, req, SPDK_NVME_SCT_GENERIC,
SPDK_NVME_SC_ABORTED_BY_REQUEST, true);
}
/* Manually abort each outstanding I/O. */
LIST_FOREACH_SAFE(tr, &qpair->outstanding_tr, list, temp) {
SPDK_ERRLOG("aborting outstanding i/o\n");
nvme_qpair_manual_complete_tracker(qpair, tr, SPDK_NVME_SCT_GENERIC,
SPDK_NVME_SC_ABORTED_BY_REQUEST, 0, true);
}
}
void
nvme_qpair_enable(struct spdk_nvme_qpair *qpair)
{
qpair->is_enabled = true;
if (nvme_qpair_is_io_queue(qpair)) {
_nvme_io_qpair_enable(qpair);
} else {
_nvme_admin_qpair_enable(qpair);
}
}
static void
_nvme_admin_qpair_disable(struct spdk_nvme_qpair *qpair)
{
qpair->is_enabled = false;
nvme_admin_qpair_abort_aers(qpair);
}
static void
_nvme_io_qpair_disable(struct spdk_nvme_qpair *qpair)
{
qpair->is_enabled = false;
qpair->transport->qpair_enable(qpair);
}
void
nvme_qpair_disable(struct spdk_nvme_qpair *qpair)
{
if (nvme_qpair_is_io_queue(qpair)) {
_nvme_io_qpair_disable(qpair);
} else {
_nvme_admin_qpair_disable(qpair);
}
qpair->is_enabled = false;
qpair->transport->qpair_disable(qpair);
}
void
nvme_qpair_fail(struct spdk_nvme_qpair *qpair)
{
struct nvme_tracker *tr;
struct nvme_request *req;
while (!STAILQ_EMPTY(&qpair->queued_req)) {
@ -1080,15 +448,5 @@ nvme_qpair_fail(struct spdk_nvme_qpair *qpair)
SPDK_NVME_SC_ABORTED_BY_REQUEST, true);
}
/* Manually abort each outstanding I/O. */
while (!LIST_EMPTY(&qpair->outstanding_tr)) {
tr = LIST_FIRST(&qpair->outstanding_tr);
/*
* Do not remove the tracker. The abort_tracker path will
* do that for us.
*/
SPDK_ERRLOG("failing outstanding i/o\n");
nvme_qpair_manual_complete_tracker(qpair, tr, SPDK_NVME_SCT_GENERIC,
SPDK_NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, true);
}
qpair->transport->qpair_fail(qpair);
}

14
test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c
View File

@ -152,6 +152,11 @@ ut_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *
return 0;
}
static void
ut_qpair_reset(struct spdk_nvme_qpair *qpair)
{
}
static const struct spdk_nvme_transport nvme_ctrlr_ut_transport = {
.ctrlr_get_pci_id = ut_ctrlr_get_pci_id,
@ -163,6 +168,8 @@ static const struct spdk_nvme_transport nvme_ctrlr_ut_transport = {
.ctrlr_create_io_qpair = ut_ctrlr_create_io_qpair,
.ctrlr_delete_io_qpair = ut_ctrlr_delete_io_qpair,
.qpair_reset = ut_qpair_reset,
};
uint16_t
@ -196,7 +203,7 @@ spdk_pci_device_compare_addr(struct spdk_pci_device *dev, struct spdk_pci_addr *
}
int nvme_qpair_construct(struct spdk_nvme_qpair *qpair, uint16_t id,
uint16_t num_entries, uint16_t num_trackers,
uint16_t num_entries,
struct spdk_nvme_ctrlr *ctrlr)
{
qpair->id = id;
@ -265,11 +272,6 @@ nvme_qpair_enable(struct spdk_nvme_qpair *qpair)
{
}
void
nvme_qpair_reset(struct spdk_nvme_qpair *qpair)
{
}
void
nvme_completion_poll_cb(void *arg, const struct spdk_nvme_cpl *cpl)
{

80
test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c
View File

@ -63,6 +63,7 @@ struct io_request {
bool invalid_second_addr;
};
#if 0 /* TODO: move to PCIe-specific unit test */
static void nvme_request_reset_sgl(void *cb_arg, uint32_t sgl_offset)
{
struct io_request *req = (struct io_request *)cb_arg;
@ -117,6 +118,7 @@ static int nvme_request_next_sge(void *cb_arg, uint64_t *address, uint32_t *leng
}
}
#endif
struct nvme_request *
nvme_allocate_request(const struct nvme_payload *payload, uint32_t payload_size,
@ -187,6 +189,39 @@ nvme_ctrlr_alloc_cmb(struct spdk_nvme_ctrlr *ctrlr, uint64_t length, uint64_t al
return -1;
}
static int
ut_qpair_construct(struct spdk_nvme_qpair *qpair)
{
return 0;
}
static void
ut_qpair_destroy(struct spdk_nvme_qpair *qpair)
{
}
static int
ut_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
{
// TODO
return 0;
}
static int32_t
ut_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions)
{
// TODO
return 0;
}
static const struct spdk_nvme_transport nvme_qpair_ut_transport = {
.qpair_construct = ut_qpair_construct,
.qpair_destroy = ut_qpair_destroy,
.qpair_submit_request = ut_qpair_submit_request,
.qpair_process_completions = ut_qpair_process_completions,
};
static void
prepare_submit_request_test(struct spdk_nvme_qpair *qpair,
struct spdk_nvme_ctrlr *ctrlr,
@ -194,9 +229,10 @@ prepare_submit_request_test(struct spdk_nvme_qpair *qpair,
{
memset(ctrlr, 0, sizeof(*ctrlr));
ctrlr->regs = regs;
ctrlr->transport = &nvme_qpair_ut_transport;
TAILQ_INIT(&ctrlr->free_io_qpairs);
TAILQ_INIT(&ctrlr->active_io_qpairs);
nvme_qpair_construct(qpair, 1, 128, 32, ctrlr);
nvme_qpair_construct(qpair, 1, 128, ctrlr);
CU_ASSERT(qpair->sq_tail == 0);
CU_ASSERT(qpair->cq_head == 0);
@ -210,6 +246,7 @@ cleanup_submit_request_test(struct spdk_nvme_qpair *qpair)
nvme_qpair_destroy(qpair);
}
#if 0 /* TODO: move to PCIe-specific unit test */
static void
ut_insert_cq_entry(struct spdk_nvme_qpair *qpair, uint32_t slot)
{
@ -232,6 +269,7 @@ ut_insert_cq_entry(struct spdk_nvme_qpair *qpair, uint32_t slot)
cpl->status.p = qpair->phase;
cpl->cid = tr->cid;
}
#endif
static void
expected_success_callback(void *arg, const struct spdk_nvme_cpl *cpl)
@ -250,10 +288,8 @@ test3(void)
{
struct spdk_nvme_qpair qpair = {};
struct nvme_request *req;
struct nvme_tracker *tr;
struct spdk_nvme_ctrlr ctrlr = {};
struct spdk_nvme_registers regs = {};
uint16_t cid;
prepare_submit_request_test(&qpair, &ctrlr, &regs);
@ -264,26 +300,12 @@ test3(void)
CU_ASSERT(nvme_qpair_submit_request(&qpair, req) == 0);
CU_ASSERT(qpair.sq_tail == 1);
/*
* Since sq_tail was 0 when the command was submitted, it is in cmd[0].
* Extract its command ID to retrieve its tracker.
*/
cid = qpair.cmd[0].cid;
tr = &qpair.tr[cid];
SPDK_CU_ASSERT_FATAL(tr != NULL);
/*
* Complete the tracker so that it is returned to the free list.
* This also frees the request.
*/
nvme_qpair_manual_complete_tracker(&qpair, tr, SPDK_NVME_SCT_GENERIC, SPDK_NVME_SC_SUCCESS, 0,
false);
nvme_free_request(req);
cleanup_submit_request_test(&qpair);
}
#if 0 /* TODO: move to PCIe-specific unit test */
static void
test4(void)
{
@ -314,7 +336,6 @@ test4(void)
cleanup_submit_request_test(&qpair);
}
static void
test_sgl_req(void)
{
@ -457,7 +478,7 @@ test_hw_sgl_req(void)
cleanup_submit_request_test(&qpair);
nvme_free_request(req);
}
#endif
static void
test_ctrlr_failed(void)
@ -498,6 +519,8 @@ static void struct_packing(void)
CU_ASSERT(offsetof(struct spdk_nvme_qpair, ctrlr) <= 128);
}
#if 0 /* TODO: move to PCIe-specific unit test */
static void test_nvme_qpair_fail(void)
{
struct spdk_nvme_qpair qpair = {};
@ -528,6 +551,7 @@ static void test_nvme_qpair_fail(void)
cleanup_submit_request_test(&qpair);
}
#endif
static void test_nvme_qpair_process_completions(void)
{
@ -543,6 +567,7 @@ static void test_nvme_qpair_process_completions(void)
cleanup_submit_request_test(&qpair);
}
#if 0 /* TODO: move to PCIe-specific unit test */
static void
test_nvme_qpair_process_completions_limit(void)
{
@ -587,11 +612,11 @@ static void test_nvme_qpair_destroy(void)
TAILQ_INIT(&ctrlr.free_io_qpairs);
TAILQ_INIT(&ctrlr.active_io_qpairs);
nvme_qpair_construct(&qpair, 1, 128, 32, &ctrlr);
nvme_qpair_construct(&qpair, 1, 128, &ctrlr);
nvme_qpair_destroy(&qpair);
nvme_qpair_construct(&qpair, 0, 128, 32, &ctrlr);
nvme_qpair_construct(&qpair, 0, 128, &ctrlr);
tr_temp = LIST_FIRST(&qpair.free_tr);
SPDK_CU_ASSERT_FATAL(tr_temp != NULL);
LIST_REMOVE(tr_temp, list);
@ -605,6 +630,7 @@ static void test_nvme_qpair_destroy(void)
nvme_qpair_destroy(&qpair);
CU_ASSERT(LIST_EMPTY(&qpair.outstanding_tr));
}
#endif
static void test_nvme_completion_is_retry(void)
{
@ -743,21 +769,29 @@ int main(int argc, char **argv)
}
if (CU_add_test(suite, "test3", test3) == NULL
#if 0
|| CU_add_test(suite, "test4", test4) == NULL
#endif
|| CU_add_test(suite, "ctrlr_failed", test_ctrlr_failed) == NULL
|| CU_add_test(suite, "struct_packing", struct_packing) == NULL
#if 0
|| CU_add_test(suite, "nvme_qpair_fail", test_nvme_qpair_fail) == NULL
#endif
|| CU_add_test(suite, "spdk_nvme_qpair_process_completions",
test_nvme_qpair_process_completions) == NULL
#if 0
|| CU_add_test(suite, "spdk_nvme_qpair_process_completions_limit",
test_nvme_qpair_process_completions_limit) == NULL
|| CU_add_test(suite, "nvme_qpair_destroy", test_nvme_qpair_destroy) == NULL
#endif
|| CU_add_test(suite, "nvme_completion_is_retry", test_nvme_completion_is_retry) == NULL
#ifdef DEBUG
|| CU_add_test(suite, "get_status_string", test_get_status_string) == NULL
#endif
#if 0
|| CU_add_test(suite, "sgl_request", test_sgl_req) == NULL
|| CU_add_test(suite, "hw_sgl_request", test_hw_sgl_req) == NULL
#endif
) {
CU_cleanup_registry();
return CU_get_error();