ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lib/idxd: replace bit arrays with lists

Browse Source 
The bit arrays were used for dynamic flow control in a previous
implementation.  They are no longer needed as flow control is
now static and managed solely in the idxd plug in module. Use
simple lists of descriptors and completion records instead.

This is a simpler implementation and will allow for some future
clean up of structures as well.

Signed-off-by: paul luse <paul.e.luse@intel.com>
Change-Id: I8c4cce12e88ac5416e3fe29a416487759214ec9f
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/8922
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ziye Yang <ziye.yang@intel.com>
This commit is contained in:
paul luse 2021-07-23 16:44:47 -04:00 committed by Tomasz Zawadzki
parent a8ca2e74f9
commit e5a19b3ea2
4 changed files with 139 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/idxd/Makefile
View File

@ -34,7 +34,7 @@
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..) SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
SO_VER := 5 SO_VER := 6
SO_MINOR := 0 SO_MINOR := 0
C_SRCS = idxd.c idxd_user.c C_SRCS = idxd.c idxd_user.c

268
lib/idxd/idxd.c
View File

@ -84,10 +84,10 @@ struct device_config g_dev_cfg1 = {
static inline void static inline void
_submit_to_hw(struct spdk_idxd_io_channel *chan, struct idxd_hw_desc *desc) _submit_to_hw(struct spdk_idxd_io_channel *chan, struct idxd_hw_desc *desc)
{ {
/* TODO: move the addition of the op to the outstanding list to here. */
movdir64b(chan->portal + chan->portal_offset, desc); movdir64b(chan->portal + chan->portal_offset, desc);
chan->portal_offset = (chan->portal_offset + chan->idxd->chan_per_device * PORTAL_STRIDE) & chan->portal_offset = (chan->portal_offset + chan->idxd->chan_per_device * PORTAL_STRIDE) &
PORTAL_MASK; PORTAL_MASK;
} }
struct spdk_idxd_io_channel * struct spdk_idxd_io_channel *
@ -128,9 +128,10 @@ spdk_idxd_get_channel(struct spdk_idxd_device *idxd)
pthread_mutex_unlock(&idxd->num_channels_lock); pthread_mutex_unlock(&idxd->num_channels_lock);
chan->idxd = idxd; chan->idxd = idxd;
TAILQ_INIT(&chan->ops_pool);
TAILQ_INIT(&chan->batches); TAILQ_INIT(&chan->batches);
TAILQ_INIT(&chan->batch_pool); TAILQ_INIT(&chan->batch_pool);
TAILQ_INIT(&chan->comp_ctx_oustanding); TAILQ_INIT(&chan->ops_outstanding);
batch = chan->batch_base; batch = chan->batch_base;
for (i = 0 ; i < NUM_BATCHES_PER_CHANNEL ; i++) { for (i = 0 ; i < NUM_BATCHES_PER_CHANNEL ; i++) {
@ -154,12 +155,11 @@ spdk_idxd_put_channel(struct spdk_idxd_io_channel *chan)
pthread_mutex_unlock(&chan->idxd->num_channels_lock); pthread_mutex_unlock(&chan->idxd->num_channels_lock);
assert(TAILQ_EMPTY(&chan->batches)); assert(TAILQ_EMPTY(&chan->batches));
spdk_free(chan->completions); spdk_free(chan->ops_base);
spdk_free(chan->desc); spdk_free(chan->desc_base);
spdk_bit_array_free(&chan->ring_slots);
while ((batch = TAILQ_FIRST(&chan->batch_pool))) { while ((batch = TAILQ_FIRST(&chan->batch_pool))) {
TAILQ_REMOVE(&chan->batch_pool, batch, link); TAILQ_REMOVE(&chan->batch_pool, batch, link);
spdk_free(batch->user_completions); spdk_free(batch->user_ops);
spdk_free(batch->user_desc); spdk_free(batch->user_desc);
} }
free(chan->batch_base); free(chan->batch_base);
@ -179,7 +179,9 @@ int
spdk_idxd_configure_chan(struct spdk_idxd_io_channel *chan) spdk_idxd_configure_chan(struct spdk_idxd_io_channel *chan)
{ {
struct idxd_batch *batch; struct idxd_batch *batch;
int rc, num_ring_slots; int rc, num_descriptors, i;
struct idxd_hw_desc *desc;
struct idxd_ops *op;
assert(chan != NULL); assert(chan != NULL);
@ -189,33 +191,30 @@ spdk_idxd_configure_chan(struct spdk_idxd_io_channel *chan)
chan->idxd->wq_id = 0; chan->idxd->wq_id = 0;
} }
num_ring_slots = chan->idxd->queues[chan->idxd->wq_id].wqcfg.wq_size / chan->idxd->chan_per_device; num_descriptors = chan->idxd->queues[chan->idxd->wq_id].wqcfg.wq_size / chan->idxd->chan_per_device;
chan->ring_slots = spdk_bit_array_create(num_ring_slots); chan->desc_base = desc = spdk_zmalloc(num_descriptors * sizeof(struct idxd_hw_desc),
if (chan->ring_slots == NULL) { 0x40, NULL,
SPDK_ERRLOG("Failed to allocate bit array for ring\n"); SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
if (chan->desc_base == NULL) {
SPDK_ERRLOG("Failed to allocate descriptor memory\n");
return -ENOMEM; return -ENOMEM;
} }
/* Store the size of the ring. */ chan->ops_base = op = spdk_zmalloc(num_descriptors * sizeof(struct idxd_ops),
chan->ring_size = num_ring_slots;
chan->desc = spdk_zmalloc(num_ring_slots * sizeof(struct idxd_hw_desc),
0x40, NULL, 0x40, NULL,
SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
if (chan->desc == NULL) { if (chan->ops_base == NULL) {
SPDK_ERRLOG("Failed to allocate descriptor memory\n");
rc = -ENOMEM;
goto err_desc;
}
chan->completions = spdk_zmalloc(num_ring_slots * sizeof(struct idxd_comp),
0x40, NULL,
SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
if (chan->completions == NULL) {
SPDK_ERRLOG("Failed to allocate completion memory\n"); SPDK_ERRLOG("Failed to allocate completion memory\n");
rc = -ENOMEM; rc = -ENOMEM;
goto err_comp; goto err_op;
}
for (i = 0; i < num_descriptors; i++) {
TAILQ_INSERT_TAIL(&chan->ops_pool, op, link);
op->desc = desc;
op++;
desc++;
} }
/* Populate the batches */ /* Populate the batches */
@ -226,16 +225,16 @@ spdk_idxd_configure_chan(struct spdk_idxd_io_channel *chan)
if (batch->user_desc == NULL) { if (batch->user_desc == NULL) {
SPDK_ERRLOG("Failed to allocate batch descriptor memory\n"); SPDK_ERRLOG("Failed to allocate batch descriptor memory\n");
rc = -ENOMEM; rc = -ENOMEM;
goto err_user_desc_or_comp; goto err_user_desc_or_op;
} }
batch->user_completions = spdk_zmalloc(DESC_PER_BATCH * sizeof(struct idxd_comp), batch->user_ops = spdk_zmalloc(DESC_PER_BATCH * sizeof(struct idxd_ops),
0x40, NULL, 0x40, NULL,
SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
if (batch->user_completions == NULL) { if (batch->user_ops == NULL) {
SPDK_ERRLOG("Failed to allocate user completion memory\n"); SPDK_ERRLOG("Failed to allocate user completion memory\n");
rc = -ENOMEM; rc = -ENOMEM;
goto err_user_desc_or_comp; goto err_user_desc_or_op;
} }
} }
@ -243,20 +242,18 @@ spdk_idxd_configure_chan(struct spdk_idxd_io_channel *chan)
return 0; return 0;
err_user_desc_or_comp: err_user_desc_or_op:
TAILQ_FOREACH(batch, &chan->batch_pool, link) { TAILQ_FOREACH(batch, &chan->batch_pool, link) {
spdk_free(batch->user_desc); spdk_free(batch->user_desc);
batch->user_desc = NULL; batch->user_desc = NULL;
spdk_free(batch->user_completions); spdk_free(batch->user_ops);
batch->user_completions = NULL; batch->user_ops = NULL;
} }
spdk_free(chan->completions); spdk_free(chan->ops_base);
chan->completions = NULL; chan->ops_base = NULL;
err_comp: err_op:
spdk_free(chan->desc); spdk_free(chan->desc_base);
chan->desc = NULL; chan->desc_base = NULL;
err_desc:
spdk_bit_array_free(&chan->ring_slots);
return rc; return rc;
} }
@ -333,23 +330,6 @@ spdk_idxd_detach(struct spdk_idxd_device *idxd)
idxd_device_destruct(idxd); idxd_device_destruct(idxd);
} }
inline static void
_track_comp(struct spdk_idxd_io_channel *chan, bool batch_op, uint32_t index,
struct idxd_comp *comp_ctx, struct idxd_hw_desc *desc, struct idxd_batch *batch)
{
comp_ctx->desc = desc;
comp_ctx->index = index;
/* Tag this as a batched operation or not so we know which bit array index to clear. */
comp_ctx->batch_op = batch_op;
/* Only add non-batch completions here. Batch completions are added when the batch is
* submitted.
*/
if (batch_op == false) {
TAILQ_INSERT_TAIL(&chan->comp_ctx_oustanding, comp_ctx, link);
}
}
inline static int inline static int
_vtophys(const void *buf, uint64_t *buf_addr, uint64_t size) _vtophys(const void *buf, uint64_t *buf_addr, uint64_t size)
{ {
@ -372,37 +352,36 @@ _vtophys(const void *buf, uint64_t *buf_addr, uint64_t size)
static int static int
_idxd_prep_command(struct spdk_idxd_io_channel *chan, spdk_idxd_req_cb cb_fn, _idxd_prep_command(struct spdk_idxd_io_channel *chan, spdk_idxd_req_cb cb_fn,
void *cb_arg, struct idxd_hw_desc **_desc, struct idxd_comp **_comp) void *cb_arg, struct idxd_hw_desc **_desc, struct idxd_ops **_op)
{ {
uint32_t index;
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_comp *comp; struct idxd_ops *op;
uint64_t comp_hw_addr; uint64_t op_hw_addr;
int rc; int rc;
index = spdk_bit_array_find_first_clear(chan->ring_slots, 0); if (!TAILQ_EMPTY(&chan->ops_pool)) {
if (index == UINT32_MAX) { op = *_op = TAILQ_FIRST(&chan->ops_pool);
/* ran out of ring slots */ desc = *_desc = op->desc;
TAILQ_REMOVE(&chan->ops_pool, op, link);
} else {
/* The application needs to handle this, violation of flow control */
return -EBUSY; return -EBUSY;
} }
spdk_bit_array_set(chan->ring_slots, index); /* TODO: pre translate this when mem is allocated. */
rc = _vtophys(&op->hw, &op_hw_addr, sizeof(struct idxd_hw_comp_record));
desc = *_desc = &chan->desc[index];
comp = *_comp = &chan->completions[index];
rc = _vtophys(&comp->hw, &comp_hw_addr, sizeof(struct idxd_hw_comp_record));
if (rc) { if (rc) {
spdk_bit_array_clear(chan->ring_slots, index); TAILQ_INSERT_TAIL(&chan->ops_pool, op, link);
return rc; return rc;
} }
_track_comp(chan, false, index, comp, desc, NULL);
desc->flags = IDXD_FLAG_COMPLETION_ADDR_VALID | IDXD_FLAG_REQUEST_COMPLETION; desc->flags = IDXD_FLAG_COMPLETION_ADDR_VALID | IDXD_FLAG_REQUEST_COMPLETION;
desc->completion_addr = comp_hw_addr; desc->completion_addr = op_hw_addr;
comp->cb_arg = cb_arg; op->cb_arg = cb_arg;
comp->cb_fn = cb_fn; op->cb_fn = cb_fn;
op->batch = NULL;
op->batch_op = false;
TAILQ_INSERT_TAIL(&chan->ops_outstanding, op, link);
return 0; return 0;
} }
@ -412,7 +391,7 @@ spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, void *dst, const void *
uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg) uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_comp *comp; struct idxd_ops *op;
uint64_t src_addr, dst_addr; uint64_t src_addr, dst_addr;
int rc; int rc;
@ -421,7 +400,7 @@ spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, void *dst, const void *
assert(src != NULL); assert(src != NULL);
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &comp); rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -455,7 +434,7 @@ spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan, void *dst1, void *d
const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg) const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_comp *comp; struct idxd_ops *op;
uint64_t src_addr, dst1_addr, dst2_addr; uint64_t src_addr, dst1_addr, dst2_addr;
int rc; int rc;
@ -470,7 +449,7 @@ spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan, void *dst1, void *d
} }
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &comp); rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -509,7 +488,7 @@ spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan, void *src1, const vo
uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg) uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_comp *comp; struct idxd_ops *op;
uint64_t src1_addr, src2_addr; uint64_t src1_addr, src2_addr;
int rc; int rc;
@ -518,7 +497,7 @@ spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan, void *src1, const vo
assert(src2 != NULL); assert(src2 != NULL);
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &comp); rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -550,7 +529,7 @@ spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan, void *dst, uint64_t fil
uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg) uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_comp *comp; struct idxd_ops *op;
uint64_t dst_addr; uint64_t dst_addr;
int rc; int rc;
@ -558,7 +537,7 @@ spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan, void *dst, uint64_t fil
assert(dst != NULL); assert(dst != NULL);
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &comp); rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -587,7 +566,7 @@ spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, uint32_t *crc_dst, vo
spdk_idxd_req_cb cb_fn, void *cb_arg) spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_comp *comp; struct idxd_ops *op;
uint64_t src_addr; uint64_t src_addr;
int rc; int rc;
@ -596,7 +575,7 @@ spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, uint32_t *crc_dst, vo
assert(src != NULL); assert(src != NULL);
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &comp); rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -613,7 +592,7 @@ spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, uint32_t *crc_dst, vo
desc->flags &= IDXD_CLEAR_CRC_FLAGS; desc->flags &= IDXD_CLEAR_CRC_FLAGS;
desc->crc32c.seed = seed; desc->crc32c.seed = seed;
desc->xfer_size = nbytes; desc->xfer_size = nbytes;
comp->crc_dst = crc_dst; op->crc_dst = crc_dst;
/* Submit operation. */ /* Submit operation. */
_submit_to_hw(chan, desc); _submit_to_hw(chan, desc);
@ -627,7 +606,7 @@ spdk_idxd_submit_copy_crc32c(struct spdk_idxd_io_channel *chan, void *dst, void
spdk_idxd_req_cb cb_fn, void *cb_arg) spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_comp *comp; struct idxd_ops *op;
uint64_t src_addr, dst_addr; uint64_t src_addr, dst_addr;
int rc; int rc;
@ -637,7 +616,7 @@ spdk_idxd_submit_copy_crc32c(struct spdk_idxd_io_channel *chan, void *dst, void
assert(crc_dst != NULL); assert(crc_dst != NULL);
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &comp); rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -659,7 +638,7 @@ spdk_idxd_submit_copy_crc32c(struct spdk_idxd_io_channel *chan, void *dst, void
desc->flags &= IDXD_CLEAR_CRC_FLAGS; desc->flags &= IDXD_CLEAR_CRC_FLAGS;
desc->crc32c.seed = seed; desc->crc32c.seed = seed;
desc->xfer_size = nbytes; desc->xfer_size = nbytes;
comp->crc_dst = crc_dst; op->crc_dst = crc_dst;
/* Submit operation. */ /* Submit operation. */
_submit_to_hw(chan, desc); _submit_to_hw(chan, desc);
@ -746,7 +725,7 @@ spdk_idxd_batch_submit(struct spdk_idxd_io_channel *chan, struct idxd_batch *bat
spdk_idxd_req_cb cb_fn, void *cb_arg) spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_comp *comp; struct idxd_ops *op;
uint64_t desc_addr; uint64_t desc_addr;
int i, rc; int i, rc;
@ -767,16 +746,17 @@ spdk_idxd_batch_submit(struct spdk_idxd_io_channel *chan, struct idxd_batch *bat
/* Add the batch elements completion contexts to the outstanding list to be polled. */ /* Add the batch elements completion contexts to the outstanding list to be polled. */
for (i = 0 ; i < batch->index; i++) { for (i = 0 ; i < batch->index; i++) {
TAILQ_INSERT_TAIL(&chan->comp_ctx_oustanding, (struct idxd_comp *)&batch->user_completions[i], TAILQ_INSERT_TAIL(&chan->ops_outstanding, (struct idxd_ops *)&batch->user_ops[i],
link); link);
} }
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &comp); rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &op);
if (rc) { if (rc) {
goto error; goto error;
} }
/* TODO: pre-tranlate these when allocated for max batch size. */
rc = _vtophys(batch->user_desc, &desc_addr, batch->index * sizeof(struct idxd_hw_desc)); rc = _vtophys(batch->user_desc, &desc_addr, batch->index * sizeof(struct idxd_hw_desc));
if (rc) { if (rc) {
rc = -EINVAL; rc = -EINVAL;
@ -787,7 +767,7 @@ spdk_idxd_batch_submit(struct spdk_idxd_io_channel *chan, struct idxd_batch *bat
desc->opcode = IDXD_OPCODE_BATCH; desc->opcode = IDXD_OPCODE_BATCH;
desc->desc_list_addr = desc_addr; desc->desc_list_addr = desc_addr;
desc->desc_count = batch->index; desc->desc_count = batch->index;
comp->batch = batch; op->batch = batch;
assert(batch->index <= DESC_PER_BATCH); assert(batch->index <= DESC_PER_BATCH);
/* Submit operation. */ /* Submit operation. */
@ -797,8 +777,8 @@ spdk_idxd_batch_submit(struct spdk_idxd_io_channel *chan, struct idxd_batch *bat
return 0; return 0;
error: error:
for (i = 0 ; i < batch->index; i++) { for (i = 0 ; i < batch->index; i++) {
comp = TAILQ_LAST(&chan->comp_ctx_oustanding, comp_head); op = TAILQ_LAST(&chan->ops_outstanding, op_head);
TAILQ_REMOVE(&chan->comp_ctx_oustanding, comp, link); TAILQ_REMOVE(&chan->ops_outstanding, op, link);
} }
return rc; return rc;
} }
@ -806,10 +786,10 @@ error:
static int static int
_idxd_prep_batch_cmd(struct spdk_idxd_io_channel *chan, spdk_idxd_req_cb cb_fn, _idxd_prep_batch_cmd(struct spdk_idxd_io_channel *chan, spdk_idxd_req_cb cb_fn,
void *cb_arg, struct idxd_batch *batch, void *cb_arg, struct idxd_batch *batch,
struct idxd_hw_desc **_desc, struct idxd_comp **_comp) struct idxd_hw_desc **_desc, struct idxd_ops **_op)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_comp *comp; struct idxd_ops *op;
if (_is_batch_valid(batch, chan) == false) { if (_is_batch_valid(batch, chan) == false) {
SPDK_ERRLOG("Attempt to add to an invalid batch.\n"); SPDK_ERRLOG("Attempt to add to an invalid batch.\n");
@ -823,17 +803,18 @@ _idxd_prep_batch_cmd(struct spdk_idxd_io_channel *chan, spdk_idxd_req_cb cb_fn,
} }
desc = *_desc = &batch->user_desc[batch->index]; desc = *_desc = &batch->user_desc[batch->index];
comp = *_comp = &batch->user_completions[batch->index]; op = *_op = &batch->user_ops[batch->index];
_track_comp(chan, true, batch->index, comp, desc, batch); op->desc = desc;
op->batch_op = true;
SPDK_DEBUGLOG(idxd, "Prep batch %p index %u\n", batch, batch->index); SPDK_DEBUGLOG(idxd, "Prep batch %p index %u\n", batch, batch->index);
batch->index++; batch->index++;
desc->flags = IDXD_FLAG_COMPLETION_ADDR_VALID | IDXD_FLAG_REQUEST_COMPLETION; desc->flags = IDXD_FLAG_COMPLETION_ADDR_VALID | IDXD_FLAG_REQUEST_COMPLETION;
desc->completion_addr = (uintptr_t)&comp->hw; desc->completion_addr = (uintptr_t)&op->hw;
comp->cb_arg = cb_arg; op->cb_arg = cb_arg;
comp->cb_fn = cb_fn; op->cb_fn = cb_fn;
comp->batch = batch; op->batch = batch;
return 0; return 0;
} }
@ -842,11 +823,11 @@ static int
_idxd_batch_prep_nop(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch) _idxd_batch_prep_nop(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_comp *comp; struct idxd_ops *op;
int rc; int rc;
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_batch_cmd(chan, NULL, NULL, batch, &desc, &comp); rc = _idxd_prep_batch_cmd(chan, NULL, NULL, batch, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -865,7 +846,7 @@ spdk_idxd_batch_prep_copy(struct spdk_idxd_io_channel *chan, struct idxd_batch *
void *dst, const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg) void *dst, const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_comp *comp; struct idxd_ops *op;
uint64_t src_addr, dst_addr; uint64_t src_addr, dst_addr;
int rc; int rc;
@ -875,7 +856,7 @@ spdk_idxd_batch_prep_copy(struct spdk_idxd_io_channel *chan, struct idxd_batch *
assert(src != NULL); assert(src != NULL);
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, batch, &desc, &comp); rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, batch, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -905,7 +886,7 @@ spdk_idxd_batch_prep_fill(struct spdk_idxd_io_channel *chan, struct idxd_batch *
spdk_idxd_req_cb cb_fn, void *cb_arg) spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_comp *comp; struct idxd_ops *op;
uint64_t dst_addr; uint64_t dst_addr;
int rc; int rc;
@ -914,7 +895,7 @@ spdk_idxd_batch_prep_fill(struct spdk_idxd_io_channel *chan, struct idxd_batch *
assert(dst != NULL); assert(dst != NULL);
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, batch, &desc, &comp); rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, batch, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -938,7 +919,7 @@ spdk_idxd_batch_prep_dualcast(struct spdk_idxd_io_channel *chan, struct idxd_bat
void *dst1, void *dst2, const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg) void *dst1, void *dst2, const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_comp *comp; struct idxd_ops *op;
uint64_t src_addr, dst1_addr, dst2_addr; uint64_t src_addr, dst1_addr, dst2_addr;
int rc; int rc;
@ -954,7 +935,7 @@ spdk_idxd_batch_prep_dualcast(struct spdk_idxd_io_channel *chan, struct idxd_bat
} }
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, batch, &desc, &comp); rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, batch, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -989,7 +970,7 @@ spdk_idxd_batch_prep_crc32c(struct spdk_idxd_io_channel *chan, struct idxd_batch
spdk_idxd_req_cb cb_fn, void *cb_arg) spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_comp *comp; struct idxd_ops *op;
uint64_t src_addr; uint64_t src_addr;
int rc; int rc;
@ -999,7 +980,7 @@ spdk_idxd_batch_prep_crc32c(struct spdk_idxd_io_channel *chan, struct idxd_batch
assert(src != NULL); assert(src != NULL);
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, batch, &desc, &comp); rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, batch, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -1016,7 +997,7 @@ spdk_idxd_batch_prep_crc32c(struct spdk_idxd_io_channel *chan, struct idxd_batch
desc->flags &= IDXD_CLEAR_CRC_FLAGS; desc->flags &= IDXD_CLEAR_CRC_FLAGS;
desc->crc32c.seed = seed; desc->crc32c.seed = seed;
desc->xfer_size = nbytes; desc->xfer_size = nbytes;
comp->crc_dst = crc_dst; op->crc_dst = crc_dst;
return 0; return 0;
} }
@ -1027,7 +1008,7 @@ spdk_idxd_batch_prep_copy_crc32c(struct spdk_idxd_io_channel *chan, struct idxd_
spdk_idxd_req_cb cb_fn, void *cb_arg) spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_comp *comp; struct idxd_ops *op;
uint64_t src_addr, dst_addr; uint64_t src_addr, dst_addr;
int rc; int rc;
@ -1037,7 +1018,7 @@ spdk_idxd_batch_prep_copy_crc32c(struct spdk_idxd_io_channel *chan, struct idxd_
assert(src != NULL); assert(src != NULL);
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, batch, &desc, &comp); rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, batch, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -1059,7 +1040,7 @@ spdk_idxd_batch_prep_copy_crc32c(struct spdk_idxd_io_channel *chan, struct idxd_
desc->flags &= IDXD_CLEAR_CRC_FLAGS; desc->flags &= IDXD_CLEAR_CRC_FLAGS;
desc->crc32c.seed = seed; desc->crc32c.seed = seed;
desc->xfer_size = nbytes; desc->xfer_size = nbytes;
comp->crc_dst = crc_dst; op->crc_dst = crc_dst;
return 0; return 0;
} }
@ -1070,7 +1051,7 @@ spdk_idxd_batch_prep_compare(struct spdk_idxd_io_channel *chan, struct idxd_batc
void *cb_arg) void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_comp *comp; struct idxd_ops *op;
uint64_t src1_addr, src2_addr; uint64_t src1_addr, src2_addr;
int rc; int rc;
@ -1080,7 +1061,7 @@ spdk_idxd_batch_prep_compare(struct spdk_idxd_io_channel *chan, struct idxd_batc
assert(src2 != NULL); assert(src2 != NULL);
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, batch, &desc, &comp); rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, batch, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -1113,73 +1094,64 @@ _dump_sw_error_reg(struct spdk_idxd_io_channel *chan)
idxd->impl->dump_sw_error(idxd, chan->portal); idxd->impl->dump_sw_error(idxd, chan->portal);
} }
/* TODO: there are multiple ways of getting completions but we can't really pick the best one without /* TODO: more performance experiments. */
* silicon (from a perf perspective at least). The current solution has a large (> cache line) comp_ctx
* struct but only has one polling loop that covers both batch and regular descriptors based on a list
* of comp_ctx that we know have outstanding commands. Another experiment would be to have a 64 byte comp_ctx
* by relying on the bit_array indicies to get all the context we need. This has been implemented in a prev
* version but has the downside of needing to poll multiple ranges of comp records (because of batches) and
* needs to look at each array bit to know whether it should even check that completion record. That may be
* faster though, need to experiment.
*/
#define IDXD_COMPLETION(x) ((x) > (0) ? (1) : (0)) #define IDXD_COMPLETION(x) ((x) > (0) ? (1) : (0))
#define IDXD_FAILURE(x) ((x) > (1) ? (1) : (0)) #define IDXD_FAILURE(x) ((x) > (1) ? (1) : (0))
#define IDXD_SW_ERROR(x) ((x) &= (0x1) ? (1) : (0)) #define IDXD_SW_ERROR(x) ((x) &= (0x1) ? (1) : (0))
int int
spdk_idxd_process_events(struct spdk_idxd_io_channel *chan) spdk_idxd_process_events(struct spdk_idxd_io_channel *chan)
{ {
struct idxd_comp *comp_ctx, *tmp; struct idxd_ops *op, *tmp;
int status = 0; int status = 0;
int rc = 0; int rc = 0;
assert(chan != NULL); assert(chan != NULL);
TAILQ_FOREACH_SAFE(comp_ctx, &chan->comp_ctx_oustanding, link, tmp) { TAILQ_FOREACH_SAFE(op, &chan->ops_outstanding, link, tmp) {
if (rc == MAX_COMPLETIONS_PER_POLL) { if (rc == MAX_COMPLETIONS_PER_POLL) {
break; break;
} }
if (IDXD_COMPLETION(comp_ctx->hw.status)) { if (IDXD_COMPLETION(op->hw.status)) {
TAILQ_REMOVE(&chan->comp_ctx_oustanding, comp_ctx, link); TAILQ_REMOVE(&chan->ops_outstanding, op, link);
rc++; rc++;
if (spdk_unlikely(IDXD_FAILURE(comp_ctx->hw.status))) { if (spdk_unlikely(IDXD_FAILURE(op->hw.status))) {
status = -EINVAL; status = -EINVAL;
_dump_sw_error_reg(chan); _dump_sw_error_reg(chan);
} }
switch (comp_ctx->desc->opcode) { switch (op->desc->opcode) {
case IDXD_OPCODE_BATCH: case IDXD_OPCODE_BATCH:
SPDK_DEBUGLOG(idxd, "Complete batch %p\n", comp_ctx->batch); SPDK_DEBUGLOG(idxd, "Complete batch %p\n", op->batch);
break; break;
case IDXD_OPCODE_CRC32C_GEN: case IDXD_OPCODE_CRC32C_GEN:
case IDXD_OPCODE_COPY_CRC: case IDXD_OPCODE_COPY_CRC:
if (spdk_likely(status == 0)) { if (spdk_likely(status == 0)) {
*comp_ctx->crc_dst = comp_ctx->hw.crc32c_val; *op->crc_dst = op->hw.crc32c_val;
*comp_ctx->crc_dst ^= ~0; *op->crc_dst ^= ~0;
} }
break; break;
case IDXD_OPCODE_COMPARE: case IDXD_OPCODE_COMPARE:
if (spdk_likely(status == 0)) { if (spdk_likely(status == 0)) {
status = comp_ctx->hw.result; status = op->hw.result;
} }
break; break;
} }
if (comp_ctx->cb_fn) { if (op->cb_fn) {
comp_ctx->cb_fn(comp_ctx->cb_arg, status); op->cb_fn(op->cb_arg, status);
} }
/* reinit the status in the completion context */ op->hw.status = status = 0;
comp_ctx->hw.status = status = 0;
if (comp_ctx->batch_op == false) { if (op->batch_op == false) {
assert(spdk_bit_array_get(chan->ring_slots, comp_ctx->index)); TAILQ_INSERT_TAIL(&chan->ops_pool, op, link);
spdk_bit_array_clear(chan->ring_slots, comp_ctx->index);
} }
if (comp_ctx->desc->opcode == IDXD_OPCODE_BATCH) { if (op->desc->opcode == IDXD_OPCODE_BATCH) {
_free_batch(comp_ctx->batch, chan); _free_batch(op->batch, chan);
} }
} else { } else {
/* /*

38
lib/idxd/idxd.h
View File

@ -39,7 +39,6 @@
#include "spdk/idxd.h" #include "spdk/idxd.h"
#include "spdk/queue.h" #include "spdk/queue.h"
#include "spdk/mmio.h" #include "spdk/mmio.h"
#include "spdk/bit_array.h"
#include "idxd_spec.h" #include "idxd_spec.h"
@ -83,7 +82,7 @@ static inline void movdir64b(void *dst, const void *src)
*/ */
struct idxd_batch { struct idxd_batch {
struct idxd_hw_desc *user_desc; struct idxd_hw_desc *user_desc;
struct idxd_comp *user_completions; struct idxd_ops *user_ops;
uint8_t index; uint8_t index;
TAILQ_ENTRY(idxd_batch) link; TAILQ_ENTRY(idxd_batch) link;
}; };
@ -95,37 +94,27 @@ struct device_config {
uint16_t total_engines; uint16_t total_engines;
}; };
struct idxd_comp ; struct idxd_ops;
struct spdk_idxd_io_channel { struct spdk_idxd_io_channel {
struct spdk_idxd_device *idxd; struct spdk_idxd_device *idxd;
/* The portal is the address that we write descriptors to for submission. */ /* The portal is the address that we write descriptors to for submission. */
void *portal; void *portal;
uint32_t portal_offset; uint32_t portal_offset;
uint16_t ring_size;
/* /*
* Descriptors and completions share the same index. User descriptors * User descriptors (those included in a batch) are managed independently from
* (those included in a batch) are managed independently from data descriptors * data descriptors and are located in the batch structure.
* and are located in the batch structure.
*/ */
struct idxd_hw_desc *desc; void *desc_base;
struct idxd_comp *completions; TAILQ_HEAD(, idxd_ops) ops_pool;
/* Current list of oustanding operations to poll. */
/* Current list of oustanding completion addresses to poll. */ TAILQ_HEAD(op_head, idxd_ops) ops_outstanding;
TAILQ_HEAD(comp_head, idxd_comp) comp_ctx_oustanding; void *ops_base;
/*
* We use one bit array to track ring slots for both
* desc and completions.
*
*/
struct spdk_bit_array *ring_slots;
/* Lists of batches, free and in use. */ /* Lists of batches, free and in use. */
TAILQ_HEAD(, idxd_batch) batch_pool; TAILQ_HEAD(, idxd_batch) batch_pool;
TAILQ_HEAD(, idxd_batch) batches; TAILQ_HEAD(, idxd_batch) batches;
void *batch_base; void *batch_base;
}; };
@ -152,19 +141,18 @@ struct idxd_group {
* This struct wraps the hardware completion record which is 32 bytes in * This struct wraps the hardware completion record which is 32 bytes in
* size and must be 32 byte aligned. * size and must be 32 byte aligned.
*/ */
struct idxd_comp { struct idxd_ops {
struct idxd_hw_comp_record hw; struct idxd_hw_comp_record hw;
void *cb_arg; void *cb_arg;
spdk_idxd_req_cb cb_fn; spdk_idxd_req_cb cb_fn;
struct idxd_batch *batch; struct idxd_batch *batch;
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
uint32_t *crc_dst; uint32_t *crc_dst;
uint32_t index;
bool batch_op; bool batch_op;
char pad[3]; char pad[7];
TAILQ_ENTRY(idxd_comp) link; TAILQ_ENTRY(idxd_ops) link;
}; };
SPDK_STATIC_ASSERT(sizeof(struct idxd_comp) == 96, "size mismatch"); SPDK_STATIC_ASSERT(sizeof(struct idxd_ops) == 96, "size mismatch");
struct idxd_wq { struct idxd_wq {
struct spdk_idxd_device *idxd; struct spdk_idxd_device *idxd;

2
mk/spdk.lib_deps.mk
View File

@ -47,7 +47,7 @@ DEPDIRS-rte_vhost :=
DEPDIRS-env_dpdk := log util DEPDIRS-env_dpdk := log util
DEPDIRS-ioat := log DEPDIRS-ioat := log
DEPDIRS-idxd := log util DEPDIRS-idxd := log
DEPDIRS-sock := log $(JSON_LIBS) DEPDIRS-sock := log $(JSON_LIBS)
DEPDIRS-util := log DEPDIRS-util := log
DEPDIRS-vmd := log DEPDIRS-vmd := log