diff --git a/doc/idxd.md b/doc/idxd.md index 9a688caed..a4ed82a89 100644 --- a/doc/idxd.md +++ b/doc/idxd.md @@ -9,10 +9,6 @@ Function | Description --------------------------------------- | ----------- spdk_idxd_probe() | @copybrief spdk_idxd_probe() -spdk_idxd_batch_get_max() | @copybrief spdk_idxd_batch_get_max() -spdk_idxd_batch_create() | @copybrief spdk_idxd_batch_create() -spdk_idxd_batch_prep_copy() | @copybrief spdk_idxd_batch_prep_copy() -spdk_idxd_batch_submit() | @copybrief spdk_idxd_batch_submit() spdk_idxd_submit_copy() | @copybrief spdk_idxd_submit_copy() spdk_idxd_submit_compare() | @copybrief spdk_idxd_submit_compare() spdk_idxd_submit_crc32c() | @copybrief spdk_idxd_submit_crc32c() diff --git a/include/spdk/idxd.h b/include/spdk/idxd.h index 2bb1ff40e..889370b7f 100644 --- a/include/spdk/idxd.h +++ b/include/spdk/idxd.h @@ -56,11 +56,6 @@ struct spdk_idxd_io_channel; */ struct spdk_idxd_device; -/** - * Opaque handle for batching. - */ -struct idxd_batch; - /** * Get the socket that this device is on * @@ -129,87 +124,6 @@ void spdk_idxd_detach(struct spdk_idxd_device *idxd); */ void spdk_idxd_set_config(uint32_t config_number, bool kernel_mode); -/** - * Return the max number of descriptors per batch for IDXD. - * - * \return max number of descriptors per batch. - */ -uint32_t spdk_idxd_batch_get_max(void); - -/** - * Create a batch sequence. - * - * \param chan IDXD channel to submit request. - * - * \return handle to use for subsequent batch requests, NULL on failure. - */ -struct idxd_batch *spdk_idxd_batch_create(struct spdk_idxd_io_channel *chan); - -/** - * Submit a batch sequence. - * - * \param chan IDXD channel to submit request. - * \param batch Handle provided when the batch was started with spdk_idxd_batch_create(). - * \param cb_fn Callback function which will be called when the request is complete. - * \param cb_arg Opaque value which will be passed back as the arg parameter in - * the completion callback. - * - * \return 0 on success, negative errno on failure. - */ -int spdk_idxd_batch_submit(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, - spdk_idxd_req_cb cb_fn, void *cb_arg); - -/** - * Cancel a batch sequence. - * - * \param chan IDXD channel to submit request. - * \param batch Handle provided when the batch was started with spdk_idxd_batch_create(). - * - * \return 0 on success, negative errno on failure. - */ -int spdk_idxd_batch_cancel(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch); - -/** - * Synchronous call to prepare a copy request into a previously initialized batch - * created with spdk_idxd_batch_create(). The callback will be called when the copy - * completes after the batch has been submitted by an asynchronous call to - * spdk_idxd_batch_submit(). - * - * \param chan IDXD channel to submit request. - * \param batch Handle provided when the batch was started with spdk_idxd_batch_create(). - * \param dst Destination virtual address. - * \param src Source virtual address. - * \param nbytes Number of bytes to copy. - * \param cb_fn Callback function which will be called when the request is complete. - * \param cb_arg Opaque value which will be passed back as the arg parameter in - * the completion callback. - * - * \return 0 on success, negative errno on failure. - */ -int spdk_idxd_batch_prep_copy(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, - void *dst, const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg); - -/** - * Synchronous call to prepare a dualcast request into a previously initialized batch - * created with spdk_idxd_batch_create(). The callback will be called when the dualcast - * completes after the batch has been submitted by an asynchronous call to - * spdk_idxd_batch_submit(). - * - * \param chan IDXD channel to submit request. - * \param batch Handle provided when the batch was started with spdk_idxd_batch_create(). - * \param dst1 First destination virtual address (must be 4K aligned). - * \param dst2 Second destination virtual address (must be 4K aligned). - * \param src Source virtual address. - * \param nbytes Number of bytes to copy. - * \param cb_fn Callback function which will be called when the request is complete. - * \param cb_arg Opaque value which will be passed back as the arg parameter in - * the completion callback. - * - * \return 0 on success, negative errno on failure. - */ -int spdk_idxd_batch_prep_dualcast(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, - void *dst1, void *dst2, const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg); - /** * Build and submit an idxd memory copy request. * @@ -253,27 +167,6 @@ int spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan, void *dst1, void *dst2, const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg); -/** - * Synchronous call to prepare a compare request into a previously initialized batch - * created with spdk_idxd_batch_create(). The callback will be called when the compare - * completes after the batch has been submitted by an asynchronous call to - * spdk_idxd_batch_submit(). - * - * \param chan IDXD channel to submit request. - * \param batch Handle provided when the batch was started with spdk_idxd_batch_create(). - * \param src1 First source to compare. - * \param src2 Second source to compare. - * \param nbytes Number of bytes to compare. - * \param cb_fn Callback function which will be called when the request is complete. - * \param cb_arg Opaque value which will be passed back as the arg parameter in - * the completion callback. - * - * \return 0 on success, negative errno on failure. - */ -int spdk_idxd_batch_prep_compare(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, - void *src1, void *src2, uint64_t nbytes, spdk_idxd_req_cb cb_fn, - void *cb_arg); - /** * Build and submit a memory compare request. * @@ -296,26 +189,6 @@ int spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan, struct iovec *siov2, size_t siov2cnt, spdk_idxd_req_cb cb_fn, void *cb_arg); -/** - * Synchronous call to prepare a fill request into a previously initialized batch - * created with spdk_idxd_batch_create(). The callback will be called when the fill - * completes after the batch has been submitted by an asynchronous call to - * spdk_idxd_batch_submit(). - * - * \param chan IDXD channel to submit request. - * \param batch Handle provided when the batch was started with spdk_idxd_batch_create(). - * \param dst Destination virtual address. - * \param fill_pattern Repeating eight-byte pattern to use for memory fill. - * \param nbytes Number of bytes to fill. - * \param cb_fn Callback function which will be called when the request is complete. - * \param cb_arg Opaque value which will be passed back as the arg parameter in - * the completion callback. - * - * \return 0 on success, negative errno on failure. - */ -int spdk_idxd_batch_prep_fill(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, - void *dst, uint64_t fill_pattern, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg); - /** * Build and submit a idxd memory fill request. * @@ -336,28 +209,6 @@ int spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan, struct iovec *diov, size_t diovcnt, uint64_t fill_pattern, spdk_idxd_req_cb cb_fn, void *cb_arg); -/** - * Synchronous call to prepare a crc32c request into a previously initialized batch - * created with spdk_idxd_batch_create(). The callback will be called when the crc32c - * completes after the batch has been submitted by an asynchronous call to - * spdk_idxd_batch_submit(). - * - * \param chan IDXD channel to submit request. - * \param batch Handle provided when the batch was started with spdk_idxd_batch_create(). - * \param crc_dst Resulting calculation. - * \param src Source virtual address. - * \param seed Four byte CRC-32C seed value. - * \param nbytes Number of bytes to calculate on. - * \param cb_fn Callback function which will be called when the request is complete. - * \param cb_arg Opaque value which will be passed back as the arg parameter in - * the completion callback. - * - * \return 0 on success, negative errno on failure. - */ -int spdk_idxd_batch_prep_crc32c(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, - uint32_t *crc_dst, void *src, uint32_t seed, uint64_t nbytes, - spdk_idxd_req_cb cb_fn, void *cb_arg); - /** * Build and submit a memory CRC32-C request. * @@ -380,29 +231,6 @@ int spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, uint32_t seed, uint32_t *crc_dst, spdk_idxd_req_cb cb_fn, void *cb_arg); -/** - * Synchronous call to prepare a copy combined with crc32c request into a previously - * initialized batch created with spdk_idxd_batch_create(). The callback will be called - * when the copy + crc32c completes after the batch has been submitted by an asynchronous - * call to spdk_idxd_batch_submit(). - * - * \param chan IDXD channel to submit request. - * \param batch Handle provided when the batch was started with spdk_idxd_batch_create(). - * \param dst Destination virtual address. - * \param src Source virtual address. - * \param crc_dst Resulting calculation. - * \param seed Four byte CRC-32C seed value. - * \param nbytes Number of bytes to calculate on. - * \param cb_fn Callback function which will be called when the request is complete. - * \param cb_arg Opaque value which will be passed back as the arg parameter in - * the completion callback. - * - * \return 0 on success, negative errno on failure. - */ -int spdk_idxd_batch_prep_copy_crc32c(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, - void *dst, void *src, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes, - spdk_idxd_req_cb cb_fn, void *cb_arg); - /** * Build and submit a copy combined with CRC32-C request. * diff --git a/lib/idxd/idxd.c b/lib/idxd/idxd.c index 16e00d9da..35d03bbde 100644 --- a/lib/idxd/idxd.c +++ b/lib/idxd/idxd.c @@ -464,8 +464,8 @@ _idxd_batch_prep_nop(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch return 0; } -struct idxd_batch * -spdk_idxd_batch_create(struct spdk_idxd_io_channel *chan) +static struct idxd_batch * +idxd_batch_create(struct spdk_idxd_io_channel *chan) { struct idxd_batch *batch; @@ -493,8 +493,8 @@ _free_batch(struct idxd_batch *batch, struct spdk_idxd_io_channel *chan) TAILQ_INSERT_TAIL(&chan->batch_pool, batch, link); } -int -spdk_idxd_batch_cancel(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch) +static int +idxd_batch_cancel(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch) { assert(chan != NULL); assert(batch != NULL); @@ -514,9 +514,9 @@ spdk_idxd_batch_cancel(struct spdk_idxd_io_channel *chan, struct idxd_batch *bat return 0; } -int -spdk_idxd_batch_submit(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, - spdk_idxd_req_cb cb_fn, void *cb_arg) +static int +idxd_batch_submit(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, + spdk_idxd_req_cb cb_fn, void *cb_arg) { struct idxd_hw_desc *desc; struct idxd_ops *op; @@ -638,7 +638,7 @@ spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, cb_fn, cb_arg); } - batch = spdk_idxd_batch_create(chan); + batch = idxd_batch_create(chan); if (!batch) { return -EBUSY; } @@ -667,10 +667,10 @@ spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, desc->xfer_size = len; } - return spdk_idxd_batch_submit(chan, batch, cb_fn, cb_arg); + return idxd_batch_submit(chan, batch, cb_fn, cb_arg); err: - spdk_idxd_batch_cancel(chan, batch); + idxd_batch_cancel(chan, batch); return rc; } @@ -801,7 +801,7 @@ spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan, cb_fn, cb_arg); } - batch = spdk_idxd_batch_create(chan); + batch = idxd_batch_create(chan); if (!batch) { return -EBUSY; } @@ -830,10 +830,10 @@ spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan, desc->xfer_size = len; } - return spdk_idxd_batch_submit(chan, batch, cb_fn, cb_arg); + return idxd_batch_submit(chan, batch, cb_fn, cb_arg); err: - spdk_idxd_batch_cancel(chan, batch); + idxd_batch_cancel(chan, batch); return rc; } @@ -892,7 +892,7 @@ spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan, diov[0].iov_len, cb_fn, cb_arg); } - batch = spdk_idxd_batch_create(chan); + batch = idxd_batch_create(chan); if (!batch) { return -EBUSY; } @@ -915,10 +915,10 @@ spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan, desc->flags |= IDXD_FLAG_CACHE_CONTROL; /* direct IO to CPU cache instead of mem */ } - return spdk_idxd_batch_submit(chan, batch, cb_fn, cb_arg); + return idxd_batch_submit(chan, batch, cb_fn, cb_arg); err: - spdk_idxd_batch_cancel(chan, batch); + idxd_batch_cancel(chan, batch); return rc; } @@ -983,7 +983,7 @@ spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, seed, siov[0].iov_len, cb_fn, cb_arg); } - batch = spdk_idxd_batch_create(chan); + batch = idxd_batch_create(chan); if (!batch) { return -EBUSY; } @@ -1018,10 +1018,10 @@ spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, op->crc_dst = crc_dst; } - return spdk_idxd_batch_submit(chan, batch, cb_fn, cb_arg); + return idxd_batch_submit(chan, batch, cb_fn, cb_arg); err: - spdk_idxd_batch_cancel(chan, batch); + idxd_batch_cancel(chan, batch); return rc; } @@ -1101,7 +1101,7 @@ spdk_idxd_submit_copy_crc32c(struct spdk_idxd_io_channel *chan, crc_dst, seed, siov[0].iov_len, cb_fn, cb_arg); } - batch = spdk_idxd_batch_create(chan); + batch = idxd_batch_create(chan); if (!batch) { return -EBUSY; } @@ -1144,264 +1144,13 @@ spdk_idxd_submit_copy_crc32c(struct spdk_idxd_io_channel *chan, op->crc_dst = crc_dst; } - return spdk_idxd_batch_submit(chan, batch, cb_fn, cb_arg); + return idxd_batch_submit(chan, batch, cb_fn, cb_arg); err: - spdk_idxd_batch_cancel(chan, batch); + idxd_batch_cancel(chan, batch); return rc; } -uint32_t -spdk_idxd_batch_get_max(void) -{ - /* TODO: consider setting this via RPC. */ - return DESC_PER_BATCH; -} - -int -spdk_idxd_batch_prep_copy(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, - void *dst, const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg) -{ - struct idxd_hw_desc *desc; - struct idxd_ops *op; - uint64_t src_addr, dst_addr; - int rc; - - assert(chan != NULL); - assert(batch != NULL); - assert(dst != NULL); - assert(src != NULL); - - /* Common prep. */ - rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, batch, &desc, &op); - if (rc) { - return rc; - } - - rc = _vtophys(src, &src_addr, nbytes); - if (rc) { - return rc; - } - - rc = _vtophys(dst, &dst_addr, nbytes); - if (rc) { - return rc; - } - - /* Command specific. */ - desc->opcode = IDXD_OPCODE_MEMMOVE; - desc->src_addr = src_addr; - desc->dst_addr = dst_addr; - desc->xfer_size = nbytes; - - return 0; -} - -int -spdk_idxd_batch_prep_fill(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, - void *dst, uint64_t fill_pattern, uint64_t nbytes, - spdk_idxd_req_cb cb_fn, void *cb_arg) -{ - struct idxd_hw_desc *desc; - struct idxd_ops *op; - uint64_t dst_addr; - int rc; - - assert(chan != NULL); - assert(batch != NULL); - assert(dst != NULL); - - /* Common prep. */ - rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, batch, &desc, &op); - if (rc) { - return rc; - } - - rc = _vtophys(dst, &dst_addr, nbytes); - if (rc) { - return rc; - } - - /* Command specific. */ - desc->opcode = IDXD_OPCODE_MEMFILL; - desc->pattern = fill_pattern; - desc->dst_addr = dst_addr; - desc->xfer_size = nbytes; - - return 0; -} - -int -spdk_idxd_batch_prep_dualcast(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, - void *dst1, void *dst2, const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg) -{ - struct idxd_hw_desc *desc; - struct idxd_ops *op; - uint64_t src_addr, dst1_addr, dst2_addr; - int rc; - - assert(chan != NULL); - assert(batch != NULL); - assert(dst1 != NULL); - assert(dst2 != NULL); - assert(src != NULL); - - if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) { - SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n"); - return -EINVAL; - } - - /* Common prep. */ - rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, batch, &desc, &op); - if (rc) { - return rc; - } - - rc = _vtophys(src, &src_addr, nbytes); - if (rc) { - return rc; - } - - rc = _vtophys(dst1, &dst1_addr, nbytes); - if (rc) { - return rc; - } - - rc = _vtophys(dst2, &dst2_addr, nbytes); - if (rc) { - return rc; - } - - desc->opcode = IDXD_OPCODE_DUALCAST; - desc->src_addr = src_addr; - desc->dst_addr = dst1_addr; - desc->dest2 = dst2_addr; - desc->xfer_size = nbytes; - - return 0; -} - -int -spdk_idxd_batch_prep_crc32c(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, - uint32_t *crc_dst, void *src, uint32_t seed, uint64_t nbytes, - spdk_idxd_req_cb cb_fn, void *cb_arg) -{ - struct idxd_hw_desc *desc; - struct idxd_ops *op; - uint64_t src_addr; - int rc; - - assert(chan != NULL); - assert(batch != NULL); - assert(crc_dst != NULL); - assert(src != NULL); - - /* Common prep. */ - rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, batch, &desc, &op); - if (rc) { - return rc; - } - - rc = _vtophys(src, &src_addr, nbytes); - if (rc) { - return rc; - } - - /* Command specific. */ - desc->opcode = IDXD_OPCODE_CRC32C_GEN; - desc->dst_addr = 0; /* per specification */ - desc->src_addr = src_addr; - desc->flags &= IDXD_CLEAR_CRC_FLAGS; - desc->crc32c.seed = seed; - desc->xfer_size = nbytes; - op->crc_dst = crc_dst; - - return 0; -} - -int -spdk_idxd_batch_prep_copy_crc32c(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, - void *dst, void *src, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes, - spdk_idxd_req_cb cb_fn, void *cb_arg) -{ - struct idxd_hw_desc *desc; - struct idxd_ops *op; - uint64_t src_addr, dst_addr; - int rc; - - assert(chan != NULL); - assert(batch != NULL); - assert(crc_dst != NULL); - assert(src != NULL); - - /* Common prep. */ - rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, batch, &desc, &op); - if (rc) { - return rc; - } - - rc = _vtophys(src, &src_addr, nbytes); - if (rc) { - return rc; - } - - rc = _vtophys(dst, &dst_addr, nbytes); - if (rc) { - return rc; - } - - /* Command specific. */ - desc->opcode = IDXD_OPCODE_COPY_CRC; - desc->dst_addr = dst_addr; - desc->src_addr = src_addr; - desc->flags &= IDXD_CLEAR_CRC_FLAGS; - desc->crc32c.seed = seed; - desc->xfer_size = nbytes; - op->crc_dst = crc_dst; - - return 0; -} - -int -spdk_idxd_batch_prep_compare(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, - void *src1, void *src2, uint64_t nbytes, spdk_idxd_req_cb cb_fn, - void *cb_arg) -{ - struct idxd_hw_desc *desc; - struct idxd_ops *op; - uint64_t src1_addr, src2_addr; - int rc; - - assert(chan != NULL); - assert(batch != NULL); - assert(src1 != NULL); - assert(src2 != NULL); - - /* Common prep. */ - rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, batch, &desc, &op); - if (rc) { - return rc; - } - - rc = _vtophys(src1, &src1_addr, nbytes); - if (rc) { - return rc; - } - - rc = _vtophys(src2, &src2_addr, nbytes); - if (rc) { - return rc; - } - - /* Command specific. */ - desc->opcode = IDXD_OPCODE_COMPARE; - desc->src_addr = src1_addr; - desc->src2_addr = src2_addr; - desc->xfer_size = nbytes; - - return 0; -} - static inline void _dump_sw_error_reg(struct spdk_idxd_io_channel *chan) {