2022-06-03 19:15:11 +00:00
|
|
|
/* SPDX-License-Identifier: BSD-3-Clause
|
2022-11-01 20:26:26 +00:00
|
|
|
* Copyright (C) 2020 Intel Corporation.
|
2022-10-04 19:16:21 +00:00
|
|
|
* Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES
|
2016-07-19 15:58:55 +00:00
|
|
|
* All rights reserved.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/** \file
|
2022-08-08 20:51:25 +00:00
|
|
|
* Acceleration Framework
|
2016-07-19 15:58:55 +00:00
|
|
|
*/
|
|
|
|
|
2022-08-08 20:51:25 +00:00
|
|
|
#ifndef SPDK_ACCEL_H
|
|
|
|
#define SPDK_ACCEL_H
|
2016-07-19 15:58:55 +00:00
|
|
|
|
2017-05-01 20:22:48 +00:00
|
|
|
#include "spdk/stdinc.h"
|
accel: initial operation chaining support
This patch introduces the concept of chaining multiple accel operations
and executing them all at once in a single step. This means that it
will be possible to schedule accel operations at different layers of the
stack (e.g. copy in NVMe-oF transport, crypto in bdev_crypto), but
execute them all in a single place. Thanks to this, we can take
advantage of hardware accelerators that supports executing multiple
operations as a single operation (e.g. copy + crypto).
This operation group is called spdk_accel_sequence and operations can be
appended to that object via one of the spdk_accel_append_* functions.
New operations are always added at the end of a sequence. Users can
specify a callback to be notified when a particular operation in a
sequence is completed, but they don't receive the status of whether it
was successful or not. This is by design, as they shouldn't care about
the status of an individual operation and should rely on other means to
receive the status of the whole sequence. It's also important to note
that any intermediate steps within a sequence may not produce observable
results. For instance, appending a copy from A to B and then a copy
from B to C, it's indeterminate whether A's data will be in B after a
sequence is executed. It is only guaranteed that A's data will be in C.
A sequence can also be reversed using spdk_accel_sequence_reverse(),
meaning that the first operation becomes last and vice versa. It's
especially useful in read paths, as it makes it possible to build the
sequence during submission, then, once the data is read from storage,
reverse the sequence and execute it.
Finally, there are two ways to terminate a sequence: aborting or
executing. It can be aborted via spdk_accel_sequence_abort() which will
execute individual operations' callbacks and free any allocated
resources. To execute it, one must use spdk_accel_sequence_finish().
For now, each operation is executed one by one and is submitted to the
appropriate accel module. Executing multiple operations as a single one
will be added in the future.
Also, currently, only fill and copy operations can be appended to a
sequence. Support for more operations will be added in subsequent
patches.
Signed-off-by: Konrad Sztyber <konrad.sztyber@intel.com>
Change-Id: Id35d093e14feb59b996f780ef77e000e10bfcd20
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15529
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@nvidia.com>
2022-11-16 07:22:55 +00:00
|
|
|
#include "spdk/dma.h"
|
2016-07-19 15:58:55 +00:00
|
|
|
|
2017-12-07 20:25:19 +00:00
|
|
|
#ifdef __cplusplus
|
|
|
|
extern "C" {
|
|
|
|
#endif
|
|
|
|
|
2022-10-04 19:16:21 +00:00
|
|
|
/** Data Encryption Key identifier */
|
|
|
|
struct spdk_accel_crypto_key;
|
|
|
|
|
2021-09-08 21:29:35 +00:00
|
|
|
/* Flags for accel operations */
|
|
|
|
#define ACCEL_FLAG_PERSISTENT (1 << 0)
|
|
|
|
|
2022-10-04 19:16:21 +00:00
|
|
|
struct spdk_accel_crypto_key_create_param {
|
|
|
|
char *cipher; /**< Cipher to be used for crypto operations */
|
|
|
|
char *hex_key; /**< Hexlified key */
|
|
|
|
char *hex_key2; /**< Hexlified key2 */
|
|
|
|
char *key_name; /**< Key name */
|
|
|
|
};
|
|
|
|
|
2022-03-15 17:43:07 +00:00
|
|
|
enum accel_opcode {
|
|
|
|
ACCEL_OPC_COPY = 0,
|
|
|
|
ACCEL_OPC_FILL = 1,
|
|
|
|
ACCEL_OPC_DUALCAST = 2,
|
|
|
|
ACCEL_OPC_COMPARE = 3,
|
|
|
|
ACCEL_OPC_CRC32C = 4,
|
|
|
|
ACCEL_OPC_COPY_CRC32C = 5,
|
2022-05-20 19:08:38 +00:00
|
|
|
ACCEL_OPC_COMPRESS = 6,
|
|
|
|
ACCEL_OPC_DECOMPRESS = 7,
|
2022-10-04 19:16:21 +00:00
|
|
|
ACCEL_OPC_ENCRYPT = 8,
|
|
|
|
ACCEL_OPC_DECRYPT = 9,
|
|
|
|
ACCEL_OPC_LAST = 10,
|
2020-04-24 16:47:55 +00:00
|
|
|
};
|
|
|
|
|
2018-04-16 04:23:59 +00:00
|
|
|
/**
|
2020-02-05 17:10:05 +00:00
|
|
|
* Acceleration operation callback.
|
2018-04-16 04:23:59 +00:00
|
|
|
*
|
2022-10-17 13:08:26 +00:00
|
|
|
* \param cb_arg Callback argument specified in the spdk_accel_submit* call.
|
2018-04-16 04:23:59 +00:00
|
|
|
* \param status 0 if it completed successfully, or negative errno if it failed.
|
|
|
|
*/
|
2022-10-17 13:08:26 +00:00
|
|
|
typedef void (*spdk_accel_completion_cb)(void *cb_arg, int status);
|
2018-04-16 04:23:59 +00:00
|
|
|
|
|
|
|
/**
|
2022-08-08 20:51:25 +00:00
|
|
|
* Acceleration framework finish callback.
|
2018-04-16 04:23:59 +00:00
|
|
|
*
|
|
|
|
* \param cb_arg Callback argument.
|
|
|
|
*/
|
2020-02-05 17:10:05 +00:00
|
|
|
typedef void (*spdk_accel_fini_cb)(void *cb_arg);
|
2016-07-19 15:58:55 +00:00
|
|
|
|
2018-03-06 08:27:30 +00:00
|
|
|
/**
|
2022-08-08 20:51:25 +00:00
|
|
|
* Initialize the acceleration framework.
|
2018-03-06 08:27:30 +00:00
|
|
|
*
|
|
|
|
* \return 0 on success.
|
|
|
|
*/
|
2022-08-08 20:51:25 +00:00
|
|
|
int spdk_accel_initialize(void);
|
2018-03-06 08:27:30 +00:00
|
|
|
|
|
|
|
/**
|
2022-08-08 20:51:25 +00:00
|
|
|
* Close the acceleration framework.
|
2018-03-06 08:27:30 +00:00
|
|
|
*
|
|
|
|
* \param cb_fn Called when the close operation completes.
|
|
|
|
* \param cb_arg Argument passed to the callback function.
|
|
|
|
*/
|
2022-08-08 20:51:25 +00:00
|
|
|
void spdk_accel_finish(spdk_accel_fini_cb cb_fn, void *cb_arg);
|
2018-03-06 08:27:30 +00:00
|
|
|
|
|
|
|
/**
|
2022-08-08 20:51:25 +00:00
|
|
|
* Get an I/O channel for the acceleration framework.
|
2018-03-06 08:27:30 +00:00
|
|
|
*
|
2022-08-08 20:51:25 +00:00
|
|
|
* This I/O channel is used to submit requests.
|
2018-03-06 08:27:30 +00:00
|
|
|
*
|
|
|
|
* \return a pointer to the I/O channel on success, or NULL on failure.
|
|
|
|
*/
|
2022-08-08 20:51:25 +00:00
|
|
|
struct spdk_io_channel *spdk_accel_get_io_channel(void);
|
2018-03-06 08:27:30 +00:00
|
|
|
|
2022-10-04 19:16:21 +00:00
|
|
|
/**
|
|
|
|
* Create a crypto key with given parameters. Accel module copies content of \b param structure
|
|
|
|
*
|
|
|
|
* \param param Key parameters
|
|
|
|
* \return 0 on success, negated errno on error
|
|
|
|
*/
|
|
|
|
int spdk_accel_crypto_key_create(const struct spdk_accel_crypto_key_create_param *param);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Destroy a crypto key
|
|
|
|
*
|
|
|
|
* \param key Key to destroy
|
|
|
|
* \return 0 on success, negated errno on error
|
|
|
|
*/
|
|
|
|
int spdk_accel_crypto_key_destroy(struct spdk_accel_crypto_key *key);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Find a crypto key structure by name
|
|
|
|
* \param name Key name
|
|
|
|
* \return Crypto key structure or NULL
|
|
|
|
*/
|
|
|
|
struct spdk_accel_crypto_key *spdk_accel_crypto_key_get(const char *name);
|
|
|
|
|
2018-03-06 08:27:30 +00:00
|
|
|
/**
|
|
|
|
* Submit a copy request.
|
|
|
|
*
|
2020-07-03 14:08:47 +00:00
|
|
|
* \param ch I/O channel associated with this call.
|
2018-03-06 08:27:30 +00:00
|
|
|
* \param dst Destination to copy to.
|
|
|
|
* \param src Source to copy from.
|
|
|
|
* \param nbytes Length in bytes to copy.
|
2021-08-24 21:08:29 +00:00
|
|
|
* \param flags Accel framework flags for operations.
|
2020-07-03 14:08:47 +00:00
|
|
|
* \param cb_fn Called when this copy operation completes.
|
|
|
|
* \param cb_arg Callback argument.
|
2018-03-06 08:27:30 +00:00
|
|
|
*
|
|
|
|
* \return 0 on success, negative errno on failure.
|
|
|
|
*/
|
2020-07-03 14:08:47 +00:00
|
|
|
int spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, uint64_t nbytes,
|
2021-08-24 21:08:29 +00:00
|
|
|
int flags, spdk_accel_completion_cb cb_fn, void *cb_arg);
|
2018-03-06 08:27:30 +00:00
|
|
|
|
2020-04-30 15:43:02 +00:00
|
|
|
/**
|
|
|
|
* Submit a dual cast copy request.
|
|
|
|
*
|
2020-07-03 14:08:47 +00:00
|
|
|
* \param ch I/O channel associated with this call.
|
2020-04-30 15:43:02 +00:00
|
|
|
* \param dst1 First destination to copy to (must be 4K aligned).
|
|
|
|
* \param dst2 Second destination to copy to (must be 4K aligned).
|
|
|
|
* \param src Source to copy from.
|
|
|
|
* \param nbytes Length in bytes to copy.
|
2021-08-24 21:08:29 +00:00
|
|
|
* \param flags Accel framework flags for operations.
|
2020-07-03 14:08:47 +00:00
|
|
|
* \param cb_fn Called when this copy operation completes.
|
|
|
|
* \param cb_arg Callback argument.
|
2020-04-30 15:43:02 +00:00
|
|
|
*
|
|
|
|
* \return 0 on success, negative errno on failure.
|
|
|
|
*/
|
2020-07-03 14:08:47 +00:00
|
|
|
int spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, void *dst2, void *src,
|
2021-08-24 21:08:29 +00:00
|
|
|
uint64_t nbytes, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg);
|
2020-04-30 15:43:02 +00:00
|
|
|
|
2020-04-29 22:27:23 +00:00
|
|
|
/**
|
|
|
|
* Submit a compare request.
|
|
|
|
*
|
2020-07-03 14:08:47 +00:00
|
|
|
* \param ch I/O channel associated with this call.
|
2020-04-29 22:27:23 +00:00
|
|
|
* \param src1 First location to perform compare on.
|
|
|
|
* \param src2 Second location to perform compare on.
|
|
|
|
* \param nbytes Length in bytes to compare.
|
2020-07-03 14:08:47 +00:00
|
|
|
* \param cb_fn Called when this compare operation completes.
|
|
|
|
* \param cb_arg Callback argument.
|
2020-04-29 22:27:23 +00:00
|
|
|
*
|
|
|
|
* \return 0 on success, any other value means there was a miscompare.
|
|
|
|
*/
|
2020-07-03 14:08:47 +00:00
|
|
|
int spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, void *src2, uint64_t nbytes,
|
|
|
|
spdk_accel_completion_cb cb_fn, void *cb_arg);
|
2020-04-29 22:27:23 +00:00
|
|
|
|
2018-03-06 08:27:30 +00:00
|
|
|
/**
|
|
|
|
* Submit a fill request.
|
|
|
|
*
|
|
|
|
* This operation will fill the destination buffer with the specified value.
|
|
|
|
*
|
2020-07-03 14:08:47 +00:00
|
|
|
* \param ch I/O channel associated with this call.
|
2018-03-06 08:27:30 +00:00
|
|
|
* \param dst Destination to fill.
|
|
|
|
* \param fill Constant byte to fill to the destination.
|
|
|
|
* \param nbytes Length in bytes to fill.
|
2021-08-24 21:08:29 +00:00
|
|
|
* \param flags Accel framework flags for operations.
|
2020-07-03 14:08:47 +00:00
|
|
|
* \param cb_fn Called when this fill operation completes.
|
|
|
|
* \param cb_arg Callback argument.
|
2018-03-06 08:27:30 +00:00
|
|
|
*
|
|
|
|
* \return 0 on success, negative errno on failure.
|
|
|
|
*/
|
2020-07-03 14:08:47 +00:00
|
|
|
int spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill, uint64_t nbytes,
|
2021-08-24 21:08:29 +00:00
|
|
|
int flags, spdk_accel_completion_cb cb_fn, void *cb_arg);
|
2018-03-06 08:27:30 +00:00
|
|
|
|
2020-04-28 22:20:57 +00:00
|
|
|
/**
|
|
|
|
* Submit a CRC-32C calculation request.
|
|
|
|
*
|
|
|
|
* This operation will calculate the 4 byte CRC32-C for the given data.
|
|
|
|
*
|
2020-07-03 14:08:47 +00:00
|
|
|
* \param ch I/O channel associated with this call.
|
2021-06-09 18:24:49 +00:00
|
|
|
* \param crc_dst Destination to write the CRC-32C to.
|
2020-04-28 22:20:57 +00:00
|
|
|
* \param src The source address for the data.
|
|
|
|
* \param seed Four byte seed value.
|
|
|
|
* \param nbytes Length in bytes.
|
2020-07-03 14:08:47 +00:00
|
|
|
* \param cb_fn Called when this CRC-32C operation completes.
|
|
|
|
* \param cb_arg Callback argument.
|
2020-04-28 22:20:57 +00:00
|
|
|
*
|
|
|
|
* \return 0 on success, negative errno on failure.
|
|
|
|
*/
|
2021-06-09 18:24:49 +00:00
|
|
|
int spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst, void *src,
|
|
|
|
uint32_t seed,
|
2020-07-03 14:08:47 +00:00
|
|
|
uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
|
2020-04-28 22:20:57 +00:00
|
|
|
|
2021-01-18 16:07:49 +00:00
|
|
|
/**
|
|
|
|
* Submit a chained CRC-32C calculation request.
|
|
|
|
*
|
|
|
|
* This operation will calculate the 4 byte CRC32-C for the given data.
|
|
|
|
*
|
|
|
|
* \param ch I/O channel associated with this call.
|
2021-06-09 18:24:49 +00:00
|
|
|
* \param crc_dst Destination to write the CRC-32C to.
|
2021-01-18 16:07:49 +00:00
|
|
|
* \param iovs The io vector array which stores the src data and len.
|
|
|
|
* \param iovcnt The size of the iov.
|
|
|
|
* \param seed Four byte seed value.
|
|
|
|
* \param cb_fn Called when this CRC-32C operation completes.
|
|
|
|
* \param cb_arg Callback argument.
|
|
|
|
*
|
|
|
|
* \return 0 on success, negative errno on failure.
|
|
|
|
*/
|
2021-06-09 18:24:49 +00:00
|
|
|
int spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, struct iovec *iovs,
|
2021-01-18 16:07:49 +00:00
|
|
|
uint32_t iovcnt, uint32_t seed, spdk_accel_completion_cb cb_fn, void *cb_arg);
|
|
|
|
|
2021-06-01 19:17:50 +00:00
|
|
|
/**
|
|
|
|
* Submit a copy with CRC-32C calculation request.
|
|
|
|
*
|
|
|
|
* This operation will copy data and calculate the 4 byte CRC32-C for the given data.
|
|
|
|
*
|
|
|
|
* \param ch I/O channel associated with this call.
|
|
|
|
* \param dst Destination to write the data to.
|
|
|
|
* \param src The source address for the data.
|
|
|
|
* \param crc_dst Destination to write the CRC-32C to.
|
|
|
|
* \param seed Four byte seed value.
|
|
|
|
* \param nbytes Length in bytes.
|
2021-08-24 21:08:29 +00:00
|
|
|
* \param flags Accel framework flags for operations.
|
2021-06-01 19:17:50 +00:00
|
|
|
* \param cb_fn Called when this CRC-32C operation completes.
|
|
|
|
* \param cb_arg Callback argument.
|
|
|
|
*
|
|
|
|
* \return 0 on success, negative errno on failure.
|
|
|
|
*/
|
|
|
|
int spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, void *src,
|
2021-08-24 21:08:29 +00:00
|
|
|
uint32_t *crc_dst, uint32_t seed, uint64_t nbytes, int flags,
|
2021-06-01 19:17:50 +00:00
|
|
|
spdk_accel_completion_cb cb_fn, void *cb_arg);
|
|
|
|
|
2021-06-07 20:00:15 +00:00
|
|
|
/**
|
|
|
|
* Submit a chained copy + CRC-32C calculation request.
|
|
|
|
*
|
|
|
|
* This operation will calculate the 4 byte CRC32-C for the given data.
|
|
|
|
*
|
|
|
|
* \param ch I/O channel associated with this call.
|
|
|
|
* \param dst Destination to write the data to.
|
|
|
|
* \param src_iovs The io vector array which stores the src data and len.
|
|
|
|
* \param iovcnt The size of the io vectors.
|
|
|
|
* \param crc_dst Destination to write the CRC-32C to.
|
|
|
|
* \param seed Four byte seed value.
|
2021-08-24 21:08:29 +00:00
|
|
|
* \param flags Accel framework flags for operations.
|
2021-06-07 20:00:15 +00:00
|
|
|
* \param cb_fn Called when this CRC-32C operation completes.
|
|
|
|
* \param cb_arg Callback argument.
|
|
|
|
*
|
|
|
|
* \return 0 on success, negative errno on failure.
|
|
|
|
*/
|
|
|
|
int spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, struct iovec *src_iovs,
|
2021-08-24 21:08:29 +00:00
|
|
|
uint32_t iovcnt, uint32_t *crc_dst, uint32_t seed,
|
|
|
|
int flags, spdk_accel_completion_cb cb_fn, void *cb_arg);
|
2021-06-07 20:00:15 +00:00
|
|
|
|
2022-05-20 19:08:38 +00:00
|
|
|
/**
|
|
|
|
* Build and submit a memory compress request.
|
|
|
|
*
|
|
|
|
* This function will build the compress descriptor and submit it.
|
|
|
|
*
|
|
|
|
* \param ch I/O channel associated with this call
|
2022-09-22 19:01:56 +00:00
|
|
|
* \param dst Destination to write the data to.
|
|
|
|
* \param nbytes Length in bytes.
|
|
|
|
* \param src_iovs The io vector array which stores the src data and len.
|
|
|
|
* \param src_iovcnt The size of the src io vectors.
|
2022-12-20 14:36:43 +00:00
|
|
|
* \param output_size The size of the compressed data (may be NULL if not desired)
|
2022-05-20 19:08:38 +00:00
|
|
|
* \param flags Flags, optional flags that can vary per operation.
|
|
|
|
* \param cb_fn Callback function which will be called when the request is complete.
|
|
|
|
* \param cb_arg Opaque value which will be passed back as the arg parameter in
|
|
|
|
* the completion callback.
|
|
|
|
*
|
|
|
|
* \return 0 on success, negative errno on failure.
|
|
|
|
*/
|
2022-09-22 19:01:56 +00:00
|
|
|
int spdk_accel_submit_compress(struct spdk_io_channel *ch, void *dst,
|
|
|
|
uint64_t nbytes, struct iovec *src_iovs,
|
|
|
|
size_t src_iovcnt, uint32_t *output_size, int flags,
|
|
|
|
spdk_accel_completion_cb cb_fn, void *cb_arg);
|
2022-05-20 19:08:38 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Build and submit a memory decompress request.
|
|
|
|
*
|
|
|
|
* This function will build the decompress descriptor and submit it.
|
|
|
|
*
|
|
|
|
* \param ch I/O channel associated with this call
|
2022-09-22 19:01:56 +00:00
|
|
|
* \param dst_iovs The io vector array which stores the dst data and len.
|
|
|
|
* \param dst_iovcnt The size of the dst io vectors.
|
|
|
|
* \param src_iovs The io vector array which stores the src data and len.
|
|
|
|
* \param src_iovcnt The size of the src io vectors.
|
2022-12-20 14:36:43 +00:00
|
|
|
* \param output_size The size of the compressed data (may be NULL if not desired)
|
2022-05-20 19:08:38 +00:00
|
|
|
* \param flags Flags, optional flags that can vary per operation.
|
|
|
|
* \param cb_fn Callback function which will be called when the request is complete.
|
|
|
|
* \param cb_arg Opaque value which will be passed back as the arg parameter in
|
|
|
|
* the completion callback.
|
|
|
|
*
|
|
|
|
* \return 0 on success, negative errno on failure.
|
|
|
|
*/
|
2022-09-22 19:01:56 +00:00
|
|
|
int spdk_accel_submit_decompress(struct spdk_io_channel *ch, struct iovec *dst_iovs,
|
|
|
|
size_t dst_iovcnt, struct iovec *src_iovs,
|
2022-12-20 14:36:43 +00:00
|
|
|
size_t src_iovcnt, uint32_t *output_size, int flags,
|
2022-05-20 19:08:38 +00:00
|
|
|
spdk_accel_completion_cb cb_fn, void *cb_arg);
|
2021-06-07 20:00:15 +00:00
|
|
|
|
accel: initial operation chaining support
This patch introduces the concept of chaining multiple accel operations
and executing them all at once in a single step. This means that it
will be possible to schedule accel operations at different layers of the
stack (e.g. copy in NVMe-oF transport, crypto in bdev_crypto), but
execute them all in a single place. Thanks to this, we can take
advantage of hardware accelerators that supports executing multiple
operations as a single operation (e.g. copy + crypto).
This operation group is called spdk_accel_sequence and operations can be
appended to that object via one of the spdk_accel_append_* functions.
New operations are always added at the end of a sequence. Users can
specify a callback to be notified when a particular operation in a
sequence is completed, but they don't receive the status of whether it
was successful or not. This is by design, as they shouldn't care about
the status of an individual operation and should rely on other means to
receive the status of the whole sequence. It's also important to note
that any intermediate steps within a sequence may not produce observable
results. For instance, appending a copy from A to B and then a copy
from B to C, it's indeterminate whether A's data will be in B after a
sequence is executed. It is only guaranteed that A's data will be in C.
A sequence can also be reversed using spdk_accel_sequence_reverse(),
meaning that the first operation becomes last and vice versa. It's
especially useful in read paths, as it makes it possible to build the
sequence during submission, then, once the data is read from storage,
reverse the sequence and execute it.
Finally, there are two ways to terminate a sequence: aborting or
executing. It can be aborted via spdk_accel_sequence_abort() which will
execute individual operations' callbacks and free any allocated
resources. To execute it, one must use spdk_accel_sequence_finish().
For now, each operation is executed one by one and is submitted to the
appropriate accel module. Executing multiple operations as a single one
will be added in the future.
Also, currently, only fill and copy operations can be appended to a
sequence. Support for more operations will be added in subsequent
patches.
Signed-off-by: Konrad Sztyber <konrad.sztyber@intel.com>
Change-Id: Id35d093e14feb59b996f780ef77e000e10bfcd20
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15529
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@nvidia.com>
2022-11-16 07:22:55 +00:00
|
|
|
/** Object grouping multiple accel operations to be executed at the same point in time */
|
|
|
|
struct spdk_accel_sequence;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Completion callback of a single operation within a sequence. After it's executed, the sequence
|
|
|
|
* object might be freed, so users should not touch it.
|
|
|
|
*/
|
|
|
|
typedef void (*spdk_accel_step_cb)(void *cb_arg);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Append a copy operation to a sequence. Copy operation in a sequence is special, as it is not
|
|
|
|
* guaranteed that the data will be actually copied. If it's possible, it will only change
|
|
|
|
* source / destination buffers of some of the operations in a sequence.
|
|
|
|
*
|
|
|
|
* \param seq Sequence object. If NULL, a new sequence object will be created.
|
|
|
|
* \param ch I/O channel.
|
|
|
|
* \param dst_iovs Destination I/O vector array.
|
|
|
|
* \param dst_iovcnt Size of the `dst_iovs` array.
|
|
|
|
* \param dst_domain Memory domain to which the destination buffers belong.
|
|
|
|
* \param dst_domain_ctx Destination buffer domain context.
|
|
|
|
* \param src_iovs Source I/O vector array.
|
|
|
|
* \param src_iovcnt Size of the `src_iovs` array.
|
|
|
|
* \param src_domain Memory domain to which the source buffers belong.
|
|
|
|
* \param src_domain_ctx Source buffer domain context.
|
|
|
|
* \param flags Accel operation flags.
|
|
|
|
* \param cb_fn Callback to be executed once this operation is completed.
|
|
|
|
* \param cb_arg Argument to be passed to `cb_fn`.
|
|
|
|
*
|
|
|
|
* \return 0 if operation was successfully added to the sequence, negative errno otherwise.
|
|
|
|
*/
|
|
|
|
int spdk_accel_append_copy(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch,
|
|
|
|
struct iovec *dst_iovs, uint32_t dst_iovcnt,
|
|
|
|
struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
|
|
|
|
struct iovec *src_iovs, uint32_t src_iovcnt,
|
|
|
|
struct spdk_memory_domain *src_domain, void *src_domain_ctx,
|
|
|
|
int flags, spdk_accel_step_cb cb_fn, void *cb_arg);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Append a fill operation to a sequence.
|
|
|
|
*
|
|
|
|
* \param seq Sequence object. If NULL, a new sequence object will be created.
|
|
|
|
* \param ch I/O channel.
|
|
|
|
* \param buf Data buffer.
|
|
|
|
* \param len Length of the data buffer.
|
|
|
|
* \param domain Memory domain to which the data buffer belongs.
|
|
|
|
* \param domain_ctx Buffer domain context.
|
|
|
|
* \param pattern Pattern to fill the buffer with.
|
|
|
|
* \param flags Accel operation flags.
|
|
|
|
* \param cb_fn Callback to be executed once this operation is completed.
|
|
|
|
* \param cb_arg Argument to be passed to `cb_fn`.
|
|
|
|
*
|
|
|
|
* \return 0 if operation was successfully added to the sequence, negative errno otherwise.
|
|
|
|
*/
|
|
|
|
int spdk_accel_append_fill(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch,
|
|
|
|
void *buf, uint64_t len,
|
|
|
|
struct spdk_memory_domain *domain, void *domain_ctx, uint8_t pattern,
|
|
|
|
int flags, spdk_accel_step_cb cb_fn, void *cb_arg);
|
|
|
|
|
2022-11-22 13:36:09 +00:00
|
|
|
/**
|
|
|
|
* Append a decompress operation to a sequence.
|
|
|
|
*
|
|
|
|
* \param seq Sequence object. If NULL, a new sequence object will be created.
|
|
|
|
* \param ch I/O channel.
|
|
|
|
* \param dst_iovs Destination I/O vector array.
|
|
|
|
* \param dst_iovcnt Size of the `dst_iovs` array.
|
|
|
|
* \param dst_domain Memory domain to which the destination buffers belong.
|
|
|
|
* \param dst_domain_ctx Destination buffer domain context.
|
|
|
|
* \param src_iovs Source I/O vector array.
|
|
|
|
* \param src_iovcnt Size of the `src_iovs` array.
|
|
|
|
* \param src_domain Memory domain to which the source buffers belong.
|
|
|
|
* \param src_domain_ctx Source buffer domain context.
|
|
|
|
* \param flags Accel operation flags.
|
|
|
|
* \param cb_fn Callback to be executed once this operation is completed.
|
|
|
|
* \param cb_arg Argument to be passed to `cb_fn`.
|
|
|
|
*
|
|
|
|
* \return 0 if operation was successfully added to the sequence, negative errno otherwise.
|
|
|
|
*/
|
|
|
|
int spdk_accel_append_decompress(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch,
|
|
|
|
struct iovec *dst_iovs, size_t dst_iovcnt,
|
|
|
|
struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
|
|
|
|
struct iovec *src_iovs, size_t src_iovcnt,
|
|
|
|
struct spdk_memory_domain *src_domain, void *src_domain_ctx,
|
|
|
|
int flags, spdk_accel_step_cb cb_fn, void *cb_arg);
|
2023-01-11 15:36:19 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Append an encrypt operation to a sequence.
|
|
|
|
*
|
|
|
|
* `nbytes` must be multiple of `block_size`. `iv` is used to encrypt the first logical block of
|
|
|
|
* size `block_size`. If `src_iovs` describes more than one logical block then `iv` will be
|
|
|
|
* incremented for each next logical block. Data Encryption Key identifier should be created before
|
|
|
|
* calling this function using methods specific to the accel module being used.
|
|
|
|
*
|
|
|
|
* \param seq Sequence object. If NULL, a new sequence object will be created.
|
|
|
|
* \param ch I/O channel.
|
|
|
|
* \param key Data Encryption Key identifier
|
|
|
|
* \param dst_iovs Destination I/O vector array.
|
|
|
|
* \param dst_iovcnt Size of the `dst_iovs` array.
|
|
|
|
* \param dst_domain Memory domain to which the destination buffers belong.
|
|
|
|
* \param dst_domain_ctx Destination buffer domain context.
|
|
|
|
* \param src_iovs Source I/O vector array.
|
|
|
|
* \param src_iovcnt Size of the `src_iovs` array.
|
|
|
|
* \param src_domain Memory domain to which the source buffers belong.
|
|
|
|
* \param src_domain_ctx Source buffer domain context.
|
|
|
|
* \param iv Initialization vector (tweak) used for encryption
|
|
|
|
* \param block_size Logical block size, if src contains more than 1 logical block, subsequent
|
|
|
|
* logical blocks will be encrypted with incremented `iv`.
|
|
|
|
* \param flags Accel operation flags.
|
|
|
|
* \param cb_fn Callback to be executed once this operation is completed.
|
|
|
|
* \param cb_arg Argument to be passed to `cb_fn`.
|
|
|
|
*
|
|
|
|
* \return 0 if operation was successfully added to the sequence, negative errno otherwise.
|
|
|
|
*/
|
|
|
|
int spdk_accel_append_encrypt(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch,
|
|
|
|
struct spdk_accel_crypto_key *key,
|
|
|
|
struct iovec *dst_iovs, uint32_t dst_iovcnt,
|
|
|
|
struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
|
|
|
|
struct iovec *src_iovs, uint32_t src_iovcnt,
|
|
|
|
struct spdk_memory_domain *src_domain, void *src_domain_ctx,
|
|
|
|
uint64_t iv, uint32_t block_size, int flags,
|
|
|
|
spdk_accel_step_cb cb_fn, void *cb_arg);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Append a decrypt operation to a sequence.
|
|
|
|
*
|
|
|
|
* `nbytes` must be multiple of `block_size`. `iv` is used to decrypt the first logical block of
|
|
|
|
* size `block_size`. If `src_iovs` describes more than one logical block then `iv` will be
|
|
|
|
* incremented for each next logical block. Data Encryption Key identifier should be created before
|
|
|
|
* calling this function using methods specific to the accel module being used.
|
|
|
|
*
|
|
|
|
* \param seq Sequence object. If NULL, a new sequence object will be created.
|
|
|
|
* \param ch I/O channel.
|
|
|
|
* \param key Data Encryption Key identifier
|
|
|
|
* \param dst_iovs Destination I/O vector array.
|
|
|
|
* \param dst_iovcnt Size of the `dst_iovs` array.
|
|
|
|
* \param dst_domain Memory domain to which the destination buffers belong.
|
|
|
|
* \param dst_domain_ctx Destination buffer domain context.
|
|
|
|
* \param src_iovs Source I/O vector array.
|
|
|
|
* \param src_iovcnt Size of the `src_iovs` array.
|
|
|
|
* \param src_domain Memory domain to which the source buffers belong.
|
|
|
|
* \param src_domain_ctx Source buffer domain context.
|
|
|
|
* \param iv Initialization vector (tweak) used for decryption. Should be the same as `iv` used for
|
|
|
|
* encryption of a data block.
|
|
|
|
* \param block_size Logical block size, if src contains more than 1 logical block, subsequent
|
|
|
|
* logical blocks will be decrypted with incremented `iv`.
|
|
|
|
* \param flags Accel operation flags.
|
|
|
|
* \param cb_fn Callback to be executed once this operation is completed.
|
|
|
|
* \param cb_arg Argument to be passed to `cb_fn`.
|
|
|
|
*
|
|
|
|
* \return 0 if operation was successfully added to the sequence, negative errno otherwise.
|
|
|
|
*/
|
|
|
|
int spdk_accel_append_decrypt(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch,
|
|
|
|
struct spdk_accel_crypto_key *key,
|
|
|
|
struct iovec *dst_iovs, uint32_t dst_iovcnt,
|
|
|
|
struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,
|
|
|
|
struct iovec *src_iovs, uint32_t src_iovcnt,
|
|
|
|
struct spdk_memory_domain *src_domain, void *src_domain_ctx,
|
|
|
|
uint64_t iv, uint32_t block_size, int flags,
|
|
|
|
spdk_accel_step_cb cb_fn, void *cb_arg);
|
2022-11-22 13:36:09 +00:00
|
|
|
|
accel: initial operation chaining support
This patch introduces the concept of chaining multiple accel operations
and executing them all at once in a single step. This means that it
will be possible to schedule accel operations at different layers of the
stack (e.g. copy in NVMe-oF transport, crypto in bdev_crypto), but
execute them all in a single place. Thanks to this, we can take
advantage of hardware accelerators that supports executing multiple
operations as a single operation (e.g. copy + crypto).
This operation group is called spdk_accel_sequence and operations can be
appended to that object via one of the spdk_accel_append_* functions.
New operations are always added at the end of a sequence. Users can
specify a callback to be notified when a particular operation in a
sequence is completed, but they don't receive the status of whether it
was successful or not. This is by design, as they shouldn't care about
the status of an individual operation and should rely on other means to
receive the status of the whole sequence. It's also important to note
that any intermediate steps within a sequence may not produce observable
results. For instance, appending a copy from A to B and then a copy
from B to C, it's indeterminate whether A's data will be in B after a
sequence is executed. It is only guaranteed that A's data will be in C.
A sequence can also be reversed using spdk_accel_sequence_reverse(),
meaning that the first operation becomes last and vice versa. It's
especially useful in read paths, as it makes it possible to build the
sequence during submission, then, once the data is read from storage,
reverse the sequence and execute it.
Finally, there are two ways to terminate a sequence: aborting or
executing. It can be aborted via spdk_accel_sequence_abort() which will
execute individual operations' callbacks and free any allocated
resources. To execute it, one must use spdk_accel_sequence_finish().
For now, each operation is executed one by one and is submitted to the
appropriate accel module. Executing multiple operations as a single one
will be added in the future.
Also, currently, only fill and copy operations can be appended to a
sequence. Support for more operations will be added in subsequent
patches.
Signed-off-by: Konrad Sztyber <konrad.sztyber@intel.com>
Change-Id: Id35d093e14feb59b996f780ef77e000e10bfcd20
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15529
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@nvidia.com>
2022-11-16 07:22:55 +00:00
|
|
|
/**
|
|
|
|
* Finish a sequence and execute all its operations. After the completion callback is executed, the
|
|
|
|
* sequence object is automatically freed.
|
|
|
|
*
|
|
|
|
* \param seq Sequence to finish.
|
|
|
|
* \param cb_fn Completion callback to be executed once all operations are executed.
|
|
|
|
* \param cb_arg Argument to be passed to `cb_fn`.
|
|
|
|
*
|
|
|
|
* \return 0 on success, negative errno otherwise.
|
|
|
|
*/
|
|
|
|
int spdk_accel_sequence_finish(struct spdk_accel_sequence *seq,
|
|
|
|
spdk_accel_completion_cb cb_fn, void *cb_arg);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Reverse a sequence, so that the last operation becomes the first and vice versa.
|
|
|
|
*
|
|
|
|
* \param seq Sequence to reverse.
|
|
|
|
*/
|
|
|
|
void spdk_accel_sequence_reverse(struct spdk_accel_sequence *seq);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Abort a sequence. This will execute the completion callbacks of all operations that were added
|
|
|
|
* to the sequence and will then free the sequence object.
|
|
|
|
*
|
|
|
|
* \param seq Sequence to abort.
|
|
|
|
*/
|
|
|
|
void spdk_accel_sequence_abort(struct spdk_accel_sequence *seq);
|
|
|
|
|
2022-11-29 11:38:53 +00:00
|
|
|
/**
|
|
|
|
* Allocate a buffer from accel domain. These buffers can be only used with operations appended to
|
|
|
|
* a sequence. The actual data buffer won't be allocated immediately, but only when it's necessary
|
|
|
|
* to execute a given operation. In some cases, this might even mean that a data buffer won't be
|
|
|
|
* allocated at all, if a sequence can be executed without it.
|
|
|
|
*
|
|
|
|
* A buffer can only be a part of one sequence, but it can be used by multiple operations within
|
|
|
|
* that sequence.
|
|
|
|
*
|
|
|
|
* \param ch I/O channel.
|
|
|
|
* \param len Length of the buffer to allocate.
|
|
|
|
* \param buf Pointer to the allocated buffer.
|
|
|
|
* \param domain Memory domain in which the buffer is allocated.
|
|
|
|
* \param domain_ctx Memory domain context related to the allocated buffer.
|
|
|
|
*
|
|
|
|
* \return 0 if a buffer was successfully allocated, negative errno otherwise.
|
|
|
|
*/
|
|
|
|
int spdk_accel_get_buf(struct spdk_io_channel *ch, uint64_t len, void **buf,
|
|
|
|
struct spdk_memory_domain **domain, void **domain_ctx);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Release a buffer allocated via `spdk_accel_get_buf()`.
|
|
|
|
*
|
|
|
|
* \param ch I/O channel.
|
|
|
|
* \param buf Buffer allocated via `spdk_accel_get_buf()`.
|
|
|
|
* \param domain Memory domain in which the buffer is allocated.
|
|
|
|
* \param domain_ctx Memory domain context related to the allocated buffer.
|
|
|
|
*/
|
|
|
|
void spdk_accel_put_buf(struct spdk_io_channel *ch, void *buf,
|
|
|
|
struct spdk_memory_domain *domain, void *domain_ctx);
|
|
|
|
|
2022-10-04 19:16:21 +00:00
|
|
|
/**
|
|
|
|
* Build and submit a data encryption request.
|
|
|
|
*
|
|
|
|
* This function will build the encryption request and submit it. \b nbytes must be multiple of \b block_size.
|
|
|
|
* \b iv is used to encrypt the first logical block of size \b block_size. If \b src_iovs describes more than
|
|
|
|
* one logical block then \b iv will be incremented for each next logical block.
|
|
|
|
* Data Encryption Key identifier should be created before calling this function using methods specific to the accel
|
|
|
|
* module being used.
|
|
|
|
*
|
|
|
|
* \param ch I/O channel associated with this call
|
|
|
|
* \param key Data Encryption Key identifier
|
|
|
|
* \param dst_iovs The io vector array which stores the dst data and len.
|
|
|
|
* \param dst_iovcnt The size of the destination io vectors.
|
|
|
|
* \param src_iovs The io vector array which stores the src data and len.
|
|
|
|
* \param src_iovcnt The size of the source io vectors.
|
|
|
|
* \param iv Initialization vector (tweak) used for encryption
|
|
|
|
* \param block_size Logical block size, if src contains more than 1 logical block, subsequent logical blocks will be
|
|
|
|
* encrypted with incremented \b iv
|
|
|
|
* \param flags Accel framework flags for operations.
|
|
|
|
* \param cb_fn Callback function which will be called when the request is complete.
|
|
|
|
* \param cb_arg Opaque value which will be passed back as the arg parameter in the completion callback.
|
|
|
|
*
|
|
|
|
* \return 0 on success, negative errno on failure.
|
|
|
|
*/
|
|
|
|
int spdk_accel_submit_encrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key,
|
|
|
|
struct iovec *dst_iovs, uint32_t dst_iovcnt,
|
|
|
|
struct iovec *src_iovs, uint32_t src_iovcnt,
|
|
|
|
uint64_t iv, uint32_t block_size, int flags,
|
|
|
|
spdk_accel_completion_cb cb_fn, void *cb_arg);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Build and submit a data decryption request.
|
|
|
|
*
|
|
|
|
* This function will build the decryption request and submit it. \b nbytes must be multiple of \b block_size.
|
|
|
|
* \b iv is used to decrypt the first logical block of size \b block_size. If \b src_iovs describes more than
|
|
|
|
* one logical block then \b iv will be incremented for each next logical block.
|
|
|
|
* Data Encryption Key identifier should be created before calling this function using methods specific to the accel
|
|
|
|
* module being used.
|
|
|
|
*
|
|
|
|
* \param ch I/O channel associated with this call
|
|
|
|
* \param key Data Encryption Key identifier
|
|
|
|
* \param dst_iovs The io vector array which stores the dst data and len.
|
|
|
|
* \param dst_iovcnt The size of the destination io vectors.
|
|
|
|
* \param src_iovs The io vector array which stores the src data and len.
|
|
|
|
* \param src_iovcnt The size of the source io vectors.
|
|
|
|
* \param iv Initialization vector (tweak) used for decryption. Should be the same as \b iv used for encryption of a
|
|
|
|
* data block
|
|
|
|
* \param block_size Logical block size, if src contains more than 1 logical block, subsequent logical blocks will be
|
|
|
|
* decrypted with incremented \b iv
|
|
|
|
* \param flags Accel framework flags for operations.
|
|
|
|
* \param cb_fn Callback function which will be called when the request is complete.
|
|
|
|
* \param cb_arg Opaque value which will be passed back as the arg parameter in the completion callback.
|
|
|
|
*
|
|
|
|
* \return 0 on success, negative errno on failure.
|
|
|
|
*/
|
|
|
|
int spdk_accel_submit_decrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key,
|
|
|
|
struct iovec *dst_iovs, uint32_t dst_iovcnt,
|
|
|
|
struct iovec *src_iovs, uint32_t src_iovcnt,
|
|
|
|
uint64_t iv, uint32_t block_size, int flags,
|
|
|
|
spdk_accel_completion_cb cb_fn, void *cb_arg);
|
|
|
|
|
2022-05-26 23:46:01 +00:00
|
|
|
/**
|
2022-11-02 15:09:36 +00:00
|
|
|
* Return the name of the module assigned to a specific opcode.
|
2022-05-26 23:46:01 +00:00
|
|
|
*
|
|
|
|
* \param opcode Accel Framework Opcode enum value. Valid codes can be retrieved using
|
|
|
|
* `accel_get_opc_assignments` or `spdk_accel_get_opc_name`.
|
2022-08-08 21:43:24 +00:00
|
|
|
* \param module_name Pointer to update with module name.
|
2022-05-26 23:46:01 +00:00
|
|
|
*
|
2022-08-08 21:43:24 +00:00
|
|
|
* \return 0 if a valid module name was provided. -EINVAL for invalid opcode
|
|
|
|
* or -ENOENT no module was found at this time for the provided opcode.
|
2022-05-26 23:46:01 +00:00
|
|
|
*/
|
2022-08-08 21:43:24 +00:00
|
|
|
int spdk_accel_get_opc_module_name(enum accel_opcode opcode, const char **module_name);
|
2022-05-26 23:46:01 +00:00
|
|
|
|
2022-06-01 00:13:04 +00:00
|
|
|
/**
|
2022-08-08 21:43:24 +00:00
|
|
|
* Override the assignment of an opcode to an module.
|
2022-06-01 00:13:04 +00:00
|
|
|
*
|
|
|
|
* \param opcode Accel Framework Opcode enum value. Valid codes can be retrieved using
|
|
|
|
* `accel_get_opc_assignments` or `spdk_accel_get_opc_name`.
|
2022-08-08 21:43:24 +00:00
|
|
|
* \param name Name of the module to assign. Valid module names may be retrieved
|
|
|
|
* with `spdk_accel_get_opc_module_name`
|
2022-06-01 00:13:04 +00:00
|
|
|
*
|
|
|
|
* \return 0 if a valid opcode name was provided. -EINVAL for invalid opcode
|
2022-08-08 21:43:24 +00:00
|
|
|
* or if the framework has started (cannot change modules after startup)
|
2022-06-01 00:13:04 +00:00
|
|
|
*/
|
|
|
|
int spdk_accel_assign_opc(enum accel_opcode opcode, const char *name);
|
|
|
|
|
2020-02-08 18:23:09 +00:00
|
|
|
struct spdk_json_write_ctx;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Write Acceleration subsystem configuration into provided JSON context.
|
|
|
|
*
|
|
|
|
* \param w JSON write context
|
|
|
|
*/
|
|
|
|
void spdk_accel_write_config_json(struct spdk_json_write_ctx *w);
|
|
|
|
|
2017-12-07 20:25:19 +00:00
|
|
|
#ifdef __cplusplus
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2016-07-19 15:58:55 +00:00
|
|
|
#endif
|