From 13f97e67374f072c6bf5d59bccd3d8a08e43db19 Mon Sep 17 00:00:00 2001 From: Alexey Marchuk Date: Wed, 31 Aug 2022 17:26:12 +0200 Subject: [PATCH] bdev/crypto: Use accel framework All DPDK related code is removed, handling of RESET command was sligthly updated. Handling of -ENOMEM was updated for cases when accel API returns -ENOMEM Crypto tests in blockdev.sh were extended with more crypto_bdevs to verify NOMEM cases - that failed with original vbdev_crypto implementation Signed-off-by: Alexey Marchuk Change-Id: If1feba2449bee852c6c4daca4b3406414db6fded Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/14860 Tested-by: SPDK CI Jenkins Reviewed-by: Paul Luse Reviewed-by: Ben Walker Reviewed-by: Jim Harris --- CHANGELOG.md | 2 + autotest.sh | 1 + doc/bdev.md | 76 +- doc/jsonrpc.md | 54 +- lib/accel/accel_rpc.c | 43 +- mk/spdk.lib_deps.mk | 2 +- .../dpdk_cryptodev/accel_dpdk_cryptodev.c | 1 + module/bdev/crypto/vbdev_crypto.c | 1513 ++--------------- module/bdev/crypto/vbdev_crypto.h | 36 +- module/bdev/crypto/vbdev_crypto_rpc.c | 283 ++- python/spdk/rpc/accel.py | 13 + python/spdk/rpc/bdev.py | 22 +- python/spdk/sma/volume/crypto_bdev.py | 32 +- scripts/rpc.py | 20 +- test/bdev/blockdev.sh | 70 +- test/json_config/config_filter.py | 4 + test/json_config/json_config.sh | 16 +- test/sma/crypto.sh | 23 +- test/sma/vfiouser_qemu.sh | 16 +- test/sma/vhost_blk.sh | 18 +- test/unit/lib/bdev/crypto.c/crypto_ut.c | 1153 +++---------- 21 files changed, 858 insertions(+), 2540 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 841ae7065..51b698df7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ Protection information is now supported by the malloc bdev module. A new API `spkd_bdev_part_submit_request_ext` was added to specify a custom completion callback. +vbdev_crypto is updated to use accel framework instead of DPDK PMDs. + ### scheduler Changing scheduler from dynamic back to static is no longer possible, diff --git a/autotest.sh b/autotest.sh index 8a9d80d50..7732bc39f 100755 --- a/autotest.sh +++ b/autotest.sh @@ -335,6 +335,7 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then if [ $SPDK_TEST_CRYPTO -eq 1 ]; then run_test "blockdev_crypto_aesni" ./test/bdev/blockdev.sh "crypto_aesni" + run_test "blockdev_crypto_sw" ./test/bdev/blockdev.sh "crypto_sw" # Proceed with the test only if QAT devices are in place if [[ $(lspci -d:37c8) ]]; then run_test "blockdev_crypto_qat" ./test/bdev/blockdev.sh "crypto_qat" diff --git a/doc/bdev.md b/doc/bdev.md index 49320fe2e..394e2aff3 100644 --- a/doc/bdev.md +++ b/doc/bdev.md @@ -162,12 +162,23 @@ all volumes, if used it will return the name or an error that the device does no ## Crypto Virtual Bdev Module {#bdev_config_crypto} The crypto virtual bdev module can be configured to provide at rest data encryption -for any underlying bdev. The module relies on the DPDK CryptoDev Framework to provide -all cryptographic functionality. The framework provides support for many different software -only cryptographic modules as well hardware assisted support for the Intel QAT board and -NVIDIA crypto enabled NICs. -The framework also provides support for cipher, hash, authentication and AEAD functions. -At this time the SPDK virtual bdev module supports cipher only as follows: +for any underlying bdev. The module relies on the SPDK Accel Framework to provide +all cryptographic functionality. +One of the accel modules, dpdk_cryptodev is implemented with the DPDK CryptoDev API, +it provides support for many different software only cryptographic modules as well hardware +assisted support for the Intel QAT board and NVIDIA crypto enabled NICs. + +For reads, the buffer provided to the crypto block device will be used as the destination buffer +for unencrypted data. For writes, however, a temporary scratch buffer is used as the +destination buffer for encryption which is then passed on to the underlying bdev as the +write buffer. This is done to avoid encrypting the data in the original source buffer which +may cause problems in some use cases. + +Below is information about accel modules which support crypto operations: + +### dpdk_cryptodev accel module + +Supports the following ciphers: - AESN-NI Multi Buffer Crypto Poll Mode Driver: RTE_CRYPTO_CIPHER_AES128_CBC - Intel(R) QuickAssist (QAT) Crypto Poll Mode Driver: RTE_CRYPTO_CIPHER_AES128_CBC, @@ -181,38 +192,61 @@ the crypto module break up all I/O into crypto operations of a size equal to the size of the underlying bdev. For example, a 4K I/O to a bdev with a 512B block size, would result in 8 cryptographic operations. -For reads, the buffer provided to the crypto module will be used as the destination buffer -for unencrypted data. For writes, however, a temporary scratch buffer is used as the -destination buffer for encryption which is then passed on to the underlying bdev as the -write buffer. This is done to avoid encrypting the data in the original source buffer which -may cause problems in some use cases. +### SW accel module -Example command +Supports the following ciphers: -`rpc.py bdev_crypto_create NVMe1n1 CryNvmeA crypto_aesni_mb 01234567891234560123456789123456` +- AES_XTS cipher with 128 or 256 bit keys implemented with ISA-L_crypto -This command will create a crypto vbdev called 'CryNvmeA' on top of the NVMe bdev -'NVMe1n1' and will use the DPDK software driver 'crypto_aesni_mb' and the key -'01234567891234560123456789123456'. +### General workflow + +- Set desired accel module to perform crypto operations, that can be done with `accel_assign_opc` RPC command +- Create a named crypto key using `accel_crypto_key_create` RPC command. The key will use the assigned accel + module. Set of parameters and supported ciphers may be different in each accel module. +- Create virtual crypto block device providing the base block device name and the crypto key name + using `bdev_crypto_create` RPC command + +#### Example + +Example command which uses dpdk_cryptodev accel module +``` +# start SPDK application with `--wait-for-rpc` parameter +rpc.py dpdk_cryptodev_scan_accel_module +rpc.py dpdk_cryptodev_set_driver crypto_aesni_mb +rpc.py accel_assign_opc -o encrypt -m dpdk_cryptodev +rpc.py accel_assign_opc -o decrypt -m dpdk_cryptodev +rpc.py framework_start_init +rpc.py accel_crypto_key_create -c AES_CBC -k 01234567891234560123456789123456 -n key_aesni_cbc_1 +rpc.py bdev_crypto_create NVMe1n1 CryNvmeA -n key_aesni_cbc_1 +``` + +These commands will create a crypto vbdev called 'CryNvmeA' on top of the NVMe bdev +'NVMe1n1' and will use a key named `key_aesni_cbc_1`. The key will work with the accel module which +has been assigned for encrypt operations, in this example it will be the dpdk_cryptodev. + +### Crypto key format Please make sure the keys are provided in hexlified format. This means string passed to rpc.py must be twice as long than the key length in binary form. -Example command +#### Example command -`rpc.py bdev_crypto_create -c AES_XTS -k2 7859243a027411e581e0c40a35c8228f NVMe1n1 CryNvmeA mlx5_pci d16a2f3a9e9f5b32daefacd7f5984f4578add84425be4a0baa489b9de8884b09` +`rpc.py accel_crypto_key_create -c AES_XTS -k2 7859243a027411e581e0c40a35c8228f -k d16a2f3a9e9f5b32daefacd7f5984f4578add84425be4a0baa489b9de8884b09 -n sample_key` -This command will create a crypto vbdev called 'CryNvmeA' on top of the NVMe bdev -'NVMe1n1' and will use the DPDK software driver 'mlx5_pci', the AES key +This command will create a key called `sample_key`, the AES key 'd16a2f3a9e9f5b32daefacd7f5984f4578add84425be4a0baa489b9de8884b09' and the XTS key '7859243a027411e581e0c40a35c8228f'. In other words, the compound AES_XTS key to be used is 'd16a2f3a9e9f5b32daefacd7f5984f4578add84425be4a0baa489b9de8884b097859243a027411e581e0c40a35c8228f' +### Delete the virtual crypto block device + To remove the vbdev use the bdev_crypto_delete command. `rpc.py bdev_crypto_delete CryNvmeA` -The MLX5 driver works with crypto enabled Nvidia NICs and requires special configuration of +### dpdk_cryptodev mlx5_pci driver configuration + +The mlx5_pci driver works with crypto enabled Nvidia NICs and requires special configuration of DPDK environment to enable crypto function. It can be done via SPDK event library by configuring `env_context` member of `spdk_app_opts` structure or by passing corresponding CLI arguments in the following form: `--allow=BDF,class=crypto,wcs_file=/full/path/to/wrapped/credentials`, e.g. diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index 4df54849b..36db786fa 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -443,6 +443,7 @@ Example response: "spdk_kill_instance", "accel_get_opc_assignments", "accel_crypto_key_create", + "accel_crypto_key_destroy", "accel_crypto_keys_get", "ioat_scan_accel_module", "dsa_scan_accel_module", @@ -1783,7 +1784,7 @@ Example response: ### accel_crypto_key_create {#rpc_accel_crypto_key_create} -Create a crypt key which will be used in accel framework +Create a crypto key which will be used in accel framework #### Parameters @@ -1822,6 +1823,41 @@ Example response: } ~~~ +### accel_crypto_key_destroy {#rpc_accel_crypto_key_destroy} + +Destroy a crypto key. The user is responsible for ensuring that the deleted key is not used by acceleration modules. + +#### Parameters + +Name | Optional | Type | Description +-----------|----------| ----------- | ----------------- +name | Required | string | The key name + +#### Example + +Example request: + +~~~json +{ + "jsonrpc": "2.0", + "method": "accel_crypto_key_destroy", + "id": 1, + "params": { + "name": "super_key" + } +} +~~~ + +Example response: + +~~~json +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + ### accel_crypto_keys_get {#rpc_accel_crypto_keys_get} Get information about existing crypto keys @@ -2039,7 +2075,7 @@ Set the DPDK cryptodev driver Name | Optional | Type | Description ----------------------- |----------|--------| ----------- -driver_name | Required | string | The driver, can be one of crypto_aesni_mb, crypto_qat or mlx5_pci +crypto_pmd | Required | string | The driver, can be one of crypto_aesni_mb, crypto_qat or mlx5_pci #### Example @@ -2051,7 +2087,7 @@ Example request: "method": "dpdk_cryptodev_set_driver", "id": 1, "params": { - "driver_name": "crypto_aesni_mb" + "crypto_pmd": "crypto_aesni_mb" } } ~~~ @@ -2693,13 +2729,15 @@ Create a new crypto bdev on a given base bdev. #### Parameters Name | Optional | Type | Description ------------------------ | -------- | ----------- | ----------- +----------------------- |----------| ----------- | ----------- base_bdev_name | Required | string | Name of the base bdev name | Required | string | Name of the crypto vbdev to create -crypto_pmd | Required | string | Name of the crypto device driver -key | Required | string | Key in hex form -cipher | Required | string | Cipher to use, AES_CBC or AES_XTS (QAT and MLX5) -key2 | Required | string | 2nd key in hex form only required for cipher AES_XTS +crypto_pmd | Optional | string | Name of the crypto device driver. Obsolete, see accel_crypto_key_create +key | Optional | string | Key in hex form. Obsolete, see accel_crypto_key_create +cipher | Optional | string | Cipher to use, AES_CBC or AES_XTS (QAT and MLX5). Obsolete, see accel_crypto_key_create +key2 | Optional | string | 2nd key in hex form only required for cipher AET_XTS. Obsolete, see accel_crypto_key_create +key_name | Optional | string | Name of the key created with accel_crypto_key_create +module | Optional | string | Name of the accel module which is used to create a key (if no key_name specified) Both key and key2 must be passed in the hexlified form. For example, 256bit AES key may look like this: afd9477abf50254219ccb75965fbe39f23ebead5676e292582a0a67f66b88215 diff --git a/lib/accel/accel_rpc.c b/lib/accel/accel_rpc.c index 8c2f56d2d..4db1be732 100644 --- a/lib/accel/accel_rpc.c +++ b/lib/accel/accel_rpc.c @@ -230,7 +230,7 @@ struct rpc_accel_crypto_keys_get_ctx { }; static const struct spdk_json_object_decoder rpc_accel_crypto_keys_get_decoders[] = { - {"key_name", offsetof(struct rpc_accel_crypto_keys_get_ctx, key_name), spdk_json_decode_string, true}, + {"key_name", offsetof(struct rpc_accel_crypto_keys_get_ctx, key_name), spdk_json_decode_string}, }; static void @@ -272,3 +272,44 @@ rpc_accel_crypto_keys_get(struct spdk_jsonrpc_request *request, spdk_jsonrpc_end_result(request, w); } SPDK_RPC_REGISTER("accel_crypto_keys_get", rpc_accel_crypto_keys_get, SPDK_RPC_RUNTIME) + +static const struct spdk_json_object_decoder rpc_accel_crypto_key_destroy_decoders[] = { + {"key_name", offsetof(struct rpc_accel_crypto_keys_get_ctx, key_name), spdk_json_decode_string, true}, +}; + +static void +rpc_accel_crypto_key_destroy(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_accel_crypto_keys_get_ctx req = {}; + struct spdk_accel_crypto_key *key = NULL; + int rc; + + if (params && spdk_json_decode_object(params, rpc_accel_crypto_key_destroy_decoders, + SPDK_COUNTOF(rpc_accel_crypto_key_destroy_decoders), + &req)) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_PARSE_ERROR, + "spdk_json_decode_object failed"); + free(req.key_name); + return; + } + + key = spdk_accel_crypto_key_get(req.key_name); + if (!key) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "No key object found"); + free(req.key_name); + return; + + } + rc = spdk_accel_crypto_key_destroy(key); + if (rc) { + spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Failed to destroy key, rc %d\n", rc); + } else { + spdk_jsonrpc_send_bool_response(request, true); + } + + free(req.key_name); +} +SPDK_RPC_REGISTER("accel_crypto_key_destroy", rpc_accel_crypto_key_destroy, SPDK_RPC_RUNTIME) diff --git a/mk/spdk.lib_deps.mk b/mk/spdk.lib_deps.mk index e08d9eb9d..2fd440500 100644 --- a/mk/spdk.lib_deps.mk +++ b/mk/spdk.lib_deps.mk @@ -127,7 +127,7 @@ DEPDIRS-bdev_split := $(BDEV_DEPS) DEPDIRS-bdev_aio := $(BDEV_DEPS_THREAD) DEPDIRS-bdev_compress := $(BDEV_DEPS_THREAD) reduce -DEPDIRS-bdev_crypto := $(BDEV_DEPS_THREAD) +DEPDIRS-bdev_crypto := $(BDEV_DEPS_THREAD) accel DEPDIRS-bdev_delay := $(BDEV_DEPS_THREAD) DEPDIRS-bdev_iscsi := $(BDEV_DEPS_THREAD) DEPDIRS-bdev_malloc := $(BDEV_DEPS_THREAD) accel diff --git a/module/accel/dpdk_cryptodev/accel_dpdk_cryptodev.c b/module/accel/dpdk_cryptodev/accel_dpdk_cryptodev.c index 7fa123990..95a6f64b7 100644 --- a/module/accel/dpdk_cryptodev/accel_dpdk_cryptodev.c +++ b/module/accel/dpdk_cryptodev/accel_dpdk_cryptodev.c @@ -601,6 +601,7 @@ accel_dpdk_cryptodev_mbuf_add_single_block(struct spdk_iov_sgl *sgl, struct rte_ spdk_iov_sgl_advance(sgl, buf_len); /* Handle the case of page boundary. */ + assert(task->base.block_size >= buf_len); remainder = task->base.block_size - buf_len; while (remainder) { buf_len = spdk_min(remainder, sgl->iov->iov_len - sgl->iov_offset); diff --git a/module/bdev/crypto/vbdev_crypto.c b/module/bdev/crypto/vbdev_crypto.c index 7dbe5d032..20113ad24 100644 --- a/module/bdev/crypto/vbdev_crypto.c +++ b/module/bdev/crypto/vbdev_crypto.c @@ -7,132 +7,14 @@ #include "vbdev_crypto.h" -#include "spdk/env.h" -#include "spdk/likely.h" -#include "spdk/endian.h" +#include "spdk_internal/assert.h" #include "spdk/thread.h" #include "spdk/bdev_module.h" -#include "spdk/log.h" -#include "spdk/hexlify.h" +#include "spdk/likely.h" -#include -#include -#include -#include -#include -#include - -/* Used to store IO context in mbuf */ -static const struct rte_mbuf_dynfield rte_mbuf_dynfield_io_context = { - .name = "context_bdev_io", - .size = sizeof(uint64_t), - .align = __alignof__(uint64_t), - .flags = 0, -}; -static int g_mbuf_offset; - -/* To add support for new device types, follow the examples of the following... - * Note that the string names are defined by the DPDK PMD in question so be - * sure to use the exact names. - */ -#define MAX_NUM_DRV_TYPES 3 - -/* The VF spread is the number of queue pairs between virtual functions, we use this to - * load balance the QAT device. - */ -#define QAT_VF_SPREAD 32 -static uint8_t g_qat_total_qp = 0; -static uint8_t g_next_qat_index; - -const char *g_driver_names[MAX_NUM_DRV_TYPES] = { AESNI_MB, QAT, MLX5 }; - -/* Global list of available crypto devices. */ -struct vbdev_dev { - struct rte_cryptodev_info cdev_info; /* includes device friendly name */ - uint8_t cdev_id; /* identifier for the device */ - TAILQ_ENTRY(vbdev_dev) link; -}; -static TAILQ_HEAD(, vbdev_dev) g_vbdev_devs = TAILQ_HEAD_INITIALIZER(g_vbdev_devs); - -/* Global list and lock for unique device/queue pair combos. We keep 1 list per supported PMD - * so that we can optimize per PMD where it make sense. For example, with QAT there an optimal - * pattern for assigning queue pairs where with AESNI there is not. - */ -struct device_qp { - struct vbdev_dev *device; /* ptr to crypto device */ - uint8_t qp; /* queue pair for this node */ - bool in_use; /* whether this node is in use or not */ - uint8_t index; /* used by QAT to load balance placement of qpairs */ - TAILQ_ENTRY(device_qp) link; -}; -static TAILQ_HEAD(, device_qp) g_device_qp_qat = TAILQ_HEAD_INITIALIZER(g_device_qp_qat); -static TAILQ_HEAD(, device_qp) g_device_qp_aesni_mb = TAILQ_HEAD_INITIALIZER(g_device_qp_aesni_mb); -static TAILQ_HEAD(, device_qp) g_device_qp_mlx5 = TAILQ_HEAD_INITIALIZER(g_device_qp_mlx5); -static pthread_mutex_t g_device_qp_lock = PTHREAD_MUTEX_INITIALIZER; - - -/* In order to limit the number of resources we need to do one crypto - * operation per LBA (we use LBA as IV), we tell the bdev layer that - * our max IO size is something reasonable. Units here are in bytes. - */ -#define CRYPTO_MAX_IO (64 * 1024) - -/* This controls how many ops will be dequeued from the crypto driver in one run - * of the poller. It is mainly a performance knob as it effectively determines how - * much work the poller has to do. However even that can vary between crypto drivers - * as the AESNI_MB driver for example does all the crypto work on dequeue whereas the - * QAT driver just dequeues what has been completed already. - */ -#define MAX_DEQUEUE_BURST_SIZE 64 - -/* When enqueueing, we need to supply the crypto driver with an array of pointers to - * operation structs. As each of these can be max 512B, we can adjust the CRYPTO_MAX_IO - * value in conjunction with the other defines to make sure we're not using crazy amounts - * of memory. All of these numbers can and probably should be adjusted based on the - * workload. By default we'll use the worst case (smallest) block size for the - * minimum number of array entries. As an example, a CRYPTO_MAX_IO size of 64K with 512B - * blocks would give us an enqueue array size of 128. - */ -#define MAX_ENQUEUE_ARRAY_SIZE (CRYPTO_MAX_IO / 512) - -/* The number of MBUFS we need must be a power of two and to support other small IOs - * in addition to the limits mentioned above, we go to the next power of two. It is - * big number because it is one mempool for source and destination mbufs. It may - * need to be bigger to support multiple crypto drivers at once. - */ -#define NUM_MBUFS 32768 -#define POOL_CACHE_SIZE 256 -#define MAX_CRYPTO_VOLUMES 128 -#define NUM_SESSIONS (2 * MAX_CRYPTO_VOLUMES) -#define SESS_MEMPOOL_CACHE_SIZE 0 -uint8_t g_number_of_claimed_volumes = 0; - -/* This is the max number of IOs we can supply to any crypto device QP at one time. - * It can vary between drivers. - */ -#define CRYPTO_QP_DESCRIPTORS 2048 - -/* At this moment DPDK descriptors allocation for mlx5 has some issues. We use 512 - * as an compromise value between performance and the time spent for initialization. */ -#define CRYPTO_QP_DESCRIPTORS_MLX5 512 - -#define AESNI_MB_NUM_QP 64 - -/* Common for supported devices. */ -#define DEFAULT_NUM_XFORMS 2 -#define IV_OFFSET (sizeof(struct rte_crypto_op) + \ - sizeof(struct rte_crypto_sym_op) + \ - (DEFAULT_NUM_XFORMS * \ - sizeof(struct rte_crypto_sym_xform))) -#define IV_LENGTH 16 -#define QUEUED_OP_OFFSET (IV_OFFSET + IV_LENGTH) - -static void _complete_internal_io(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg); -static void _complete_internal_read(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg); -static void _complete_internal_write(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg); -static void vbdev_crypto_examine(struct spdk_bdev *bdev); -static int vbdev_crypto_claim(const char *bdev_name); -static void vbdev_crypto_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io); +/* Limit the max IO size by some reasonable value. Since in write operation we use aux buffer, + * let's set the limit to the bdev bounce aux buffer size */ +#define CRYPTO_MAX_IO SPDK_BDEV_LARGE_BUF_MAX_SIZE struct bdev_names { struct vbdev_crypto_opts *opts; @@ -146,11 +28,7 @@ struct vbdev_crypto { struct spdk_bdev *base_bdev; /* the thing we're attaching to */ struct spdk_bdev_desc *base_desc; /* its descriptor we get from open */ struct spdk_bdev crypto_bdev; /* the crypto virtual bdev */ - struct vbdev_crypto_opts *opts; /* crypto options such as key, cipher */ - uint32_t qp_desc_nr; /* number of qp descriptors */ - void *session_encrypt; /* encryption session for this bdev */ - void *session_decrypt; /* decryption session for this bdev */ - struct rte_crypto_sym_xform cipher_xform; /* crypto control struct for this bdev */ + struct vbdev_crypto_opts *opts; /* crypto options such as names and DEK */ TAILQ_ENTRY(vbdev_crypto) link; struct spdk_thread *thread; /* thread where base device is opened */ }; @@ -160,48 +38,30 @@ struct vbdev_crypto { */ static TAILQ_HEAD(, vbdev_crypto) g_vbdev_crypto = TAILQ_HEAD_INITIALIZER(g_vbdev_crypto); -/* Shared mempools between all devices on this system */ -static struct rte_mempool *g_session_mp = NULL; -static struct rte_mempool *g_session_mp_priv = NULL; -static struct rte_mempool *g_mbuf_mp = NULL; /* mbuf mempool */ -static struct rte_mempool *g_crypto_op_mp = NULL; /* crypto operations, must be rte* mempool */ - -static struct rte_mbuf_ext_shared_info g_shinfo = {}; /* used by DPDK mbuf macro */ - -/* For queueing up crypto operations that we can't submit for some reason */ -struct vbdev_crypto_op { - uint8_t cdev_id; - uint8_t qp; - struct rte_crypto_op *crypto_op; - struct spdk_bdev_io *bdev_io; - TAILQ_ENTRY(vbdev_crypto_op) link; -}; -#define QUEUED_OP_LENGTH (sizeof(struct vbdev_crypto_op)) - /* The crypto vbdev channel struct. It is allocated and freed on my behalf by the io channel code. - * We store things in here that are needed on per thread basis like the base_channel for this thread, - * and the poller for this thread. + * We store things in here that are needed on per thread basis like the base_channel for this thread. */ struct crypto_io_channel { - struct spdk_io_channel *base_ch; /* IO channel of base device */ - struct spdk_poller *poller; /* completion poller */ - struct device_qp *device_qp; /* unique device/qp combination for this channel */ - TAILQ_HEAD(, spdk_bdev_io) pending_cry_ios; /* outstanding operations to the crypto device */ - struct spdk_io_channel_iter *iter; /* used with for_each_channel in reset */ - TAILQ_HEAD(, vbdev_crypto_op) queued_cry_ops; /* queued for re-submission to CryptoDev */ + struct spdk_io_channel *base_ch; /* IO channel of base device */ + struct spdk_io_channel *accel_channel; /* Accel engine channel used for crypto ops */ + struct spdk_accel_crypto_key *crypto_key; + TAILQ_HEAD(, spdk_bdev_io) in_accel_fw; /* request submitted to accel fw */ + struct spdk_io_channel_iter *reset_iter; /* used with for_each_channel in reset */ +}; + +enum crypto_io_resubmit_state { + CRYPTO_IO_NEW, /* Resubmit IO from the scratch */ + CRYPTO_IO_READ_DONE, /* Need to decrypt */ + CRYPTO_IO_ENCRYPT_DONE, /* Need to write */ }; /* This is the crypto per IO context that the bdev layer allocates for us opaquely and attaches to * each IO for us. */ struct crypto_bdev_io { - int cryop_cnt_remaining; /* counter used when completing crypto ops */ struct crypto_io_channel *crypto_ch; /* need to store for crypto completion handling */ struct vbdev_crypto *crypto_bdev; /* the crypto node struct associated with this IO */ - struct spdk_bdev_io *orig_io; /* the original IO */ struct spdk_bdev_io *read_io; /* the read IO we issued */ - int8_t bdev_io_status; /* the status we'll report back on the bdev IO */ - bool on_pending_list; /* Used for the single contiguous buffer that serves as the crypto destination target for writes */ uint64_t aux_num_blocks; /* num of blocks for the contiguous buffer */ uint64_t aux_offset_blocks; /* block offset on media */ @@ -210,892 +70,129 @@ struct crypto_bdev_io { /* for bdev_io_wait */ struct spdk_bdev_io_wait_entry bdev_io_wait; - struct spdk_io_channel *ch; + enum crypto_io_resubmit_state resubmit_state; }; -/* Called by vbdev_crypto_init_crypto_drivers() to init each discovered crypto device */ -static int -create_vbdev_dev(uint8_t index, uint16_t num_lcores) -{ - struct vbdev_dev *device; - uint8_t j, cdev_id, cdrv_id; - struct device_qp *dev_qp; - struct device_qp *tmp_qp; - uint32_t qp_desc_nr; - int rc; - TAILQ_HEAD(device_qps, device_qp) *dev_qp_head; - - device = calloc(1, sizeof(struct vbdev_dev)); - if (!device) { - return -ENOMEM; - } - - /* Get details about this device. */ - rte_cryptodev_info_get(index, &device->cdev_info); - cdrv_id = device->cdev_info.driver_id; - cdev_id = device->cdev_id = index; - - /* QAT_ASYM devices are not supported at this time. */ - if (strcmp(device->cdev_info.driver_name, QAT_ASYM) == 0) { - free(device); - return 0; - } - - /* Before going any further, make sure we have enough resources for this - * device type to function. We need a unique queue pair per core across each - * device type to remain lockless.... - */ - if ((rte_cryptodev_device_count_by_driver(cdrv_id) * - device->cdev_info.max_nb_queue_pairs) < num_lcores) { - SPDK_ERRLOG("Insufficient unique queue pairs available for %s\n", - device->cdev_info.driver_name); - SPDK_ERRLOG("Either add more crypto devices or decrease core count\n"); - rc = -EINVAL; - goto err; - } - - /* Setup queue pairs. */ - struct rte_cryptodev_config conf = { - .nb_queue_pairs = device->cdev_info.max_nb_queue_pairs, - .socket_id = SPDK_ENV_SOCKET_ID_ANY - }; - - rc = rte_cryptodev_configure(cdev_id, &conf); - if (rc < 0) { - SPDK_ERRLOG("Failed to configure cryptodev %u: error %d\n", - cdev_id, rc); - rc = -EINVAL; - goto err; - } - - /* Select the right device/qp list based on driver name - * or error if it does not exist. - */ - if (strcmp(device->cdev_info.driver_name, QAT) == 0) { - dev_qp_head = (struct device_qps *)&g_device_qp_qat; - qp_desc_nr = CRYPTO_QP_DESCRIPTORS; - } else if (strcmp(device->cdev_info.driver_name, AESNI_MB) == 0) { - dev_qp_head = (struct device_qps *)&g_device_qp_aesni_mb; - qp_desc_nr = CRYPTO_QP_DESCRIPTORS; - } else if (strcmp(device->cdev_info.driver_name, MLX5) == 0) { - dev_qp_head = (struct device_qps *)&g_device_qp_mlx5; - qp_desc_nr = CRYPTO_QP_DESCRIPTORS_MLX5; - } else { - SPDK_ERRLOG("Failed to start device %u. Invalid driver name \"%s\"\n", - cdev_id, device->cdev_info.driver_name); - rc = -EINVAL; - goto err_qp_setup; - } - - struct rte_cryptodev_qp_conf qp_conf = { - .nb_descriptors = qp_desc_nr, - .mp_session = g_session_mp, -#if RTE_VERSION < RTE_VERSION_NUM(22, 11, 0, 0) - .mp_session_private = g_session_mp_priv, -#endif - }; - - /* Pre-setup all potential qpairs now and assign them in the channel - * callback. If we were to create them there, we'd have to stop the - * entire device affecting all other threads that might be using it - * even on other queue pairs. - */ - for (j = 0; j < device->cdev_info.max_nb_queue_pairs; j++) { - rc = rte_cryptodev_queue_pair_setup(cdev_id, j, &qp_conf, SOCKET_ID_ANY); - if (rc < 0) { - SPDK_ERRLOG("Failed to setup queue pair %u on " - "cryptodev %u: error %d\n", j, cdev_id, rc); - rc = -EINVAL; - goto err_qp_setup; - } - } - - rc = rte_cryptodev_start(cdev_id); - if (rc < 0) { - SPDK_ERRLOG("Failed to start device %u: error %d\n", - cdev_id, rc); - rc = -EINVAL; - goto err_dev_start; - } - - /* Build up lists of device/qp combinations per PMD */ - for (j = 0; j < device->cdev_info.max_nb_queue_pairs; j++) { - dev_qp = calloc(1, sizeof(struct device_qp)); - if (!dev_qp) { - rc = -ENOMEM; - goto err_qp_alloc; - } - dev_qp->device = device; - dev_qp->qp = j; - dev_qp->in_use = false; - if (strcmp(device->cdev_info.driver_name, QAT) == 0) { - g_qat_total_qp++; - } - TAILQ_INSERT_TAIL(dev_qp_head, dev_qp, link); - } - - /* Add to our list of available crypto devices. */ - TAILQ_INSERT_TAIL(&g_vbdev_devs, device, link); - - return 0; -err_qp_alloc: - TAILQ_FOREACH_SAFE(dev_qp, dev_qp_head, link, tmp_qp) { - if (dev_qp->device->cdev_id != device->cdev_id) { - continue; - } - TAILQ_REMOVE(dev_qp_head, dev_qp, link); - if (dev_qp_head == (struct device_qps *)&g_device_qp_qat) { - g_qat_total_qp--; - } - free(dev_qp); - } - rte_cryptodev_stop(cdev_id); -err_dev_start: -err_qp_setup: - rte_cryptodev_close(cdev_id); -err: - free(device); - - return rc; -} - -static void -release_vbdev_dev(struct vbdev_dev *device) -{ - struct device_qp *dev_qp; - struct device_qp *tmp_qp; - TAILQ_HEAD(device_qps, device_qp) *dev_qp_head = NULL; - - assert(device); - - /* Select the right device/qp list based on driver name. */ - if (strcmp(device->cdev_info.driver_name, QAT) == 0) { - dev_qp_head = (struct device_qps *)&g_device_qp_qat; - } else if (strcmp(device->cdev_info.driver_name, AESNI_MB) == 0) { - dev_qp_head = (struct device_qps *)&g_device_qp_aesni_mb; - } else if (strcmp(device->cdev_info.driver_name, MLX5) == 0) { - dev_qp_head = (struct device_qps *)&g_device_qp_mlx5; - } - if (dev_qp_head) { - TAILQ_FOREACH_SAFE(dev_qp, dev_qp_head, link, tmp_qp) { - /* Remove only qps of our device even if the driver names matches. */ - if (dev_qp->device->cdev_id != device->cdev_id) { - continue; - } - TAILQ_REMOVE(dev_qp_head, dev_qp, link); - if (dev_qp_head == (struct device_qps *)&g_device_qp_qat) { - g_qat_total_qp--; - } - free(dev_qp); - } - } - rte_cryptodev_stop(device->cdev_id); - rte_cryptodev_close(device->cdev_id); - free(device); -} - -/* Dummy function used by DPDK to free ext attached buffers to mbufs, we free them ourselves but - * this callback has to be here. */ -static void -shinfo_free_cb(void *arg1, void *arg2) -{ -} - -/* This is called from the module's init function. We setup all crypto devices early on as we are unable - * to easily dynamically configure queue pairs after the drivers are up and running. So, here, we - * configure the max capabilities of each device and assign threads to queue pairs as channels are - * requested. - */ -static int -vbdev_crypto_init_crypto_drivers(void) -{ - uint8_t cdev_count; - uint8_t cdev_id; - int i, rc; - struct vbdev_dev *device; - struct vbdev_dev *tmp_dev; - struct device_qp *dev_qp; - unsigned int max_sess_size = 0, sess_size; - uint16_t num_lcores = rte_lcore_count(); - char aesni_args[32]; - - /* Only the first call, via RPC or module init should init the crypto drivers. */ - if (g_session_mp != NULL) { - return 0; - } - - /* We always init AESNI_MB */ - snprintf(aesni_args, sizeof(aesni_args), "max_nb_queue_pairs=%d", AESNI_MB_NUM_QP); - rc = rte_vdev_init(AESNI_MB, aesni_args); - if (rc) { - SPDK_NOTICELOG("Failed to create virtual PMD %s: error %d. " - "Possibly %s is not supported by DPDK library. " - "Keep going...\n", AESNI_MB, rc, AESNI_MB); - } - - /* If we have no crypto devices, there's no reason to continue. */ - cdev_count = rte_cryptodev_count(); - SPDK_NOTICELOG("Found crypto devices: %d\n", (int)cdev_count); - if (cdev_count == 0) { - return 0; - } - - g_mbuf_offset = rte_mbuf_dynfield_register(&rte_mbuf_dynfield_io_context); - if (g_mbuf_offset < 0) { - SPDK_ERRLOG("error registering dynamic field with DPDK\n"); - return -EINVAL; - } - - /* - * Create global mempools, shared by all devices regardless of type. - */ - - /* First determine max session size, most pools are shared by all the devices, - * so we need to find the global max sessions size. - */ - for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { - sess_size = rte_cryptodev_sym_get_private_session_size(cdev_id); - if (sess_size > max_sess_size) { - max_sess_size = sess_size; - } - } - -#if RTE_VERSION < RTE_VERSION_NUM(22, 11, 0, 0) - g_session_mp_priv = rte_mempool_create("session_mp_priv", NUM_SESSIONS, max_sess_size, - SESS_MEMPOOL_CACHE_SIZE, 0, NULL, NULL, NULL, - NULL, SOCKET_ID_ANY, 0); - if (g_session_mp_priv == NULL) { - SPDK_ERRLOG("Cannot create private session pool max size 0x%x\n", max_sess_size); - return -ENOMEM; - } - /* When session private data mempool allocated, the element size for the session mempool - * should be 0. */ - max_sess_size = 0; -#endif - - g_session_mp = rte_cryptodev_sym_session_pool_create( - "session_mp", - NUM_SESSIONS, max_sess_size, SESS_MEMPOOL_CACHE_SIZE, 0, - SOCKET_ID_ANY); - if (g_session_mp == NULL) { - SPDK_ERRLOG("Cannot create session pool max size 0x%x\n", max_sess_size); - rc = -ENOMEM; - goto error_create_session_mp; - } - - g_mbuf_mp = rte_pktmbuf_pool_create("mbuf_mp", NUM_MBUFS, POOL_CACHE_SIZE, - 0, 0, SPDK_ENV_SOCKET_ID_ANY); - if (g_mbuf_mp == NULL) { - SPDK_ERRLOG("Cannot create mbuf pool\n"); - rc = -ENOMEM; - goto error_create_mbuf; - } - - /* We use per op private data as suggested by DPDK and to store the IV and - * our own struct for queueing ops. - */ - g_crypto_op_mp = rte_crypto_op_pool_create("op_mp", - RTE_CRYPTO_OP_TYPE_SYMMETRIC, - NUM_MBUFS, - POOL_CACHE_SIZE, - (DEFAULT_NUM_XFORMS * - sizeof(struct rte_crypto_sym_xform)) + - IV_LENGTH + QUEUED_OP_LENGTH, - rte_socket_id()); - - if (g_crypto_op_mp == NULL) { - SPDK_ERRLOG("Cannot create op pool\n"); - rc = -ENOMEM; - goto error_create_op; - } - - /* Init all devices */ - for (i = 0; i < cdev_count; i++) { - rc = create_vbdev_dev(i, num_lcores); - if (rc) { - goto err; - } - } - - /* Assign index values to the QAT device qp nodes so that we can - * assign them for optimal performance. - */ - i = 0; - TAILQ_FOREACH(dev_qp, &g_device_qp_qat, link) { - dev_qp->index = i++; - } - - g_shinfo.free_cb = shinfo_free_cb; - return 0; - - /* Error cleanup paths. */ -err: - TAILQ_FOREACH_SAFE(device, &g_vbdev_devs, link, tmp_dev) { - TAILQ_REMOVE(&g_vbdev_devs, device, link); - release_vbdev_dev(device); - } - rte_mempool_free(g_crypto_op_mp); - g_crypto_op_mp = NULL; -error_create_op: - rte_mempool_free(g_mbuf_mp); - g_mbuf_mp = NULL; -error_create_mbuf: - rte_mempool_free(g_session_mp); - g_session_mp = NULL; -error_create_session_mp: - if (g_session_mp_priv != NULL) { - rte_mempool_free(g_session_mp_priv); - g_session_mp_priv = NULL; - } - return rc; -} +static void vbdev_crypto_queue_io(struct spdk_bdev_io *bdev_io, + enum crypto_io_resubmit_state state); +static void _complete_internal_io(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg); +static void _complete_internal_read(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg); +static void _complete_internal_write(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg); +static void vbdev_crypto_examine(struct spdk_bdev *bdev); +static int vbdev_crypto_claim(const char *bdev_name); +static void vbdev_crypto_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io); /* Following an encrypt or decrypt we need to then either write the encrypted data or finish * the read on decrypted data. Do that here. */ static void -_crypto_operation_complete(struct spdk_bdev_io *bdev_io) +_crypto_operation_complete(void *ref, int status) { + struct spdk_bdev_io *bdev_io = ref; struct vbdev_crypto *crypto_bdev = SPDK_CONTAINEROF(bdev_io->bdev, struct vbdev_crypto, crypto_bdev); - struct crypto_bdev_io *io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; - struct crypto_io_channel *crypto_ch = io_ctx->crypto_ch; - struct spdk_bdev_io *free_me = io_ctx->read_io; + struct crypto_bdev_io *crypto_io = (struct crypto_bdev_io *)bdev_io->driver_ctx; + struct crypto_io_channel *crypto_ch = crypto_io->crypto_ch; + struct spdk_bdev_io *free_me = crypto_io->read_io; int rc = 0; - /* Can also be called from the crypto_dev_poller() to fail the stuck re-enqueue ops IO. */ - if (io_ctx->on_pending_list) { - TAILQ_REMOVE(&crypto_ch->pending_cry_ios, bdev_io, module_link); - io_ctx->on_pending_list = false; + if (status || crypto_ch->reset_iter) { + /* If we're completing this with an outstanding reset we need to fail it */ + rc = -EINVAL; } - if (bdev_io->type == SPDK_BDEV_IO_TYPE_READ) { + TAILQ_REMOVE(&crypto_ch->in_accel_fw, bdev_io, module_link); + if (bdev_io->type == SPDK_BDEV_IO_TYPE_READ) { /* Complete the original IO and then free the one that we created * as a result of issuing an IO via submit_request. */ - if (io_ctx->bdev_io_status != SPDK_BDEV_IO_STATUS_FAILED) { + if (!rc) { spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); } else { SPDK_ERRLOG("Issue with decryption on bdev_io %p\n", bdev_io); - rc = -EINVAL; } spdk_bdev_free_io(free_me); } else if (bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE) { - - if (io_ctx->bdev_io_status != SPDK_BDEV_IO_STATUS_FAILED) { + if (!rc) { /* Write the encrypted data. */ rc = spdk_bdev_writev_blocks(crypto_bdev->base_desc, crypto_ch->base_ch, - &io_ctx->aux_buf_iov, 1, io_ctx->aux_offset_blocks, - io_ctx->aux_num_blocks, _complete_internal_write, + &crypto_io->aux_buf_iov, 1, crypto_io->aux_offset_blocks, + crypto_io->aux_num_blocks, _complete_internal_write, bdev_io); + if (rc == -ENOMEM) { + vbdev_crypto_queue_io(bdev_io, CRYPTO_IO_ENCRYPT_DONE); + goto check_reset; + } } else { SPDK_ERRLOG("Issue with encryption on bdev_io %p\n", bdev_io); - rc = -EINVAL; } - } else { - SPDK_ERRLOG("Unknown bdev type %u on crypto operation completion\n", - bdev_io->type); + SPDK_ERRLOG("Unknown bdev type %u on crypto operation completion\n", bdev_io->type); rc = -EINVAL; } if (rc) { spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); } -} -static void -cancel_queued_crypto_ops(struct crypto_io_channel *crypto_ch, struct spdk_bdev_io *bdev_io) -{ - struct rte_mbuf *mbufs_to_free[2 * MAX_DEQUEUE_BURST_SIZE]; - struct rte_crypto_op *dequeued_ops[MAX_DEQUEUE_BURST_SIZE]; - struct vbdev_crypto_op *op_to_cancel, *tmp_op; - struct rte_crypto_op *crypto_op; - int num_mbufs, num_dequeued_ops; - - /* Remove all ops from the failed IO. Since we don't know the - * order we have to check them all. */ - num_mbufs = 0; - num_dequeued_ops = 0; - TAILQ_FOREACH_SAFE(op_to_cancel, &crypto_ch->queued_cry_ops, link, tmp_op) { - /* Checking if this is our op. One IO contains multiple ops. */ - if (bdev_io == op_to_cancel->bdev_io) { - crypto_op = op_to_cancel->crypto_op; - TAILQ_REMOVE(&crypto_ch->queued_cry_ops, op_to_cancel, link); - - /* Populating lists for freeing mbufs and ops. */ - mbufs_to_free[num_mbufs++] = (void *)crypto_op->sym->m_src; - if (crypto_op->sym->m_dst) { - mbufs_to_free[num_mbufs++] = (void *)crypto_op->sym->m_dst; - } - dequeued_ops[num_dequeued_ops++] = crypto_op; - } - } - - /* Now bulk free both mbufs and crypto operations. */ - if (num_dequeued_ops > 0) { - rte_mempool_put_bulk(g_crypto_op_mp, (void **)dequeued_ops, - num_dequeued_ops); - assert(num_mbufs > 0); - /* This also releases chained mbufs if any. */ - rte_pktmbuf_free_bulk(mbufs_to_free, num_mbufs); - } -} - -static int _crypto_operation(struct spdk_bdev_io *bdev_io, - enum rte_crypto_cipher_operation crypto_op, - void *aux_buf); - -/* This is the poller for the crypto device. It uses a single API to dequeue whatever is ready at - * the device. Then we need to decide if what we've got so far (including previous poller - * runs) totals up to one or more complete bdev_ios and if so continue with the bdev_io - * accordingly. This means either completing a read or issuing a new write. - */ -static int -crypto_dev_poller(void *args) -{ - struct crypto_io_channel *crypto_ch = args; - uint8_t cdev_id = crypto_ch->device_qp->device->cdev_id; - int i, num_dequeued_ops, num_enqueued_ops; - struct spdk_bdev_io *bdev_io = NULL; - struct crypto_bdev_io *io_ctx = NULL; - struct rte_crypto_op *dequeued_ops[MAX_DEQUEUE_BURST_SIZE]; - struct rte_mbuf *mbufs_to_free[2 * MAX_DEQUEUE_BURST_SIZE]; - int num_mbufs = 0; - struct vbdev_crypto_op *op_to_resubmit; - - /* Each run of the poller will get just what the device has available - * at the moment we call it, we don't check again after draining the - * first batch. - */ - num_dequeued_ops = rte_cryptodev_dequeue_burst(cdev_id, crypto_ch->device_qp->qp, - dequeued_ops, MAX_DEQUEUE_BURST_SIZE); - - /* Check if operation was processed successfully */ - for (i = 0; i < num_dequeued_ops; i++) { - - /* We don't know the order or association of the crypto ops wrt any - * particular bdev_io so need to look at each and determine if it's - * the last one for it's bdev_io or not. - */ - bdev_io = (struct spdk_bdev_io *)*RTE_MBUF_DYNFIELD(dequeued_ops[i]->sym->m_src, g_mbuf_offset, - uint64_t *); - assert(bdev_io != NULL); - io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; - - if (dequeued_ops[i]->status != RTE_CRYPTO_OP_STATUS_SUCCESS) { - SPDK_ERRLOG("error with op %d status %u\n", i, - dequeued_ops[i]->status); - /* Update the bdev status to error, we'll still process the - * rest of the crypto ops for this bdev_io though so they - * aren't left hanging. - */ - io_ctx->bdev_io_status = SPDK_BDEV_IO_STATUS_FAILED; - } - - assert(io_ctx->cryop_cnt_remaining > 0); - - /* Return the associated src and dst mbufs by collecting them into - * an array that we can use the bulk API to free after the loop. - */ - *RTE_MBUF_DYNFIELD(dequeued_ops[i]->sym->m_src, g_mbuf_offset, uint64_t *) = 0; - mbufs_to_free[num_mbufs++] = (void *)dequeued_ops[i]->sym->m_src; - if (dequeued_ops[i]->sym->m_dst) { - mbufs_to_free[num_mbufs++] = (void *)dequeued_ops[i]->sym->m_dst; - } - - /* done encrypting, complete the bdev_io */ - if (--io_ctx->cryop_cnt_remaining == 0) { - - /* If we're completing this with an outstanding reset we need - * to fail it. - */ - if (crypto_ch->iter) { - io_ctx->bdev_io_status = SPDK_BDEV_IO_STATUS_FAILED; - } - - /* Complete the IO */ - _crypto_operation_complete(bdev_io); - } - } - - /* Now bulk free both mbufs and crypto operations. */ - if (num_dequeued_ops > 0) { - rte_mempool_put_bulk(g_crypto_op_mp, - (void **)dequeued_ops, - num_dequeued_ops); - assert(num_mbufs > 0); - /* This also releases chained mbufs if any. */ - rte_pktmbuf_free_bulk(mbufs_to_free, num_mbufs); - } - - /* Check if there are any pending crypto ops to process */ - while (!TAILQ_EMPTY(&crypto_ch->queued_cry_ops)) { - op_to_resubmit = TAILQ_FIRST(&crypto_ch->queued_cry_ops); - bdev_io = op_to_resubmit->bdev_io; - io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; - num_enqueued_ops = rte_cryptodev_enqueue_burst(op_to_resubmit->cdev_id, - op_to_resubmit->qp, - &op_to_resubmit->crypto_op, - 1); - if (num_enqueued_ops == 1) { - /* Make sure we don't put this on twice as one bdev_io is made up - * of many crypto ops. - */ - if (io_ctx->on_pending_list == false) { - TAILQ_INSERT_TAIL(&crypto_ch->pending_cry_ios, bdev_io, module_link); - io_ctx->on_pending_list = true; - } - TAILQ_REMOVE(&crypto_ch->queued_cry_ops, op_to_resubmit, link); - } else { - if (op_to_resubmit->crypto_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) { - /* If we couldn't get one, just break and try again later. */ - break; - } else { - /* Something is really wrong with the op. Most probably the - * mbuf is broken or the HW is not able to process the request. - * Fail the IO and remove its ops from the queued ops list. */ - io_ctx->bdev_io_status = SPDK_BDEV_IO_STATUS_FAILED; - - cancel_queued_crypto_ops(crypto_ch, bdev_io); - - /* Fail the IO if there is nothing left on device. */ - if (--io_ctx->cryop_cnt_remaining == 0) { - _crypto_operation_complete(bdev_io); - } - } - - } - } - - /* If the channel iter is not NULL, we need to continue to poll +check_reset: + /* If the channel iter is not NULL, we need to wait * until the pending list is empty, then we can move on to the * next channel. */ - if (crypto_ch->iter && TAILQ_EMPTY(&crypto_ch->pending_cry_ios)) { + if (crypto_ch->reset_iter && TAILQ_EMPTY(&crypto_ch->in_accel_fw)) { SPDK_NOTICELOG("Channel %p has been quiesced.\n", crypto_ch); - spdk_for_each_channel_continue(crypto_ch->iter, 0); - crypto_ch->iter = NULL; + spdk_for_each_channel_continue(crypto_ch->reset_iter, 0); + crypto_ch->reset_iter = NULL; } - - return num_dequeued_ops; -} - -/* Allocate the new mbuf of @remainder size with data pointed by @addr and attach - * it to the @orig_mbuf. */ -static int -mbuf_chain_remainder(struct spdk_bdev_io *bdev_io, struct rte_mbuf *orig_mbuf, - uint8_t *addr, uint32_t remainder) -{ - uint64_t phys_addr, phys_len; - struct rte_mbuf *chain_mbuf; - int rc; - - phys_len = remainder; - phys_addr = spdk_vtophys((void *)addr, &phys_len); - if (spdk_unlikely(phys_addr == SPDK_VTOPHYS_ERROR || phys_len != remainder)) { - return -EFAULT; - } - rc = rte_pktmbuf_alloc_bulk(g_mbuf_mp, (struct rte_mbuf **)&chain_mbuf, 1); - if (spdk_unlikely(rc)) { - return -ENOMEM; - } - /* Store context in every mbuf as we don't know anything about completion order */ - *RTE_MBUF_DYNFIELD(chain_mbuf, g_mbuf_offset, uint64_t *) = (uint64_t)bdev_io; - rte_pktmbuf_attach_extbuf(chain_mbuf, addr, phys_addr, phys_len, &g_shinfo); - rte_pktmbuf_append(chain_mbuf, phys_len); - - /* Chained buffer is released by rte_pktbuf_free_bulk() automagicaly. */ - rte_pktmbuf_chain(orig_mbuf, chain_mbuf); - return 0; -} - -/* Attach data buffer pointed by @addr to @mbuf. Return utilized len of the - * contiguous space that was physically available. */ -static uint64_t -mbuf_attach_buf(struct spdk_bdev_io *bdev_io, struct rte_mbuf *mbuf, - uint8_t *addr, uint32_t len) -{ - uint64_t phys_addr, phys_len; - - /* Store context in every mbuf as we don't know anything about completion order */ - *RTE_MBUF_DYNFIELD(mbuf, g_mbuf_offset, uint64_t *) = (uint64_t)bdev_io; - - phys_len = len; - phys_addr = spdk_vtophys((void *)addr, &phys_len); - if (spdk_unlikely(phys_addr == SPDK_VTOPHYS_ERROR || phys_len == 0)) { - return 0; - } - assert(phys_len <= len); - - /* Set the mbuf elements address and length. */ - rte_pktmbuf_attach_extbuf(mbuf, addr, phys_addr, phys_len, &g_shinfo); - rte_pktmbuf_append(mbuf, phys_len); - - return phys_len; } /* We're either encrypting on the way down or decrypting on the way back. */ static int -_crypto_operation(struct spdk_bdev_io *bdev_io, enum rte_crypto_cipher_operation crypto_op, - void *aux_buf) +_crypto_operation(struct spdk_bdev_io *bdev_io, bool encrypt, void *aux_buf) { - uint16_t num_enqueued_ops = 0; - uint32_t cryop_cnt = bdev_io->u.bdev.num_blocks; - struct crypto_bdev_io *io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; - struct crypto_io_channel *crypto_ch = io_ctx->crypto_ch; - uint8_t cdev_id = crypto_ch->device_qp->device->cdev_id; - uint32_t crypto_len = io_ctx->crypto_bdev->crypto_bdev.blocklen; - uint64_t total_length = bdev_io->u.bdev.num_blocks * crypto_len; + struct crypto_bdev_io *crypto_io = (struct crypto_bdev_io *)bdev_io->driver_ctx; + struct crypto_io_channel *crypto_ch = crypto_io->crypto_ch; + uint32_t crypto_len = crypto_io->crypto_bdev->crypto_bdev.blocklen; + uint64_t total_length; + uint64_t alignment; int rc; - uint32_t iov_index = 0; - uint32_t allocated = 0; - uint8_t *current_iov = NULL; - uint64_t total_remaining = 0; - uint64_t current_iov_remaining = 0; - uint32_t crypto_index = 0; - uint32_t en_offset = 0; - struct rte_crypto_op *crypto_ops[MAX_ENQUEUE_ARRAY_SIZE]; - struct rte_mbuf *src_mbufs[MAX_ENQUEUE_ARRAY_SIZE]; - struct rte_mbuf *dst_mbufs[MAX_ENQUEUE_ARRAY_SIZE]; - int burst; - struct vbdev_crypto_op *op_to_queue; - uint64_t alignment = spdk_bdev_get_buf_align(&io_ctx->crypto_bdev->crypto_bdev); - - assert((bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen) <= CRYPTO_MAX_IO); - - /* Get the number of source mbufs that we need. These will always be 1:1 because we - * don't support chaining. The reason we don't is because of our decision to use - * LBA as IV, there can be no case where we'd need >1 mbuf per crypto op or the - * op would be > 1 LBA. - */ - rc = rte_pktmbuf_alloc_bulk(g_mbuf_mp, src_mbufs, cryop_cnt); - if (rc) { - SPDK_ERRLOG("Failed to get src_mbufs!\n"); - return -ENOMEM; - } - - /* Get the same amount but these buffers to describe the encrypted data location (dst). */ - if (crypto_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { - rc = rte_pktmbuf_alloc_bulk(g_mbuf_mp, dst_mbufs, cryop_cnt); - if (rc) { - SPDK_ERRLOG("Failed to get dst_mbufs!\n"); - rc = -ENOMEM; - goto error_get_dst; - } - } - -#ifdef __clang_analyzer__ - /* silence scan-build false positive */ - SPDK_CLANG_ANALYZER_PREINIT_PTR_ARRAY(crypto_ops, MAX_ENQUEUE_ARRAY_SIZE, 0x1000); -#endif - /* Allocate crypto operations. */ - allocated = rte_crypto_op_bulk_alloc(g_crypto_op_mp, - RTE_CRYPTO_OP_TYPE_SYMMETRIC, - crypto_ops, cryop_cnt); - if (allocated < cryop_cnt) { - SPDK_ERRLOG("Failed to allocate crypto ops!\n"); - rc = -ENOMEM; - goto error_get_ops; - } /* For encryption, we need to prepare a single contiguous buffer as the encryption * destination, we'll then pass that along for the write after encryption is done. * This is done to avoiding encrypting the provided write buffer which may be * undesirable in some use cases. */ - if (crypto_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { - io_ctx->aux_buf_iov.iov_len = total_length; - io_ctx->aux_buf_raw = aux_buf; - io_ctx->aux_buf_iov.iov_base = (void *)(((uintptr_t)aux_buf + (alignment - 1)) & ~(alignment - 1)); - io_ctx->aux_offset_blocks = bdev_io->u.bdev.offset_blocks; - io_ctx->aux_num_blocks = bdev_io->u.bdev.num_blocks; + if (encrypt) { + total_length = bdev_io->u.bdev.num_blocks * crypto_len; + alignment = spdk_bdev_get_buf_align(&crypto_io->crypto_bdev->crypto_bdev); + crypto_io->aux_buf_iov.iov_len = total_length; + crypto_io->aux_buf_raw = aux_buf; + crypto_io->aux_buf_iov.iov_base = (void *)(((uintptr_t)aux_buf + (alignment - 1)) & ~ + (alignment - 1)); + crypto_io->aux_offset_blocks = bdev_io->u.bdev.offset_blocks; + crypto_io->aux_num_blocks = bdev_io->u.bdev.num_blocks; + + rc = spdk_accel_submit_encrypt(crypto_ch->accel_channel, crypto_ch->crypto_key, + &crypto_io->aux_buf_iov, 1, + bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, + bdev_io->u.bdev.offset_blocks, crypto_len, 0, + _crypto_operation_complete, bdev_io); + } else { + rc = spdk_accel_submit_decrypt(crypto_ch->accel_channel, crypto_ch->crypto_key, + bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, bdev_io->u.bdev.iovs, + bdev_io->u.bdev.iovcnt, bdev_io->u.bdev.offset_blocks, + crypto_len, 0, + _crypto_operation_complete, bdev_io); } - /* This value is used in the completion callback to determine when the bdev_io is - * complete. - */ - io_ctx->cryop_cnt_remaining = cryop_cnt; - - /* As we don't support chaining because of a decision to use LBA as IV, construction - * of crypto operations is straightforward. We build both the op, the mbuf and the - * dst_mbuf in our local arrays by looping through the length of the bdev IO and - * picking off LBA sized blocks of memory from the IOVs as we walk through them. Each - * LBA sized chunk of memory will correspond 1:1 to a crypto operation and a single - * mbuf per crypto operation. - */ - total_remaining = total_length; - current_iov = bdev_io->u.bdev.iovs[iov_index].iov_base; - current_iov_remaining = bdev_io->u.bdev.iovs[iov_index].iov_len; - do { - uint8_t *iv_ptr; - uint8_t *buf_addr; - uint64_t phys_len; - uint32_t remainder; - uint64_t op_block_offset; - - phys_len = mbuf_attach_buf(bdev_io, src_mbufs[crypto_index], - current_iov, crypto_len); - if (spdk_unlikely(phys_len == 0)) { - goto error_attach_session; - rc = -EFAULT; - } - - /* Handle the case of page boundary. */ - remainder = crypto_len - phys_len; - if (spdk_unlikely(remainder > 0)) { - rc = mbuf_chain_remainder(bdev_io, src_mbufs[crypto_index], - current_iov + phys_len, remainder); - if (spdk_unlikely(rc)) { - goto error_attach_session; - } - } - - /* Set the IV - we use the LBA of the crypto_op */ - iv_ptr = rte_crypto_op_ctod_offset(crypto_ops[crypto_index], uint8_t *, - IV_OFFSET); - memset(iv_ptr, 0, IV_LENGTH); - op_block_offset = bdev_io->u.bdev.offset_blocks + crypto_index; - rte_memcpy(iv_ptr, &op_block_offset, sizeof(uint64_t)); - - /* Set the data to encrypt/decrypt length */ - crypto_ops[crypto_index]->sym->cipher.data.length = crypto_len; - crypto_ops[crypto_index]->sym->cipher.data.offset = 0; - - /* link the mbuf to the crypto op. */ - crypto_ops[crypto_index]->sym->m_src = src_mbufs[crypto_index]; - - /* For encrypt, point the destination to a buffer we allocate and redirect the bdev_io - * that will be used to process the write on completion to the same buffer. Setting - * up the en_buffer is a little simpler as we know the destination buffer is single IOV. - */ - if (crypto_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { - buf_addr = io_ctx->aux_buf_iov.iov_base + en_offset; - phys_len = mbuf_attach_buf(bdev_io, dst_mbufs[crypto_index], - buf_addr, crypto_len); - if (spdk_unlikely(phys_len == 0)) { - rc = -EFAULT; - goto error_attach_session; - } - - crypto_ops[crypto_index]->sym->m_dst = dst_mbufs[crypto_index]; - en_offset += phys_len; - - /* Handle the case of page boundary. */ - remainder = crypto_len - phys_len; - if (spdk_unlikely(remainder > 0)) { - rc = mbuf_chain_remainder(bdev_io, dst_mbufs[crypto_index], - buf_addr + phys_len, remainder); - if (spdk_unlikely(rc)) { - goto error_attach_session; - } - en_offset += remainder; - } - - /* Attach the crypto session to the operation */ - rc = rte_crypto_op_attach_sym_session(crypto_ops[crypto_index], - io_ctx->crypto_bdev->session_encrypt); - if (rc) { - rc = -EINVAL; - goto error_attach_session; - } - } else { - crypto_ops[crypto_index]->sym->m_dst = NULL; - - /* Attach the crypto session to the operation */ - rc = rte_crypto_op_attach_sym_session(crypto_ops[crypto_index], - io_ctx->crypto_bdev->session_decrypt); - if (rc) { - rc = -EINVAL; - goto error_attach_session; - } - } - - /* Subtract our running totals for the op in progress and the overall bdev io */ - total_remaining -= crypto_len; - current_iov_remaining -= crypto_len; - - /* move our current IOV pointer accordingly. */ - current_iov += crypto_len; - - /* move on to the next crypto operation */ - crypto_index++; - - /* If we're done with this IOV, move to the next one. */ - if (current_iov_remaining == 0 && total_remaining > 0) { - iov_index++; - current_iov = bdev_io->u.bdev.iovs[iov_index].iov_base; - current_iov_remaining = bdev_io->u.bdev.iovs[iov_index].iov_len; - } - } while (total_remaining > 0); - - /* Enqueue everything we've got but limit by the max number of descriptors we - * configured the crypto device for. - */ - burst = spdk_min(cryop_cnt, io_ctx->crypto_bdev->qp_desc_nr); - num_enqueued_ops = rte_cryptodev_enqueue_burst(cdev_id, crypto_ch->device_qp->qp, - &crypto_ops[0], - burst); - - /* Add this bdev_io to our outstanding list if any of its crypto ops made it. */ - if (num_enqueued_ops > 0) { - TAILQ_INSERT_TAIL(&crypto_ch->pending_cry_ios, bdev_io, module_link); - io_ctx->on_pending_list = true; - } - /* We were unable to enqueue everything but did get some, so need to decide what - * to do based on the status of the last op. - */ - if (num_enqueued_ops < cryop_cnt) { - switch (crypto_ops[num_enqueued_ops]->status) { - case RTE_CRYPTO_OP_STATUS_NOT_PROCESSED: - /* Queue them up on a linked list to be resubmitted via the poller. */ - for (crypto_index = num_enqueued_ops; crypto_index < cryop_cnt; crypto_index++) { - op_to_queue = (struct vbdev_crypto_op *)rte_crypto_op_ctod_offset(crypto_ops[crypto_index], - uint8_t *, QUEUED_OP_OFFSET); - op_to_queue->cdev_id = cdev_id; - op_to_queue->qp = crypto_ch->device_qp->qp; - op_to_queue->crypto_op = crypto_ops[crypto_index]; - op_to_queue->bdev_io = bdev_io; - TAILQ_INSERT_TAIL(&crypto_ch->queued_cry_ops, - op_to_queue, - link); - } - break; - default: - /* For all other statuses, set the io_ctx bdev_io status so that - * the poller will pick the failure up for the overall bdev status. - */ - io_ctx->bdev_io_status = SPDK_BDEV_IO_STATUS_FAILED; - if (num_enqueued_ops == 0) { - /* If nothing was enqueued, but the last one wasn't because of - * busy, fail it now as the poller won't know anything about it. - */ - rc = -EINVAL; - goto error_attach_session; - } - break; - } + if (!rc) { + TAILQ_INSERT_TAIL(&crypto_ch->in_accel_fw, bdev_io, module_link); } return rc; - - /* Error cleanup paths. */ -error_attach_session: -error_get_ops: - if (crypto_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { - /* This also releases chained mbufs if any. */ - rte_pktmbuf_free_bulk(dst_mbufs, cryop_cnt); - } - if (allocated > 0) { - rte_mempool_put_bulk(g_crypto_op_mp, (void **)crypto_ops, - allocated); - } -error_get_dst: - /* This also releases chained mbufs if any. */ - rte_pktmbuf_free_bulk(src_mbufs, cryop_cnt); - return rc; } /* This function is called after all channels have been quiesced following @@ -1104,27 +201,26 @@ error_get_dst: static void _ch_quiesce_done(struct spdk_io_channel_iter *i, int status) { - struct crypto_bdev_io *io_ctx = spdk_io_channel_iter_get_ctx(i); + struct crypto_bdev_io *crypto_io = spdk_io_channel_iter_get_ctx(i); + struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(crypto_io); - assert(TAILQ_EMPTY(&io_ctx->crypto_ch->pending_cry_ios)); - assert(io_ctx->orig_io != NULL); + assert(TAILQ_EMPTY(&crypto_io->crypto_ch->in_accel_fw)); - spdk_bdev_io_complete(io_ctx->orig_io, SPDK_BDEV_IO_STATUS_SUCCESS); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); } -/* This function is called per channel to quiesce IOs before completing a - * bdev reset that we received. - */ static void _ch_quiesce(struct spdk_io_channel_iter *i) { struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); struct crypto_io_channel *crypto_ch = spdk_io_channel_get_ctx(ch); - crypto_ch->iter = i; - /* When the poller runs, it will see the non-NULL iter and handle - * the quiesce. - */ + if (TAILQ_EMPTY(&crypto_ch->in_accel_fw)) { + spdk_for_each_channel_continue(i, 0); + } else { + /* In accel completion callback we will see the non-NULL iter and handle the quiesce */ + crypto_ch->reset_iter = i; + } } /* Completion callback for IO that were issued from this bdev other than read/write. @@ -1139,8 +235,6 @@ _complete_internal_io(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) if (bdev_io->type == SPDK_BDEV_IO_TYPE_RESET) { struct crypto_bdev_io *orig_ctx = (struct crypto_bdev_io *)orig_io->driver_ctx; - assert(orig_io == orig_ctx->orig_io); - spdk_bdev_free_io(bdev_io); spdk_for_each_channel(orig_ctx->crypto_bdev, @@ -1174,16 +268,23 @@ _complete_internal_read(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg { struct spdk_bdev_io *orig_io = cb_arg; struct crypto_bdev_io *orig_ctx = (struct crypto_bdev_io *)orig_io->driver_ctx; + int rc; if (success) { - /* Save off this bdev_io so it can be freed after decryption. */ orig_ctx->read_io = bdev_io; - - if (!_crypto_operation(orig_io, RTE_CRYPTO_CIPHER_OP_DECRYPT, NULL)) { + rc = _crypto_operation(orig_io, false, NULL); + if (!rc) { return; } else { - SPDK_ERRLOG("Failed to decrypt!\n"); + if (rc == -ENOMEM) { + SPDK_DEBUGLOG(vbdev_crypto, "No memory, queue the IO.\n"); + /* We will repeat crypto operation later */ + vbdev_crypto_queue_io(orig_io, CRYPTO_IO_READ_DONE); + return; + } else { + SPDK_ERRLOG("Failed to decrypt, rc %d\n", rc); + } } } else { SPDK_ERRLOG("Failed to read prior to decrypting!\n"); @@ -1197,22 +298,39 @@ static void vbdev_crypto_resubmit_io(void *arg) { struct spdk_bdev_io *bdev_io = (struct spdk_bdev_io *)arg; - struct crypto_bdev_io *io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; + struct crypto_bdev_io *crypto_io = (struct crypto_bdev_io *)bdev_io->driver_ctx; + struct spdk_io_channel *ch; - vbdev_crypto_submit_request(io_ctx->ch, bdev_io); + switch (crypto_io->resubmit_state) { + case CRYPTO_IO_NEW: + assert(crypto_io->crypto_ch); + ch = spdk_io_channel_from_ctx(crypto_io->crypto_ch); + vbdev_crypto_submit_request(ch, bdev_io); + break; + case CRYPTO_IO_ENCRYPT_DONE: + _crypto_operation_complete(bdev_io, 0); + break; + case CRYPTO_IO_READ_DONE: + _complete_internal_read(crypto_io->read_io, true, bdev_io); + break; + default: + SPDK_UNREACHABLE(); + } } static void -vbdev_crypto_queue_io(struct spdk_bdev_io *bdev_io) +vbdev_crypto_queue_io(struct spdk_bdev_io *bdev_io, enum crypto_io_resubmit_state state) { - struct crypto_bdev_io *io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; + struct crypto_bdev_io *crypto_io = (struct crypto_bdev_io *)bdev_io->driver_ctx; int rc; - io_ctx->bdev_io_wait.bdev = bdev_io->bdev; - io_ctx->bdev_io_wait.cb_fn = vbdev_crypto_resubmit_io; - io_ctx->bdev_io_wait.cb_arg = bdev_io; + crypto_io->bdev_io_wait.bdev = bdev_io->bdev; + crypto_io->bdev_io_wait.cb_fn = vbdev_crypto_resubmit_io; + crypto_io->bdev_io_wait.cb_arg = bdev_io; + crypto_io->resubmit_state = state; - rc = spdk_bdev_queue_io_wait(bdev_io->bdev, io_ctx->crypto_ch->base_ch, &io_ctx->bdev_io_wait); + rc = spdk_bdev_queue_io_wait(bdev_io->bdev, crypto_io->crypto_ch->base_ch, + &crypto_io->bdev_io_wait); if (rc != 0) { SPDK_ERRLOG("Queue io failed in vbdev_crypto_queue_io, rc=%d.\n", rc); spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); @@ -1230,7 +348,6 @@ crypto_read_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, struct vbdev_crypto *crypto_bdev = SPDK_CONTAINEROF(bdev_io->bdev, struct vbdev_crypto, crypto_bdev); struct crypto_io_channel *crypto_ch = spdk_io_channel_get_ctx(ch); - struct crypto_bdev_io *io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; int rc; if (!success) { @@ -1245,8 +362,7 @@ crypto_read_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, if (rc != 0) { if (rc == -ENOMEM) { SPDK_DEBUGLOG(vbdev_crypto, "No memory, queue the IO.\n"); - io_ctx->ch = ch; - vbdev_crypto_queue_io(bdev_io); + vbdev_crypto_queue_io(bdev_io, CRYPTO_IO_NEW); } else { SPDK_ERRLOG("Failed to submit bdev_io!\n"); spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); @@ -1262,18 +378,21 @@ static void crypto_write_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, void *aux_buf) { - struct crypto_bdev_io *io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; - int rc = 0; + int rc; - rc = _crypto_operation(bdev_io, RTE_CRYPTO_CIPHER_OP_ENCRYPT, aux_buf); + if (spdk_unlikely(!aux_buf)) { + SPDK_ERRLOG("Failed to get aux buffer!\n"); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + return; + } + rc = _crypto_operation(bdev_io, true, aux_buf); if (rc != 0) { spdk_bdev_io_put_aux_buf(bdev_io, aux_buf); if (rc == -ENOMEM) { SPDK_DEBUGLOG(vbdev_crypto, "No memory, queue the IO.\n"); - io_ctx->ch = ch; - vbdev_crypto_queue_io(bdev_io); + vbdev_crypto_queue_io(bdev_io, CRYPTO_IO_NEW); } else { - SPDK_ERRLOG("Failed to submit bdev_io!\n"); + SPDK_ERRLOG("Failed to submit crypto operation!\n"); spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); } } @@ -1292,14 +411,12 @@ vbdev_crypto_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bde struct vbdev_crypto *crypto_bdev = SPDK_CONTAINEROF(bdev_io->bdev, struct vbdev_crypto, crypto_bdev); struct crypto_io_channel *crypto_ch = spdk_io_channel_get_ctx(ch); - struct crypto_bdev_io *io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; + struct crypto_bdev_io *crypto_io = (struct crypto_bdev_io *)bdev_io->driver_ctx; int rc = 0; - memset(io_ctx, 0, sizeof(struct crypto_bdev_io)); - io_ctx->crypto_bdev = crypto_bdev; - io_ctx->crypto_ch = crypto_ch; - io_ctx->orig_io = bdev_io; - io_ctx->bdev_io_status = SPDK_BDEV_IO_STATUS_SUCCESS; + memset(crypto_io, 0, sizeof(struct crypto_bdev_io)); + crypto_io->crypto_bdev = crypto_bdev; + crypto_io->crypto_ch = crypto_ch; switch (bdev_io->type) { case SPDK_BDEV_IO_TYPE_READ: @@ -1338,8 +455,7 @@ vbdev_crypto_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bde if (rc != 0) { if (rc == -ENOMEM) { SPDK_DEBUGLOG(vbdev_crypto, "No memory, queue the IO.\n"); - io_ctx->ch = ch; - vbdev_crypto_queue_io(bdev_io); + vbdev_crypto_queue_io(bdev_io, CRYPTO_IO_NEW); } else { SPDK_ERRLOG("Failed to submit bdev_io!\n"); spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); @@ -1372,62 +488,6 @@ vbdev_crypto_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) } } -static struct vbdev_dev * -_vdev_dev_get(struct vbdev_crypto *vbdev) -{ - struct vbdev_dev *device; - - TAILQ_FOREACH(device, &g_vbdev_devs, link) { - if (strcmp(device->cdev_info.driver_name, vbdev->opts->drv_name) == 0) { - return device; - } - } - return NULL; -} - -static void -_cryptodev_sym_session_free(struct vbdev_crypto *vbdev, void *session) -{ -#if RTE_VERSION >= RTE_VERSION_NUM(22, 11, 0, 0) - struct vbdev_dev *device = _vdev_dev_get(vbdev); - - assert(device != NULL); - - rte_cryptodev_sym_session_free(device->cdev_id, session); -#else - rte_cryptodev_sym_session_free(session); -#endif -} - -static void * -_cryptodev_sym_session_create(struct vbdev_crypto *vbdev, struct rte_crypto_sym_xform *xforms) -{ - void *session; - struct vbdev_dev *device; - - device = _vdev_dev_get(vbdev); - if (!device) { - SPDK_ERRLOG("Failed to match crypto device driver to crypto vbdev.\n"); - return NULL; - } - -#if RTE_VERSION >= RTE_VERSION_NUM(22, 11, 0, 0) - session = rte_cryptodev_sym_session_create(device->cdev_id, xforms, g_session_mp); -#else - session = rte_cryptodev_sym_session_create(g_session_mp); - if (!session) { - return NULL; - } - - if (rte_cryptodev_sym_session_init(device->cdev_id, session, xforms, g_session_mp_priv) < 0) { - _cryptodev_sym_session_free(vbdev, session); - return NULL; - } -#endif - - return session; -} - /* Callback for unregistering the IO device. */ static void _device_unregister_cb(void *io_device) @@ -1435,8 +495,6 @@ _device_unregister_cb(void *io_device) struct vbdev_crypto *crypto_bdev = io_device; /* Done with this crypto_bdev. */ - _cryptodev_sym_session_free(crypto_bdev, crypto_bdev->session_decrypt); - _cryptodev_sym_session_free(crypto_bdev, crypto_bdev->session_encrypt); crypto_bdev->opts = NULL; spdk_bdev_destruct_done(&crypto_bdev->crypto_bdev, 0); @@ -1477,8 +535,6 @@ vbdev_crypto_destruct(void *ctx) /* Unregister the io_device. */ spdk_io_device_unregister(crypto_bdev, _device_unregister_cb); - g_number_of_claimed_volumes--; - return 1; } @@ -1507,45 +563,15 @@ static int vbdev_crypto_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) { struct vbdev_crypto *crypto_bdev = (struct vbdev_crypto *)ctx; - char *hexkey = NULL, *hexkey2 = NULL; - int rc = 0; - - hexkey = spdk_hexlify(crypto_bdev->opts->key, - crypto_bdev->opts->key_size); - if (!hexkey) { - return -ENOMEM; - } - - if (crypto_bdev->opts->key2) { - hexkey2 = spdk_hexlify(crypto_bdev->opts->key2, - crypto_bdev->opts->key2_size); - if (!hexkey2) { - rc = -ENOMEM; - goto out_err; - } - } spdk_json_write_name(w, "crypto"); spdk_json_write_object_begin(w); spdk_json_write_named_string(w, "base_bdev_name", spdk_bdev_get_name(crypto_bdev->base_bdev)); spdk_json_write_named_string(w, "name", spdk_bdev_get_name(&crypto_bdev->crypto_bdev)); - spdk_json_write_named_string(w, "crypto_pmd", crypto_bdev->opts->drv_name); - spdk_json_write_named_string(w, "key", hexkey); - if (hexkey2) { - spdk_json_write_named_string(w, "key2", hexkey2); - } - spdk_json_write_named_string(w, "cipher", crypto_bdev->opts->cipher); + spdk_json_write_named_string(w, "key_name", crypto_bdev->opts->key->param.key_name); spdk_json_write_object_end(w); -out_err: - if (hexkey) { - memset(hexkey, 0, strlen(hexkey)); - free(hexkey); - } - if (hexkey2) { - memset(hexkey2, 0, strlen(hexkey2)); - free(hexkey2); - } - return rc; + + return 0; } static int @@ -1554,95 +580,18 @@ vbdev_crypto_config_json(struct spdk_json_write_ctx *w) struct vbdev_crypto *crypto_bdev; TAILQ_FOREACH(crypto_bdev, &g_vbdev_crypto, link) { - char *hexkey = NULL, *hexkey2 = NULL; - - hexkey = spdk_hexlify(crypto_bdev->opts->key, - crypto_bdev->opts->key_size); - if (!hexkey) { - return -ENOMEM; - } - - if (crypto_bdev->opts->key2) { - hexkey2 = spdk_hexlify(crypto_bdev->opts->key2, - crypto_bdev->opts->key2_size); - if (!hexkey2) { - memset(hexkey, 0, strlen(hexkey)); - free(hexkey); - return -ENOMEM; - } - } - spdk_json_write_object_begin(w); spdk_json_write_named_string(w, "method", "bdev_crypto_create"); spdk_json_write_named_object_begin(w, "params"); spdk_json_write_named_string(w, "base_bdev_name", spdk_bdev_get_name(crypto_bdev->base_bdev)); spdk_json_write_named_string(w, "name", spdk_bdev_get_name(&crypto_bdev->crypto_bdev)); - spdk_json_write_named_string(w, "crypto_pmd", crypto_bdev->opts->drv_name); - spdk_json_write_named_string(w, "key", hexkey); - if (hexkey2) { - spdk_json_write_named_string(w, "key2", hexkey2); - } - spdk_json_write_named_string(w, "cipher", crypto_bdev->opts->cipher); + spdk_json_write_named_string(w, "key_name", crypto_bdev->opts->key->param.key_name); spdk_json_write_object_end(w); spdk_json_write_object_end(w); - - if (hexkey) { - memset(hexkey, 0, strlen(hexkey)); - free(hexkey); - } - if (hexkey2) { - memset(hexkey2, 0, strlen(hexkey2)); - free(hexkey2); - } } return 0; } -/* Helper function for the channel creation callback. */ -static void -_assign_device_qp(struct vbdev_crypto *crypto_bdev, struct device_qp *device_qp, - struct crypto_io_channel *crypto_ch) -{ - pthread_mutex_lock(&g_device_qp_lock); - if (strcmp(crypto_bdev->opts->drv_name, QAT) == 0) { - /* For some QAT devices, the optimal qp to use is every 32nd as this spreads the - * workload out over the multiple virtual functions in the device. For the devices - * where this isn't the case, it doesn't hurt. - */ - TAILQ_FOREACH(device_qp, &g_device_qp_qat, link) { - if (device_qp->index != g_next_qat_index) { - continue; - } - if (device_qp->in_use == false) { - crypto_ch->device_qp = device_qp; - device_qp->in_use = true; - g_next_qat_index = (g_next_qat_index + QAT_VF_SPREAD) % g_qat_total_qp; - break; - } else { - /* if the preferred index is used, skip to the next one in this set. */ - g_next_qat_index = (g_next_qat_index + 1) % g_qat_total_qp; - } - } - } else if (strcmp(crypto_bdev->opts->drv_name, AESNI_MB) == 0) { - TAILQ_FOREACH(device_qp, &g_device_qp_aesni_mb, link) { - if (device_qp->in_use == false) { - crypto_ch->device_qp = device_qp; - device_qp->in_use = true; - break; - } - } - } else if (strcmp(crypto_bdev->opts->drv_name, MLX5) == 0) { - TAILQ_FOREACH(device_qp, &g_device_qp_mlx5, link) { - if (device_qp->in_use == false) { - crypto_ch->device_qp = device_qp; - device_qp->in_use = true; - break; - } - } - } - pthread_mutex_unlock(&g_device_qp_lock); -} - /* We provide this callback for the SPDK channel code to create a channel using * the channel struct we provided in our module get_io_channel() entry point. Here * we get and save off an underlying base channel of the device below us so that @@ -1654,21 +603,13 @@ crypto_bdev_ch_create_cb(void *io_device, void *ctx_buf) { struct crypto_io_channel *crypto_ch = ctx_buf; struct vbdev_crypto *crypto_bdev = io_device; - struct device_qp *device_qp = NULL; crypto_ch->base_ch = spdk_bdev_get_io_channel(crypto_bdev->base_desc); - crypto_ch->poller = SPDK_POLLER_REGISTER(crypto_dev_poller, crypto_ch, 0); - crypto_ch->device_qp = NULL; - - /* Assign a device/qp combination that is unique per channel per PMD. */ - _assign_device_qp(crypto_bdev, device_qp, crypto_ch); - assert(crypto_ch->device_qp); + crypto_ch->accel_channel = spdk_accel_get_io_channel(); + crypto_ch->crypto_key = crypto_bdev->opts->key; /* We use this queue to track outstanding IO in our layer. */ - TAILQ_INIT(&crypto_ch->pending_cry_ios); - - /* We use this to queue up crypto ops when the device is busy. */ - TAILQ_INIT(&crypto_ch->queued_cry_ops); + TAILQ_INIT(&crypto_ch->in_accel_fw); return 0; } @@ -1682,12 +623,8 @@ crypto_bdev_ch_destroy_cb(void *io_device, void *ctx_buf) { struct crypto_io_channel *crypto_ch = ctx_buf; - pthread_mutex_lock(&g_device_qp_lock); - crypto_ch->device_qp->in_use = false; - pthread_mutex_unlock(&g_device_qp_lock); - - spdk_poller_unregister(&crypto_ch->poller); spdk_put_io_channel(crypto_ch->base_ch); + spdk_put_io_channel(crypto_ch->accel_channel); } /* Create the association from the bdev and vbdev name and insert @@ -1696,8 +633,6 @@ static int vbdev_crypto_insert_name(struct vbdev_crypto_opts *opts, struct bdev_names **out) { struct bdev_names *name; - bool found = false; - int j; assert(opts); assert(out); @@ -1709,17 +644,6 @@ vbdev_crypto_insert_name(struct vbdev_crypto_opts *opts, struct bdev_names **out } } - for (j = 0; j < MAX_NUM_DRV_TYPES ; j++) { - if (strcmp(opts->drv_name, g_driver_names[j]) == 0) { - found = true; - break; - } - } - if (!found) { - SPDK_ERRLOG("Crypto PMD type %s is not supported.\n", opts->drv_name); - return -EINVAL; - } - name = calloc(1, sizeof(struct bdev_names)); if (!name) { SPDK_ERRLOG("Failed to allocate memory for bdev_names.\n"); @@ -1738,20 +662,6 @@ free_crypto_opts(struct vbdev_crypto_opts *opts) { free(opts->bdev_name); free(opts->vbdev_name); - free(opts->drv_name); - if (opts->xts_key) { - memset(opts->xts_key, 0, - opts->key_size + opts->key2_size); - free(opts->xts_key); - } - memset(opts->key, 0, opts->key_size); - free(opts->key); - opts->key_size = 0; - if (opts->key2) { - memset(opts->key2, 0, opts->key2_size); - free(opts->key2); - } - opts->key2_size = 0; free(opts); } @@ -1760,6 +670,9 @@ vbdev_crypto_delete_name(struct bdev_names *name) { TAILQ_REMOVE(&g_bdev_names, name, link); if (name->opts) { + if (name->opts->key_owner && name->opts->key) { + spdk_accel_crypto_key_destroy(name->opts->key); + } free_crypto_opts(name->opts); name->opts = NULL; } @@ -1787,7 +700,7 @@ create_crypto_disk(struct vbdev_crypto_opts *opts) if (rc) { assert(name != NULL); /* In case of error we let the caller function to deallocate @opts - * since it is its responsibiltiy. Setting name->opts = NULL let's + * since it is its responsibility. Setting name->opts = NULL let's * vbdev_crypto_delete_name() know it does not have to do anything * about @opts. */ @@ -1803,15 +716,7 @@ create_crypto_disk(struct vbdev_crypto_opts *opts) static int vbdev_crypto_init(void) { - int rc = 0; - - /* Fully configure both SW and HW drivers. */ - rc = vbdev_crypto_init_crypto_drivers(); - if (rc) { - SPDK_ERRLOG("Error setting up crypto devices\n"); - } - - return rc; + return 0; } /* Called when the entire module is being torn down. */ @@ -1819,29 +724,10 @@ static void vbdev_crypto_finish(void) { struct bdev_names *name; - struct vbdev_dev *device; while ((name = TAILQ_FIRST(&g_bdev_names))) { vbdev_crypto_delete_name(name); } - - while ((device = TAILQ_FIRST(&g_vbdev_devs))) { - TAILQ_REMOVE(&g_vbdev_devs, device, link); - release_vbdev_dev(device); - } - rte_vdev_uninit(AESNI_MB); - - /* These are removed in release_vbdev_dev() */ - assert(TAILQ_EMPTY(&g_device_qp_qat)); - assert(TAILQ_EMPTY(&g_device_qp_aesni_mb)); - assert(TAILQ_EMPTY(&g_device_qp_mlx5)); - - rte_mempool_free(g_crypto_op_mp); - rte_mempool_free(g_mbuf_mp); - rte_mempool_free(g_session_mp); - if (g_session_mp_priv != NULL) { - rte_mempool_free(g_session_mp_priv); - } } /* During init we'll be asked how much memory we'd like passed to us @@ -1881,12 +767,6 @@ vbdev_crypto_base_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev } } -static void -vbdev_crypto_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) -{ - /* No config per bdev needed */ -} - /* When we register our bdev this is how we specify our entry points. */ static const struct spdk_bdev_fn_table vbdev_crypto_fn_table = { .destruct = vbdev_crypto_destruct, @@ -1894,7 +774,6 @@ static const struct spdk_bdev_fn_table vbdev_crypto_fn_table = { .io_type_supported = vbdev_crypto_io_type_supported, .get_io_channel = vbdev_crypto_get_io_channel, .dump_info_json = vbdev_crypto_dump_info_json, - .write_config_json = vbdev_crypto_write_config_json }; static struct spdk_bdev_module crypto_if = { @@ -1914,15 +793,8 @@ vbdev_crypto_claim(const char *bdev_name) struct bdev_names *name; struct vbdev_crypto *vbdev; struct spdk_bdev *bdev; - uint8_t key_size; int rc = 0; - if (g_number_of_claimed_volumes >= MAX_CRYPTO_VOLUMES) { - SPDK_DEBUGLOG(vbdev_crypto, "Reached max number of claimed volumes\n"); - return -EINVAL; - } - g_number_of_claimed_volumes++; - /* Check our list of names from config versus this bdev and if * there's a match, create the crypto_bdev & bdev accordingly. */ @@ -1935,8 +807,7 @@ vbdev_crypto_claim(const char *bdev_name) vbdev = calloc(1, sizeof(struct vbdev_crypto)); if (!vbdev) { SPDK_ERRLOG("Failed to allocate memory for crypto_bdev.\n"); - rc = -ENOMEM; - goto error_vbdev_alloc; + return -ENOMEM; } vbdev->crypto_bdev.product_name = "crypto"; @@ -1959,31 +830,7 @@ vbdev_crypto_claim(const char *bdev_name) bdev = spdk_bdev_desc_get_bdev(vbdev->base_desc); vbdev->base_bdev = bdev; - if (strcmp(name->opts->drv_name, MLX5) == 0) { - vbdev->qp_desc_nr = CRYPTO_QP_DESCRIPTORS_MLX5; - } else { - vbdev->qp_desc_nr = CRYPTO_QP_DESCRIPTORS; - } - vbdev->crypto_bdev.write_cache = bdev->write_cache; - if (strcmp(name->opts->drv_name, QAT) == 0) { - vbdev->crypto_bdev.required_alignment = - spdk_max(spdk_u32log2(bdev->blocklen), bdev->required_alignment); - SPDK_NOTICELOG("QAT in use: Required alignment set to %u\n", - vbdev->crypto_bdev.required_alignment); - SPDK_NOTICELOG("QAT using cipher: %s\n", name->opts->cipher); - } else if (strcmp(name->opts->drv_name, MLX5) == 0) { - vbdev->crypto_bdev.required_alignment = bdev->required_alignment; - SPDK_NOTICELOG("MLX5 using cipher: %s\n", name->opts->cipher); - } else { - vbdev->crypto_bdev.required_alignment = bdev->required_alignment; - SPDK_NOTICELOG("AESNI_MB using cipher: %s\n", name->opts->cipher); - } - vbdev->cipher_xform.cipher.iv.length = IV_LENGTH; - - /* Note: CRYPTO_MAX_IO is in units of bytes, optimal_io_boundary is - * in units of blocks. - */ if (bdev->optimal_io_boundary > 0) { vbdev->crypto_bdev.optimal_io_boundary = spdk_min((CRYPTO_MAX_IO / bdev->blocklen), bdev->optimal_io_boundary); @@ -1991,6 +838,14 @@ vbdev_crypto_claim(const char *bdev_name) vbdev->crypto_bdev.optimal_io_boundary = (CRYPTO_MAX_IO / bdev->blocklen); } vbdev->crypto_bdev.split_on_optimal_io_boundary = true; + if (bdev->required_alignment > 0) { + vbdev->crypto_bdev.required_alignment = bdev->required_alignment; + } else { + /* Some accel modules may not support SGL input or output, if this module works with physical + * addresses, unaligned buffer may cross huge page boundary which leads to scattered payload. + * To avoid such cases, set required_alignment to the block size */ + vbdev->crypto_bdev.required_alignment = spdk_u32log2(bdev->blocklen); + } vbdev->crypto_bdev.blocklen = bdev->blocklen; vbdev->crypto_bdev.blockcnt = bdev->blockcnt; @@ -2019,41 +874,6 @@ vbdev_crypto_claim(const char *bdev_name) goto error_claim; } - /* Init our per vbdev xform with the desired cipher options. */ - vbdev->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; - vbdev->cipher_xform.cipher.iv.offset = IV_OFFSET; - if (strcmp(vbdev->opts->cipher, AES_CBC) == 0) { - vbdev->cipher_xform.cipher.key.data = vbdev->opts->key; - vbdev->cipher_xform.cipher.key.length = vbdev->opts->key_size; - vbdev->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC; - } else if (strcmp(vbdev->opts->cipher, AES_XTS) == 0) { - key_size = vbdev->opts->key_size + vbdev->opts->key2_size; - vbdev->cipher_xform.cipher.key.data = vbdev->opts->xts_key; - vbdev->cipher_xform.cipher.key.length = key_size; - vbdev->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_XTS; - } else { - SPDK_ERRLOG("Invalid cipher name %s.\n", vbdev->opts->cipher); - rc = -EINVAL; - goto error_session_de_create; - } - vbdev->cipher_xform.cipher.iv.length = IV_LENGTH; - - vbdev->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; - vbdev->session_encrypt = _cryptodev_sym_session_create(vbdev, &vbdev->cipher_xform); - if (NULL == vbdev->session_encrypt) { - SPDK_ERRLOG("Failed to create encrypt crypto session.\n"); - rc = -EINVAL; - goto error_session_en_create; - } - - vbdev->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT; - vbdev->session_decrypt = _cryptodev_sym_session_create(vbdev, &vbdev->cipher_xform); - if (NULL == vbdev->session_decrypt) { - SPDK_ERRLOG("Failed to create decrypt crypto session.\n"); - rc = -EINVAL; - goto error_session_de_create; - } - rc = spdk_bdev_register(&vbdev->crypto_bdev); if (rc < 0) { SPDK_ERRLOG("Failed to register vbdev: error %d\n", rc); @@ -2069,10 +889,6 @@ vbdev_crypto_claim(const char *bdev_name) /* Error cleanup paths. */ error_bdev_register: - _cryptodev_sym_session_free(vbdev, vbdev->session_decrypt); -error_session_de_create: - _cryptodev_sym_session_free(vbdev, vbdev->session_encrypt); -error_session_en_create: spdk_bdev_module_release_bdev(vbdev->base_bdev); error_claim: TAILQ_REMOVE(&g_vbdev_crypto, vbdev, link); @@ -2082,8 +898,7 @@ error_open: free(vbdev->crypto_bdev.name); error_bdev_name: free(vbdev); -error_vbdev_alloc: - g_number_of_claimed_volumes--; + return rc; } diff --git a/module/bdev/crypto/vbdev_crypto.h b/module/bdev/crypto/vbdev_crypto.h index 810181bfa..27079faa4 100644 --- a/module/bdev/crypto/vbdev_crypto.h +++ b/module/bdev/crypto/vbdev_crypto.h @@ -12,43 +12,19 @@ #include "spdk/util.h" #include "spdk/string.h" #include "spdk/log.h" +#include "spdk/accel.h" +#include "spdk_internal/accel_module.h" #include "spdk/bdev.h" -#define AESNI_MB "crypto_aesni_mb" -#define QAT "crypto_qat" -#define QAT_ASYM "crypto_qat_asym" -#define MLX5 "mlx5_pci" +#define BDEV_CRYPTO_DEFAULT_CIPHER "AES_CBC" /* QAT and AESNI_MB */ -/* Supported ciphers */ -#define AES_CBC "AES_CBC" /* QAT and AESNI_MB */ -#define AES_XTS "AES_XTS" /* QAT and MLX5 */ - -/* Specific to AES_CBC. */ -#define AES_CBC_KEY_LENGTH 16 - -#define AES_XTS_128_BLOCK_KEY_LENGTH 16 /* AES-XTS-128 block key size. */ -#define AES_XTS_256_BLOCK_KEY_LENGTH 32 /* AES-XTS-256 block key size. */ -#define AES_XTS_512_BLOCK_KEY_LENGTH 64 /* AES-XTS-512 block key size. */ - -#define AES_XTS_TWEAK_KEY_LENGTH 16 /* XTS part key size is always 128 bit. */ - -/* Structure to hold crypto options for crypto pmd setup. */ +/* Structure to hold crypto options */ struct vbdev_crypto_opts { char *vbdev_name; /* name of the vbdev to create */ char *bdev_name; /* base bdev name */ - - char *drv_name; /* name of the crypto device driver */ - char *cipher; /* AES_CBC or AES_XTS */ - - /* Note, for dev/test we allow use of key in the config file, for production - * use, you must use an RPC to specify the key for security reasons. - */ - uint8_t *key; /* key per bdev */ - uint8_t key_size; /* key size */ - uint8_t *key2; /* key #2 for AES_XTS, per bdev */ - uint8_t key2_size; /* key #2 size */ - uint8_t *xts_key; /* key + key 2 */ + struct spdk_accel_crypto_key *key; /* crypto key */ + bool key_owner; /* If wet to true then the key was created by RPC and needs to be destroyed */ }; typedef void (*spdk_delete_crypto_complete)(void *cb_arg, int bdeverrno); diff --git a/module/bdev/crypto/vbdev_crypto_rpc.c b/module/bdev/crypto/vbdev_crypto_rpc.c index 8a88f090c..1653b25aa 100644 --- a/module/bdev/crypto/vbdev_crypto_rpc.c +++ b/module/bdev/crypto/vbdev_crypto_rpc.c @@ -9,14 +9,15 @@ #include "spdk/hexlify.h" +/* Reasonable bdev name length + cipher's name len */ +#define MAX_KEY_NAME_LEN 128 + /* Structure to hold the parameters for this RPC method. */ struct rpc_construct_crypto { char *base_bdev_name; char *name; char *crypto_pmd; - char *key; - char *cipher; - char *key2; + struct spdk_accel_crypto_key_create_param param; }; /* Free the allocated memory resource after the RPC handling. */ @@ -26,192 +27,54 @@ free_rpc_construct_crypto(struct rpc_construct_crypto *r) free(r->base_bdev_name); free(r->name); free(r->crypto_pmd); - free(r->key); - free(r->cipher); - free(r->key2); + free(r->param.cipher); + if (r->param.hex_key) { + memset(r->param.hex_key, 0, strnlen(r->param.hex_key, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH)); + free(r->param.hex_key); + } + if (r->param.hex_key2) { + memset(r->param.hex_key2, 0, strnlen(r->param.hex_key2, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH)); + free(r->param.hex_key2); + } + free(r->param.key_name); } /* Structure to decode the input parameters for this RPC method. */ static const struct spdk_json_object_decoder rpc_construct_crypto_decoders[] = { {"base_bdev_name", offsetof(struct rpc_construct_crypto, base_bdev_name), spdk_json_decode_string}, {"name", offsetof(struct rpc_construct_crypto, name), spdk_json_decode_string}, - {"crypto_pmd", offsetof(struct rpc_construct_crypto, crypto_pmd), spdk_json_decode_string}, - {"key", offsetof(struct rpc_construct_crypto, key), spdk_json_decode_string}, - {"cipher", offsetof(struct rpc_construct_crypto, cipher), spdk_json_decode_string, true}, - {"key2", offsetof(struct rpc_construct_crypto, key2), spdk_json_decode_string, true}, + {"crypto_pmd", offsetof(struct rpc_construct_crypto, crypto_pmd), spdk_json_decode_string, true}, + {"key", offsetof(struct rpc_construct_crypto, param.hex_key), spdk_json_decode_string, true}, + {"cipher", offsetof(struct rpc_construct_crypto, param.cipher), spdk_json_decode_string, true}, + {"key2", offsetof(struct rpc_construct_crypto, param.hex_key2), spdk_json_decode_string, true}, + {"key_name", offsetof(struct rpc_construct_crypto, param.key_name), spdk_json_decode_string, true}, }; -/** - * Create crypto opts from rpc @req. Validate req fields and populate the - * correspoending fields in @opts. - * - * \param rpc Pointer to the rpc req. - * \param request Pointer to json request. - * \return Allocated and populated crypto opts or NULL on failure. - */ static struct vbdev_crypto_opts * -create_crypto_opts(struct rpc_construct_crypto *rpc, - struct spdk_jsonrpc_request *request) +create_crypto_opts(struct rpc_construct_crypto *rpc, struct spdk_accel_crypto_key *key, + bool key_owner) { - struct vbdev_crypto_opts *opts; - int key_size, key2_size; + struct vbdev_crypto_opts *opts = calloc(1, sizeof(*opts)); - if (strcmp(rpc->crypto_pmd, AESNI_MB) == 0 && strcmp(rpc->cipher, AES_XTS) == 0) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid cipher. AES_XTS is not available on AESNI_MB."); - return NULL; - } - - if (strcmp(rpc->crypto_pmd, MLX5) == 0 && strcmp(rpc->cipher, AES_XTS) != 0) { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid cipher. %s is not available on MLX5.", - rpc->cipher); - return NULL; - } - - if (strcmp(rpc->cipher, AES_XTS) == 0 && rpc->key2 == NULL) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid key. A 2nd key is needed for AES_XTS."); - return NULL; - } - - if (strcmp(rpc->cipher, AES_CBC) == 0 && rpc->key2 != NULL) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid key. A 2nd key is needed only for AES_XTS."); - return NULL; - } - - opts = calloc(1, sizeof(struct vbdev_crypto_opts)); if (!opts) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Failed to allocate memory for crypto_opts."); return NULL; } opts->bdev_name = strdup(rpc->base_bdev_name); if (!opts->bdev_name) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Failed to allocate memory for bdev_name."); - goto error_alloc_bname; + free_crypto_opts(opts); + return NULL; } - opts->vbdev_name = strdup(rpc->name); if (!opts->vbdev_name) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Failed to allocate memory for vbdev_name."); - goto error_alloc_vname; + free_crypto_opts(opts); + return NULL; } - opts->drv_name = strdup(rpc->crypto_pmd); - if (!opts->drv_name) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Failed to allocate memory for drv_name."); - goto error_alloc_dname; - } + opts->key = key; + opts->key_owner = key_owner; - if (strcmp(opts->drv_name, MLX5) == 0) { - /* Only AES-XTS supported. */ - - /* We cannot use strlen() after spdk_unhexlify() because of possible \0 chars - * used in the key. Hexlified version of key is twice as longer. */ - key_size = strnlen(rpc->key, (AES_XTS_512_BLOCK_KEY_LENGTH * 2) + 1); - if (key_size != AES_XTS_256_BLOCK_KEY_LENGTH * 2 && - key_size != AES_XTS_512_BLOCK_KEY_LENGTH * 2) { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid AES_XTS key string length for mlx5: %d. " - "Supported sizes in hex form: %d or %d.", - key_size, AES_XTS_256_BLOCK_KEY_LENGTH * 2, - AES_XTS_512_BLOCK_KEY_LENGTH * 2); - goto error_invalid_key; - } - } else { - if (strncmp(rpc->cipher, AES_XTS, sizeof(AES_XTS)) == 0) { - /* AES_XTS for qat uses 128bit key. */ - key_size = strnlen(rpc->key, (AES_XTS_128_BLOCK_KEY_LENGTH * 2) + 1); - if (key_size != AES_XTS_128_BLOCK_KEY_LENGTH * 2) { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid AES_XTS key string length: %d. " - "Supported size in hex form: %d.", - key_size, AES_XTS_128_BLOCK_KEY_LENGTH * 2); - goto error_invalid_key; - } - } else { - key_size = strnlen(rpc->key, (AES_CBC_KEY_LENGTH * 2) + 1); - if (key_size != AES_CBC_KEY_LENGTH * 2) { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid AES_CBC key string length: %d. " - "Supported size in hex form: %d.", - key_size, AES_CBC_KEY_LENGTH * 2); - goto error_invalid_key; - } - } - } - opts->key = spdk_unhexlify(rpc->key); - if (!opts->key) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Failed to unhexlify key."); - goto error_alloc_key; - } - opts->key_size = key_size / 2; - - if (strncmp(rpc->cipher, AES_XTS, sizeof(AES_XTS)) == 0) { - opts->cipher = AES_XTS; - assert(rpc->key2); - key2_size = strnlen(rpc->key2, (AES_XTS_TWEAK_KEY_LENGTH * 2) + 1); - if (key2_size != AES_XTS_TWEAK_KEY_LENGTH * 2) { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid AES_XTS key2 length %d. " - "Supported size in hex form: %d.", - key2_size, AES_XTS_TWEAK_KEY_LENGTH * 2); - goto error_invalid_key2; - } - opts->key2 = spdk_unhexlify(rpc->key2); - if (!opts->key2) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Failed to unhexlify key2."); - goto error_alloc_key2; - } - opts->key2_size = key2_size / 2; - - /* DPDK expects the keys to be concatenated together. */ - opts->xts_key = calloc(1, opts->key_size + opts->key2_size + 1); - if (opts->xts_key == NULL) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Failed to allocate memory for XTS key."); - goto error_alloc_xts; - } - memcpy(opts->xts_key, opts->key, opts->key_size); - memcpy(opts->xts_key + opts->key_size, opts->key2, opts->key2_size); - } else if (strncmp(rpc->cipher, AES_CBC, sizeof(AES_CBC)) == 0) { - opts->cipher = AES_CBC; - } else { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid param. Cipher %s is not supported.", - rpc->cipher); - goto error_cipher; - } return opts; - - /* Error cleanup paths. */ -error_cipher: -error_alloc_xts: -error_alloc_key2: -error_invalid_key2: - if (opts->key) { - memset(opts->key, 0, opts->key_size); - free(opts->key); - } - opts->key_size = 0; -error_alloc_key: -error_invalid_key: - free(opts->drv_name); -error_alloc_dname: - free(opts->vbdev_name); -error_alloc_vname: - free(opts->bdev_name); -error_alloc_bname: - free(opts); - return NULL; } /* Decode the parameters for this RPC method and properly construct the crypto @@ -221,30 +84,92 @@ static void rpc_bdev_crypto_create(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) { - struct rpc_construct_crypto req = {NULL}; - struct vbdev_crypto_opts *crypto_opts; + struct rpc_construct_crypto req = {}; + struct vbdev_crypto_opts *crypto_opts = NULL; struct spdk_json_write_ctx *w; - int rc; + struct spdk_accel_crypto_key *key = NULL; + struct spdk_accel_crypto_key *created_key = NULL; + int rc = 0; if (spdk_json_decode_object(params, rpc_construct_crypto_decoders, SPDK_COUNTOF(rpc_construct_crypto_decoders), &req)) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_PARSE_ERROR, "Failed to decode crypto disk create parameters."); goto cleanup; } - if (req.cipher == NULL) { - req.cipher = strdup(AES_CBC); - if (req.cipher == NULL) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Unable to allocate memory for req.cipher"); - goto cleanup; + if (!req.name) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "crypto_bdev name is missing"); + goto cleanup; + } + + if (req.param.key_name) { + /* New config version */ + key = spdk_accel_crypto_key_get(req.param.key_name); + if (key) { + if (req.param.hex_key || req.param.cipher || req.crypto_pmd) { + SPDK_NOTICELOG("Key name specified, other parameters are ignored\n"); + } + SPDK_NOTICELOG("Found key \"%s\"\n", req.param.key_name); } } - crypto_opts = create_crypto_opts(&req, request); - if (crypto_opts == NULL) { + /* No key_name. Support legacy configuration */ + if (!key) { + if (req.param.key_name) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Key was not found"); + goto cleanup; + } + + if (req.param.cipher == NULL) { + req.param.cipher = strdup(BDEV_CRYPTO_DEFAULT_CIPHER); + if (req.param.cipher == NULL) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Unable to allocate memory for req.cipher"); + goto cleanup; + } + } + if (req.crypto_pmd) { + SPDK_WARNLOG("\"crypto_pmd\" parameters is obsolete and ignored\n"); + } + + req.param.key_name = calloc(1, MAX_KEY_NAME_LEN); + if (!req.param.key_name) { + /* The new API requires key name. Create it as pmd_name + cipher */ + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Unable to allocate memory for key_name"); + goto cleanup; + } + snprintf(req.param.key_name, MAX_KEY_NAME_LEN, "%s_%s", req.name, req.param.cipher); + + /* Try to find a key with generated name, we may be loading from a json config where crypto_bdev had no key_name parameter */ + key = spdk_accel_crypto_key_get(req.param.key_name); + if (key) { + SPDK_NOTICELOG("Found key \"%s\"\n", req.param.key_name); + } else { + rc = spdk_accel_crypto_key_create(&req.param); + if (!rc) { + key = spdk_accel_crypto_key_get(req.param.key_name); + created_key = key; + } + } + } + + if (!key) { + /* We haven't found an existing key or were not able to create a new one */ + SPDK_ERRLOG("No key was found\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "No key was found"); + goto cleanup; + } + + crypto_opts = create_crypto_opts(&req, key, created_key != NULL); + if (!crypto_opts) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Memory allocation failed"); goto cleanup; } @@ -258,7 +183,11 @@ rpc_bdev_crypto_create(struct spdk_jsonrpc_request *request, w = spdk_jsonrpc_begin_result(request); spdk_json_write_string(w, req.name); spdk_jsonrpc_end_result(request, w); + cleanup: + if (rc && created_key) { + spdk_accel_crypto_key_destroy(created_key); + } free_rpc_construct_crypto(&req); } SPDK_RPC_REGISTER("bdev_crypto_create", rpc_bdev_crypto_create, SPDK_RPC_RUNTIME) diff --git a/python/spdk/rpc/accel.py b/python/spdk/rpc/accel.py index 00035b9f5..e4cccc208 100644 --- a/python/spdk/rpc/accel.py +++ b/python/spdk/rpc/accel.py @@ -54,6 +54,19 @@ def accel_crypto_key_create(client, cipher, key, key2, name): return client.call('accel_crypto_key_create', params) +def accel_crypto_key_destroy(client, name): + """Destroy Data Encryption Key. + + Args: + name: key name + """ + params = { + 'name': name + } + + return client.call('accel_crypto_key_destroy', params) + + def accel_crypto_keys_get(client, key_name): """Get a list of the crypto keys. diff --git a/python/spdk/rpc/bdev.py b/python/spdk/rpc/bdev.py index 4f4567557..58a08e899 100644 --- a/python/spdk/rpc/bdev.py +++ b/python/spdk/rpc/bdev.py @@ -104,23 +104,33 @@ def bdev_compress_get_orphans(client, name=None): return client.call('bdev_compress_get_orphans', params) -def bdev_crypto_create(client, base_bdev_name, name, crypto_pmd, key, cipher=None, key2=None): +def bdev_crypto_create(client, base_bdev_name, name, crypto_pmd=None, key=None, cipher=None, key2=None, key_name=None): """Construct a crypto virtual block device. Args: base_bdev_name: name of the underlying base bdev name: name for the crypto vbdev - crypto_pmd: name of of the DPDK crypto driver to use + crypto_pmd: name of the DPDK crypto driver to use key: key + cipher: crypto algorithm to use + key2: Optional second part of the key + key_name: The key name to use in crypto operations Returns: Name of created virtual block device. """ - params = {'base_bdev_name': base_bdev_name, 'name': name, 'crypto_pmd': crypto_pmd, 'key': key} - if cipher: - params['cipher'] = cipher - if key2: + params = {'base_bdev_name': base_bdev_name, 'name': name} + + if crypto_pmd is not None: + params['crypto_pmd'] = crypto_pmd + if key is not None: + params['key'] = key + if key2 is not None: params['key2'] = key2 + if cipher is not None: + params['cipher'] = cipher + if key_name is not None: + params['key_name'] = key_name return client.call('bdev_crypto_create', params) diff --git a/python/spdk/sma/volume/crypto_bdev.py b/python/spdk/sma/volume/crypto_bdev.py index 300f976d4..9911a6ce9 100644 --- a/python/spdk/sma/volume/crypto_bdev.py +++ b/python/spdk/sma/volume/crypto_bdev.py @@ -23,10 +23,8 @@ class CryptoEngineBdev(crypto.CryptoEngine): def init(self, client, params): super().init(client, params) - driver = params.get('driver') - if driver is None: - raise ValueError('Crypto driver must be configured for bdev_crypto') - self._driver = driver + # _driver can be None + self._driver = params.get('driver') def setup(self, volume_id, key, cipher, key2=None): try: @@ -37,9 +35,10 @@ class CryptoEngineBdev(crypto.CryptoEngine): 'Invalid volume crypto configuration: bad cipher') params = {'base_bdev_name': volume_id, 'name': str(uuid.uuid4()), - 'crypto_pmd': self._driver, 'key': key, 'cipher': cipher} + if self._driver is not None: + params['crypto_pmd'] = self._driver if key2 is not None: params['key2'] = key2 log.info('Creating crypto bdev: {} on volume: {}'.format( @@ -72,19 +71,26 @@ class CryptoEngineBdev(crypto.CryptoEngine): if key is None: return params = crypto_bdev['driver_specific']['crypto'] + crypto_key = self._get_crypto_key(params['key_name']) + if crypto_key is None: + raise crypto.CryptoException(grpc.StatusCode.INVALID_ARGUMENT, + 'No key object found') cipher = self._ciphers.get(cipher) if cipher is None: raise crypto.CryptoException(grpc.StatusCode.INVALID_ARGUMENT, 'Invalid volume crypto configuration: bad cipher') - if params['cipher'].lower() != cipher.lower(): + if crypto_key['cipher'].lower() != cipher.lower(): raise crypto.CryptoException(grpc.StatusCode.INVALID_ARGUMENT, 'Invalid volume crypto configuration: bad cipher') - if params['key'].lower() != key.lower(): + if crypto_key['key'].lower() != key.lower(): raise crypto.CryptoException(grpc.StatusCode.INVALID_ARGUMENT, 'Invalid volume crypto configuration: bad key') - if key2 is not None and params.get('key2', '').lower() != key2.lower(): + if key2 is not None and crypto_key.get('key2', '').lower() != key2.lower(): raise crypto.CryptoException(grpc.StatusCode.INVALID_ARGUMENT, 'Invalid volume crypto configuration: bad key2') + if crypto_key['name'].lower() != params['key_name'].lower(): + raise crypto.CryptoException(grpc.StatusCode.INVALID_ARGUMENT, + 'Invalid volume crypto configuration: key name does not match') def _get_crypto_bdev(self, volume_id): try: @@ -106,6 +112,16 @@ class CryptoEngineBdev(crypto.CryptoEngine): raise crypto.CryptoException(grpc.StatusCode.INTERNAL, f'Failed to get bdev_crypto for volume: {volume_id}') + def _get_crypto_key(self, key_name): + try: + with self._client() as client: + _keys = client.call('accel_crypto_keys_get', {'key_name': key_name}) + if _keys is not None: + return _keys[0] + return None + except JSONRPCException: + pass + def get_crypto_bdev(self, volume_id): bdev = self._get_crypto_bdev(volume_id) if bdev is not None: diff --git a/scripts/rpc.py b/scripts/rpc.py index 958f04acc..6a3a36d47 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -277,14 +277,16 @@ if __name__ == "__main__": crypto_pmd=args.crypto_pmd, key=args.key, cipher=args.cipher, - key2=args.key2)) + key2=args.key2, + key_name=args.key_name)) p = subparsers.add_parser('bdev_crypto_create', help='Add a crypto vbdev') p.add_argument('base_bdev_name', help="Name of the base bdev") p.add_argument('name', help="Name of the crypto vbdev") - p.add_argument('crypto_pmd', help="Name of the crypto device driver") - p.add_argument('key', help="Key") - p.add_argument('-c', '--cipher', help="cipher to use, AES_CBC or AES_XTS (QAT only)") - p.add_argument('-k2', '--key2', help="2nd key for cipher AES_XTS", default=None) + p.add_argument('-p', '--crypto-pmd', help="Name of the crypto device driver. Obsolete, see dpdk_cryptodev_set_driver", required=False) + p.add_argument('-k', '--key', help="Key. Obsolete, see accel_crypto_key_create", required=False) + p.add_argument('-c', '--cipher', help="cipher to use. Obsolete, see accel_crypto_key_create", required=False) + p.add_argument('-k2', '--key2', help="2nd key for cipher AES_XTS. Obsolete, see accel_crypto_key_create", default=None) + p.add_argument('-n', '--key-name', help="Key name to use, see accel_crypto_key_create", required=False) p.set_defaults(func=bdev_crypto_create) def bdev_crypto_delete(args): @@ -2835,6 +2837,14 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse p.add_argument('-n', '--name', help='key name', required=True, type=str) p.set_defaults(func=accel_crypto_key_create) + def accel_crypto_key_destroy(args): + print_dict(rpc.accel.accel_crypto_key_destroy(args.client, + name=args.name)) + + p = subparsers.add_parser('accel_crypto_key_destroy', help='Destroy encryption key') + p.add_argument('-n', '--name', help='key name', required=True, type=str) + p.set_defaults(func=accel_crypto_key_destroy) + def accel_crypto_keys_get(args): print_dict(rpc.accel.accel_crypto_keys_get(args.client, key_name=args.key_name)) diff --git a/test/bdev/blockdev.sh b/test/bdev/blockdev.sh index 6210d95ac..28dc7bc25 100755 --- a/test/bdev/blockdev.sh +++ b/test/bdev/blockdev.sh @@ -48,7 +48,7 @@ function cleanup() { } function start_spdk_tgt() { - "$SPDK_BIN_DIR/spdk_tgt" "$env_ctx" & + "$SPDK_BIN_DIR/spdk_tgt" "$env_ctx" "$wait_for_rpc" & spdk_tgt_pid=$! trap 'killprocess "$spdk_tgt_pid"; exit 1' SIGINT SIGTERM EXIT waitforlisten "$spdk_tgt_pid" @@ -137,10 +137,23 @@ function setup_gpt_conf() { function setup_crypto_aesni_conf() { # Malloc0 and Malloc1 use AESNI "$rpc_py" <<- RPC - bdev_malloc_create -b Malloc0 16 512 - bdev_malloc_create -b Malloc1 16 512 - bdev_crypto_create Malloc0 crypto_ram crypto_aesni_mb 01234567891234560123456789123456 - bdev_crypto_create Malloc1 crypto_ram2 crypto_aesni_mb 90123456789123459012345678912345 + dpdk_cryptodev_scan_accel_module + dpdk_cryptodev_set_driver -d crypto_aesni_mb + accel_assign_opc -o encrypt -m dpdk_cryptodev + accel_assign_opc -o decrypt -m dpdk_cryptodev + framework_start_init + accel_crypto_key_create -c AES_CBC -k 01234567891234560123456789123456 -n test_dek_aesni_cbc_1 + accel_crypto_key_create -c AES_CBC -k 12345678912345601234567891234560 -n test_dek_aesni_cbc_2 + accel_crypto_key_create -c AES_CBC -k 23456789123456012345678912345601 -n test_dek_aesni_cbc_3 + accel_crypto_key_create -c AES_CBC -k 34567891234560123456789123456012 -n test_dek_aesni_cbc_4 + bdev_malloc_create -b Malloc0 32 512 + bdev_malloc_create -b Malloc1 32 512 + bdev_malloc_create -b Malloc2 32 4096 + bdev_malloc_create -b Malloc3 32 4096 + bdev_crypto_create Malloc0 crypto_ram -n test_dek_aesni_cbc_1 + bdev_crypto_create Malloc1 crypto_ram2 -n test_dek_aesni_cbc_2 + bdev_crypto_create Malloc2 crypto_ram3 -n test_dek_aesni_cbc_3 + bdev_crypto_create Malloc3 crypto_ram4 -n test_dek_aesni_cbc_4 RPC } @@ -148,10 +161,36 @@ function setup_crypto_qat_conf() { # Malloc0 will use QAT AES_CBC # Malloc1 will use QAT AES_XTS "$rpc_py" <<- RPC + dpdk_cryptodev_scan_accel_module + dpdk_cryptodev_set_driver -d crypto_qat + accel_assign_opc -o encrypt -m dpdk_cryptodev + accel_assign_opc -o decrypt -m dpdk_cryptodev + framework_start_init + accel_crypto_key_create -c AES_CBC -k 01234567891234560123456789123456 -n test_dek_qat_cbc + accel_crypto_key_create -c AES_XTS -k 00112233445566778899001122334455 -e 12345678912345601234567891234560 -n test_dek_qat_xts + accel_crypto_key_create -c AES_CBC -k 23456789123456012345678912345601 -n test_dek_qat_cbc2 + accel_crypto_key_create -c AES_XTS -k 22334455667788990011223344550011 -e 34567891234560123456789123456012 -n test_dek_qat_xts2 + bdev_malloc_create -b Malloc0 32 512 + bdev_malloc_create -b Malloc1 32 512 + bdev_malloc_create -b Malloc2 32 4096 + bdev_malloc_create -b Malloc3 32 4096 + bdev_crypto_create Malloc0 crypto_ram -n test_dek_qat_cbc + bdev_crypto_create Malloc1 crypto_ram1 -n test_dek_qat_xts + bdev_crypto_create Malloc2 crypto_ram2 -n test_dek_qat_cbc2 + bdev_crypto_create Malloc3 crypto_ram3 -n test_dek_qat_xts2 + bdev_get_bdevs -b Malloc1 + RPC +} + +function setup_crypto_sw_conf() { + "$rpc_py" <<- RPC + framework_start_init bdev_malloc_create -b Malloc0 16 512 - bdev_malloc_create -b Malloc1 16 512 - bdev_crypto_create Malloc0 crypto_ram crypto_qat 01234567891234560123456789123456 - bdev_crypto_create -c AES_XTS -k2 01234567891234560123456789123456 Malloc1 crypto_ram3 crypto_qat 01234567891234560123456789123456 + bdev_malloc_create -b Malloc1 16 4096 + accel_crypto_key_create -c AES_XTS -k 00112233445566778899001122334455 -e 11223344556677889900112233445500 -n test_dek_sw + accel_crypto_key_create -c AES_XTS -k 22334455667788990011223344550011 -e 33445566778899001122334455001122 -n test_dek_sw2 + bdev_crypto_create Malloc0 crypto_ram -n test_dek_sw + bdev_crypto_create Malloc1 crypto_ram2 -n test_dek_sw2 bdev_get_bdevs -b Malloc1 RPC } @@ -182,8 +221,13 @@ function setup_crypto_mlx5_conf() { # Malloc0 will use MLX5 AES_XTS "$rpc_py" <<- RPC + dpdk_cryptodev_scan_accel_module + dpdk_cryptodev_set_driver -d mlx5_pci + accel_assign_opc -o encrypt -m dpdk_cryptodev + accel_assign_opc -o decrypt -m dpdk_cryptodev + framework_start_init bdev_malloc_create -b Malloc0 16 512 - bdev_crypto_create -c AES_XTS -k2 $tweak_key Malloc0 crypto_ram4 mlx5_pci $block_key + bdev_crypto_create Malloc0 crypto_ram4 -k $block_key -c AES_XTS -k2 $tweak_key bdev_get_bdevs -b Malloc0 RPC } @@ -570,6 +614,7 @@ crypto_device=$2 wcs_file=$3 dek=$4 env_ctx="" +wait_for_rpc="" if [ -n "$crypto_device" ] && [ -n "$wcs_file" ]; then # We need full path here since fio perf test does 'pushd' to the test dir # and crypto login of fio plugin test can fail. @@ -581,6 +626,9 @@ if [ -n "$crypto_device" ] && [ -n "$wcs_file" ]; then exit 1 fi fi +if [[ $test_type == crypto_* ]]; then + wait_for_rpc="--wait-for-rpc" +fi start_spdk_tgt case "$test_type" in bdev) @@ -598,6 +646,9 @@ case "$test_type" in crypto_qat) setup_crypto_qat_conf ;; + crypto_sw) + setup_crypto_sw_conf + ;; crypto_mlx5) setup_crypto_mlx5_conf $dek ;; @@ -627,6 +678,7 @@ esac # Generate json config and use it throughout all the tests cat <<- CONF > "$conf_file" {"subsystems":[ + $("$rpc_py" save_subsystem_config -n accel), $("$rpc_py" save_subsystem_config -n bdev) ]} CONF diff --git a/test/json_config/config_filter.py b/test/json_config/config_filter.py index f0b473926..c345209c4 100755 --- a/test/json_config/config_filter.py +++ b/test/json_config/config_filter.py @@ -41,6 +41,10 @@ def filter_methods(do_remove_global_rpcs): 'sock_impl_set_options', 'sock_set_default_impl', 'framework_set_scheduler', + 'accel_crypto_key_create', + 'accel_assign_opc', + 'dpdk_cryptodev_scan_accel_module', + 'dpdk_cryptodev_set_driver', ] data = json.loads(sys.stdin.read()) diff --git a/test/json_config/json_config.sh b/test/json_config/json_config.sh index c82022636..1727537cf 100755 --- a/test/json_config/json_config.sh +++ b/test/json_config/json_config.sh @@ -143,6 +143,18 @@ function json_config_test_shutdown_app() { echo "SPDK $app shutdown done" } +function create_accel_config() { + timing_enter "${FUNCNAME[0]}" + + if [[ $SPDK_TEST_CRYPTO -eq 1 ]]; then + tgt_rpc dpdk_cryptodev_scan_accel_module + tgt_rpc accel_assign_opc -o encrypt -m dpdk_cryptodev + tgt_rpc accel_assign_opc -o decrypt -m dpdk_cryptodev + fi + + timing_exit "${FUNCNAME[0]}" +} + function create_bdev_subsystem_config() { timing_enter "${FUNCNAME[0]}" @@ -206,7 +218,7 @@ function create_bdev_subsystem_config() { local crypto_driver=crypto_qat fi - tgt_rpc bdev_crypto_create MallocForCryptoBdev CryptoMallocBdev $crypto_driver 01234567891234560123456789123456 + tgt_rpc bdev_crypto_create MallocForCryptoBdev CryptoMallocBdev -p $crypto_driver -k 01234567891234560123456789123456 expected_notifications+=( bdev_register:MallocForCryptoBdev bdev_register:CryptoMallocBdev @@ -324,6 +336,8 @@ function json_config_test_init() { #TODO: global subsystem params + create_accel_config + # Load nvme configuration. The load_config will issue framework_start_init automatically ( $rootdir/scripts/gen_nvme.sh --json-with-subsystems diff --git a/test/sma/crypto.sh b/test/sma/crypto.sh index a9e97cb5b..6d01394ba 100755 --- a/test/sma/crypto.sh +++ b/test/sma/crypto.sh @@ -144,9 +144,18 @@ verify_crypto_volume() { trap "cleanup; exit 1" SIGINT SIGTERM EXIT -"$rootdir/build/bin/spdk_tgt" -m 0x1 & +"$rootdir/build/bin/spdk_tgt" -m 0x1 --wait-for-rpc & hostpid=$! +waitforlisten $hostpid + +# Configure host with accel crypto parameters +$rpc_py dpdk_cryptodev_scan_accel_module +rpc_cmd dpdk_cryptodev_set_driver -d crypto_aesni_mb +$rpc_py accel_assign_opc -o encrypt -m dpdk_cryptodev +$rpc_py accel_assign_opc -o decrypt -m dpdk_cryptodev +$rpc_py framework_start_init + "$rootdir/build/bin/spdk_tgt" -r "$tgtsock" -m 0x2 & tgtpid=$! @@ -158,8 +167,6 @@ $rootdir/scripts/sma.py -c <( - name: 'nvmf_tcp' crypto: name: 'bdev_crypto' - params: - driver: 'crypto_aesni_mb' CONFIG ) & smapid=$! @@ -202,7 +209,10 @@ attach_volume $device $uuid AES_CBC $key0 verify_crypto_volume $localnqn $uuid # Check that it's using correct key crypto_bdev=$(rpc_cmd bdev_get_bdevs | jq -r '.[] | select(.product_name == "crypto")') -[[ $(jq -r '.driver_specific.crypto.key' <<< "$crypto_bdev") == "$key0" ]] +key_name=$(jq -r '.driver_specific.crypto.key_name' <<< "$crypto_bdev") +key_obj=$(rpc_cmd accel_crypto_keys_get -k $key_name) +[[ $(jq -r '.[0].key' <<< "$key_obj") == "$key0" ]] +[[ $(jq -r '.[0].cipher' <<< "$key_obj") == "AES_CBC" ]] # Attach the same volume again attach_volume $device $uuid AES_CBC $key0 @@ -213,7 +223,10 @@ attach_volume $device $uuid AES_CBC $key0 verify_crypto_volume $localnqn $uuid crypto_bdev2=$(rpc_cmd bdev_get_bdevs | jq -r '.[] | select(.product_name == "crypto")') [[ $(jq -r '.name' <<< "$crypto_bdev") == $(jq -r '.name' <<< "$crypto_bdev2") ]] -[[ $(jq -r '.driver_specific.crypto.key' <<< "$crypto_bdev2") == "$key0" ]] +key_name=$(jq -r '.driver_specific.crypto.key_name' <<< "$crypto_bdev2") +key_obj=$(rpc_cmd accel_crypto_keys_get -k $key_name) +[[ $(jq -r '.[0].key' <<< "$key_obj") == "$key0" ]] +[[ $(jq -r '.[0].cipher' <<< "$key_obj") == "AES_CBC" ]] # Try to do attach it again, but this time use a different crypto algorithm NOT attach_volume $device $uuid AES_XTS $key0 diff --git a/test/sma/vfiouser_qemu.sh b/test/sma/vfiouser_qemu.sh index c0c4cf634..d016d0c89 100755 --- a/test/sma/vfiouser_qemu.sh +++ b/test/sma/vfiouser_qemu.sh @@ -125,10 +125,17 @@ vm_run $vm_no vm_wait_for_boot 300 $vm_no # Start SPDK -$rootdir/build/bin/spdk_tgt & +$rootdir/build/bin/spdk_tgt --wait-for-rpc & tgtpid=$! waitforlisten $tgtpid +# Configure accel crypto module & operations +rpc_cmd dpdk_cryptodev_scan_accel_module +rpc_cmd dpdk_cryptodev_set_driver -d crypto_aesni_mb +rpc_cmd accel_assign_opc -o encrypt -m dpdk_cryptodev +rpc_cmd accel_assign_opc -o decrypt -m dpdk_cryptodev +rpc_cmd framework_start_init + # Prepare the target rpc_cmd bdev_null_create null0 100 4096 rpc_cmd bdev_null_create null1 100 4096 @@ -148,8 +155,6 @@ $rootdir/scripts/sma.py -c <( qmp_port: 10005 crypto: name: 'bdev_crypto' - params: - driver: 'crypto_aesni_mb' EOF ) & smapid=$! @@ -304,7 +309,10 @@ ns_bdev=$(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[ crypto_bdev=$(rpc_cmd bdev_get_bdevs -b "$ns_bdev" | jq -r '.[] | select(.product_name == "crypto")') [[ $(rpc_cmd bdev_get_bdevs | jq -r '[.[] | select(.product_name == "crypto")] | length') -eq 1 ]] -[[ $(jq -r '.driver_specific.crypto.key' <<< "$crypto_bdev") == "$key0" ]] +key_name=$(jq -r '.driver_specific.crypto.key_name' <<< "$crypto_bdev") +key_obj=$(rpc_cmd accel_crypto_keys_get -k $key_name) +[[ $(jq -r '.[0].key' <<< "$key_obj") == "$key0" ]] +[[ $(jq -r '.[0].cipher' <<< "$key_obj") == "AES_CBC" ]] detach_volume "$device0" "$uuid0" delete_device "$device0" diff --git a/test/sma/vhost_blk.sh b/test/sma/vhost_blk.sh index a3c897378..3ad525ada 100755 --- a/test/sma/vhost_blk.sh +++ b/test/sma/vhost_blk.sh @@ -58,9 +58,18 @@ vm_run $vm_no vm_wait_for_boot 300 $vm_no timing_exit setup_vm -$rootdir/build/bin/vhost -S /var/tmp -m 0x3 & +$rootdir/build/bin/vhost -S /var/tmp -m 0x3 --wait-for-rpc & vhostpid=$! +waitforlisten $vhostpid + +# Configure accel crypto module & operations +rpc_cmd dpdk_cryptodev_scan_accel_module +rpc_cmd dpdk_cryptodev_set_driver -d crypto_aesni_mb +rpc_cmd accel_assign_opc -o encrypt -m dpdk_cryptodev +rpc_cmd accel_assign_opc -o decrypt -m dpdk_cryptodev +rpc_cmd framework_start_init + $rootdir/scripts/sma.py -c <( cat <<- EOF address: 127.0.0.1 @@ -77,8 +86,6 @@ $rootdir/scripts/sma.py -c <( qmp_port: 9090 crypto: name: 'bdev_crypto' - params: - driver: 'crypto_aesni_mb' EOF ) & smapid=$! @@ -191,8 +198,11 @@ devid0=$( bdev=$(rpc_cmd vhost_get_controllers | jq -r '.[].backend_specific.block.bdev') crypto_bdev=$(rpc_cmd bdev_get_bdevs | jq -r '.[] | select(.product_name == "crypto")') -[[ $(jq -r '.driver_specific.crypto.key' <<< "$crypto_bdev") == "$key0" ]] [[ $(jq -r '.driver_specific.crypto.name' <<< "$crypto_bdev") == "$bdev" ]] +key_name=$(jq -r '.driver_specific.crypto.key_name' <<< "$crypto_bdev") +key_obj=$(rpc_cmd accel_crypto_keys_get -k $key_name) +[[ $(jq -r '.[0].key' <<< "$key_obj") == "$key0" ]] +[[ $(jq -r '.[0].cipher' <<< "$key_obj") == "AES_CBC" ]] # Delete crypto device and check if it's gone delete_device $devid0 diff --git a/test/unit/lib/bdev/crypto.c/crypto_ut.c b/test/unit/lib/bdev/crypto.c/crypto_ut.c index c999b6c65..2e972e20c 100644 --- a/test/unit/lib/bdev/crypto.c/crypto_ut.c +++ b/test/unit/lib/bdev/crypto.c/crypto_ut.c @@ -1,11 +1,12 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (C) 2018 Intel Corporation. + * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. * All rights reserved. */ #include "spdk_cunit.h" -#include "common/lib/test_env.c" +#include "common/lib/ut_multithread.c" #include "spdk_internal/mock.h" #include "thread/thread_internal.h" #include "unit/lib/json_mock.c" @@ -14,12 +15,6 @@ #include #include -#define MAX_TEST_BLOCKS 8192 -struct rte_crypto_op *g_test_crypto_ops[MAX_TEST_BLOCKS]; -struct rte_crypto_op *g_test_dev_full_ops[MAX_TEST_BLOCKS]; - -uint16_t g_dequeue_mock; -uint16_t g_enqueue_mock; unsigned ut_rte_crypto_op_bulk_alloc; int ut_rte_crypto_op_attach_sym_session = 0; #define MOCK_INFO_GET_1QP_AESNI 0 @@ -29,195 +24,6 @@ int ut_rte_crypto_op_attach_sym_session = 0; int ut_rte_cryptodev_info_get = 0; bool ut_rte_cryptodev_info_get_mocked = false; -void mock_rte_pktmbuf_free_bulk(struct rte_mbuf **m, unsigned int cnt); -#define rte_pktmbuf_free_bulk mock_rte_pktmbuf_free_bulk -void -mock_rte_pktmbuf_free_bulk(struct rte_mbuf **m, unsigned int cnt) -{ - spdk_mempool_put_bulk((struct spdk_mempool *)m[0]->pool, (void **)m, cnt); -} - -void mock_rte_pktmbuf_free(struct rte_mbuf *m); -#define rte_pktmbuf_free mock_rte_pktmbuf_free -void -mock_rte_pktmbuf_free(struct rte_mbuf *m) -{ - spdk_mempool_put((struct spdk_mempool *)m->pool, (void *)m); -} - -void -rte_mempool_free(struct rte_mempool *mp) -{ - spdk_mempool_free((struct spdk_mempool *)mp); -} - -int mock_rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, struct rte_mbuf **mbufs, - unsigned count); -#define rte_pktmbuf_alloc_bulk mock_rte_pktmbuf_alloc_bulk -int -mock_rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, struct rte_mbuf **mbufs, - unsigned count) -{ - int rc; - - rc = spdk_mempool_get_bulk((struct spdk_mempool *)pool, (void **)mbufs, count); - if (rc) { - return rc; - } - for (unsigned i = 0; i < count; i++) { - rte_pktmbuf_reset(mbufs[i]); - mbufs[i]->pool = pool; - } - return rc; -} - -struct rte_mempool * -rte_cryptodev_sym_session_pool_create(const char *name, uint32_t nb_elts, - uint32_t elt_size, uint32_t cache_size, - uint16_t priv_size, int socket_id) -{ - struct spdk_mempool *tmp; - - tmp = spdk_mempool_create(name, nb_elts, elt_size + priv_size, - cache_size, socket_id); - - return (struct rte_mempool *)tmp; - -} - -struct rte_mempool * -rte_pktmbuf_pool_create(const char *name, unsigned n, unsigned cache_size, - uint16_t priv_size, uint16_t data_room_size, int socket_id) -{ - struct spdk_mempool *tmp; - - tmp = spdk_mempool_create(name, n, sizeof(struct rte_mbuf) + priv_size, - cache_size, socket_id); - - return (struct rte_mempool *)tmp; -} - -struct rte_mempool * -rte_mempool_create(const char *name, unsigned n, unsigned elt_size, - unsigned cache_size, unsigned private_data_size, - rte_mempool_ctor_t *mp_init, void *mp_init_arg, - rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, - int socket_id, unsigned flags) -{ - struct spdk_mempool *tmp; - - tmp = spdk_mempool_create(name, n, elt_size + private_data_size, - cache_size, socket_id); - - return (struct rte_mempool *)tmp; -} - -DEFINE_RETURN_MOCK(rte_crypto_op_pool_create, struct rte_mempool *); -struct rte_mempool * -rte_crypto_op_pool_create(const char *name, enum rte_crypto_op_type type, - unsigned nb_elts, unsigned cache_size, - uint16_t priv_size, int socket_id) -{ - struct spdk_mempool *tmp; - - HANDLE_RETURN_MOCK(rte_crypto_op_pool_create); - - tmp = spdk_mempool_create(name, nb_elts, - sizeof(struct rte_crypto_op) + priv_size, - cache_size, socket_id); - - return (struct rte_mempool *)tmp; - -} - -/* Those functions are defined as static inline in DPDK, so we can't - * mock them straight away. We use defines to redirect them into - * our custom functions. - */ -static bool g_resubmit_test = false; -#define rte_cryptodev_enqueue_burst mock_rte_cryptodev_enqueue_burst -static inline uint16_t -mock_rte_cryptodev_enqueue_burst(uint8_t dev_id, uint16_t qp_id, - struct rte_crypto_op **ops, uint16_t nb_ops) -{ - int i; - - CU_ASSERT(nb_ops > 0); - - for (i = 0; i < nb_ops; i++) { - /* Use this empty (til now) array of pointers to store - * enqueued operations for assertion in dev_full test. - */ - g_test_dev_full_ops[i] = *ops++; - if (g_resubmit_test == true) { - CU_ASSERT(g_test_dev_full_ops[i] == (void *)0xDEADBEEF); - } - } - - return g_enqueue_mock; -} - -#define rte_cryptodev_dequeue_burst mock_rte_cryptodev_dequeue_burst -static inline uint16_t -mock_rte_cryptodev_dequeue_burst(uint8_t dev_id, uint16_t qp_id, - struct rte_crypto_op **ops, uint16_t nb_ops) -{ - int i; - - CU_ASSERT(nb_ops > 0); - - for (i = 0; i < g_dequeue_mock; i++) { - *ops++ = g_test_crypto_ops[i]; - } - - return g_dequeue_mock; -} - -/* Instead of allocating real memory, assign the allocations to our - * test array for assertion in tests. - */ -#define rte_crypto_op_bulk_alloc mock_rte_crypto_op_bulk_alloc -static inline unsigned -mock_rte_crypto_op_bulk_alloc(struct rte_mempool *mempool, - enum rte_crypto_op_type type, - struct rte_crypto_op **ops, uint16_t nb_ops) -{ - int i; - - for (i = 0; i < nb_ops; i++) { - *ops++ = g_test_crypto_ops[i]; - } - return ut_rte_crypto_op_bulk_alloc; -} - -#define rte_mempool_put_bulk mock_rte_mempool_put_bulk -static __rte_always_inline void -mock_rte_mempool_put_bulk(struct rte_mempool *mp, void *const *obj_table, - unsigned int n) -{ - return; -} - -#define rte_crypto_op_attach_sym_session mock_rte_crypto_op_attach_sym_session -#if RTE_VERSION >= RTE_VERSION_NUM(22, 11, 0, 0) -static inline int -mock_rte_crypto_op_attach_sym_session(struct rte_crypto_op *op, void *sess) -#else -static inline int -mock_rte_crypto_op_attach_sym_session(struct rte_crypto_op *op, - struct rte_cryptodev_sym_session *sess) -#endif -{ - return ut_rte_crypto_op_attach_sym_session; -} - -#define rte_lcore_count mock_rte_lcore_count -static inline unsigned -mock_rte_lcore_count(void) -{ - return 1; -} - #include "bdev/crypto/vbdev_crypto.c" /* SPDK stubs */ @@ -248,67 +54,15 @@ DEFINE_STUB_V(spdk_bdev_module_examine_done, (struct spdk_bdev_module *module)); DEFINE_STUB(spdk_bdev_register, int, (struct spdk_bdev *vbdev), 0); DEFINE_STUB_V(spdk_bdev_destruct_done, (struct spdk_bdev *bdev, int bdeverrno)); -/* DPDK stubs */ -#define DPDK_DYNFIELD_OFFSET offsetof(struct rte_mbuf, dynfield1[1]) -DEFINE_STUB(rte_mbuf_dynfield_register, int, (const struct rte_mbuf_dynfield *params), - DPDK_DYNFIELD_OFFSET); -DEFINE_STUB(rte_cryptodev_count, uint8_t, (void), 0); -DEFINE_STUB(rte_socket_id, unsigned, (void), 0); -DEFINE_STUB(rte_cryptodev_device_count_by_driver, uint8_t, (uint8_t driver_id), 0); -DEFINE_STUB(rte_cryptodev_configure, int, (uint8_t dev_id, struct rte_cryptodev_config *config), 0); -DEFINE_STUB(rte_cryptodev_queue_pair_setup, int, (uint8_t dev_id, uint16_t queue_pair_id, - const struct rte_cryptodev_qp_conf *qp_conf, int socket_id), 0); -DEFINE_STUB(rte_cryptodev_start, int, (uint8_t dev_id), 0); -DEFINE_STUB_V(rte_cryptodev_stop, (uint8_t dev_id)); -DEFINE_STUB(rte_cryptodev_close, int, (uint8_t dev_id), 0); -DEFINE_STUB(rte_vdev_init, int, (const char *name, const char *args), 0); -DEFINE_STUB(rte_vdev_uninit, int, (const char *name), 0); - -#if RTE_VERSION >= RTE_VERSION_NUM(22, 11, 0, 0) -DEFINE_STUB(rte_cryptodev_sym_session_create, void *, - (uint8_t dev_id, struct rte_crypto_sym_xform *xforms, struct rte_mempool *mempool), (void *)1); -DEFINE_STUB(rte_cryptodev_sym_session_free, int, (uint8_t dev_id, void *sess), 0); -#else -DEFINE_STUB(rte_cryptodev_sym_session_create, struct rte_cryptodev_sym_session *, - (struct rte_mempool *mempool), (void *)1); -DEFINE_STUB(rte_cryptodev_sym_session_init, int, (uint8_t dev_id, - struct rte_cryptodev_sym_session *sess, - struct rte_crypto_sym_xform *xforms, struct rte_mempool *mempool), 0); -DEFINE_STUB(rte_cryptodev_sym_session_free, int, (struct rte_cryptodev_sym_session *sess), 0); -#endif - -struct rte_cryptodev *rte_cryptodevs; +DEFINE_STUB(spdk_accel_crypto_key_destroy, int, (struct spdk_accel_crypto_key *key), 0); /* global vars and setup/cleanup functions used for all test functions */ struct spdk_bdev_io *g_bdev_io; struct crypto_bdev_io *g_io_ctx; struct crypto_io_channel *g_crypto_ch; struct spdk_io_channel *g_io_ch; -struct vbdev_dev g_device; struct vbdev_crypto g_crypto_bdev; struct vbdev_crypto_opts g_crypto_bdev_opts; -struct device_qp g_dev_qp; - -void -rte_cryptodev_info_get(uint8_t dev_id, struct rte_cryptodev_info *dev_info) -{ - dev_info->max_nb_queue_pairs = 1; - if (ut_rte_cryptodev_info_get == MOCK_INFO_GET_1QP_AESNI) { - dev_info->driver_name = g_driver_names[0]; - } else if (ut_rte_cryptodev_info_get == MOCK_INFO_GET_1QP_QAT) { - dev_info->driver_name = g_driver_names[1]; - } else if (ut_rte_cryptodev_info_get == MOCK_INFO_GET_1QP_MLX5) { - dev_info->driver_name = g_driver_names[2]; - } else if (ut_rte_cryptodev_info_get == MOCK_INFO_GET_1QP_BOGUS_PMD) { - dev_info->driver_name = "junk"; - } -} - -unsigned int -rte_cryptodev_sym_get_private_session_size(uint8_t dev_id) -{ - return (unsigned int)dev_id; -} void spdk_bdev_io_get_aux_buf(struct spdk_bdev_io *bdev_io, spdk_bdev_io_get_aux_buf_cb cb) @@ -322,61 +76,92 @@ spdk_bdev_io_get_buf(struct spdk_bdev_io *bdev_io, spdk_bdev_io_get_buf_cb cb, u cb(g_io_ch, g_bdev_io, true); } +struct ut_vbdev_crypto_bdev_cpl_args { + spdk_bdev_io_completion_cb cb_fn; + struct spdk_bdev_io *bdev_io; + void *cb_arg; + bool result; +}; + +static void +_ut_vbdev_crypto_bdev_cpl(void *arg) +{ + struct ut_vbdev_crypto_bdev_cpl_args *cpl_args = arg; + + cpl_args->cb_fn(cpl_args->bdev_io, cpl_args->result, cpl_args->cb_arg); + free(cpl_args); +} + +static void +ut_vbdev_crypto_bdev_cpl(spdk_bdev_io_completion_cb cb_fn, struct spdk_bdev_io *bdev_io, + bool result, void *cb_arg) +{ + struct ut_vbdev_crypto_bdev_cpl_args *cpl_args = calloc(1, sizeof(*cpl_args)); + + SPDK_CU_ASSERT_FATAL(cpl_args); + cpl_args->cb_fn = cb_fn; + cpl_args->bdev_io = bdev_io; + cpl_args->result = result; + cpl_args->cb_arg = cb_arg; + + spdk_thread_send_msg(spdk_get_thread(), _ut_vbdev_crypto_bdev_cpl, cpl_args); +} + /* Mock these functions to call the callback and then return the value we require */ -int ut_spdk_bdev_readv_blocks = 0; -bool ut_spdk_bdev_readv_blocks_mocked = false; +DEFINE_RETURN_MOCK(spdk_bdev_readv_blocks, int); int spdk_bdev_readv_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, struct iovec *iov, int iovcnt, uint64_t offset_blocks, uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg) { - cb(g_bdev_io, !ut_spdk_bdev_readv_blocks, cb_arg); - return ut_spdk_bdev_readv_blocks; + HANDLE_RETURN_MOCK(spdk_bdev_readv_blocks); + ut_vbdev_crypto_bdev_cpl(cb, g_bdev_io, !ut_spdk_bdev_readv_blocks, cb_arg); + return 0; } -int ut_spdk_bdev_writev_blocks = 0; -bool ut_spdk_bdev_writev_blocks_mocked = false; +DEFINE_RETURN_MOCK(spdk_bdev_writev_blocks, int); int spdk_bdev_writev_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, struct iovec *iov, int iovcnt, uint64_t offset_blocks, uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg) { - cb(g_bdev_io, !ut_spdk_bdev_writev_blocks, cb_arg); - return ut_spdk_bdev_writev_blocks; + HANDLE_RETURN_MOCK(spdk_bdev_writev_blocks); + ut_vbdev_crypto_bdev_cpl(cb, g_bdev_io, !ut_spdk_bdev_writev_blocks, cb_arg); + return 0; } -int ut_spdk_bdev_unmap_blocks = 0; -bool ut_spdk_bdev_unmap_blocks_mocked = false; +DEFINE_RETURN_MOCK(spdk_bdev_unmap_blocks, int); int spdk_bdev_unmap_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, uint64_t offset_blocks, uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg) { - cb(g_bdev_io, !ut_spdk_bdev_unmap_blocks, cb_arg); - return ut_spdk_bdev_unmap_blocks; + HANDLE_RETURN_MOCK(spdk_bdev_unmap_blocks); + ut_vbdev_crypto_bdev_cpl(cb, g_bdev_io, !ut_spdk_bdev_unmap_blocks, cb_arg); + return 0; } -int ut_spdk_bdev_flush_blocks = 0; -bool ut_spdk_bdev_flush_blocks_mocked = false; +DEFINE_RETURN_MOCK(spdk_bdev_flush_blocks, int); int spdk_bdev_flush_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, uint64_t offset_blocks, uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg) { - cb(g_bdev_io, !ut_spdk_bdev_flush_blocks, cb_arg); - return ut_spdk_bdev_flush_blocks; + HANDLE_RETURN_MOCK(spdk_bdev_flush_blocks); + ut_vbdev_crypto_bdev_cpl(cb, g_bdev_io, !ut_spdk_bdev_flush_blocks, cb_arg); + return 0; } -int ut_spdk_bdev_reset = 0; -bool ut_spdk_bdev_reset_mocked = false; +DEFINE_RETURN_MOCK(spdk_bdev_reset, int); int spdk_bdev_reset(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, spdk_bdev_io_completion_cb cb, void *cb_arg) { - cb(g_bdev_io, !ut_spdk_bdev_reset, cb_arg); - return ut_spdk_bdev_reset; + HANDLE_RETURN_MOCK(spdk_bdev_reset); + ut_vbdev_crypto_bdev_cpl(cb, g_bdev_io, !ut_spdk_bdev_reset, cb_arg); + return 0; } bool g_completion_called = false; @@ -387,12 +172,75 @@ spdk_bdev_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status sta g_completion_called = true; } +struct ut_vbdev_crypto_accel_cpl_args { + spdk_accel_completion_cb cb_fn; + void *cb_arg; + int rc; +}; + +static void +_vbdev_crypto_ut_accel_cpl(void *arg) +{ + struct ut_vbdev_crypto_accel_cpl_args *cpl_args = arg; + + cpl_args->cb_fn(cpl_args->cb_arg, cpl_args->rc); + free(cpl_args); +} + +static void +vbdev_crypto_ut_accel_cpl(spdk_accel_completion_cb cb_fn, void *cb_arg, int rc) +{ + struct ut_vbdev_crypto_accel_cpl_args *cpl_args = calloc(1, sizeof(*cpl_args)); + + SPDK_CU_ASSERT_FATAL(cpl_args); + cpl_args->cb_fn = cb_fn; + cpl_args->cb_arg = cb_arg; + cpl_args->rc = rc; + + spdk_thread_send_msg(spdk_get_thread(), _vbdev_crypto_ut_accel_cpl, cpl_args); +} + +DEFINE_RETURN_MOCK(spdk_accel_submit_encrypt, int); +int ut_spdk_accel_submit_encrypt_cb_rc = 0; +int +spdk_accel_submit_encrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key, + struct iovec *dst_iovs, uint32_t dst_iovcnt, + struct iovec *src_iovs, uint32_t src_iovcnt, + uint64_t iv, uint32_t block_size, int flags, + spdk_accel_completion_cb cb_fn, void *cb_arg) +{ + HANDLE_RETURN_MOCK(spdk_accel_submit_encrypt); + /* We must not call cb_fn immediately */ + vbdev_crypto_ut_accel_cpl(cb_fn, cb_arg, ut_spdk_accel_submit_encrypt_cb_rc); + + return 0; +} + +int ut_spdk_accel_submit_decrypt_cb_rc; +DEFINE_RETURN_MOCK(spdk_accel_submit_decrypt, int); +int +spdk_accel_submit_decrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key, + struct iovec *dst_iovs, uint32_t dst_iovcnt, + struct iovec *src_iovs, uint32_t src_iovcnt, + uint64_t iv, uint32_t block_size, int flags, + spdk_accel_completion_cb cb_fn, void *cb_arg) +{ + HANDLE_RETURN_MOCK(spdk_accel_submit_decrypt); + /* We must not call cb_fn immediately */ + vbdev_crypto_ut_accel_cpl(cb_fn, cb_arg, ut_spdk_accel_submit_decrypt_cb_rc); + + return 0; +} + +struct spdk_io_channel *spdk_accel_get_io_channel(void) +{ + return (struct spdk_io_channel *)0xfeedbeef; +} + /* Global setup for all tests that share a bunch of preparation... */ static int test_setup(void) { - int i, rc; - /* Prepare essential variables for test routines */ g_bdev_io = calloc(1, sizeof(struct spdk_bdev_io) + sizeof(struct crypto_bdev_io)); g_bdev_io->u.bdev.iovs = calloc(1, sizeof(struct iovec) * 128); @@ -400,34 +248,13 @@ test_setup(void) g_io_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct crypto_io_channel)); g_crypto_ch = (struct crypto_io_channel *)spdk_io_channel_get_ctx(g_io_ch); g_io_ctx = (struct crypto_bdev_io *)g_bdev_io->driver_ctx; - memset(&g_device, 0, sizeof(struct vbdev_dev)); memset(&g_crypto_bdev, 0, sizeof(struct vbdev_crypto)); memset(&g_crypto_bdev_opts, 0, sizeof(struct vbdev_crypto_opts)); - g_dev_qp.device = &g_device; + g_crypto_bdev.crypto_bdev.blocklen = 512; g_io_ctx->crypto_ch = g_crypto_ch; g_io_ctx->crypto_bdev = &g_crypto_bdev; - g_io_ctx->crypto_bdev->qp_desc_nr = CRYPTO_QP_DESCRIPTORS; g_io_ctx->crypto_bdev->opts = &g_crypto_bdev_opts; - g_crypto_ch->device_qp = &g_dev_qp; - TAILQ_INIT(&g_crypto_ch->pending_cry_ios); - TAILQ_INIT(&g_crypto_ch->queued_cry_ops); - - /* Allocate a real mbuf pool so we can test error paths */ - g_mbuf_mp = rte_pktmbuf_pool_create("mbuf_mp", NUM_MBUFS, - (unsigned)SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, - 0, 0, SPDK_ENV_SOCKET_ID_ANY); - /* Instead of allocating real rte mempools for these, it's easier and provides the - * same coverage just calloc them here. - */ - for (i = 0; i < MAX_TEST_BLOCKS; i++) { - size_t size = IV_OFFSET + IV_LENGTH + QUEUED_OP_LENGTH; - rc = posix_memalign((void **)&g_test_crypto_ops[i], 64, size); - if (rc != 0) { - assert(false); - } - memset(g_test_crypto_ops[i], 0, IV_OFFSET + QUEUED_OP_LENGTH); - } - g_mbuf_offset = DPDK_DYNFIELD_OFFSET; + TAILQ_INIT(&g_crypto_ch->in_accel_fw); return 0; } @@ -436,29 +263,6 @@ test_setup(void) static int test_cleanup(void) { - int i; - - if (g_crypto_op_mp) { - rte_mempool_free(g_crypto_op_mp); - g_crypto_op_mp = NULL; - } - if (g_mbuf_mp) { - rte_mempool_free(g_mbuf_mp); - g_mbuf_mp = NULL; - } - if (g_session_mp) { - rte_mempool_free(g_session_mp); - g_session_mp = NULL; - } - if (g_session_mp_priv != NULL) { - /* g_session_mp_priv may or may not be set depending on the DPDK version */ - rte_mempool_free(g_session_mp_priv); - g_session_mp_priv = NULL; - } - - for (i = 0; i < MAX_TEST_BLOCKS; i++) { - free(g_test_crypto_ops[i]); - } free(g_bdev_io->u.bdev.iovs); free(g_bdev_io); free(g_io_ch); @@ -468,54 +272,118 @@ test_cleanup(void) static void test_error_paths(void) { - /* Single element block size write, just to test error paths - * in vbdev_crypto_submit_request(). - */ + g_crypto_bdev.crypto_bdev.blocklen = 512; + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; g_bdev_io->u.bdev.iovcnt = 1; g_bdev_io->u.bdev.num_blocks = 1; g_bdev_io->u.bdev.iovs[0].iov_len = 512; g_bdev_io->u.bdev.iovs[0].iov_base = (void *)0xDEADBEEF; - g_crypto_bdev.crypto_bdev.blocklen = 512; g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; - g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = 1; - /* test failure of spdk_mempool_get_bulk(), will result in success because it - * will get queued. - */ - g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; - MOCK_SET(spdk_mempool_get, NULL); + /* test error returned by accel fw */ + MOCK_SET(spdk_accel_submit_encrypt, -ENOMEM); vbdev_crypto_submit_request(g_io_ch, g_bdev_io); CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_io_ctx->bdev_io_wait.bdev == &g_crypto_bdev.crypto_bdev); + CU_ASSERT(g_io_ctx->bdev_io_wait.cb_fn == vbdev_crypto_resubmit_io); + CU_ASSERT(g_io_ctx->bdev_io_wait.cb_arg == g_bdev_io); + CU_ASSERT(g_io_ctx->resubmit_state == CRYPTO_IO_NEW); + memset(&g_io_ctx->bdev_io_wait, 0, sizeof(g_io_ctx->bdev_io_wait)); - /* same thing but switch to reads to test error path in _crypto_complete_io() */ + MOCK_SET(spdk_accel_submit_encrypt, -EINVAL); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + MOCK_CLEAR(spdk_accel_submit_encrypt); + + /* Test error returned in accel cpl cb */ + ut_spdk_accel_submit_encrypt_cb_rc = -EINVAL; + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + poll_threads(); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + ut_spdk_accel_submit_encrypt_cb_rc = 0; + + /* Test error returned from bdev */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + MOCK_SET(spdk_bdev_writev_blocks, -ENOMEM); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + poll_threads(); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_io_ctx->bdev_io_wait.bdev == &g_crypto_bdev.crypto_bdev); + CU_ASSERT(g_io_ctx->bdev_io_wait.cb_fn == vbdev_crypto_resubmit_io); + CU_ASSERT(g_io_ctx->bdev_io_wait.cb_arg == g_bdev_io); + CU_ASSERT(g_io_ctx->resubmit_state == CRYPTO_IO_ENCRYPT_DONE); + memset(&g_io_ctx->bdev_io_wait, 0, sizeof(g_io_ctx->bdev_io_wait)); + MOCK_CLEAR(spdk_bdev_readv_blocks); + + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + MOCK_SET(spdk_bdev_writev_blocks, -EINVAL); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + poll_threads(); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + MOCK_CLEAR(spdk_bdev_writev_blocks); + + /* Test error returned in bdev cpl */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + ut_spdk_bdev_writev_blocks = -EINVAL; + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + poll_threads(); + poll_threads(); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + ut_spdk_bdev_writev_blocks = 0; + + /* the same for read path */ + /* Test error returned from bdev */ g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; - TAILQ_INSERT_TAIL(&g_crypto_ch->pending_cry_ios, g_bdev_io, module_link); - vbdev_crypto_submit_request(g_io_ch, g_bdev_io); - CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); - /* Now with the read_blocks failing */ - g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; - g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; - MOCK_SET(spdk_bdev_readv_blocks, -1); - vbdev_crypto_submit_request(g_io_ch, g_bdev_io); - CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); - MOCK_SET(spdk_bdev_readv_blocks, 0); - MOCK_CLEAR(spdk_mempool_get); - /* test failure of rte_crypto_op_bulk_alloc() */ - g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; - ut_rte_crypto_op_bulk_alloc = 0; + MOCK_SET(spdk_bdev_readv_blocks, -ENOMEM); vbdev_crypto_submit_request(g_io_ch, g_bdev_io); - CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); - ut_rte_crypto_op_bulk_alloc = 1; + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_io_ctx->bdev_io_wait.bdev == &g_crypto_bdev.crypto_bdev); + CU_ASSERT(g_io_ctx->bdev_io_wait.cb_fn == vbdev_crypto_resubmit_io); + CU_ASSERT(g_io_ctx->bdev_io_wait.cb_arg == g_bdev_io); + CU_ASSERT(g_io_ctx->resubmit_state == CRYPTO_IO_NEW); + memset(&g_io_ctx->bdev_io_wait, 0, sizeof(g_io_ctx->bdev_io_wait)); + MOCK_CLEAR(spdk_bdev_readv_blocks); - /* test failure of rte_crypto_op_attach_sym_session() */ g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; - ut_rte_crypto_op_attach_sym_session = -1; + MOCK_SET(spdk_bdev_readv_blocks, -EINVAL); vbdev_crypto_submit_request(g_io_ch, g_bdev_io); CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); - ut_rte_crypto_op_attach_sym_session = 0; + MOCK_CLEAR(spdk_bdev_readv_blocks); + + /* Test error returned in bdev cpl */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + ut_spdk_bdev_readv_blocks = -EINVAL; + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + poll_threads(); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + ut_spdk_bdev_readv_blocks = 0; + + /* test error returned by accel fw */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + MOCK_SET(spdk_accel_submit_decrypt, -ENOMEM); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + poll_threads(); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_io_ctx->bdev_io_wait.bdev == &g_crypto_bdev.crypto_bdev); + CU_ASSERT(g_io_ctx->bdev_io_wait.cb_fn == vbdev_crypto_resubmit_io); + CU_ASSERT(g_io_ctx->bdev_io_wait.cb_arg == g_bdev_io); + CU_ASSERT(g_io_ctx->resubmit_state == CRYPTO_IO_READ_DONE); + memset(&g_io_ctx->bdev_io_wait, 0, sizeof(g_io_ctx->bdev_io_wait)); + MOCK_CLEAR(spdk_accel_submit_decrypt); + + /* test error returned in accel cpl */ + ut_spdk_accel_submit_decrypt_cb_rc = -EINVAL; + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + poll_threads(); + poll_threads(); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + ut_spdk_accel_submit_decrypt_cb_rc = 0; + } static void @@ -530,27 +398,16 @@ test_simple_write(void) g_bdev_io->u.bdev.iovs[0].iov_base = &test_simple_write; g_crypto_bdev.crypto_bdev.blocklen = 512; g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; - g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = 1; vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + /* 1st poll to trigger accel completions, 2nd for bdev */ + poll_threads(); + poll_threads(); CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); - CU_ASSERT(g_io_ctx->cryop_cnt_remaining == 1); CU_ASSERT(g_io_ctx->aux_buf_iov.iov_len == 512); CU_ASSERT(g_io_ctx->aux_buf_iov.iov_base != NULL); CU_ASSERT(g_io_ctx->aux_offset_blocks == 0); CU_ASSERT(g_io_ctx->aux_num_blocks == 1); - CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->buf_addr == &test_simple_write); - CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->data_len == 512); - CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->next == NULL); - CU_ASSERT(g_test_crypto_ops[0]->sym->cipher.data.length == 512); - CU_ASSERT(g_test_crypto_ops[0]->sym->cipher.data.offset == 0); - CU_ASSERT(*RTE_MBUF_DYNFIELD(g_test_crypto_ops[0]->sym->m_src, g_mbuf_offset, - uint64_t *) == (uint64_t)g_bdev_io); - CU_ASSERT(g_test_crypto_ops[0]->sym->m_dst->buf_addr != NULL); - CU_ASSERT(g_test_crypto_ops[0]->sym->m_dst->data_len == 512); - - rte_pktmbuf_free(g_test_crypto_ops[0]->sym->m_src); - rte_pktmbuf_free(g_test_crypto_ops[0]->sym->m_dst); } static void @@ -564,225 +421,12 @@ test_simple_read(void) g_bdev_io->u.bdev.iovs[0].iov_base = &test_simple_read; g_crypto_bdev.crypto_bdev.blocklen = 512; g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; - g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = 1; vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + /* 1st poll to trigger dev completions, 2nd for accel */ + poll_threads(); + poll_threads(); CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); - CU_ASSERT(g_io_ctx->cryop_cnt_remaining == 1); - CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->buf_addr == &test_simple_read); - CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->data_len == 512); - CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->next == NULL); - CU_ASSERT(g_test_crypto_ops[0]->sym->cipher.data.length == 512); - CU_ASSERT(g_test_crypto_ops[0]->sym->cipher.data.offset == 0); - CU_ASSERT(*RTE_MBUF_DYNFIELD(g_test_crypto_ops[0]->sym->m_src, g_mbuf_offset, - uint64_t *) == (uint64_t)g_bdev_io); - CU_ASSERT(g_test_crypto_ops[0]->sym->m_dst == NULL); - - rte_pktmbuf_free(g_test_crypto_ops[0]->sym->m_src); -} - -static void -test_large_rw(void) -{ - unsigned block_len = 512; - unsigned num_blocks = CRYPTO_MAX_IO / block_len; - unsigned io_len = block_len * num_blocks; - unsigned i; - - /* Multi block size read, multi-element */ - g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; - g_bdev_io->u.bdev.iovcnt = 1; - g_bdev_io->u.bdev.num_blocks = num_blocks; - g_bdev_io->u.bdev.iovs[0].iov_len = io_len; - g_bdev_io->u.bdev.iovs[0].iov_base = &test_large_rw; - g_crypto_bdev.crypto_bdev.blocklen = block_len; - g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; - g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = num_blocks; - - vbdev_crypto_submit_request(g_io_ch, g_bdev_io); - CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); - CU_ASSERT(g_io_ctx->cryop_cnt_remaining == (int)num_blocks); - - for (i = 0; i < num_blocks; i++) { - CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->buf_addr == &test_large_rw + (i * block_len)); - CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->data_len == block_len); - CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->next == NULL); - CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.length == block_len); - CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.offset == 0); - CU_ASSERT(*RTE_MBUF_DYNFIELD(g_test_crypto_ops[i]->sym->m_src, g_mbuf_offset, - uint64_t *) == (uint64_t)g_bdev_io); - CU_ASSERT(g_test_crypto_ops[i]->sym->m_dst == NULL); - rte_pktmbuf_free(g_test_crypto_ops[i]->sym->m_src); - } - - /* Multi block size write, multi-element */ - g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; - g_bdev_io->u.bdev.iovcnt = 1; - g_bdev_io->u.bdev.num_blocks = num_blocks; - g_bdev_io->u.bdev.iovs[0].iov_len = io_len; - g_bdev_io->u.bdev.iovs[0].iov_base = &test_large_rw; - g_crypto_bdev.crypto_bdev.blocklen = block_len; - g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; - g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = num_blocks; - - vbdev_crypto_submit_request(g_io_ch, g_bdev_io); - CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); - CU_ASSERT(g_io_ctx->cryop_cnt_remaining == (int)num_blocks); - - for (i = 0; i < num_blocks; i++) { - CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->buf_addr == &test_large_rw + (i * block_len)); - CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->data_len == block_len); - CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->next == NULL); - CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.length == block_len); - CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.offset == 0); - CU_ASSERT(*RTE_MBUF_DYNFIELD(g_test_crypto_ops[i]->sym->m_src, g_mbuf_offset, - uint64_t *) == (uint64_t)g_bdev_io); - CU_ASSERT(g_io_ctx->aux_buf_iov.iov_len == io_len); - CU_ASSERT(g_io_ctx->aux_buf_iov.iov_base != NULL); - CU_ASSERT(g_io_ctx->aux_offset_blocks == 0); - CU_ASSERT(g_io_ctx->aux_num_blocks == num_blocks); - CU_ASSERT(g_test_crypto_ops[i]->sym->m_dst->buf_addr != NULL); - CU_ASSERT(g_test_crypto_ops[i]->sym->m_dst->data_len == block_len); - rte_pktmbuf_free(g_test_crypto_ops[i]->sym->m_src); - rte_pktmbuf_free(g_test_crypto_ops[i]->sym->m_dst); - } -} - -static void -test_dev_full(void) -{ - struct vbdev_crypto_op *queued_op; - struct rte_crypto_sym_op *sym_op; - struct crypto_bdev_io *io_ctx; - - /* Two element block size read */ - g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; - g_bdev_io->u.bdev.iovcnt = 1; - g_bdev_io->u.bdev.num_blocks = 2; - g_bdev_io->u.bdev.iovs[0].iov_len = 512; - g_bdev_io->u.bdev.iovs[0].iov_base = (void *)0xDEADBEEF; - g_bdev_io->u.bdev.iovs[1].iov_len = 512; - g_bdev_io->u.bdev.iovs[1].iov_base = (void *)0xFEEDBEEF; - g_crypto_bdev.crypto_bdev.blocklen = 512; - g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; - g_enqueue_mock = g_dequeue_mock = 1; - ut_rte_crypto_op_bulk_alloc = 2; - - g_test_crypto_ops[1]->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; - CU_ASSERT(TAILQ_EMPTY(&g_crypto_ch->queued_cry_ops) == true); - - vbdev_crypto_submit_request(g_io_ch, g_bdev_io); - CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); - CU_ASSERT(g_io_ctx->cryop_cnt_remaining == 2); - sym_op = g_test_crypto_ops[0]->sym; - CU_ASSERT(sym_op->m_src->buf_addr == (void *)0xDEADBEEF); - CU_ASSERT(sym_op->m_src->data_len == 512); - CU_ASSERT(sym_op->m_src->next == NULL); - CU_ASSERT(sym_op->cipher.data.length == 512); - CU_ASSERT(sym_op->cipher.data.offset == 0); - CU_ASSERT(*RTE_MBUF_DYNFIELD(sym_op->m_src, g_mbuf_offset, uint64_t *) == (uint64_t)g_bdev_io); - CU_ASSERT(sym_op->m_dst == NULL); - - /* make sure one got queued and confirm its values */ - CU_ASSERT(TAILQ_EMPTY(&g_crypto_ch->queued_cry_ops) == false); - queued_op = TAILQ_FIRST(&g_crypto_ch->queued_cry_ops); - sym_op = queued_op->crypto_op->sym; - TAILQ_REMOVE(&g_crypto_ch->queued_cry_ops, queued_op, link); - CU_ASSERT(queued_op->bdev_io == g_bdev_io); - CU_ASSERT(queued_op->crypto_op == g_test_crypto_ops[1]); - CU_ASSERT(sym_op->m_src->buf_addr == (void *)0xFEEDBEEF); - CU_ASSERT(sym_op->m_src->data_len == 512); - CU_ASSERT(sym_op->m_src->next == NULL); - CU_ASSERT(sym_op->cipher.data.length == 512); - CU_ASSERT(sym_op->cipher.data.offset == 0); - CU_ASSERT(*RTE_MBUF_DYNFIELD(sym_op->m_src, g_mbuf_offset, uint64_t *) == (uint64_t)g_bdev_io); - CU_ASSERT(sym_op->m_dst == NULL); - CU_ASSERT(TAILQ_EMPTY(&g_crypto_ch->queued_cry_ops) == true); - rte_pktmbuf_free(g_test_crypto_ops[0]->sym->m_src); - rte_pktmbuf_free(g_test_crypto_ops[1]->sym->m_src); - - /* Non-busy reason for enqueue failure, all were rejected. */ - g_enqueue_mock = 0; - g_test_crypto_ops[0]->status = RTE_CRYPTO_OP_STATUS_ERROR; - vbdev_crypto_submit_request(g_io_ch, g_bdev_io); - io_ctx = (struct crypto_bdev_io *)g_bdev_io->driver_ctx; - CU_ASSERT(io_ctx->bdev_io_status == SPDK_BDEV_IO_STATUS_FAILED); -} - -static void -test_crazy_rw(void) -{ - unsigned block_len = 512; - int num_blocks = 4; - int i; - - /* Multi block size read, single element, strange IOV makeup */ - g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; - g_bdev_io->u.bdev.iovcnt = 3; - g_bdev_io->u.bdev.num_blocks = num_blocks; - g_bdev_io->u.bdev.iovs[0].iov_len = 512; - g_bdev_io->u.bdev.iovs[0].iov_base = &test_crazy_rw; - g_bdev_io->u.bdev.iovs[1].iov_len = 1024; - g_bdev_io->u.bdev.iovs[1].iov_base = &test_crazy_rw + 512; - g_bdev_io->u.bdev.iovs[2].iov_len = 512; - g_bdev_io->u.bdev.iovs[2].iov_base = &test_crazy_rw + 512 + 1024; - - g_crypto_bdev.crypto_bdev.blocklen = block_len; - g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; - g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = num_blocks; - - vbdev_crypto_submit_request(g_io_ch, g_bdev_io); - CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); - CU_ASSERT(g_io_ctx->cryop_cnt_remaining == num_blocks); - - for (i = 0; i < num_blocks; i++) { - CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->buf_addr == &test_crazy_rw + (i * block_len)); - CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->data_len == block_len); - CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->next == NULL); - CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.length == block_len); - CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.offset == 0); - CU_ASSERT(*RTE_MBUF_DYNFIELD(g_test_crypto_ops[i]->sym->m_src, g_mbuf_offset, - uint64_t *) == (uint64_t)g_bdev_io); - CU_ASSERT(g_test_crypto_ops[i]->sym->m_src == g_test_crypto_ops[i]->sym->m_src); - CU_ASSERT(g_test_crypto_ops[i]->sym->m_dst == NULL); - rte_pktmbuf_free(g_test_crypto_ops[i]->sym->m_src); - } - - /* Multi block size write, single element strange IOV makeup */ - num_blocks = 8; - g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; - g_bdev_io->u.bdev.iovcnt = 4; - g_bdev_io->u.bdev.num_blocks = num_blocks; - g_bdev_io->u.bdev.iovs[0].iov_len = 2048; - g_bdev_io->u.bdev.iovs[0].iov_base = &test_crazy_rw; - g_bdev_io->u.bdev.iovs[1].iov_len = 512; - g_bdev_io->u.bdev.iovs[1].iov_base = &test_crazy_rw + 2048; - g_bdev_io->u.bdev.iovs[2].iov_len = 512; - g_bdev_io->u.bdev.iovs[2].iov_base = &test_crazy_rw + 2048 + 512; - g_bdev_io->u.bdev.iovs[3].iov_len = 1024; - g_bdev_io->u.bdev.iovs[3].iov_base = &test_crazy_rw + 2048 + 512 + 512; - - g_crypto_bdev.crypto_bdev.blocklen = block_len; - g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; - g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = num_blocks; - - vbdev_crypto_submit_request(g_io_ch, g_bdev_io); - CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); - CU_ASSERT(g_io_ctx->cryop_cnt_remaining == num_blocks); - - for (i = 0; i < num_blocks; i++) { - CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->buf_addr == &test_crazy_rw + (i * block_len)); - CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->data_len == block_len); - CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->next == NULL); - CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.length == block_len); - CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.offset == 0); - CU_ASSERT(*RTE_MBUF_DYNFIELD(g_test_crypto_ops[i]->sym->m_src, g_mbuf_offset, - uint64_t *) == (uint64_t)g_bdev_io); - CU_ASSERT(g_test_crypto_ops[i]->sym->m_src == g_test_crypto_ops[i]->sym->m_src); - CU_ASSERT(g_test_crypto_ops[i]->sym->m_dst == g_test_crypto_ops[i]->sym->m_dst); - rte_pktmbuf_free(g_test_crypto_ops[i]->sym->m_src); - rte_pktmbuf_free(g_test_crypto_ops[i]->sym->m_dst); - } } static void @@ -790,19 +434,21 @@ test_passthru(void) { /* Make sure these follow our completion callback, test success & fail. */ g_bdev_io->type = SPDK_BDEV_IO_TYPE_UNMAP; - MOCK_SET(spdk_bdev_unmap_blocks, 0); + MOCK_CLEAR(spdk_bdev_unmap_blocks); vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + poll_threads(); CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); - MOCK_SET(spdk_bdev_unmap_blocks, -1); + MOCK_SET(spdk_bdev_unmap_blocks, -EINVAL); vbdev_crypto_submit_request(g_io_ch, g_bdev_io); CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); MOCK_CLEAR(spdk_bdev_unmap_blocks); g_bdev_io->type = SPDK_BDEV_IO_TYPE_FLUSH; - MOCK_SET(spdk_bdev_flush_blocks, 0); + MOCK_CLEAR(spdk_bdev_flush_blocks); vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + poll_threads(); CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); - MOCK_SET(spdk_bdev_flush_blocks, -1); + MOCK_SET(spdk_bdev_flush_blocks, -EINVAL); vbdev_crypto_submit_request(g_io_ch, g_bdev_io); CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); MOCK_CLEAR(spdk_bdev_flush_blocks); @@ -823,175 +469,12 @@ test_reset(void) */ } -static void -init_cleanup(void) -{ - if (g_crypto_op_mp) { - rte_mempool_free(g_crypto_op_mp); - g_crypto_op_mp = NULL; - } - if (g_mbuf_mp) { - rte_mempool_free(g_mbuf_mp); - g_mbuf_mp = NULL; - } - if (g_session_mp) { - rte_mempool_free(g_session_mp); - g_session_mp = NULL; - } - if (g_session_mp_priv != NULL) { - /* g_session_mp_priv may or may not be set depending on the DPDK version */ - rte_mempool_free(g_session_mp_priv); - g_session_mp_priv = NULL; - } -} - -static void -test_initdrivers(void) -{ - int rc; - static struct rte_mempool *orig_mbuf_mp; - static struct rte_mempool *orig_session_mp; - static struct rte_mempool *orig_session_mp_priv; - - /* These tests will alloc and free our g_mbuf_mp - * so save that off here and restore it after each test is over. - */ - orig_mbuf_mp = g_mbuf_mp; - orig_session_mp = g_session_mp; - orig_session_mp_priv = g_session_mp_priv; - - g_session_mp_priv = NULL; - g_session_mp = NULL; - g_mbuf_mp = NULL; - - /* No drivers available, not an error though */ - MOCK_SET(rte_cryptodev_count, 0); - rc = vbdev_crypto_init_crypto_drivers(); - CU_ASSERT(rc == 0); - CU_ASSERT(g_mbuf_mp == NULL); - CU_ASSERT(g_session_mp == NULL); - CU_ASSERT(g_session_mp_priv == NULL); - - /* Can't create session pool. */ - MOCK_SET(rte_cryptodev_count, 2); - MOCK_SET(spdk_mempool_create, NULL); - rc = vbdev_crypto_init_crypto_drivers(); - CU_ASSERT(rc == -ENOMEM); - CU_ASSERT(g_mbuf_mp == NULL); - CU_ASSERT(g_session_mp == NULL); - CU_ASSERT(g_session_mp_priv == NULL); - MOCK_CLEAR(spdk_mempool_create); - - /* Can't create op pool. */ - MOCK_SET(rte_crypto_op_pool_create, NULL); - rc = vbdev_crypto_init_crypto_drivers(); - CU_ASSERT(rc == -ENOMEM); - CU_ASSERT(g_mbuf_mp == NULL); - CU_ASSERT(g_session_mp == NULL); - CU_ASSERT(g_session_mp_priv == NULL); - MOCK_CLEAR(rte_crypto_op_pool_create); - - /* Check resources are not sufficient */ - MOCK_CLEARED_ASSERT(spdk_mempool_create); - rc = vbdev_crypto_init_crypto_drivers(); - CU_ASSERT(rc == -EINVAL); - - /* Test crypto dev configure failure. */ - MOCK_SET(rte_cryptodev_device_count_by_driver, 2); - MOCK_SET(rte_cryptodev_info_get, MOCK_INFO_GET_1QP_AESNI); - MOCK_SET(rte_cryptodev_configure, -1); - MOCK_CLEARED_ASSERT(spdk_mempool_create); - rc = vbdev_crypto_init_crypto_drivers(); - MOCK_SET(rte_cryptodev_configure, 0); - CU_ASSERT(g_mbuf_mp == NULL); - CU_ASSERT(g_session_mp == NULL); - CU_ASSERT(g_session_mp_priv == NULL); - CU_ASSERT(rc == -EINVAL); - - /* Test failure of qp setup. */ - MOCK_SET(rte_cryptodev_queue_pair_setup, -1); - MOCK_CLEARED_ASSERT(spdk_mempool_create); - rc = vbdev_crypto_init_crypto_drivers(); - CU_ASSERT(rc == -EINVAL); - CU_ASSERT(g_mbuf_mp == NULL); - CU_ASSERT(g_session_mp == NULL); - CU_ASSERT(g_session_mp_priv == NULL); - MOCK_SET(rte_cryptodev_queue_pair_setup, 0); - - /* Test failure of dev start. */ - MOCK_SET(rte_cryptodev_start, -1); - MOCK_CLEARED_ASSERT(spdk_mempool_create); - rc = vbdev_crypto_init_crypto_drivers(); - CU_ASSERT(rc == -EINVAL); - CU_ASSERT(g_mbuf_mp == NULL); - CU_ASSERT(g_session_mp == NULL); - CU_ASSERT(g_session_mp_priv == NULL); - MOCK_SET(rte_cryptodev_start, 0); - - /* Test bogus PMD */ - MOCK_CLEARED_ASSERT(spdk_mempool_create); - MOCK_SET(rte_cryptodev_info_get, MOCK_INFO_GET_1QP_BOGUS_PMD); - rc = vbdev_crypto_init_crypto_drivers(); - CU_ASSERT(g_mbuf_mp == NULL); - CU_ASSERT(g_session_mp == NULL); - CU_ASSERT(rc == -EINVAL); - - /* Test happy path QAT. */ - MOCK_CLEARED_ASSERT(spdk_mempool_create); - MOCK_SET(rte_cryptodev_info_get, MOCK_INFO_GET_1QP_QAT); - rc = vbdev_crypto_init_crypto_drivers(); - CU_ASSERT(g_mbuf_mp != NULL); - CU_ASSERT(g_session_mp != NULL); - init_cleanup(); - CU_ASSERT(rc == 0); - - /* Test happy path AESNI. */ - MOCK_CLEARED_ASSERT(spdk_mempool_create); - MOCK_SET(rte_cryptodev_info_get, MOCK_INFO_GET_1QP_AESNI); - rc = vbdev_crypto_init_crypto_drivers(); - CU_ASSERT(g_mbuf_offset == DPDK_DYNFIELD_OFFSET); - init_cleanup(); - CU_ASSERT(rc == 0); - - /* Test happy path MLX5. */ - MOCK_CLEARED_ASSERT(spdk_mempool_create); - MOCK_SET(rte_cryptodev_info_get, MOCK_INFO_GET_1QP_MLX5); - rc = vbdev_crypto_init_crypto_drivers(); - CU_ASSERT(g_mbuf_offset == DPDK_DYNFIELD_OFFSET); - init_cleanup(); - CU_ASSERT(rc == 0); - - /* Test failure of DPDK dev init. By now it is not longer an error - * situation for entire crypto framework. */ - MOCK_SET(rte_cryptodev_count, 2); - MOCK_SET(rte_cryptodev_device_count_by_driver, 2); - MOCK_SET(rte_vdev_init, -1); - MOCK_CLEARED_ASSERT(spdk_mempool_create); - MOCK_SET(rte_cryptodev_info_get, MOCK_INFO_GET_1QP_QAT); - rc = vbdev_crypto_init_crypto_drivers(); - CU_ASSERT(rc == 0); - CU_ASSERT(g_mbuf_mp != NULL); - CU_ASSERT(g_session_mp != NULL); -#if RTE_VERSION < RTE_VERSION_NUM(22, 11, 0, 0) - CU_ASSERT(g_session_mp_priv != NULL); -#endif - init_cleanup(); - MOCK_SET(rte_vdev_init, 0); - MOCK_CLEAR(rte_cryptodev_device_count_by_driver); - - /* restore our initial values. */ - g_mbuf_mp = orig_mbuf_mp; - g_session_mp = orig_session_mp; - g_session_mp_priv = orig_session_mp_priv; -} - static void test_crypto_op_complete(void) { /* Make sure completion code respects failure. */ - g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED; g_completion_called = false; - _crypto_operation_complete(g_bdev_io); + _crypto_operation_complete(g_bdev_io, -1); CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); CU_ASSERT(g_completion_called == true); @@ -999,7 +482,7 @@ test_crypto_op_complete(void) g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; g_completion_called = false; - _crypto_operation_complete(g_bdev_io); + _crypto_operation_complete(g_bdev_io, 0); CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); CU_ASSERT(g_completion_called == true); @@ -1007,8 +490,9 @@ test_crypto_op_complete(void) g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; g_completion_called = false; - MOCK_SET(spdk_bdev_writev_blocks, 0); - _crypto_operation_complete(g_bdev_io); + MOCK_CLEAR(spdk_bdev_writev_blocks); + _crypto_operation_complete(g_bdev_io, 0); + poll_threads(); CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); CU_ASSERT(g_completion_called == true); @@ -1016,16 +500,17 @@ test_crypto_op_complete(void) g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; g_completion_called = false; - MOCK_SET(spdk_bdev_writev_blocks, -1); - _crypto_operation_complete(g_bdev_io); + MOCK_SET(spdk_bdev_writev_blocks, -EINVAL); + _crypto_operation_complete(g_bdev_io, 0); CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); CU_ASSERT(g_completion_called == true); + MOCK_CLEAR(spdk_bdev_writev_blocks); /* Test bogus type for this completion. */ g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; g_bdev_io->type = SPDK_BDEV_IO_TYPE_RESET; g_completion_called = false; - _crypto_operation_complete(g_bdev_io); + _crypto_operation_complete(g_bdev_io, 0); CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); CU_ASSERT(g_completion_called == true); } @@ -1042,160 +527,6 @@ test_supported_io(void) rc = vbdev_crypto_io_type_supported(ctx, SPDK_BDEV_IO_TYPE_WRITE_ZEROES); CU_ASSERT(rc == false); } - -static void -test_poller(void) -{ - int rc; - struct rte_mbuf *src_mbufs[2]; - struct vbdev_crypto_op *op_to_resubmit; - - /* test regular 1 op to dequeue and complete */ - g_dequeue_mock = g_enqueue_mock = 1; - rte_pktmbuf_alloc_bulk(g_mbuf_mp, src_mbufs, 1); - g_test_crypto_ops[0]->sym->m_src = src_mbufs[0]; - *RTE_MBUF_DYNFIELD(g_test_crypto_ops[0]->sym->m_src, g_mbuf_offset, - uint64_t *) = (uintptr_t)g_bdev_io; - g_test_crypto_ops[0]->sym->m_dst = NULL; - g_io_ctx->cryop_cnt_remaining = 1; - g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; - rc = crypto_dev_poller(g_crypto_ch); - CU_ASSERT(rc == 1); - - /* We have nothing dequeued but have some to resubmit */ - g_dequeue_mock = 0; - CU_ASSERT(TAILQ_EMPTY(&g_crypto_ch->queued_cry_ops) == true); - - /* add an op to the queued list. */ - g_resubmit_test = true; - op_to_resubmit = (struct vbdev_crypto_op *)((uint8_t *)g_test_crypto_ops[0] + QUEUED_OP_OFFSET); - op_to_resubmit->crypto_op = (void *)0xDEADBEEF; - op_to_resubmit->bdev_io = g_bdev_io; - TAILQ_INSERT_TAIL(&g_crypto_ch->queued_cry_ops, - op_to_resubmit, - link); - CU_ASSERT(TAILQ_EMPTY(&g_crypto_ch->queued_cry_ops) == false); - rc = crypto_dev_poller(g_crypto_ch); - g_resubmit_test = false; - CU_ASSERT(rc == 0); - CU_ASSERT(TAILQ_EMPTY(&g_crypto_ch->queued_cry_ops) == true); - - /* 2 to dequeue but 2nd one failed */ - g_dequeue_mock = g_enqueue_mock = 2; - g_io_ctx->cryop_cnt_remaining = 2; - rte_pktmbuf_alloc_bulk(g_mbuf_mp, src_mbufs, 2); - g_test_crypto_ops[0]->sym->m_src = src_mbufs[0]; - *RTE_MBUF_DYNFIELD(g_test_crypto_ops[0]->sym->m_src, g_mbuf_offset, - uint64_t *) = (uint64_t)g_bdev_io; - g_test_crypto_ops[0]->sym->m_dst = NULL; - g_test_crypto_ops[0]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; - g_test_crypto_ops[1]->sym->m_src = src_mbufs[1]; - *RTE_MBUF_DYNFIELD(g_test_crypto_ops[1]->sym->m_src, g_mbuf_offset, - uint64_t *) = (uint64_t)g_bdev_io; - g_test_crypto_ops[1]->sym->m_dst = NULL; - g_test_crypto_ops[1]->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; - g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; - rc = crypto_dev_poller(g_crypto_ch); - CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); - CU_ASSERT(rc == 2); -} - -/* Helper function for test_assign_device_qp() */ -static void -_clear_device_qp_lists(void) -{ - struct device_qp *device_qp = NULL; - - while (!TAILQ_EMPTY(&g_device_qp_qat)) { - device_qp = TAILQ_FIRST(&g_device_qp_qat); - TAILQ_REMOVE(&g_device_qp_qat, device_qp, link); - free(device_qp); - - } - CU_ASSERT(TAILQ_EMPTY(&g_device_qp_qat) == true); - while (!TAILQ_EMPTY(&g_device_qp_aesni_mb)) { - device_qp = TAILQ_FIRST(&g_device_qp_aesni_mb); - TAILQ_REMOVE(&g_device_qp_aesni_mb, device_qp, link); - free(device_qp); - } - CU_ASSERT(TAILQ_EMPTY(&g_device_qp_aesni_mb) == true); - while (!TAILQ_EMPTY(&g_device_qp_mlx5)) { - device_qp = TAILQ_FIRST(&g_device_qp_mlx5); - TAILQ_REMOVE(&g_device_qp_mlx5, device_qp, link); - free(device_qp); - } - CU_ASSERT(TAILQ_EMPTY(&g_device_qp_mlx5) == true); -} - -/* Helper function for test_assign_device_qp() */ -static void -_check_expected_values(struct vbdev_crypto *crypto_bdev, struct device_qp *device_qp, - struct crypto_io_channel *crypto_ch, uint8_t expected_index, - uint8_t current_index) -{ - _assign_device_qp(&g_crypto_bdev, device_qp, g_crypto_ch); - CU_ASSERT(g_crypto_ch->device_qp->index == expected_index); - CU_ASSERT(g_next_qat_index == current_index); -} - -static void -test_assign_device_qp(void) -{ - struct device_qp *device_qp = NULL; - int i; - - /* start with a known state, clear the device/qp lists */ - _clear_device_qp_lists(); - - /* make sure that one AESNI_MB qp is found */ - device_qp = calloc(1, sizeof(struct device_qp)); - TAILQ_INSERT_TAIL(&g_device_qp_aesni_mb, device_qp, link); - g_crypto_ch->device_qp = NULL; - g_crypto_bdev.opts->drv_name = AESNI_MB; - _assign_device_qp(&g_crypto_bdev, device_qp, g_crypto_ch); - CU_ASSERT(g_crypto_ch->device_qp != NULL); - - /* QAT testing is more complex as the code under test load balances by - * assigning each subsequent device/qp to every QAT_VF_SPREAD modulo - * g_qat_total_qp. For the current latest QAT we'll have 48 virtual functions - * each with 2 qp so the "spread" between assignments is 32. - */ - g_qat_total_qp = 96; - for (i = 0; i < g_qat_total_qp; i++) { - device_qp = calloc(1, sizeof(struct device_qp)); - device_qp->index = i; - TAILQ_INSERT_TAIL(&g_device_qp_qat, device_qp, link); - } - g_crypto_ch->device_qp = NULL; - g_crypto_bdev.opts->drv_name = QAT; - - /* First assignment will assign to 0 and next at 32. */ - _check_expected_values(&g_crypto_bdev, device_qp, g_crypto_ch, - 0, QAT_VF_SPREAD); - - /* Second assignment will assign to 32 and next at 64. */ - _check_expected_values(&g_crypto_bdev, device_qp, g_crypto_ch, - QAT_VF_SPREAD, QAT_VF_SPREAD * 2); - - /* Third assignment will assign to 64 and next at 0. */ - _check_expected_values(&g_crypto_bdev, device_qp, g_crypto_ch, - QAT_VF_SPREAD * 2, 0); - - /* Fourth assignment will assign to 1 and next at 33. */ - _check_expected_values(&g_crypto_bdev, device_qp, g_crypto_ch, - 1, QAT_VF_SPREAD + 1); - - /* make sure that one MLX5 qp is found */ - device_qp = calloc(1, sizeof(struct device_qp)); - TAILQ_INSERT_TAIL(&g_device_qp_mlx5, device_qp, link); - g_crypto_ch->device_qp = NULL; - g_crypto_bdev.opts->drv_name = MLX5; - _assign_device_qp(&g_crypto_bdev, device_qp, g_crypto_ch); - CU_ASSERT(g_crypto_ch->device_qp == device_qp); - - _clear_device_qp_lists(); -} - int main(int argc, char **argv) { @@ -1209,19 +540,19 @@ main(int argc, char **argv) CU_ADD_TEST(suite, test_error_paths); CU_ADD_TEST(suite, test_simple_write); CU_ADD_TEST(suite, test_simple_read); - CU_ADD_TEST(suite, test_large_rw); - CU_ADD_TEST(suite, test_dev_full); - CU_ADD_TEST(suite, test_crazy_rw); CU_ADD_TEST(suite, test_passthru); - CU_ADD_TEST(suite, test_initdrivers); CU_ADD_TEST(suite, test_crypto_op_complete); CU_ADD_TEST(suite, test_supported_io); CU_ADD_TEST(suite, test_reset); - CU_ADD_TEST(suite, test_poller); - CU_ADD_TEST(suite, test_assign_device_qp); + + allocate_threads(1); + set_thread(0); CU_basic_set_mode(CU_BRM_VERBOSE); CU_basic_run_tests(); + + free_threads(); + num_failures = CU_get_number_of_failures(); CU_cleanup_registry(); return num_failures;