From f5d1a924a1dcfac15289d1d9705f8ead23d1114b Mon Sep 17 00:00:00 2001 From: Alexey Marchuk Date: Mon, 9 Jan 2023 10:30:08 +0100 Subject: [PATCH] accel/dpdk_cryptodev: start with copy of vbdev_crypto This patch is just a copy of vbdev_crypto.c and the corresponding UT file. It makes it easier to review the next patch which adds accel operations Signed-off-by: Alexey Marchuk Change-Id: Ib88b45d573b011b1acb35da9bf4dab922d8fb183 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/16182 Reviewed-by: Jim Harris Reviewed-by: Shuhei Matsumoto Reviewed-by: Paul Luse Tested-by: SPDK CI Jenkins --- .../dpdk_cryptodev/accel_dpdk_cryptodev.c | 2128 +++++++++++++++++ test/common/skipped_build_files.txt | 4 + .../accel_dpdk_cryptodev_ut.c | 1227 ++++++++++ 3 files changed, 3359 insertions(+) create mode 100644 module/accel/dpdk_cryptodev/accel_dpdk_cryptodev.c create mode 100644 test/unit/lib/accel/dpdk_cryptodev.c/accel_dpdk_cryptodev_ut.c diff --git a/module/accel/dpdk_cryptodev/accel_dpdk_cryptodev.c b/module/accel/dpdk_cryptodev/accel_dpdk_cryptodev.c new file mode 100644 index 000000000..fa52169aa --- /dev/null +++ b/module/accel/dpdk_cryptodev/accel_dpdk_cryptodev.c @@ -0,0 +1,2128 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (C) 2018 Intel Corporation. + * All rights reserved. + * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. + * All rights reserved. + */ + +#include "vbdev_crypto.h" + +#include "spdk/env.h" +#include "spdk/likely.h" +#include "spdk/endian.h" +#include "spdk/thread.h" +#include "spdk/bdev_module.h" +#include "spdk/log.h" +#include "spdk/hexlify.h" + +#include +#include +#include +#include +#include +#include + +/* Used to store IO context in mbuf */ +static const struct rte_mbuf_dynfield rte_mbuf_dynfield_io_context = { + .name = "context_bdev_io", + .size = sizeof(uint64_t), + .align = __alignof__(uint64_t), + .flags = 0, +}; +static int g_mbuf_offset; + +/* To add support for new device types, follow the examples of the following... + * Note that the string names are defined by the DPDK PMD in question so be + * sure to use the exact names. + */ +#define MAX_NUM_DRV_TYPES 3 + +/* The VF spread is the number of queue pairs between virtual functions, we use this to + * load balance the QAT device. + */ +#define QAT_VF_SPREAD 32 +static uint8_t g_qat_total_qp = 0; +static uint8_t g_next_qat_index; + +const char *g_driver_names[MAX_NUM_DRV_TYPES] = { AESNI_MB, QAT, MLX5 }; + +/* Global list of available crypto devices. */ +struct vbdev_dev { + struct rte_cryptodev_info cdev_info; /* includes device friendly name */ + uint8_t cdev_id; /* identifier for the device */ + TAILQ_ENTRY(vbdev_dev) link; +}; +static TAILQ_HEAD(, vbdev_dev) g_vbdev_devs = TAILQ_HEAD_INITIALIZER(g_vbdev_devs); + +/* Global list and lock for unique device/queue pair combos. We keep 1 list per supported PMD + * so that we can optimize per PMD where it make sense. For example, with QAT there an optimal + * pattern for assigning queue pairs where with AESNI there is not. + */ +struct device_qp { + struct vbdev_dev *device; /* ptr to crypto device */ + uint8_t qp; /* queue pair for this node */ + bool in_use; /* whether this node is in use or not */ + uint8_t index; /* used by QAT to load balance placement of qpairs */ + TAILQ_ENTRY(device_qp) link; +}; +static TAILQ_HEAD(, device_qp) g_device_qp_qat = TAILQ_HEAD_INITIALIZER(g_device_qp_qat); +static TAILQ_HEAD(, device_qp) g_device_qp_aesni_mb = TAILQ_HEAD_INITIALIZER(g_device_qp_aesni_mb); +static TAILQ_HEAD(, device_qp) g_device_qp_mlx5 = TAILQ_HEAD_INITIALIZER(g_device_qp_mlx5); +static pthread_mutex_t g_device_qp_lock = PTHREAD_MUTEX_INITIALIZER; + + +/* In order to limit the number of resources we need to do one crypto + * operation per LBA (we use LBA as IV), we tell the bdev layer that + * our max IO size is something reasonable. Units here are in bytes. + */ +#define CRYPTO_MAX_IO (64 * 1024) + +/* This controls how many ops will be dequeued from the crypto driver in one run + * of the poller. It is mainly a performance knob as it effectively determines how + * much work the poller has to do. However even that can vary between crypto drivers + * as the AESNI_MB driver for example does all the crypto work on dequeue whereas the + * QAT driver just dequeues what has been completed already. + */ +#define MAX_DEQUEUE_BURST_SIZE 64 + +/* When enqueueing, we need to supply the crypto driver with an array of pointers to + * operation structs. As each of these can be max 512B, we can adjust the CRYPTO_MAX_IO + * value in conjunction with the other defines to make sure we're not using crazy amounts + * of memory. All of these numbers can and probably should be adjusted based on the + * workload. By default we'll use the worst case (smallest) block size for the + * minimum number of array entries. As an example, a CRYPTO_MAX_IO size of 64K with 512B + * blocks would give us an enqueue array size of 128. + */ +#define MAX_ENQUEUE_ARRAY_SIZE (CRYPTO_MAX_IO / 512) + +/* The number of MBUFS we need must be a power of two and to support other small IOs + * in addition to the limits mentioned above, we go to the next power of two. It is + * big number because it is one mempool for source and destination mbufs. It may + * need to be bigger to support multiple crypto drivers at once. + */ +#define NUM_MBUFS 32768 +#define POOL_CACHE_SIZE 256 +#define MAX_CRYPTO_VOLUMES 128 +#define NUM_SESSIONS (2 * MAX_CRYPTO_VOLUMES) +#define SESS_MEMPOOL_CACHE_SIZE 0 +uint8_t g_number_of_claimed_volumes = 0; + +/* This is the max number of IOs we can supply to any crypto device QP at one time. + * It can vary between drivers. + */ +#define CRYPTO_QP_DESCRIPTORS 2048 + +/* At this moment DPDK descriptors allocation for mlx5 has some issues. We use 512 + * as an compromise value between performance and the time spent for initialization. */ +#define CRYPTO_QP_DESCRIPTORS_MLX5 512 + +#define AESNI_MB_NUM_QP 64 + +/* Common for supported devices. */ +#define DEFAULT_NUM_XFORMS 2 +#define IV_OFFSET (sizeof(struct rte_crypto_op) + \ + sizeof(struct rte_crypto_sym_op) + \ + (DEFAULT_NUM_XFORMS * \ + sizeof(struct rte_crypto_sym_xform))) +#define IV_LENGTH 16 +#define QUEUED_OP_OFFSET (IV_OFFSET + IV_LENGTH) + +static void _complete_internal_io(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg); +static void _complete_internal_read(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg); +static void _complete_internal_write(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg); +static void vbdev_crypto_examine(struct spdk_bdev *bdev); +static int vbdev_crypto_claim(const char *bdev_name); +static void vbdev_crypto_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io); + +struct bdev_names { + struct vbdev_crypto_opts *opts; + TAILQ_ENTRY(bdev_names) link; +}; + +/* List of crypto_bdev names and their base bdevs via configuration file. */ +static TAILQ_HEAD(, bdev_names) g_bdev_names = TAILQ_HEAD_INITIALIZER(g_bdev_names); + +struct vbdev_crypto { + struct spdk_bdev *base_bdev; /* the thing we're attaching to */ + struct spdk_bdev_desc *base_desc; /* its descriptor we get from open */ + struct spdk_bdev crypto_bdev; /* the crypto virtual bdev */ + struct vbdev_crypto_opts *opts; /* crypto options such as key, cipher */ + uint32_t qp_desc_nr; /* number of qp descriptors */ + void *session_encrypt; /* encryption session for this bdev */ + void *session_decrypt; /* decryption session for this bdev */ + struct rte_crypto_sym_xform cipher_xform; /* crypto control struct for this bdev */ + TAILQ_ENTRY(vbdev_crypto) link; + struct spdk_thread *thread; /* thread where base device is opened */ +}; + +/* List of virtual bdevs and associated info for each. We keep the device friendly name here even + * though its also in the device struct because we use it early on. + */ +static TAILQ_HEAD(, vbdev_crypto) g_vbdev_crypto = TAILQ_HEAD_INITIALIZER(g_vbdev_crypto); + +/* Shared mempools between all devices on this system */ +static struct rte_mempool *g_session_mp = NULL; +static struct rte_mempool *g_session_mp_priv = NULL; +static struct rte_mempool *g_mbuf_mp = NULL; /* mbuf mempool */ +static struct rte_mempool *g_crypto_op_mp = NULL; /* crypto operations, must be rte* mempool */ + +static struct rte_mbuf_ext_shared_info g_shinfo = {}; /* used by DPDK mbuf macro */ + +/* For queueing up crypto operations that we can't submit for some reason */ +struct vbdev_crypto_op { + uint8_t cdev_id; + uint8_t qp; + struct rte_crypto_op *crypto_op; + struct spdk_bdev_io *bdev_io; + TAILQ_ENTRY(vbdev_crypto_op) link; +}; +#define QUEUED_OP_LENGTH (sizeof(struct vbdev_crypto_op)) + +/* The crypto vbdev channel struct. It is allocated and freed on my behalf by the io channel code. + * We store things in here that are needed on per thread basis like the base_channel for this thread, + * and the poller for this thread. + */ +struct crypto_io_channel { + struct spdk_io_channel *base_ch; /* IO channel of base device */ + struct spdk_poller *poller; /* completion poller */ + struct device_qp *device_qp; /* unique device/qp combination for this channel */ + TAILQ_HEAD(, spdk_bdev_io) pending_cry_ios; /* outstanding operations to the crypto device */ + struct spdk_io_channel_iter *iter; /* used with for_each_channel in reset */ + TAILQ_HEAD(, vbdev_crypto_op) queued_cry_ops; /* queued for re-submission to CryptoDev */ +}; + +/* This is the crypto per IO context that the bdev layer allocates for us opaquely and attaches to + * each IO for us. + */ +struct crypto_bdev_io { + int cryop_cnt_remaining; /* counter used when completing crypto ops */ + struct crypto_io_channel *crypto_ch; /* need to store for crypto completion handling */ + struct vbdev_crypto *crypto_bdev; /* the crypto node struct associated with this IO */ + struct spdk_bdev_io *orig_io; /* the original IO */ + struct spdk_bdev_io *read_io; /* the read IO we issued */ + int8_t bdev_io_status; /* the status we'll report back on the bdev IO */ + bool on_pending_list; + /* Used for the single contiguous buffer that serves as the crypto destination target for writes */ + uint64_t aux_num_blocks; /* num of blocks for the contiguous buffer */ + uint64_t aux_offset_blocks; /* block offset on media */ + void *aux_buf_raw; /* raw buffer that the bdev layer gave us for write buffer */ + struct iovec aux_buf_iov; /* iov representing aligned contig write buffer */ + + /* for bdev_io_wait */ + struct spdk_bdev_io_wait_entry bdev_io_wait; + struct spdk_io_channel *ch; +}; + +/* Called by vbdev_crypto_init_crypto_drivers() to init each discovered crypto device */ +static int +create_vbdev_dev(uint8_t index, uint16_t num_lcores) +{ + struct vbdev_dev *device; + uint8_t j, cdev_id, cdrv_id; + struct device_qp *dev_qp; + struct device_qp *tmp_qp; + uint32_t qp_desc_nr; + int rc; + TAILQ_HEAD(device_qps, device_qp) *dev_qp_head; + + device = calloc(1, sizeof(struct vbdev_dev)); + if (!device) { + return -ENOMEM; + } + + /* Get details about this device. */ + rte_cryptodev_info_get(index, &device->cdev_info); + cdrv_id = device->cdev_info.driver_id; + cdev_id = device->cdev_id = index; + + /* QAT_ASYM devices are not supported at this time. */ + if (strcmp(device->cdev_info.driver_name, QAT_ASYM) == 0) { + free(device); + return 0; + } + + /* Before going any further, make sure we have enough resources for this + * device type to function. We need a unique queue pair per core across each + * device type to remain lockless.... + */ + if ((rte_cryptodev_device_count_by_driver(cdrv_id) * + device->cdev_info.max_nb_queue_pairs) < num_lcores) { + SPDK_ERRLOG("Insufficient unique queue pairs available for %s\n", + device->cdev_info.driver_name); + SPDK_ERRLOG("Either add more crypto devices or decrease core count\n"); + rc = -EINVAL; + goto err; + } + + /* Setup queue pairs. */ + struct rte_cryptodev_config conf = { + .nb_queue_pairs = device->cdev_info.max_nb_queue_pairs, + .socket_id = SPDK_ENV_SOCKET_ID_ANY + }; + + rc = rte_cryptodev_configure(cdev_id, &conf); + if (rc < 0) { + SPDK_ERRLOG("Failed to configure cryptodev %u: error %d\n", + cdev_id, rc); + rc = -EINVAL; + goto err; + } + + /* Select the right device/qp list based on driver name + * or error if it does not exist. + */ + if (strcmp(device->cdev_info.driver_name, QAT) == 0) { + dev_qp_head = (struct device_qps *)&g_device_qp_qat; + qp_desc_nr = CRYPTO_QP_DESCRIPTORS; + } else if (strcmp(device->cdev_info.driver_name, AESNI_MB) == 0) { + dev_qp_head = (struct device_qps *)&g_device_qp_aesni_mb; + qp_desc_nr = CRYPTO_QP_DESCRIPTORS; + } else if (strcmp(device->cdev_info.driver_name, MLX5) == 0) { + dev_qp_head = (struct device_qps *)&g_device_qp_mlx5; + qp_desc_nr = CRYPTO_QP_DESCRIPTORS_MLX5; + } else { + SPDK_ERRLOG("Failed to start device %u. Invalid driver name \"%s\"\n", + cdev_id, device->cdev_info.driver_name); + rc = -EINVAL; + goto err_qp_setup; + } + + struct rte_cryptodev_qp_conf qp_conf = { + .nb_descriptors = qp_desc_nr, + .mp_session = g_session_mp, +#if RTE_VERSION < RTE_VERSION_NUM(22, 11, 0, 0) + .mp_session_private = g_session_mp_priv, +#endif + }; + + /* Pre-setup all potential qpairs now and assign them in the channel + * callback. If we were to create them there, we'd have to stop the + * entire device affecting all other threads that might be using it + * even on other queue pairs. + */ + for (j = 0; j < device->cdev_info.max_nb_queue_pairs; j++) { + rc = rte_cryptodev_queue_pair_setup(cdev_id, j, &qp_conf, SOCKET_ID_ANY); + if (rc < 0) { + SPDK_ERRLOG("Failed to setup queue pair %u on " + "cryptodev %u: error %d\n", j, cdev_id, rc); + rc = -EINVAL; + goto err_qp_setup; + } + } + + rc = rte_cryptodev_start(cdev_id); + if (rc < 0) { + SPDK_ERRLOG("Failed to start device %u: error %d\n", + cdev_id, rc); + rc = -EINVAL; + goto err_dev_start; + } + + /* Build up lists of device/qp combinations per PMD */ + for (j = 0; j < device->cdev_info.max_nb_queue_pairs; j++) { + dev_qp = calloc(1, sizeof(struct device_qp)); + if (!dev_qp) { + rc = -ENOMEM; + goto err_qp_alloc; + } + dev_qp->device = device; + dev_qp->qp = j; + dev_qp->in_use = false; + if (strcmp(device->cdev_info.driver_name, QAT) == 0) { + g_qat_total_qp++; + } + TAILQ_INSERT_TAIL(dev_qp_head, dev_qp, link); + } + + /* Add to our list of available crypto devices. */ + TAILQ_INSERT_TAIL(&g_vbdev_devs, device, link); + + return 0; +err_qp_alloc: + TAILQ_FOREACH_SAFE(dev_qp, dev_qp_head, link, tmp_qp) { + if (dev_qp->device->cdev_id != device->cdev_id) { + continue; + } + TAILQ_REMOVE(dev_qp_head, dev_qp, link); + if (dev_qp_head == (struct device_qps *)&g_device_qp_qat) { + g_qat_total_qp--; + } + free(dev_qp); + } + rte_cryptodev_stop(cdev_id); +err_dev_start: +err_qp_setup: + rte_cryptodev_close(cdev_id); +err: + free(device); + + return rc; +} + +static void +release_vbdev_dev(struct vbdev_dev *device) +{ + struct device_qp *dev_qp; + struct device_qp *tmp_qp; + TAILQ_HEAD(device_qps, device_qp) *dev_qp_head = NULL; + + assert(device); + + /* Select the right device/qp list based on driver name. */ + if (strcmp(device->cdev_info.driver_name, QAT) == 0) { + dev_qp_head = (struct device_qps *)&g_device_qp_qat; + } else if (strcmp(device->cdev_info.driver_name, AESNI_MB) == 0) { + dev_qp_head = (struct device_qps *)&g_device_qp_aesni_mb; + } else if (strcmp(device->cdev_info.driver_name, MLX5) == 0) { + dev_qp_head = (struct device_qps *)&g_device_qp_mlx5; + } + if (dev_qp_head) { + TAILQ_FOREACH_SAFE(dev_qp, dev_qp_head, link, tmp_qp) { + /* Remove only qps of our device even if the driver names matches. */ + if (dev_qp->device->cdev_id != device->cdev_id) { + continue; + } + TAILQ_REMOVE(dev_qp_head, dev_qp, link); + if (dev_qp_head == (struct device_qps *)&g_device_qp_qat) { + g_qat_total_qp--; + } + free(dev_qp); + } + } + rte_cryptodev_stop(device->cdev_id); + rte_cryptodev_close(device->cdev_id); + free(device); +} + +/* Dummy function used by DPDK to free ext attached buffers to mbufs, we free them ourselves but + * this callback has to be here. */ +static void +shinfo_free_cb(void *arg1, void *arg2) +{ +} + +/* This is called from the module's init function. We setup all crypto devices early on as we are unable + * to easily dynamically configure queue pairs after the drivers are up and running. So, here, we + * configure the max capabilities of each device and assign threads to queue pairs as channels are + * requested. + */ +static int +vbdev_crypto_init_crypto_drivers(void) +{ + uint8_t cdev_count; + uint8_t cdev_id; + int i, rc; + struct vbdev_dev *device; + struct vbdev_dev *tmp_dev; + struct device_qp *dev_qp; + unsigned int max_sess_size = 0, sess_size; + uint16_t num_lcores = rte_lcore_count(); + char aesni_args[32]; + + /* Only the first call, via RPC or module init should init the crypto drivers. */ + if (g_session_mp != NULL) { + return 0; + } + + /* We always init AESNI_MB */ + snprintf(aesni_args, sizeof(aesni_args), "max_nb_queue_pairs=%d", AESNI_MB_NUM_QP); + rc = rte_vdev_init(AESNI_MB, aesni_args); + if (rc) { + SPDK_NOTICELOG("Failed to create virtual PMD %s: error %d. " + "Possibly %s is not supported by DPDK library. " + "Keep going...\n", AESNI_MB, rc, AESNI_MB); + } + + /* If we have no crypto devices, there's no reason to continue. */ + cdev_count = rte_cryptodev_count(); + SPDK_NOTICELOG("Found crypto devices: %d\n", (int)cdev_count); + if (cdev_count == 0) { + return 0; + } + + g_mbuf_offset = rte_mbuf_dynfield_register(&rte_mbuf_dynfield_io_context); + if (g_mbuf_offset < 0) { + SPDK_ERRLOG("error registering dynamic field with DPDK\n"); + return -EINVAL; + } + + /* + * Create global mempools, shared by all devices regardless of type. + */ + + /* First determine max session size, most pools are shared by all the devices, + * so we need to find the global max sessions size. + */ + for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { + sess_size = rte_cryptodev_sym_get_private_session_size(cdev_id); + if (sess_size > max_sess_size) { + max_sess_size = sess_size; + } + } + +#if RTE_VERSION < RTE_VERSION_NUM(22, 11, 0, 0) + g_session_mp_priv = rte_mempool_create("session_mp_priv", NUM_SESSIONS, max_sess_size, + SESS_MEMPOOL_CACHE_SIZE, 0, NULL, NULL, NULL, + NULL, SOCKET_ID_ANY, 0); + if (g_session_mp_priv == NULL) { + SPDK_ERRLOG("Cannot create private session pool max size 0x%x\n", max_sess_size); + return -ENOMEM; + } + /* When session private data mempool allocated, the element size for the session mempool + * should be 0. */ + max_sess_size = 0; +#endif + + g_session_mp = rte_cryptodev_sym_session_pool_create( + "session_mp", + NUM_SESSIONS, max_sess_size, SESS_MEMPOOL_CACHE_SIZE, 0, + SOCKET_ID_ANY); + if (g_session_mp == NULL) { + SPDK_ERRLOG("Cannot create session pool max size 0x%x\n", max_sess_size); + rc = -ENOMEM; + goto error_create_session_mp; + } + + g_mbuf_mp = rte_pktmbuf_pool_create("mbuf_mp", NUM_MBUFS, POOL_CACHE_SIZE, + 0, 0, SPDK_ENV_SOCKET_ID_ANY); + if (g_mbuf_mp == NULL) { + SPDK_ERRLOG("Cannot create mbuf pool\n"); + rc = -ENOMEM; + goto error_create_mbuf; + } + + /* We use per op private data as suggested by DPDK and to store the IV and + * our own struct for queueing ops. + */ + g_crypto_op_mp = rte_crypto_op_pool_create("op_mp", + RTE_CRYPTO_OP_TYPE_SYMMETRIC, + NUM_MBUFS, + POOL_CACHE_SIZE, + (DEFAULT_NUM_XFORMS * + sizeof(struct rte_crypto_sym_xform)) + + IV_LENGTH + QUEUED_OP_LENGTH, + rte_socket_id()); + + if (g_crypto_op_mp == NULL) { + SPDK_ERRLOG("Cannot create op pool\n"); + rc = -ENOMEM; + goto error_create_op; + } + + /* Init all devices */ + for (i = 0; i < cdev_count; i++) { + rc = create_vbdev_dev(i, num_lcores); + if (rc) { + goto err; + } + } + + /* Assign index values to the QAT device qp nodes so that we can + * assign them for optimal performance. + */ + i = 0; + TAILQ_FOREACH(dev_qp, &g_device_qp_qat, link) { + dev_qp->index = i++; + } + + g_shinfo.free_cb = shinfo_free_cb; + return 0; + + /* Error cleanup paths. */ +err: + TAILQ_FOREACH_SAFE(device, &g_vbdev_devs, link, tmp_dev) { + TAILQ_REMOVE(&g_vbdev_devs, device, link); + release_vbdev_dev(device); + } + rte_mempool_free(g_crypto_op_mp); + g_crypto_op_mp = NULL; +error_create_op: + rte_mempool_free(g_mbuf_mp); + g_mbuf_mp = NULL; +error_create_mbuf: + rte_mempool_free(g_session_mp); + g_session_mp = NULL; +error_create_session_mp: + if (g_session_mp_priv != NULL) { + rte_mempool_free(g_session_mp_priv); + g_session_mp_priv = NULL; + } + return rc; +} + +/* Following an encrypt or decrypt we need to then either write the encrypted data or finish + * the read on decrypted data. Do that here. + */ +static void +_crypto_operation_complete(struct spdk_bdev_io *bdev_io) +{ + struct vbdev_crypto *crypto_bdev = SPDK_CONTAINEROF(bdev_io->bdev, struct vbdev_crypto, + crypto_bdev); + struct crypto_bdev_io *io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; + struct crypto_io_channel *crypto_ch = io_ctx->crypto_ch; + struct spdk_bdev_io *free_me = io_ctx->read_io; + int rc = 0; + + /* Can also be called from the crypto_dev_poller() to fail the stuck re-enqueue ops IO. */ + if (io_ctx->on_pending_list) { + TAILQ_REMOVE(&crypto_ch->pending_cry_ios, bdev_io, module_link); + io_ctx->on_pending_list = false; + } + + if (bdev_io->type == SPDK_BDEV_IO_TYPE_READ) { + + /* Complete the original IO and then free the one that we created + * as a result of issuing an IO via submit_request. + */ + if (io_ctx->bdev_io_status != SPDK_BDEV_IO_STATUS_FAILED) { + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); + } else { + SPDK_ERRLOG("Issue with decryption on bdev_io %p\n", bdev_io); + rc = -EINVAL; + } + spdk_bdev_free_io(free_me); + + } else if (bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE) { + + if (io_ctx->bdev_io_status != SPDK_BDEV_IO_STATUS_FAILED) { + /* Write the encrypted data. */ + rc = spdk_bdev_writev_blocks(crypto_bdev->base_desc, crypto_ch->base_ch, + &io_ctx->aux_buf_iov, 1, io_ctx->aux_offset_blocks, + io_ctx->aux_num_blocks, _complete_internal_write, + bdev_io); + } else { + SPDK_ERRLOG("Issue with encryption on bdev_io %p\n", bdev_io); + rc = -EINVAL; + } + + } else { + SPDK_ERRLOG("Unknown bdev type %u on crypto operation completion\n", + bdev_io->type); + rc = -EINVAL; + } + + if (rc) { + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + } +} + +static void +cancel_queued_crypto_ops(struct crypto_io_channel *crypto_ch, struct spdk_bdev_io *bdev_io) +{ + struct rte_mbuf *mbufs_to_free[2 * MAX_DEQUEUE_BURST_SIZE]; + struct rte_crypto_op *dequeued_ops[MAX_DEQUEUE_BURST_SIZE]; + struct vbdev_crypto_op *op_to_cancel, *tmp_op; + struct rte_crypto_op *crypto_op; + int num_mbufs, num_dequeued_ops; + + /* Remove all ops from the failed IO. Since we don't know the + * order we have to check them all. */ + num_mbufs = 0; + num_dequeued_ops = 0; + TAILQ_FOREACH_SAFE(op_to_cancel, &crypto_ch->queued_cry_ops, link, tmp_op) { + /* Checking if this is our op. One IO contains multiple ops. */ + if (bdev_io == op_to_cancel->bdev_io) { + crypto_op = op_to_cancel->crypto_op; + TAILQ_REMOVE(&crypto_ch->queued_cry_ops, op_to_cancel, link); + + /* Populating lists for freeing mbufs and ops. */ + mbufs_to_free[num_mbufs++] = (void *)crypto_op->sym->m_src; + if (crypto_op->sym->m_dst) { + mbufs_to_free[num_mbufs++] = (void *)crypto_op->sym->m_dst; + } + dequeued_ops[num_dequeued_ops++] = crypto_op; + } + } + + /* Now bulk free both mbufs and crypto operations. */ + if (num_dequeued_ops > 0) { + rte_mempool_put_bulk(g_crypto_op_mp, (void **)dequeued_ops, + num_dequeued_ops); + assert(num_mbufs > 0); + /* This also releases chained mbufs if any. */ + rte_pktmbuf_free_bulk(mbufs_to_free, num_mbufs); + } +} + +static int _crypto_operation(struct spdk_bdev_io *bdev_io, + enum rte_crypto_cipher_operation crypto_op, + void *aux_buf); + +/* This is the poller for the crypto device. It uses a single API to dequeue whatever is ready at + * the device. Then we need to decide if what we've got so far (including previous poller + * runs) totals up to one or more complete bdev_ios and if so continue with the bdev_io + * accordingly. This means either completing a read or issuing a new write. + */ +static int +crypto_dev_poller(void *args) +{ + struct crypto_io_channel *crypto_ch = args; + uint8_t cdev_id = crypto_ch->device_qp->device->cdev_id; + int i, num_dequeued_ops, num_enqueued_ops; + struct spdk_bdev_io *bdev_io = NULL; + struct crypto_bdev_io *io_ctx = NULL; + struct rte_crypto_op *dequeued_ops[MAX_DEQUEUE_BURST_SIZE]; + struct rte_mbuf *mbufs_to_free[2 * MAX_DEQUEUE_BURST_SIZE]; + int num_mbufs = 0; + struct vbdev_crypto_op *op_to_resubmit; + + /* Each run of the poller will get just what the device has available + * at the moment we call it, we don't check again after draining the + * first batch. + */ + num_dequeued_ops = rte_cryptodev_dequeue_burst(cdev_id, crypto_ch->device_qp->qp, + dequeued_ops, MAX_DEQUEUE_BURST_SIZE); + + /* Check if operation was processed successfully */ + for (i = 0; i < num_dequeued_ops; i++) { + + /* We don't know the order or association of the crypto ops wrt any + * particular bdev_io so need to look at each and determine if it's + * the last one for it's bdev_io or not. + */ + bdev_io = (struct spdk_bdev_io *)*RTE_MBUF_DYNFIELD(dequeued_ops[i]->sym->m_src, g_mbuf_offset, + uint64_t *); + assert(bdev_io != NULL); + io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; + + if (dequeued_ops[i]->status != RTE_CRYPTO_OP_STATUS_SUCCESS) { + SPDK_ERRLOG("error with op %d status %u\n", i, + dequeued_ops[i]->status); + /* Update the bdev status to error, we'll still process the + * rest of the crypto ops for this bdev_io though so they + * aren't left hanging. + */ + io_ctx->bdev_io_status = SPDK_BDEV_IO_STATUS_FAILED; + } + + assert(io_ctx->cryop_cnt_remaining > 0); + + /* Return the associated src and dst mbufs by collecting them into + * an array that we can use the bulk API to free after the loop. + */ + *RTE_MBUF_DYNFIELD(dequeued_ops[i]->sym->m_src, g_mbuf_offset, uint64_t *) = 0; + mbufs_to_free[num_mbufs++] = (void *)dequeued_ops[i]->sym->m_src; + if (dequeued_ops[i]->sym->m_dst) { + mbufs_to_free[num_mbufs++] = (void *)dequeued_ops[i]->sym->m_dst; + } + + /* done encrypting, complete the bdev_io */ + if (--io_ctx->cryop_cnt_remaining == 0) { + + /* If we're completing this with an outstanding reset we need + * to fail it. + */ + if (crypto_ch->iter) { + io_ctx->bdev_io_status = SPDK_BDEV_IO_STATUS_FAILED; + } + + /* Complete the IO */ + _crypto_operation_complete(bdev_io); + } + } + + /* Now bulk free both mbufs and crypto operations. */ + if (num_dequeued_ops > 0) { + rte_mempool_put_bulk(g_crypto_op_mp, + (void **)dequeued_ops, + num_dequeued_ops); + assert(num_mbufs > 0); + /* This also releases chained mbufs if any. */ + rte_pktmbuf_free_bulk(mbufs_to_free, num_mbufs); + } + + /* Check if there are any pending crypto ops to process */ + while (!TAILQ_EMPTY(&crypto_ch->queued_cry_ops)) { + op_to_resubmit = TAILQ_FIRST(&crypto_ch->queued_cry_ops); + bdev_io = op_to_resubmit->bdev_io; + io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; + num_enqueued_ops = rte_cryptodev_enqueue_burst(op_to_resubmit->cdev_id, + op_to_resubmit->qp, + &op_to_resubmit->crypto_op, + 1); + if (num_enqueued_ops == 1) { + /* Make sure we don't put this on twice as one bdev_io is made up + * of many crypto ops. + */ + if (io_ctx->on_pending_list == false) { + TAILQ_INSERT_TAIL(&crypto_ch->pending_cry_ios, bdev_io, module_link); + io_ctx->on_pending_list = true; + } + TAILQ_REMOVE(&crypto_ch->queued_cry_ops, op_to_resubmit, link); + } else { + if (op_to_resubmit->crypto_op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) { + /* If we couldn't get one, just break and try again later. */ + break; + } else { + /* Something is really wrong with the op. Most probably the + * mbuf is broken or the HW is not able to process the request. + * Fail the IO and remove its ops from the queued ops list. */ + io_ctx->bdev_io_status = SPDK_BDEV_IO_STATUS_FAILED; + + cancel_queued_crypto_ops(crypto_ch, bdev_io); + + /* Fail the IO if there is nothing left on device. */ + if (--io_ctx->cryop_cnt_remaining == 0) { + _crypto_operation_complete(bdev_io); + } + } + + } + } + + /* If the channel iter is not NULL, we need to continue to poll + * until the pending list is empty, then we can move on to the + * next channel. + */ + if (crypto_ch->iter && TAILQ_EMPTY(&crypto_ch->pending_cry_ios)) { + SPDK_NOTICELOG("Channel %p has been quiesced.\n", crypto_ch); + spdk_for_each_channel_continue(crypto_ch->iter, 0); + crypto_ch->iter = NULL; + } + + return num_dequeued_ops; +} + +/* Allocate the new mbuf of @remainder size with data pointed by @addr and attach + * it to the @orig_mbuf. */ +static int +mbuf_chain_remainder(struct spdk_bdev_io *bdev_io, struct rte_mbuf *orig_mbuf, + uint8_t *addr, uint32_t remainder) +{ + uint64_t phys_addr, phys_len; + struct rte_mbuf *chain_mbuf; + int rc; + + phys_len = remainder; + phys_addr = spdk_vtophys((void *)addr, &phys_len); + if (spdk_unlikely(phys_addr == SPDK_VTOPHYS_ERROR || phys_len != remainder)) { + return -EFAULT; + } + rc = rte_pktmbuf_alloc_bulk(g_mbuf_mp, (struct rte_mbuf **)&chain_mbuf, 1); + if (spdk_unlikely(rc)) { + return -ENOMEM; + } + /* Store context in every mbuf as we don't know anything about completion order */ + *RTE_MBUF_DYNFIELD(chain_mbuf, g_mbuf_offset, uint64_t *) = (uint64_t)bdev_io; + rte_pktmbuf_attach_extbuf(chain_mbuf, addr, phys_addr, phys_len, &g_shinfo); + rte_pktmbuf_append(chain_mbuf, phys_len); + + /* Chained buffer is released by rte_pktbuf_free_bulk() automagicaly. */ + rte_pktmbuf_chain(orig_mbuf, chain_mbuf); + return 0; +} + +/* Attach data buffer pointed by @addr to @mbuf. Return utilized len of the + * contiguous space that was physically available. */ +static uint64_t +mbuf_attach_buf(struct spdk_bdev_io *bdev_io, struct rte_mbuf *mbuf, + uint8_t *addr, uint32_t len) +{ + uint64_t phys_addr, phys_len; + + /* Store context in every mbuf as we don't know anything about completion order */ + *RTE_MBUF_DYNFIELD(mbuf, g_mbuf_offset, uint64_t *) = (uint64_t)bdev_io; + + phys_len = len; + phys_addr = spdk_vtophys((void *)addr, &phys_len); + if (spdk_unlikely(phys_addr == SPDK_VTOPHYS_ERROR || phys_len == 0)) { + return 0; + } + assert(phys_len <= len); + + /* Set the mbuf elements address and length. */ + rte_pktmbuf_attach_extbuf(mbuf, addr, phys_addr, phys_len, &g_shinfo); + rte_pktmbuf_append(mbuf, phys_len); + + return phys_len; +} + +/* We're either encrypting on the way down or decrypting on the way back. */ +static int +_crypto_operation(struct spdk_bdev_io *bdev_io, enum rte_crypto_cipher_operation crypto_op, + void *aux_buf) +{ + uint16_t num_enqueued_ops = 0; + uint32_t cryop_cnt = bdev_io->u.bdev.num_blocks; + struct crypto_bdev_io *io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; + struct crypto_io_channel *crypto_ch = io_ctx->crypto_ch; + uint8_t cdev_id = crypto_ch->device_qp->device->cdev_id; + uint32_t crypto_len = io_ctx->crypto_bdev->crypto_bdev.blocklen; + uint64_t total_length = bdev_io->u.bdev.num_blocks * crypto_len; + int rc; + uint32_t iov_index = 0; + uint32_t allocated = 0; + uint8_t *current_iov = NULL; + uint64_t total_remaining = 0; + uint64_t current_iov_remaining = 0; + uint32_t crypto_index = 0; + uint32_t en_offset = 0; + struct rte_crypto_op *crypto_ops[MAX_ENQUEUE_ARRAY_SIZE]; + struct rte_mbuf *src_mbufs[MAX_ENQUEUE_ARRAY_SIZE]; + struct rte_mbuf *dst_mbufs[MAX_ENQUEUE_ARRAY_SIZE]; + int burst; + struct vbdev_crypto_op *op_to_queue; + uint64_t alignment = spdk_bdev_get_buf_align(&io_ctx->crypto_bdev->crypto_bdev); + + assert((bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen) <= CRYPTO_MAX_IO); + + /* Get the number of source mbufs that we need. These will always be 1:1 because we + * don't support chaining. The reason we don't is because of our decision to use + * LBA as IV, there can be no case where we'd need >1 mbuf per crypto op or the + * op would be > 1 LBA. + */ + rc = rte_pktmbuf_alloc_bulk(g_mbuf_mp, src_mbufs, cryop_cnt); + if (rc) { + SPDK_ERRLOG("Failed to get src_mbufs!\n"); + return -ENOMEM; + } + + /* Get the same amount but these buffers to describe the encrypted data location (dst). */ + if (crypto_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + rc = rte_pktmbuf_alloc_bulk(g_mbuf_mp, dst_mbufs, cryop_cnt); + if (rc) { + SPDK_ERRLOG("Failed to get dst_mbufs!\n"); + rc = -ENOMEM; + goto error_get_dst; + } + } + +#ifdef __clang_analyzer__ + /* silence scan-build false positive */ + SPDK_CLANG_ANALYZER_PREINIT_PTR_ARRAY(crypto_ops, MAX_ENQUEUE_ARRAY_SIZE, 0x1000); +#endif + /* Allocate crypto operations. */ + allocated = rte_crypto_op_bulk_alloc(g_crypto_op_mp, + RTE_CRYPTO_OP_TYPE_SYMMETRIC, + crypto_ops, cryop_cnt); + if (allocated < cryop_cnt) { + SPDK_ERRLOG("Failed to allocate crypto ops!\n"); + rc = -ENOMEM; + goto error_get_ops; + } + + /* For encryption, we need to prepare a single contiguous buffer as the encryption + * destination, we'll then pass that along for the write after encryption is done. + * This is done to avoiding encrypting the provided write buffer which may be + * undesirable in some use cases. + */ + if (crypto_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + io_ctx->aux_buf_iov.iov_len = total_length; + io_ctx->aux_buf_raw = aux_buf; + io_ctx->aux_buf_iov.iov_base = (void *)(((uintptr_t)aux_buf + (alignment - 1)) & ~(alignment - 1)); + io_ctx->aux_offset_blocks = bdev_io->u.bdev.offset_blocks; + io_ctx->aux_num_blocks = bdev_io->u.bdev.num_blocks; + } + + /* This value is used in the completion callback to determine when the bdev_io is + * complete. + */ + io_ctx->cryop_cnt_remaining = cryop_cnt; + + /* As we don't support chaining because of a decision to use LBA as IV, construction + * of crypto operations is straightforward. We build both the op, the mbuf and the + * dst_mbuf in our local arrays by looping through the length of the bdev IO and + * picking off LBA sized blocks of memory from the IOVs as we walk through them. Each + * LBA sized chunk of memory will correspond 1:1 to a crypto operation and a single + * mbuf per crypto operation. + */ + total_remaining = total_length; + current_iov = bdev_io->u.bdev.iovs[iov_index].iov_base; + current_iov_remaining = bdev_io->u.bdev.iovs[iov_index].iov_len; + do { + uint8_t *iv_ptr; + uint8_t *buf_addr; + uint64_t phys_len; + uint32_t remainder; + uint64_t op_block_offset; + + phys_len = mbuf_attach_buf(bdev_io, src_mbufs[crypto_index], + current_iov, crypto_len); + if (spdk_unlikely(phys_len == 0)) { + goto error_attach_session; + rc = -EFAULT; + } + + /* Handle the case of page boundary. */ + remainder = crypto_len - phys_len; + if (spdk_unlikely(remainder > 0)) { + rc = mbuf_chain_remainder(bdev_io, src_mbufs[crypto_index], + current_iov + phys_len, remainder); + if (spdk_unlikely(rc)) { + goto error_attach_session; + } + } + + /* Set the IV - we use the LBA of the crypto_op */ + iv_ptr = rte_crypto_op_ctod_offset(crypto_ops[crypto_index], uint8_t *, + IV_OFFSET); + memset(iv_ptr, 0, IV_LENGTH); + op_block_offset = bdev_io->u.bdev.offset_blocks + crypto_index; + rte_memcpy(iv_ptr, &op_block_offset, sizeof(uint64_t)); + + /* Set the data to encrypt/decrypt length */ + crypto_ops[crypto_index]->sym->cipher.data.length = crypto_len; + crypto_ops[crypto_index]->sym->cipher.data.offset = 0; + + /* link the mbuf to the crypto op. */ + crypto_ops[crypto_index]->sym->m_src = src_mbufs[crypto_index]; + + /* For encrypt, point the destination to a buffer we allocate and redirect the bdev_io + * that will be used to process the write on completion to the same buffer. Setting + * up the en_buffer is a little simpler as we know the destination buffer is single IOV. + */ + if (crypto_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + buf_addr = io_ctx->aux_buf_iov.iov_base + en_offset; + phys_len = mbuf_attach_buf(bdev_io, dst_mbufs[crypto_index], + buf_addr, crypto_len); + if (spdk_unlikely(phys_len == 0)) { + rc = -EFAULT; + goto error_attach_session; + } + + crypto_ops[crypto_index]->sym->m_dst = dst_mbufs[crypto_index]; + en_offset += phys_len; + + /* Handle the case of page boundary. */ + remainder = crypto_len - phys_len; + if (spdk_unlikely(remainder > 0)) { + rc = mbuf_chain_remainder(bdev_io, dst_mbufs[crypto_index], + buf_addr + phys_len, remainder); + if (spdk_unlikely(rc)) { + goto error_attach_session; + } + en_offset += remainder; + } + + /* Attach the crypto session to the operation */ + rc = rte_crypto_op_attach_sym_session(crypto_ops[crypto_index], + io_ctx->crypto_bdev->session_encrypt); + if (rc) { + rc = -EINVAL; + goto error_attach_session; + } + } else { + crypto_ops[crypto_index]->sym->m_dst = NULL; + + /* Attach the crypto session to the operation */ + rc = rte_crypto_op_attach_sym_session(crypto_ops[crypto_index], + io_ctx->crypto_bdev->session_decrypt); + if (rc) { + rc = -EINVAL; + goto error_attach_session; + } + } + + /* Subtract our running totals for the op in progress and the overall bdev io */ + total_remaining -= crypto_len; + current_iov_remaining -= crypto_len; + + /* move our current IOV pointer accordingly. */ + current_iov += crypto_len; + + /* move on to the next crypto operation */ + crypto_index++; + + /* If we're done with this IOV, move to the next one. */ + if (current_iov_remaining == 0 && total_remaining > 0) { + iov_index++; + current_iov = bdev_io->u.bdev.iovs[iov_index].iov_base; + current_iov_remaining = bdev_io->u.bdev.iovs[iov_index].iov_len; + } + } while (total_remaining > 0); + + /* Enqueue everything we've got but limit by the max number of descriptors we + * configured the crypto device for. + */ + burst = spdk_min(cryop_cnt, io_ctx->crypto_bdev->qp_desc_nr); + num_enqueued_ops = rte_cryptodev_enqueue_burst(cdev_id, crypto_ch->device_qp->qp, + &crypto_ops[0], + burst); + + /* Add this bdev_io to our outstanding list if any of its crypto ops made it. */ + if (num_enqueued_ops > 0) { + TAILQ_INSERT_TAIL(&crypto_ch->pending_cry_ios, bdev_io, module_link); + io_ctx->on_pending_list = true; + } + /* We were unable to enqueue everything but did get some, so need to decide what + * to do based on the status of the last op. + */ + if (num_enqueued_ops < cryop_cnt) { + switch (crypto_ops[num_enqueued_ops]->status) { + case RTE_CRYPTO_OP_STATUS_NOT_PROCESSED: + /* Queue them up on a linked list to be resubmitted via the poller. */ + for (crypto_index = num_enqueued_ops; crypto_index < cryop_cnt; crypto_index++) { + op_to_queue = (struct vbdev_crypto_op *)rte_crypto_op_ctod_offset(crypto_ops[crypto_index], + uint8_t *, QUEUED_OP_OFFSET); + op_to_queue->cdev_id = cdev_id; + op_to_queue->qp = crypto_ch->device_qp->qp; + op_to_queue->crypto_op = crypto_ops[crypto_index]; + op_to_queue->bdev_io = bdev_io; + TAILQ_INSERT_TAIL(&crypto_ch->queued_cry_ops, + op_to_queue, + link); + } + break; + default: + /* For all other statuses, set the io_ctx bdev_io status so that + * the poller will pick the failure up for the overall bdev status. + */ + io_ctx->bdev_io_status = SPDK_BDEV_IO_STATUS_FAILED; + if (num_enqueued_ops == 0) { + /* If nothing was enqueued, but the last one wasn't because of + * busy, fail it now as the poller won't know anything about it. + */ + rc = -EINVAL; + goto error_attach_session; + } + break; + } + } + + return rc; + + /* Error cleanup paths. */ +error_attach_session: +error_get_ops: + if (crypto_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + /* This also releases chained mbufs if any. */ + rte_pktmbuf_free_bulk(dst_mbufs, cryop_cnt); + } + if (allocated > 0) { + rte_mempool_put_bulk(g_crypto_op_mp, (void **)crypto_ops, + allocated); + } +error_get_dst: + /* This also releases chained mbufs if any. */ + rte_pktmbuf_free_bulk(src_mbufs, cryop_cnt); + return rc; +} + +/* This function is called after all channels have been quiesced following + * a bdev reset. + */ +static void +_ch_quiesce_done(struct spdk_io_channel_iter *i, int status) +{ + struct crypto_bdev_io *io_ctx = spdk_io_channel_iter_get_ctx(i); + + assert(TAILQ_EMPTY(&io_ctx->crypto_ch->pending_cry_ios)); + assert(io_ctx->orig_io != NULL); + + spdk_bdev_io_complete(io_ctx->orig_io, SPDK_BDEV_IO_STATUS_SUCCESS); +} + +/* This function is called per channel to quiesce IOs before completing a + * bdev reset that we received. + */ +static void +_ch_quiesce(struct spdk_io_channel_iter *i) +{ + struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); + struct crypto_io_channel *crypto_ch = spdk_io_channel_get_ctx(ch); + + crypto_ch->iter = i; + /* When the poller runs, it will see the non-NULL iter and handle + * the quiesce. + */ +} + +/* Completion callback for IO that were issued from this bdev other than read/write. + * They have their own for readability. + */ +static void +_complete_internal_io(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct spdk_bdev_io *orig_io = cb_arg; + int status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; + + if (bdev_io->type == SPDK_BDEV_IO_TYPE_RESET) { + struct crypto_bdev_io *orig_ctx = (struct crypto_bdev_io *)orig_io->driver_ctx; + + assert(orig_io == orig_ctx->orig_io); + + spdk_bdev_free_io(bdev_io); + + spdk_for_each_channel(orig_ctx->crypto_bdev, + _ch_quiesce, + orig_ctx, + _ch_quiesce_done); + return; + } + + spdk_bdev_io_complete(orig_io, status); + spdk_bdev_free_io(bdev_io); +} + +/* Completion callback for writes that were issued from this bdev. */ +static void +_complete_internal_write(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct spdk_bdev_io *orig_io = cb_arg; + int status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; + struct crypto_bdev_io *orig_ctx = (struct crypto_bdev_io *)orig_io->driver_ctx; + + spdk_bdev_io_put_aux_buf(orig_io, orig_ctx->aux_buf_raw); + + spdk_bdev_io_complete(orig_io, status); + spdk_bdev_free_io(bdev_io); +} + +/* Completion callback for reads that were issued from this bdev. */ +static void +_complete_internal_read(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct spdk_bdev_io *orig_io = cb_arg; + struct crypto_bdev_io *orig_ctx = (struct crypto_bdev_io *)orig_io->driver_ctx; + + if (success) { + + /* Save off this bdev_io so it can be freed after decryption. */ + orig_ctx->read_io = bdev_io; + + if (!_crypto_operation(orig_io, RTE_CRYPTO_CIPHER_OP_DECRYPT, NULL)) { + return; + } else { + SPDK_ERRLOG("Failed to decrypt!\n"); + } + } else { + SPDK_ERRLOG("Failed to read prior to decrypting!\n"); + } + + spdk_bdev_io_complete(orig_io, SPDK_BDEV_IO_STATUS_FAILED); + spdk_bdev_free_io(bdev_io); +} + +static void +vbdev_crypto_resubmit_io(void *arg) +{ + struct spdk_bdev_io *bdev_io = (struct spdk_bdev_io *)arg; + struct crypto_bdev_io *io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; + + vbdev_crypto_submit_request(io_ctx->ch, bdev_io); +} + +static void +vbdev_crypto_queue_io(struct spdk_bdev_io *bdev_io) +{ + struct crypto_bdev_io *io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; + int rc; + + io_ctx->bdev_io_wait.bdev = bdev_io->bdev; + io_ctx->bdev_io_wait.cb_fn = vbdev_crypto_resubmit_io; + io_ctx->bdev_io_wait.cb_arg = bdev_io; + + rc = spdk_bdev_queue_io_wait(bdev_io->bdev, io_ctx->crypto_ch->base_ch, &io_ctx->bdev_io_wait); + if (rc != 0) { + SPDK_ERRLOG("Queue io failed in vbdev_crypto_queue_io, rc=%d.\n", rc); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + } +} + +/* Callback for getting a buf from the bdev pool in the event that the caller passed + * in NULL, we need to own the buffer so it doesn't get freed by another vbdev module + * beneath us before we're done with it. + */ +static void +crypto_read_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, + bool success) +{ + struct vbdev_crypto *crypto_bdev = SPDK_CONTAINEROF(bdev_io->bdev, struct vbdev_crypto, + crypto_bdev); + struct crypto_io_channel *crypto_ch = spdk_io_channel_get_ctx(ch); + struct crypto_bdev_io *io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; + int rc; + + if (!success) { + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + return; + } + + rc = spdk_bdev_readv_blocks(crypto_bdev->base_desc, crypto_ch->base_ch, bdev_io->u.bdev.iovs, + bdev_io->u.bdev.iovcnt, bdev_io->u.bdev.offset_blocks, + bdev_io->u.bdev.num_blocks, _complete_internal_read, + bdev_io); + if (rc != 0) { + if (rc == -ENOMEM) { + SPDK_DEBUGLOG(vbdev_crypto, "No memory, queue the IO.\n"); + io_ctx->ch = ch; + vbdev_crypto_queue_io(bdev_io); + } else { + SPDK_ERRLOG("Failed to submit bdev_io!\n"); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + } + } +} + +/* For encryption we don't want to encrypt the data in place as the host isn't + * expecting us to mangle its data buffers so we need to encrypt into the bdev + * aux buffer, then we can use that as the source for the disk data transfer. + */ +static void +crypto_write_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, + void *aux_buf) +{ + struct crypto_bdev_io *io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; + int rc = 0; + + rc = _crypto_operation(bdev_io, RTE_CRYPTO_CIPHER_OP_ENCRYPT, aux_buf); + if (rc != 0) { + spdk_bdev_io_put_aux_buf(bdev_io, aux_buf); + if (rc == -ENOMEM) { + SPDK_DEBUGLOG(vbdev_crypto, "No memory, queue the IO.\n"); + io_ctx->ch = ch; + vbdev_crypto_queue_io(bdev_io); + } else { + SPDK_ERRLOG("Failed to submit bdev_io!\n"); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + } + } +} + +/* Called when someone submits IO to this crypto vbdev. For IO's not relevant to crypto, + * we're simply passing it on here via SPDK IO calls which in turn allocate another bdev IO + * and call our cpl callback provided below along with the original bdev_io so that we can + * complete it once this IO completes. For crypto operations, we'll either encrypt it first + * (writes) then call back into bdev to submit it or we'll submit a read and then catch it + * on the way back for decryption. + */ +static void +vbdev_crypto_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) +{ + struct vbdev_crypto *crypto_bdev = SPDK_CONTAINEROF(bdev_io->bdev, struct vbdev_crypto, + crypto_bdev); + struct crypto_io_channel *crypto_ch = spdk_io_channel_get_ctx(ch); + struct crypto_bdev_io *io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; + int rc = 0; + + memset(io_ctx, 0, sizeof(struct crypto_bdev_io)); + io_ctx->crypto_bdev = crypto_bdev; + io_ctx->crypto_ch = crypto_ch; + io_ctx->orig_io = bdev_io; + io_ctx->bdev_io_status = SPDK_BDEV_IO_STATUS_SUCCESS; + + switch (bdev_io->type) { + case SPDK_BDEV_IO_TYPE_READ: + spdk_bdev_io_get_buf(bdev_io, crypto_read_get_buf_cb, + bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); + break; + case SPDK_BDEV_IO_TYPE_WRITE: + /* Tell the bdev layer that we need an aux buf in addition to the data + * buf already associated with the bdev. + */ + spdk_bdev_io_get_aux_buf(bdev_io, crypto_write_get_buf_cb); + break; + case SPDK_BDEV_IO_TYPE_UNMAP: + rc = spdk_bdev_unmap_blocks(crypto_bdev->base_desc, crypto_ch->base_ch, + bdev_io->u.bdev.offset_blocks, + bdev_io->u.bdev.num_blocks, + _complete_internal_io, bdev_io); + break; + case SPDK_BDEV_IO_TYPE_FLUSH: + rc = spdk_bdev_flush_blocks(crypto_bdev->base_desc, crypto_ch->base_ch, + bdev_io->u.bdev.offset_blocks, + bdev_io->u.bdev.num_blocks, + _complete_internal_io, bdev_io); + break; + case SPDK_BDEV_IO_TYPE_RESET: + rc = spdk_bdev_reset(crypto_bdev->base_desc, crypto_ch->base_ch, + _complete_internal_io, bdev_io); + break; + case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: + default: + SPDK_ERRLOG("crypto: unknown I/O type %d\n", bdev_io->type); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + return; + } + + if (rc != 0) { + if (rc == -ENOMEM) { + SPDK_DEBUGLOG(vbdev_crypto, "No memory, queue the IO.\n"); + io_ctx->ch = ch; + vbdev_crypto_queue_io(bdev_io); + } else { + SPDK_ERRLOG("Failed to submit bdev_io!\n"); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + } + } +} + +/* We'll just call the base bdev and let it answer except for WZ command which + * we always say we don't support so that the bdev layer will actually send us + * real writes that we can encrypt. + */ +static bool +vbdev_crypto_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) +{ + struct vbdev_crypto *crypto_bdev = (struct vbdev_crypto *)ctx; + + switch (io_type) { + case SPDK_BDEV_IO_TYPE_WRITE: + case SPDK_BDEV_IO_TYPE_UNMAP: + case SPDK_BDEV_IO_TYPE_RESET: + case SPDK_BDEV_IO_TYPE_READ: + case SPDK_BDEV_IO_TYPE_FLUSH: + return spdk_bdev_io_type_supported(crypto_bdev->base_bdev, io_type); + case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: + /* Force the bdev layer to issue actual writes of zeroes so we can + * encrypt them as regular writes. + */ + default: + return false; + } +} + +static struct vbdev_dev * +_vdev_dev_get(struct vbdev_crypto *vbdev) +{ + struct vbdev_dev *device; + + TAILQ_FOREACH(device, &g_vbdev_devs, link) { + if (strcmp(device->cdev_info.driver_name, vbdev->opts->drv_name) == 0) { + return device; + } + } + return NULL; +} + +static void +_cryptodev_sym_session_free(struct vbdev_crypto *vbdev, void *session) +{ +#if RTE_VERSION >= RTE_VERSION_NUM(22, 11, 0, 0) + struct vbdev_dev *device = _vdev_dev_get(vbdev); + + assert(device != NULL); + + rte_cryptodev_sym_session_free(device->cdev_id, session); +#else + rte_cryptodev_sym_session_free(session); +#endif +} + +static void * +_cryptodev_sym_session_create(struct vbdev_crypto *vbdev, struct rte_crypto_sym_xform *xforms) +{ + void *session; + struct vbdev_dev *device; + + device = _vdev_dev_get(vbdev); + if (!device) { + SPDK_ERRLOG("Failed to match crypto device driver to crypto vbdev.\n"); + return NULL; + } + +#if RTE_VERSION >= RTE_VERSION_NUM(22, 11, 0, 0) + session = rte_cryptodev_sym_session_create(device->cdev_id, xforms, g_session_mp); +#else + session = rte_cryptodev_sym_session_create(g_session_mp); + if (!session) { + return NULL; + } + + if (rte_cryptodev_sym_session_init(device->cdev_id, session, xforms, g_session_mp_priv) < 0) { + _cryptodev_sym_session_free(vbdev, session); + return NULL; + } +#endif + + return session; +} + +/* Callback for unregistering the IO device. */ +static void +_device_unregister_cb(void *io_device) +{ + struct vbdev_crypto *crypto_bdev = io_device; + + /* Done with this crypto_bdev. */ + _cryptodev_sym_session_free(crypto_bdev, crypto_bdev->session_decrypt); + _cryptodev_sym_session_free(crypto_bdev, crypto_bdev->session_encrypt); + crypto_bdev->opts = NULL; + free(crypto_bdev->crypto_bdev.name); + free(crypto_bdev); +} + +/* Wrapper for the bdev close operation. */ +static void +_vbdev_crypto_destruct(void *ctx) +{ + struct spdk_bdev_desc *desc = ctx; + + spdk_bdev_close(desc); +} + +/* Called after we've unregistered following a hot remove callback. + * Our finish entry point will be called next. + */ +static int +vbdev_crypto_destruct(void *ctx) +{ + struct vbdev_crypto *crypto_bdev = (struct vbdev_crypto *)ctx; + + /* Remove this device from the internal list */ + TAILQ_REMOVE(&g_vbdev_crypto, crypto_bdev, link); + + /* Unclaim the underlying bdev. */ + spdk_bdev_module_release_bdev(crypto_bdev->base_bdev); + + /* Close the underlying bdev on its same opened thread. */ + if (crypto_bdev->thread && crypto_bdev->thread != spdk_get_thread()) { + spdk_thread_send_msg(crypto_bdev->thread, _vbdev_crypto_destruct, crypto_bdev->base_desc); + } else { + spdk_bdev_close(crypto_bdev->base_desc); + } + + /* Unregister the io_device. */ + spdk_io_device_unregister(crypto_bdev, _device_unregister_cb); + + g_number_of_claimed_volumes--; + + return 0; +} + +/* We supplied this as an entry point for upper layers who want to communicate to this + * bdev. This is how they get a channel. We are passed the same context we provided when + * we created our crypto vbdev in examine() which, for this bdev, is the address of one of + * our context nodes. From here we'll ask the SPDK channel code to fill out our channel + * struct and we'll keep it in our crypto node. + */ +static struct spdk_io_channel * +vbdev_crypto_get_io_channel(void *ctx) +{ + struct vbdev_crypto *crypto_bdev = (struct vbdev_crypto *)ctx; + + /* The IO channel code will allocate a channel for us which consists of + * the SPDK channel structure plus the size of our crypto_io_channel struct + * that we passed in when we registered our IO device. It will then call + * our channel create callback to populate any elements that we need to + * update. + */ + return spdk_get_io_channel(crypto_bdev); +} + +/* This is the output for bdev_get_bdevs() for this vbdev */ +static int +vbdev_crypto_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) +{ + struct vbdev_crypto *crypto_bdev = (struct vbdev_crypto *)ctx; + char *hexkey = NULL, *hexkey2 = NULL; + int rc = 0; + + hexkey = spdk_hexlify(crypto_bdev->opts->key, + crypto_bdev->opts->key_size); + if (!hexkey) { + return -ENOMEM; + } + + if (crypto_bdev->opts->key2) { + hexkey2 = spdk_hexlify(crypto_bdev->opts->key2, + crypto_bdev->opts->key2_size); + if (!hexkey2) { + rc = -ENOMEM; + goto out_err; + } + } + + spdk_json_write_name(w, "crypto"); + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "base_bdev_name", spdk_bdev_get_name(crypto_bdev->base_bdev)); + spdk_json_write_named_string(w, "name", spdk_bdev_get_name(&crypto_bdev->crypto_bdev)); + spdk_json_write_named_string(w, "crypto_pmd", crypto_bdev->opts->drv_name); + spdk_json_write_named_string(w, "key", hexkey); + if (hexkey2) { + spdk_json_write_named_string(w, "key2", hexkey2); + } + spdk_json_write_named_string(w, "cipher", crypto_bdev->opts->cipher); + spdk_json_write_object_end(w); +out_err: + if (hexkey) { + memset(hexkey, 0, strlen(hexkey)); + free(hexkey); + } + if (hexkey2) { + memset(hexkey2, 0, strlen(hexkey2)); + free(hexkey2); + } + return rc; +} + +static int +vbdev_crypto_config_json(struct spdk_json_write_ctx *w) +{ + struct vbdev_crypto *crypto_bdev; + + TAILQ_FOREACH(crypto_bdev, &g_vbdev_crypto, link) { + char *hexkey = NULL, *hexkey2 = NULL; + + hexkey = spdk_hexlify(crypto_bdev->opts->key, + crypto_bdev->opts->key_size); + if (!hexkey) { + return -ENOMEM; + } + + if (crypto_bdev->opts->key2) { + hexkey2 = spdk_hexlify(crypto_bdev->opts->key2, + crypto_bdev->opts->key2_size); + if (!hexkey2) { + memset(hexkey, 0, strlen(hexkey)); + free(hexkey); + return -ENOMEM; + } + } + + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "method", "bdev_crypto_create"); + spdk_json_write_named_object_begin(w, "params"); + spdk_json_write_named_string(w, "base_bdev_name", spdk_bdev_get_name(crypto_bdev->base_bdev)); + spdk_json_write_named_string(w, "name", spdk_bdev_get_name(&crypto_bdev->crypto_bdev)); + spdk_json_write_named_string(w, "crypto_pmd", crypto_bdev->opts->drv_name); + spdk_json_write_named_string(w, "key", hexkey); + if (hexkey2) { + spdk_json_write_named_string(w, "key2", hexkey2); + } + spdk_json_write_named_string(w, "cipher", crypto_bdev->opts->cipher); + spdk_json_write_object_end(w); + spdk_json_write_object_end(w); + + if (hexkey) { + memset(hexkey, 0, strlen(hexkey)); + free(hexkey); + } + if (hexkey2) { + memset(hexkey2, 0, strlen(hexkey2)); + free(hexkey2); + } + } + return 0; +} + +/* Helper function for the channel creation callback. */ +static void +_assign_device_qp(struct vbdev_crypto *crypto_bdev, struct device_qp *device_qp, + struct crypto_io_channel *crypto_ch) +{ + pthread_mutex_lock(&g_device_qp_lock); + if (strcmp(crypto_bdev->opts->drv_name, QAT) == 0) { + /* For some QAT devices, the optimal qp to use is every 32nd as this spreads the + * workload out over the multiple virtual functions in the device. For the devices + * where this isn't the case, it doesn't hurt. + */ + TAILQ_FOREACH(device_qp, &g_device_qp_qat, link) { + if (device_qp->index != g_next_qat_index) { + continue; + } + if (device_qp->in_use == false) { + crypto_ch->device_qp = device_qp; + device_qp->in_use = true; + g_next_qat_index = (g_next_qat_index + QAT_VF_SPREAD) % g_qat_total_qp; + break; + } else { + /* if the preferred index is used, skip to the next one in this set. */ + g_next_qat_index = (g_next_qat_index + 1) % g_qat_total_qp; + } + } + } else if (strcmp(crypto_bdev->opts->drv_name, AESNI_MB) == 0) { + TAILQ_FOREACH(device_qp, &g_device_qp_aesni_mb, link) { + if (device_qp->in_use == false) { + crypto_ch->device_qp = device_qp; + device_qp->in_use = true; + break; + } + } + } else if (strcmp(crypto_bdev->opts->drv_name, MLX5) == 0) { + TAILQ_FOREACH(device_qp, &g_device_qp_mlx5, link) { + if (device_qp->in_use == false) { + crypto_ch->device_qp = device_qp; + device_qp->in_use = true; + break; + } + } + } + pthread_mutex_unlock(&g_device_qp_lock); +} + +/* We provide this callback for the SPDK channel code to create a channel using + * the channel struct we provided in our module get_io_channel() entry point. Here + * we get and save off an underlying base channel of the device below us so that + * we can communicate with the base bdev on a per channel basis. We also register the + * poller used to complete crypto operations from the device. + */ +static int +crypto_bdev_ch_create_cb(void *io_device, void *ctx_buf) +{ + struct crypto_io_channel *crypto_ch = ctx_buf; + struct vbdev_crypto *crypto_bdev = io_device; + struct device_qp *device_qp = NULL; + + crypto_ch->base_ch = spdk_bdev_get_io_channel(crypto_bdev->base_desc); + crypto_ch->poller = SPDK_POLLER_REGISTER(crypto_dev_poller, crypto_ch, 0); + crypto_ch->device_qp = NULL; + + /* Assign a device/qp combination that is unique per channel per PMD. */ + _assign_device_qp(crypto_bdev, device_qp, crypto_ch); + assert(crypto_ch->device_qp); + + /* We use this queue to track outstanding IO in our layer. */ + TAILQ_INIT(&crypto_ch->pending_cry_ios); + + /* We use this to queue up crypto ops when the device is busy. */ + TAILQ_INIT(&crypto_ch->queued_cry_ops); + + return 0; +} + +/* We provide this callback for the SPDK channel code to destroy a channel + * created with our create callback. We just need to undo anything we did + * when we created. + */ +static void +crypto_bdev_ch_destroy_cb(void *io_device, void *ctx_buf) +{ + struct crypto_io_channel *crypto_ch = ctx_buf; + + pthread_mutex_lock(&g_device_qp_lock); + crypto_ch->device_qp->in_use = false; + pthread_mutex_unlock(&g_device_qp_lock); + + spdk_poller_unregister(&crypto_ch->poller); + spdk_put_io_channel(crypto_ch->base_ch); +} + +/* Create the association from the bdev and vbdev name and insert + * on the global list. */ +static int +vbdev_crypto_insert_name(struct vbdev_crypto_opts *opts, struct bdev_names **out) +{ + struct bdev_names *name; + bool found = false; + int j; + + assert(opts); + assert(out); + + TAILQ_FOREACH(name, &g_bdev_names, link) { + if (strcmp(opts->vbdev_name, name->opts->vbdev_name) == 0) { + SPDK_ERRLOG("Crypto bdev %s already exists\n", opts->vbdev_name); + return -EEXIST; + } + } + + for (j = 0; j < MAX_NUM_DRV_TYPES ; j++) { + if (strcmp(opts->drv_name, g_driver_names[j]) == 0) { + found = true; + break; + } + } + if (!found) { + SPDK_ERRLOG("Crypto PMD type %s is not supported.\n", opts->drv_name); + return -EINVAL; + } + + name = calloc(1, sizeof(struct bdev_names)); + if (!name) { + SPDK_ERRLOG("Failed to allocate memory for bdev_names.\n"); + return -ENOMEM; + } + + name->opts = opts; + TAILQ_INSERT_TAIL(&g_bdev_names, name, link); + *out = name; + + return 0; +} + +void +free_crypto_opts(struct vbdev_crypto_opts *opts) +{ + free(opts->bdev_name); + free(opts->vbdev_name); + free(opts->drv_name); + if (opts->xts_key) { + memset(opts->xts_key, 0, + opts->key_size + opts->key2_size); + free(opts->xts_key); + } + memset(opts->key, 0, opts->key_size); + free(opts->key); + opts->key_size = 0; + if (opts->key2) { + memset(opts->key2, 0, opts->key2_size); + free(opts->key2); + } + opts->key2_size = 0; + free(opts); +} + +static void +vbdev_crypto_delete_name(struct bdev_names *name) +{ + TAILQ_REMOVE(&g_bdev_names, name, link); + if (name->opts) { + free_crypto_opts(name->opts); + name->opts = NULL; + } + free(name); +} + +/* RPC entry point for crypto creation. */ +int +create_crypto_disk(struct vbdev_crypto_opts *opts) +{ + struct bdev_names *name = NULL; + int rc; + + rc = vbdev_crypto_insert_name(opts, &name); + if (rc) { + return rc; + } + + rc = vbdev_crypto_claim(opts->bdev_name); + if (rc == -ENODEV) { + SPDK_NOTICELOG("vbdev creation deferred pending base bdev arrival\n"); + rc = 0; + } + + if (rc) { + assert(name != NULL); + /* In case of error we let the caller function to deallocate @opts + * since it is its responsibiltiy. Setting name->opts = NULL let's + * vbdev_crypto_delete_name() know it does not have to do anything + * about @opts. + */ + name->opts = NULL; + vbdev_crypto_delete_name(name); + } + return rc; +} + +/* Called at driver init time, parses config file to prepare for examine calls, + * also fully initializes the crypto drivers. + */ +static int +vbdev_crypto_init(void) +{ + int rc = 0; + + /* Fully configure both SW and HW drivers. */ + rc = vbdev_crypto_init_crypto_drivers(); + if (rc) { + SPDK_ERRLOG("Error setting up crypto devices\n"); + } + + return rc; +} + +/* Called when the entire module is being torn down. */ +static void +vbdev_crypto_finish(void) +{ + struct bdev_names *name; + struct vbdev_dev *device; + + while ((name = TAILQ_FIRST(&g_bdev_names))) { + vbdev_crypto_delete_name(name); + } + + while ((device = TAILQ_FIRST(&g_vbdev_devs))) { + TAILQ_REMOVE(&g_vbdev_devs, device, link); + release_vbdev_dev(device); + } + rte_vdev_uninit(AESNI_MB); + + /* These are removed in release_vbdev_dev() */ + assert(TAILQ_EMPTY(&g_device_qp_qat)); + assert(TAILQ_EMPTY(&g_device_qp_aesni_mb)); + assert(TAILQ_EMPTY(&g_device_qp_mlx5)); + + rte_mempool_free(g_crypto_op_mp); + rte_mempool_free(g_mbuf_mp); + rte_mempool_free(g_session_mp); + if (g_session_mp_priv != NULL) { + rte_mempool_free(g_session_mp_priv); + } +} + +/* During init we'll be asked how much memory we'd like passed to us + * in bev_io structures as context. Here's where we specify how + * much context we want per IO. + */ +static int +vbdev_crypto_get_ctx_size(void) +{ + return sizeof(struct crypto_bdev_io); +} + +static void +vbdev_crypto_base_bdev_hotremove_cb(struct spdk_bdev *bdev_find) +{ + struct vbdev_crypto *crypto_bdev, *tmp; + + TAILQ_FOREACH_SAFE(crypto_bdev, &g_vbdev_crypto, link, tmp) { + if (bdev_find == crypto_bdev->base_bdev) { + spdk_bdev_unregister(&crypto_bdev->crypto_bdev, NULL, NULL); + } + } +} + +/* Called when the underlying base bdev triggers asynchronous event such as bdev removal. */ +static void +vbdev_crypto_base_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, + void *event_ctx) +{ + switch (type) { + case SPDK_BDEV_EVENT_REMOVE: + vbdev_crypto_base_bdev_hotremove_cb(bdev); + break; + default: + SPDK_NOTICELOG("Unsupported bdev event: type %d\n", type); + break; + } +} + +static void +vbdev_crypto_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) +{ + /* No config per bdev needed */ +} + +/* When we register our bdev this is how we specify our entry points. */ +static const struct spdk_bdev_fn_table vbdev_crypto_fn_table = { + .destruct = vbdev_crypto_destruct, + .submit_request = vbdev_crypto_submit_request, + .io_type_supported = vbdev_crypto_io_type_supported, + .get_io_channel = vbdev_crypto_get_io_channel, + .dump_info_json = vbdev_crypto_dump_info_json, + .write_config_json = vbdev_crypto_write_config_json +}; + +static struct spdk_bdev_module crypto_if = { + .name = "crypto", + .module_init = vbdev_crypto_init, + .get_ctx_size = vbdev_crypto_get_ctx_size, + .examine_config = vbdev_crypto_examine, + .module_fini = vbdev_crypto_finish, + .config_json = vbdev_crypto_config_json +}; + +SPDK_BDEV_MODULE_REGISTER(crypto, &crypto_if) + +static int +vbdev_crypto_claim(const char *bdev_name) +{ + struct bdev_names *name; + struct vbdev_crypto *vbdev; + struct spdk_bdev *bdev; + uint8_t key_size; + int rc = 0; + + if (g_number_of_claimed_volumes >= MAX_CRYPTO_VOLUMES) { + SPDK_DEBUGLOG(vbdev_crypto, "Reached max number of claimed volumes\n"); + return -EINVAL; + } + g_number_of_claimed_volumes++; + + /* Check our list of names from config versus this bdev and if + * there's a match, create the crypto_bdev & bdev accordingly. + */ + TAILQ_FOREACH(name, &g_bdev_names, link) { + if (strcmp(name->opts->bdev_name, bdev_name) != 0) { + continue; + } + SPDK_DEBUGLOG(vbdev_crypto, "Match on %s\n", bdev_name); + + vbdev = calloc(1, sizeof(struct vbdev_crypto)); + if (!vbdev) { + SPDK_ERRLOG("Failed to allocate memory for crypto_bdev.\n"); + rc = -ENOMEM; + goto error_vbdev_alloc; + } + vbdev->crypto_bdev.product_name = "crypto"; + + vbdev->crypto_bdev.name = strdup(name->opts->vbdev_name); + if (!vbdev->crypto_bdev.name) { + SPDK_ERRLOG("Failed to allocate memory for crypto_bdev name.\n"); + rc = -ENOMEM; + goto error_bdev_name; + } + + rc = spdk_bdev_open_ext(bdev_name, true, vbdev_crypto_base_bdev_event_cb, + NULL, &vbdev->base_desc); + if (rc) { + if (rc != -ENODEV) { + SPDK_ERRLOG("Failed to open bdev %s: error %d\n", bdev_name, rc); + } + goto error_open; + } + + bdev = spdk_bdev_desc_get_bdev(vbdev->base_desc); + vbdev->base_bdev = bdev; + + if (strcmp(name->opts->drv_name, MLX5) == 0) { + vbdev->qp_desc_nr = CRYPTO_QP_DESCRIPTORS_MLX5; + } else { + vbdev->qp_desc_nr = CRYPTO_QP_DESCRIPTORS; + } + + vbdev->crypto_bdev.write_cache = bdev->write_cache; + if (strcmp(name->opts->drv_name, QAT) == 0) { + vbdev->crypto_bdev.required_alignment = + spdk_max(spdk_u32log2(bdev->blocklen), bdev->required_alignment); + SPDK_NOTICELOG("QAT in use: Required alignment set to %u\n", + vbdev->crypto_bdev.required_alignment); + SPDK_NOTICELOG("QAT using cipher: %s\n", name->opts->cipher); + } else if (strcmp(name->opts->drv_name, MLX5) == 0) { + vbdev->crypto_bdev.required_alignment = bdev->required_alignment; + SPDK_NOTICELOG("MLX5 using cipher: %s\n", name->opts->cipher); + } else { + vbdev->crypto_bdev.required_alignment = bdev->required_alignment; + SPDK_NOTICELOG("AESNI_MB using cipher: %s\n", name->opts->cipher); + } + vbdev->cipher_xform.cipher.iv.length = IV_LENGTH; + + /* Note: CRYPTO_MAX_IO is in units of bytes, optimal_io_boundary is + * in units of blocks. + */ + if (bdev->optimal_io_boundary > 0) { + vbdev->crypto_bdev.optimal_io_boundary = + spdk_min((CRYPTO_MAX_IO / bdev->blocklen), bdev->optimal_io_boundary); + } else { + vbdev->crypto_bdev.optimal_io_boundary = (CRYPTO_MAX_IO / bdev->blocklen); + } + vbdev->crypto_bdev.split_on_optimal_io_boundary = true; + vbdev->crypto_bdev.blocklen = bdev->blocklen; + vbdev->crypto_bdev.blockcnt = bdev->blockcnt; + + /* This is the context that is passed to us when the bdev + * layer calls in so we'll save our crypto_bdev node here. + */ + vbdev->crypto_bdev.ctxt = vbdev; + vbdev->crypto_bdev.fn_table = &vbdev_crypto_fn_table; + vbdev->crypto_bdev.module = &crypto_if; + + /* Assign crypto opts from the name. The pointer is valid up to the point + * the module is unloaded and all names removed from the list. */ + vbdev->opts = name->opts; + + TAILQ_INSERT_TAIL(&g_vbdev_crypto, vbdev, link); + + spdk_io_device_register(vbdev, crypto_bdev_ch_create_cb, crypto_bdev_ch_destroy_cb, + sizeof(struct crypto_io_channel), vbdev->crypto_bdev.name); + + /* Save the thread where the base device is opened */ + vbdev->thread = spdk_get_thread(); + + rc = spdk_bdev_module_claim_bdev(bdev, vbdev->base_desc, vbdev->crypto_bdev.module); + if (rc) { + SPDK_ERRLOG("Failed to claim bdev %s\n", spdk_bdev_get_name(bdev)); + goto error_claim; + } + + /* Init our per vbdev xform with the desired cipher options. */ + vbdev->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + vbdev->cipher_xform.cipher.iv.offset = IV_OFFSET; + if (strcmp(vbdev->opts->cipher, AES_CBC) == 0) { + vbdev->cipher_xform.cipher.key.data = vbdev->opts->key; + vbdev->cipher_xform.cipher.key.length = vbdev->opts->key_size; + vbdev->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC; + } else if (strcmp(vbdev->opts->cipher, AES_XTS) == 0) { + key_size = vbdev->opts->key_size + vbdev->opts->key2_size; + vbdev->cipher_xform.cipher.key.data = vbdev->opts->xts_key; + vbdev->cipher_xform.cipher.key.length = key_size; + vbdev->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_XTS; + } else { + SPDK_ERRLOG("Invalid cipher name %s.\n", vbdev->opts->cipher); + rc = -EINVAL; + goto error_session_de_create; + } + vbdev->cipher_xform.cipher.iv.length = IV_LENGTH; + + vbdev->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; + vbdev->session_encrypt = _cryptodev_sym_session_create(vbdev, &vbdev->cipher_xform); + if (NULL == vbdev->session_encrypt) { + SPDK_ERRLOG("Failed to create encrypt crypto session.\n"); + rc = -EINVAL; + goto error_session_en_create; + } + + vbdev->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT; + vbdev->session_decrypt = _cryptodev_sym_session_create(vbdev, &vbdev->cipher_xform); + if (NULL == vbdev->session_decrypt) { + SPDK_ERRLOG("Failed to create decrypt crypto session.\n"); + rc = -EINVAL; + goto error_session_de_create; + } + + rc = spdk_bdev_register(&vbdev->crypto_bdev); + if (rc < 0) { + SPDK_ERRLOG("Failed to register vbdev: error %d\n", rc); + rc = -EINVAL; + goto error_bdev_register; + } + SPDK_DEBUGLOG(vbdev_crypto, "Registered io_device and virtual bdev for: %s\n", + vbdev->opts->vbdev_name); + break; + } + + return rc; + + /* Error cleanup paths. */ +error_bdev_register: + _cryptodev_sym_session_free(vbdev, vbdev->session_decrypt); +error_session_de_create: + _cryptodev_sym_session_free(vbdev, vbdev->session_encrypt); +error_session_en_create: + spdk_bdev_module_release_bdev(vbdev->base_bdev); +error_claim: + TAILQ_REMOVE(&g_vbdev_crypto, vbdev, link); + spdk_io_device_unregister(vbdev, NULL); + spdk_bdev_close(vbdev->base_desc); +error_open: + free(vbdev->crypto_bdev.name); +error_bdev_name: + free(vbdev); +error_vbdev_alloc: + g_number_of_claimed_volumes--; + return rc; +} + +/* RPC entry for deleting a crypto vbdev. */ +void +delete_crypto_disk(const char *bdev_name, spdk_delete_crypto_complete cb_fn, + void *cb_arg) +{ + struct bdev_names *name; + int rc; + + /* Some cleanup happens in the destruct callback. */ + rc = spdk_bdev_unregister_by_name(bdev_name, &crypto_if, cb_fn, cb_arg); + if (rc == 0) { + /* Remove the association (vbdev, bdev) from g_bdev_names. This is required so that the + * vbdev does not get re-created if the same bdev is constructed at some other time, + * unless the underlying bdev was hot-removed. + */ + TAILQ_FOREACH(name, &g_bdev_names, link) { + if (strcmp(name->opts->vbdev_name, bdev_name) == 0) { + vbdev_crypto_delete_name(name); + break; + } + } + } else { + cb_fn(cb_arg, rc); + } +} + +/* Because we specified this function in our crypto bdev function table when we + * registered our crypto bdev, we'll get this call anytime a new bdev shows up. + * Here we need to decide if we care about it and if so what to do. We + * parsed the config file at init so we check the new bdev against the list + * we built up at that time and if the user configured us to attach to this + * bdev, here's where we do it. + */ +static void +vbdev_crypto_examine(struct spdk_bdev *bdev) +{ + vbdev_crypto_claim(spdk_bdev_get_name(bdev)); + spdk_bdev_module_examine_done(&crypto_if); +} + +SPDK_LOG_REGISTER_COMPONENT(vbdev_crypto) diff --git a/test/common/skipped_build_files.txt b/test/common/skipped_build_files.txt index 08842fad5..3ea3d8ad6 100644 --- a/test/common/skipped_build_files.txt +++ b/test/common/skipped_build_files.txt @@ -56,3 +56,7 @@ module/bdev/daos/bdev_daos_rpc # Not configured to test xNVMe bdev module/bdev/xnvme/bdev_xnvme module/bdev/xnvme/bdev_xnvme_rpc + +# Temporary added, will be remove in the next patch +module/accel/dpdk_cryptodev/accel_dpdk_cryptodev +test/unit/lib/accel/dpdk_cryptodev.c/accel_dpdk_cryptodev_ut diff --git a/test/unit/lib/accel/dpdk_cryptodev.c/accel_dpdk_cryptodev_ut.c b/test/unit/lib/accel/dpdk_cryptodev.c/accel_dpdk_cryptodev_ut.c new file mode 100644 index 000000000..aca77e64e --- /dev/null +++ b/test/unit/lib/accel/dpdk_cryptodev.c/accel_dpdk_cryptodev_ut.c @@ -0,0 +1,1227 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (C) 2018 Intel Corporation. + * All rights reserved. + */ + +#include "spdk_cunit.h" + +#include "common/lib/test_env.c" +#include "spdk_internal/mock.h" +#include "thread/thread_internal.h" +#include "unit/lib/json_mock.c" + +#include +#include +#include + +#define MAX_TEST_BLOCKS 8192 +struct rte_crypto_op *g_test_crypto_ops[MAX_TEST_BLOCKS]; +struct rte_crypto_op *g_test_dev_full_ops[MAX_TEST_BLOCKS]; + +uint16_t g_dequeue_mock; +uint16_t g_enqueue_mock; +unsigned ut_rte_crypto_op_bulk_alloc; +int ut_rte_crypto_op_attach_sym_session = 0; +#define MOCK_INFO_GET_1QP_AESNI 0 +#define MOCK_INFO_GET_1QP_QAT 1 +#define MOCK_INFO_GET_1QP_MLX5 2 +#define MOCK_INFO_GET_1QP_BOGUS_PMD 3 +int ut_rte_cryptodev_info_get = 0; +bool ut_rte_cryptodev_info_get_mocked = false; + +void mock_rte_pktmbuf_free_bulk(struct rte_mbuf **m, unsigned int cnt); +#define rte_pktmbuf_free_bulk mock_rte_pktmbuf_free_bulk +void +mock_rte_pktmbuf_free_bulk(struct rte_mbuf **m, unsigned int cnt) +{ + spdk_mempool_put_bulk((struct spdk_mempool *)m[0]->pool, (void **)m, cnt); +} + +void mock_rte_pktmbuf_free(struct rte_mbuf *m); +#define rte_pktmbuf_free mock_rte_pktmbuf_free +void +mock_rte_pktmbuf_free(struct rte_mbuf *m) +{ + spdk_mempool_put((struct spdk_mempool *)m->pool, (void *)m); +} + +void +rte_mempool_free(struct rte_mempool *mp) +{ + spdk_mempool_free((struct spdk_mempool *)mp); +} + +int mock_rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, struct rte_mbuf **mbufs, + unsigned count); +#define rte_pktmbuf_alloc_bulk mock_rte_pktmbuf_alloc_bulk +int +mock_rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, struct rte_mbuf **mbufs, + unsigned count) +{ + int rc; + + rc = spdk_mempool_get_bulk((struct spdk_mempool *)pool, (void **)mbufs, count); + if (rc) { + return rc; + } + for (unsigned i = 0; i < count; i++) { + rte_pktmbuf_reset(mbufs[i]); + mbufs[i]->pool = pool; + } + return rc; +} + +struct rte_mempool * +rte_cryptodev_sym_session_pool_create(const char *name, uint32_t nb_elts, + uint32_t elt_size, uint32_t cache_size, + uint16_t priv_size, int socket_id) +{ + struct spdk_mempool *tmp; + + tmp = spdk_mempool_create(name, nb_elts, elt_size + priv_size, + cache_size, socket_id); + + return (struct rte_mempool *)tmp; + +} + +struct rte_mempool * +rte_pktmbuf_pool_create(const char *name, unsigned n, unsigned cache_size, + uint16_t priv_size, uint16_t data_room_size, int socket_id) +{ + struct spdk_mempool *tmp; + + tmp = spdk_mempool_create(name, n, sizeof(struct rte_mbuf) + priv_size, + cache_size, socket_id); + + return (struct rte_mempool *)tmp; +} + +struct rte_mempool * +rte_mempool_create(const char *name, unsigned n, unsigned elt_size, + unsigned cache_size, unsigned private_data_size, + rte_mempool_ctor_t *mp_init, void *mp_init_arg, + rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, + int socket_id, unsigned flags) +{ + struct spdk_mempool *tmp; + + tmp = spdk_mempool_create(name, n, elt_size + private_data_size, + cache_size, socket_id); + + return (struct rte_mempool *)tmp; +} + +DEFINE_RETURN_MOCK(rte_crypto_op_pool_create, struct rte_mempool *); +struct rte_mempool * +rte_crypto_op_pool_create(const char *name, enum rte_crypto_op_type type, + unsigned nb_elts, unsigned cache_size, + uint16_t priv_size, int socket_id) +{ + struct spdk_mempool *tmp; + + HANDLE_RETURN_MOCK(rte_crypto_op_pool_create); + + tmp = spdk_mempool_create(name, nb_elts, + sizeof(struct rte_crypto_op) + priv_size, + cache_size, socket_id); + + return (struct rte_mempool *)tmp; + +} + +/* Those functions are defined as static inline in DPDK, so we can't + * mock them straight away. We use defines to redirect them into + * our custom functions. + */ +static bool g_resubmit_test = false; +#define rte_cryptodev_enqueue_burst mock_rte_cryptodev_enqueue_burst +static inline uint16_t +mock_rte_cryptodev_enqueue_burst(uint8_t dev_id, uint16_t qp_id, + struct rte_crypto_op **ops, uint16_t nb_ops) +{ + int i; + + CU_ASSERT(nb_ops > 0); + + for (i = 0; i < nb_ops; i++) { + /* Use this empty (til now) array of pointers to store + * enqueued operations for assertion in dev_full test. + */ + g_test_dev_full_ops[i] = *ops++; + if (g_resubmit_test == true) { + CU_ASSERT(g_test_dev_full_ops[i] == (void *)0xDEADBEEF); + } + } + + return g_enqueue_mock; +} + +#define rte_cryptodev_dequeue_burst mock_rte_cryptodev_dequeue_burst +static inline uint16_t +mock_rte_cryptodev_dequeue_burst(uint8_t dev_id, uint16_t qp_id, + struct rte_crypto_op **ops, uint16_t nb_ops) +{ + int i; + + CU_ASSERT(nb_ops > 0); + + for (i = 0; i < g_dequeue_mock; i++) { + *ops++ = g_test_crypto_ops[i]; + } + + return g_dequeue_mock; +} + +/* Instead of allocating real memory, assign the allocations to our + * test array for assertion in tests. + */ +#define rte_crypto_op_bulk_alloc mock_rte_crypto_op_bulk_alloc +static inline unsigned +mock_rte_crypto_op_bulk_alloc(struct rte_mempool *mempool, + enum rte_crypto_op_type type, + struct rte_crypto_op **ops, uint16_t nb_ops) +{ + int i; + + for (i = 0; i < nb_ops; i++) { + *ops++ = g_test_crypto_ops[i]; + } + return ut_rte_crypto_op_bulk_alloc; +} + +#define rte_mempool_put_bulk mock_rte_mempool_put_bulk +static __rte_always_inline void +mock_rte_mempool_put_bulk(struct rte_mempool *mp, void *const *obj_table, + unsigned int n) +{ + return; +} + +#define rte_crypto_op_attach_sym_session mock_rte_crypto_op_attach_sym_session +#if RTE_VERSION >= RTE_VERSION_NUM(22, 11, 0, 0) +static inline int +mock_rte_crypto_op_attach_sym_session(struct rte_crypto_op *op, void *sess) +#else +static inline int +mock_rte_crypto_op_attach_sym_session(struct rte_crypto_op *op, + struct rte_cryptodev_sym_session *sess) +#endif +{ + return ut_rte_crypto_op_attach_sym_session; +} + +#define rte_lcore_count mock_rte_lcore_count +static inline unsigned +mock_rte_lcore_count(void) +{ + return 1; +} + +#include "bdev/crypto/vbdev_crypto.c" + +/* SPDK stubs */ +DEFINE_STUB(spdk_bdev_queue_io_wait, int, (struct spdk_bdev *bdev, struct spdk_io_channel *ch, + struct spdk_bdev_io_wait_entry *entry), 0); +DEFINE_STUB_V(spdk_bdev_module_list_add, (struct spdk_bdev_module *bdev_module)); +DEFINE_STUB_V(spdk_bdev_free_io, (struct spdk_bdev_io *g_bdev_io)); +DEFINE_STUB_V(spdk_bdev_io_put_aux_buf, (struct spdk_bdev_io *bdev_io, void *aux_buf)); +DEFINE_STUB(spdk_bdev_io_type_supported, bool, (struct spdk_bdev *bdev, + enum spdk_bdev_io_type io_type), 0); +DEFINE_STUB_V(spdk_bdev_module_release_bdev, (struct spdk_bdev *bdev)); +DEFINE_STUB_V(spdk_bdev_close, (struct spdk_bdev_desc *desc)); +DEFINE_STUB(spdk_bdev_get_name, const char *, (const struct spdk_bdev *bdev), 0); +DEFINE_STUB(spdk_bdev_get_buf_align, size_t, (const struct spdk_bdev *bdev), 64); +DEFINE_STUB(spdk_bdev_get_io_channel, struct spdk_io_channel *, (struct spdk_bdev_desc *desc), 0); +DEFINE_STUB_V(spdk_bdev_unregister, (struct spdk_bdev *bdev, spdk_bdev_unregister_cb cb_fn, + void *cb_arg)); +DEFINE_STUB(spdk_bdev_unregister_by_name, int, (const char *bdev_name, + struct spdk_bdev_module *module, + spdk_bdev_unregister_cb cb_fn, void *cb_arg), 0); +DEFINE_STUB(spdk_bdev_open_ext, int, (const char *bdev_name, bool write, + spdk_bdev_event_cb_t event_cb, + void *event_ctx, struct spdk_bdev_desc **_desc), 0); +DEFINE_STUB(spdk_bdev_desc_get_bdev, struct spdk_bdev *, (struct spdk_bdev_desc *desc), NULL); +DEFINE_STUB(spdk_bdev_module_claim_bdev, int, (struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, + struct spdk_bdev_module *module), 0); +DEFINE_STUB_V(spdk_bdev_module_examine_done, (struct spdk_bdev_module *module)); +DEFINE_STUB(spdk_bdev_register, int, (struct spdk_bdev *vbdev), 0); + +/* DPDK stubs */ +#define DPDK_DYNFIELD_OFFSET offsetof(struct rte_mbuf, dynfield1[1]) +DEFINE_STUB(rte_mbuf_dynfield_register, int, (const struct rte_mbuf_dynfield *params), + DPDK_DYNFIELD_OFFSET); +DEFINE_STUB(rte_cryptodev_count, uint8_t, (void), 0); +DEFINE_STUB(rte_socket_id, unsigned, (void), 0); +DEFINE_STUB(rte_cryptodev_device_count_by_driver, uint8_t, (uint8_t driver_id), 0); +DEFINE_STUB(rte_cryptodev_configure, int, (uint8_t dev_id, struct rte_cryptodev_config *config), 0); +DEFINE_STUB(rte_cryptodev_queue_pair_setup, int, (uint8_t dev_id, uint16_t queue_pair_id, + const struct rte_cryptodev_qp_conf *qp_conf, int socket_id), 0); +DEFINE_STUB(rte_cryptodev_start, int, (uint8_t dev_id), 0); +DEFINE_STUB_V(rte_cryptodev_stop, (uint8_t dev_id)); +DEFINE_STUB(rte_cryptodev_close, int, (uint8_t dev_id), 0); +DEFINE_STUB(rte_vdev_init, int, (const char *name, const char *args), 0); +DEFINE_STUB(rte_vdev_uninit, int, (const char *name), 0); + +#if RTE_VERSION >= RTE_VERSION_NUM(22, 11, 0, 0) +DEFINE_STUB(rte_cryptodev_sym_session_create, void *, + (uint8_t dev_id, struct rte_crypto_sym_xform *xforms, struct rte_mempool *mempool), (void *)1); +DEFINE_STUB(rte_cryptodev_sym_session_free, int, (uint8_t dev_id, void *sess), 0); +#else +DEFINE_STUB(rte_cryptodev_sym_session_create, struct rte_cryptodev_sym_session *, + (struct rte_mempool *mempool), (void *)1); +DEFINE_STUB(rte_cryptodev_sym_session_init, int, (uint8_t dev_id, + struct rte_cryptodev_sym_session *sess, + struct rte_crypto_sym_xform *xforms, struct rte_mempool *mempool), 0); +DEFINE_STUB(rte_cryptodev_sym_session_free, int, (struct rte_cryptodev_sym_session *sess), 0); +#endif + +struct rte_cryptodev *rte_cryptodevs; + +/* global vars and setup/cleanup functions used for all test functions */ +struct spdk_bdev_io *g_bdev_io; +struct crypto_bdev_io *g_io_ctx; +struct crypto_io_channel *g_crypto_ch; +struct spdk_io_channel *g_io_ch; +struct vbdev_dev g_device; +struct vbdev_crypto g_crypto_bdev; +struct vbdev_crypto_opts g_crypto_bdev_opts; +struct device_qp g_dev_qp; + +void +rte_cryptodev_info_get(uint8_t dev_id, struct rte_cryptodev_info *dev_info) +{ + dev_info->max_nb_queue_pairs = 1; + if (ut_rte_cryptodev_info_get == MOCK_INFO_GET_1QP_AESNI) { + dev_info->driver_name = g_driver_names[0]; + } else if (ut_rte_cryptodev_info_get == MOCK_INFO_GET_1QP_QAT) { + dev_info->driver_name = g_driver_names[1]; + } else if (ut_rte_cryptodev_info_get == MOCK_INFO_GET_1QP_MLX5) { + dev_info->driver_name = g_driver_names[2]; + } else if (ut_rte_cryptodev_info_get == MOCK_INFO_GET_1QP_BOGUS_PMD) { + dev_info->driver_name = "junk"; + } +} + +unsigned int +rte_cryptodev_sym_get_private_session_size(uint8_t dev_id) +{ + return (unsigned int)dev_id; +} + +void +spdk_bdev_io_get_aux_buf(struct spdk_bdev_io *bdev_io, spdk_bdev_io_get_aux_buf_cb cb) +{ + cb(g_io_ch, g_bdev_io, (void *)0xDEADBEEF); +} + +void +spdk_bdev_io_get_buf(struct spdk_bdev_io *bdev_io, spdk_bdev_io_get_buf_cb cb, uint64_t len) +{ + cb(g_io_ch, g_bdev_io, true); +} + +/* Mock these functions to call the callback and then return the value we require */ +int ut_spdk_bdev_readv_blocks = 0; +bool ut_spdk_bdev_readv_blocks_mocked = false; +int +spdk_bdev_readv_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + struct iovec *iov, int iovcnt, + uint64_t offset_blocks, uint64_t num_blocks, + spdk_bdev_io_completion_cb cb, void *cb_arg) +{ + cb(g_bdev_io, !ut_spdk_bdev_readv_blocks, cb_arg); + return ut_spdk_bdev_readv_blocks; +} + +int ut_spdk_bdev_writev_blocks = 0; +bool ut_spdk_bdev_writev_blocks_mocked = false; +int +spdk_bdev_writev_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + struct iovec *iov, int iovcnt, + uint64_t offset_blocks, uint64_t num_blocks, + spdk_bdev_io_completion_cb cb, void *cb_arg) +{ + cb(g_bdev_io, !ut_spdk_bdev_writev_blocks, cb_arg); + return ut_spdk_bdev_writev_blocks; +} + +int ut_spdk_bdev_unmap_blocks = 0; +bool ut_spdk_bdev_unmap_blocks_mocked = false; +int +spdk_bdev_unmap_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + uint64_t offset_blocks, uint64_t num_blocks, + spdk_bdev_io_completion_cb cb, void *cb_arg) +{ + cb(g_bdev_io, !ut_spdk_bdev_unmap_blocks, cb_arg); + return ut_spdk_bdev_unmap_blocks; +} + +int ut_spdk_bdev_flush_blocks = 0; +bool ut_spdk_bdev_flush_blocks_mocked = false; +int +spdk_bdev_flush_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + uint64_t offset_blocks, uint64_t num_blocks, spdk_bdev_io_completion_cb cb, + void *cb_arg) +{ + cb(g_bdev_io, !ut_spdk_bdev_flush_blocks, cb_arg); + return ut_spdk_bdev_flush_blocks; +} + +int ut_spdk_bdev_reset = 0; +bool ut_spdk_bdev_reset_mocked = false; +int +spdk_bdev_reset(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + spdk_bdev_io_completion_cb cb, void *cb_arg) +{ + cb(g_bdev_io, !ut_spdk_bdev_reset, cb_arg); + return ut_spdk_bdev_reset; +} + +bool g_completion_called = false; +void +spdk_bdev_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status status) +{ + bdev_io->internal.status = status; + g_completion_called = true; +} + +/* Global setup for all tests that share a bunch of preparation... */ +static int +test_setup(void) +{ + int i, rc; + + /* Prepare essential variables for test routines */ + g_bdev_io = calloc(1, sizeof(struct spdk_bdev_io) + sizeof(struct crypto_bdev_io)); + g_bdev_io->u.bdev.iovs = calloc(1, sizeof(struct iovec) * 128); + g_bdev_io->bdev = &g_crypto_bdev.crypto_bdev; + g_io_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct crypto_io_channel)); + g_crypto_ch = (struct crypto_io_channel *)spdk_io_channel_get_ctx(g_io_ch); + g_io_ctx = (struct crypto_bdev_io *)g_bdev_io->driver_ctx; + memset(&g_device, 0, sizeof(struct vbdev_dev)); + memset(&g_crypto_bdev, 0, sizeof(struct vbdev_crypto)); + memset(&g_crypto_bdev_opts, 0, sizeof(struct vbdev_crypto_opts)); + g_dev_qp.device = &g_device; + g_io_ctx->crypto_ch = g_crypto_ch; + g_io_ctx->crypto_bdev = &g_crypto_bdev; + g_io_ctx->crypto_bdev->qp_desc_nr = CRYPTO_QP_DESCRIPTORS; + g_io_ctx->crypto_bdev->opts = &g_crypto_bdev_opts; + g_crypto_ch->device_qp = &g_dev_qp; + TAILQ_INIT(&g_crypto_ch->pending_cry_ios); + TAILQ_INIT(&g_crypto_ch->queued_cry_ops); + + /* Allocate a real mbuf pool so we can test error paths */ + g_mbuf_mp = rte_pktmbuf_pool_create("mbuf_mp", NUM_MBUFS, + (unsigned)SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, + 0, 0, SPDK_ENV_SOCKET_ID_ANY); + /* Instead of allocating real rte mempools for these, it's easier and provides the + * same coverage just calloc them here. + */ + for (i = 0; i < MAX_TEST_BLOCKS; i++) { + size_t size = IV_OFFSET + IV_LENGTH + QUEUED_OP_LENGTH; + rc = posix_memalign((void **)&g_test_crypto_ops[i], 64, size); + if (rc != 0) { + assert(false); + } + memset(g_test_crypto_ops[i], 0, IV_OFFSET + QUEUED_OP_LENGTH); + } + g_mbuf_offset = DPDK_DYNFIELD_OFFSET; + + return 0; +} + +/* Global teardown for all tests */ +static int +test_cleanup(void) +{ + int i; + + if (g_crypto_op_mp) { + rte_mempool_free(g_crypto_op_mp); + g_crypto_op_mp = NULL; + } + if (g_mbuf_mp) { + rte_mempool_free(g_mbuf_mp); + g_mbuf_mp = NULL; + } + if (g_session_mp) { + rte_mempool_free(g_session_mp); + g_session_mp = NULL; + } + if (g_session_mp_priv != NULL) { + /* g_session_mp_priv may or may not be set depending on the DPDK version */ + rte_mempool_free(g_session_mp_priv); + g_session_mp_priv = NULL; + } + + for (i = 0; i < MAX_TEST_BLOCKS; i++) { + free(g_test_crypto_ops[i]); + } + free(g_bdev_io->u.bdev.iovs); + free(g_bdev_io); + free(g_io_ch); + return 0; +} + +static void +test_error_paths(void) +{ + /* Single element block size write, just to test error paths + * in vbdev_crypto_submit_request(). + */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->u.bdev.iovcnt = 1; + g_bdev_io->u.bdev.num_blocks = 1; + g_bdev_io->u.bdev.iovs[0].iov_len = 512; + g_bdev_io->u.bdev.iovs[0].iov_base = (void *)0xDEADBEEF; + g_crypto_bdev.crypto_bdev.blocklen = 512; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; + g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = 1; + + /* test failure of spdk_mempool_get_bulk(), will result in success because it + * will get queued. + */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + MOCK_SET(spdk_mempool_get, NULL); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + + /* same thing but switch to reads to test error path in _crypto_complete_io() */ + g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + TAILQ_INSERT_TAIL(&g_crypto_ch->pending_cry_ios, g_bdev_io, module_link); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + /* Now with the read_blocks failing */ + g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + MOCK_SET(spdk_bdev_readv_blocks, -1); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + MOCK_SET(spdk_bdev_readv_blocks, 0); + MOCK_CLEAR(spdk_mempool_get); + + /* test failure of rte_crypto_op_bulk_alloc() */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + ut_rte_crypto_op_bulk_alloc = 0; + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + ut_rte_crypto_op_bulk_alloc = 1; + + /* test failure of rte_crypto_op_attach_sym_session() */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + ut_rte_crypto_op_attach_sym_session = -1; + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + ut_rte_crypto_op_attach_sym_session = 0; +} + +static void +test_simple_write(void) +{ + /* Single element block size write */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->u.bdev.iovcnt = 1; + g_bdev_io->u.bdev.num_blocks = 1; + g_bdev_io->u.bdev.offset_blocks = 0; + g_bdev_io->u.bdev.iovs[0].iov_len = 512; + g_bdev_io->u.bdev.iovs[0].iov_base = &test_simple_write; + g_crypto_bdev.crypto_bdev.blocklen = 512; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; + g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = 1; + + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_io_ctx->cryop_cnt_remaining == 1); + CU_ASSERT(g_io_ctx->aux_buf_iov.iov_len == 512); + CU_ASSERT(g_io_ctx->aux_buf_iov.iov_base != NULL); + CU_ASSERT(g_io_ctx->aux_offset_blocks == 0); + CU_ASSERT(g_io_ctx->aux_num_blocks == 1); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->buf_addr == &test_simple_write); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->data_len == 512); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->next == NULL); + CU_ASSERT(g_test_crypto_ops[0]->sym->cipher.data.length == 512); + CU_ASSERT(g_test_crypto_ops[0]->sym->cipher.data.offset == 0); + CU_ASSERT(*RTE_MBUF_DYNFIELD(g_test_crypto_ops[0]->sym->m_src, g_mbuf_offset, + uint64_t *) == (uint64_t)g_bdev_io); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_dst->buf_addr != NULL); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_dst->data_len == 512); + + rte_pktmbuf_free(g_test_crypto_ops[0]->sym->m_src); + rte_pktmbuf_free(g_test_crypto_ops[0]->sym->m_dst); +} + +static void +test_simple_read(void) +{ + /* Single element block size read */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->u.bdev.iovcnt = 1; + g_bdev_io->u.bdev.num_blocks = 1; + g_bdev_io->u.bdev.iovs[0].iov_len = 512; + g_bdev_io->u.bdev.iovs[0].iov_base = &test_simple_read; + g_crypto_bdev.crypto_bdev.blocklen = 512; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; + g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = 1; + + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_io_ctx->cryop_cnt_remaining == 1); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->buf_addr == &test_simple_read); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->data_len == 512); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->next == NULL); + CU_ASSERT(g_test_crypto_ops[0]->sym->cipher.data.length == 512); + CU_ASSERT(g_test_crypto_ops[0]->sym->cipher.data.offset == 0); + CU_ASSERT(*RTE_MBUF_DYNFIELD(g_test_crypto_ops[0]->sym->m_src, g_mbuf_offset, + uint64_t *) == (uint64_t)g_bdev_io); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_dst == NULL); + + rte_pktmbuf_free(g_test_crypto_ops[0]->sym->m_src); +} + +static void +test_large_rw(void) +{ + unsigned block_len = 512; + unsigned num_blocks = CRYPTO_MAX_IO / block_len; + unsigned io_len = block_len * num_blocks; + unsigned i; + + /* Multi block size read, multi-element */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->u.bdev.iovcnt = 1; + g_bdev_io->u.bdev.num_blocks = num_blocks; + g_bdev_io->u.bdev.iovs[0].iov_len = io_len; + g_bdev_io->u.bdev.iovs[0].iov_base = &test_large_rw; + g_crypto_bdev.crypto_bdev.blocklen = block_len; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; + g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = num_blocks; + + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_io_ctx->cryop_cnt_remaining == (int)num_blocks); + + for (i = 0; i < num_blocks; i++) { + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->buf_addr == &test_large_rw + (i * block_len)); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->data_len == block_len); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->next == NULL); + CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.length == block_len); + CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.offset == 0); + CU_ASSERT(*RTE_MBUF_DYNFIELD(g_test_crypto_ops[i]->sym->m_src, g_mbuf_offset, + uint64_t *) == (uint64_t)g_bdev_io); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_dst == NULL); + rte_pktmbuf_free(g_test_crypto_ops[i]->sym->m_src); + } + + /* Multi block size write, multi-element */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->u.bdev.iovcnt = 1; + g_bdev_io->u.bdev.num_blocks = num_blocks; + g_bdev_io->u.bdev.iovs[0].iov_len = io_len; + g_bdev_io->u.bdev.iovs[0].iov_base = &test_large_rw; + g_crypto_bdev.crypto_bdev.blocklen = block_len; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; + g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = num_blocks; + + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_io_ctx->cryop_cnt_remaining == (int)num_blocks); + + for (i = 0; i < num_blocks; i++) { + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->buf_addr == &test_large_rw + (i * block_len)); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->data_len == block_len); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->next == NULL); + CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.length == block_len); + CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.offset == 0); + CU_ASSERT(*RTE_MBUF_DYNFIELD(g_test_crypto_ops[i]->sym->m_src, g_mbuf_offset, + uint64_t *) == (uint64_t)g_bdev_io); + CU_ASSERT(g_io_ctx->aux_buf_iov.iov_len == io_len); + CU_ASSERT(g_io_ctx->aux_buf_iov.iov_base != NULL); + CU_ASSERT(g_io_ctx->aux_offset_blocks == 0); + CU_ASSERT(g_io_ctx->aux_num_blocks == num_blocks); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_dst->buf_addr != NULL); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_dst->data_len == block_len); + rte_pktmbuf_free(g_test_crypto_ops[i]->sym->m_src); + rte_pktmbuf_free(g_test_crypto_ops[i]->sym->m_dst); + } +} + +static void +test_dev_full(void) +{ + struct vbdev_crypto_op *queued_op; + struct rte_crypto_sym_op *sym_op; + struct crypto_bdev_io *io_ctx; + + /* Two element block size read */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->u.bdev.iovcnt = 1; + g_bdev_io->u.bdev.num_blocks = 2; + g_bdev_io->u.bdev.iovs[0].iov_len = 512; + g_bdev_io->u.bdev.iovs[0].iov_base = (void *)0xDEADBEEF; + g_bdev_io->u.bdev.iovs[1].iov_len = 512; + g_bdev_io->u.bdev.iovs[1].iov_base = (void *)0xFEEDBEEF; + g_crypto_bdev.crypto_bdev.blocklen = 512; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; + g_enqueue_mock = g_dequeue_mock = 1; + ut_rte_crypto_op_bulk_alloc = 2; + + g_test_crypto_ops[1]->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + CU_ASSERT(TAILQ_EMPTY(&g_crypto_ch->queued_cry_ops) == true); + + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_io_ctx->cryop_cnt_remaining == 2); + sym_op = g_test_crypto_ops[0]->sym; + CU_ASSERT(sym_op->m_src->buf_addr == (void *)0xDEADBEEF); + CU_ASSERT(sym_op->m_src->data_len == 512); + CU_ASSERT(sym_op->m_src->next == NULL); + CU_ASSERT(sym_op->cipher.data.length == 512); + CU_ASSERT(sym_op->cipher.data.offset == 0); + CU_ASSERT(*RTE_MBUF_DYNFIELD(sym_op->m_src, g_mbuf_offset, uint64_t *) == (uint64_t)g_bdev_io); + CU_ASSERT(sym_op->m_dst == NULL); + + /* make sure one got queued and confirm its values */ + CU_ASSERT(TAILQ_EMPTY(&g_crypto_ch->queued_cry_ops) == false); + queued_op = TAILQ_FIRST(&g_crypto_ch->queued_cry_ops); + sym_op = queued_op->crypto_op->sym; + TAILQ_REMOVE(&g_crypto_ch->queued_cry_ops, queued_op, link); + CU_ASSERT(queued_op->bdev_io == g_bdev_io); + CU_ASSERT(queued_op->crypto_op == g_test_crypto_ops[1]); + CU_ASSERT(sym_op->m_src->buf_addr == (void *)0xFEEDBEEF); + CU_ASSERT(sym_op->m_src->data_len == 512); + CU_ASSERT(sym_op->m_src->next == NULL); + CU_ASSERT(sym_op->cipher.data.length == 512); + CU_ASSERT(sym_op->cipher.data.offset == 0); + CU_ASSERT(*RTE_MBUF_DYNFIELD(sym_op->m_src, g_mbuf_offset, uint64_t *) == (uint64_t)g_bdev_io); + CU_ASSERT(sym_op->m_dst == NULL); + CU_ASSERT(TAILQ_EMPTY(&g_crypto_ch->queued_cry_ops) == true); + rte_pktmbuf_free(g_test_crypto_ops[0]->sym->m_src); + rte_pktmbuf_free(g_test_crypto_ops[1]->sym->m_src); + + /* Non-busy reason for enqueue failure, all were rejected. */ + g_enqueue_mock = 0; + g_test_crypto_ops[0]->status = RTE_CRYPTO_OP_STATUS_ERROR; + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + io_ctx = (struct crypto_bdev_io *)g_bdev_io->driver_ctx; + CU_ASSERT(io_ctx->bdev_io_status == SPDK_BDEV_IO_STATUS_FAILED); +} + +static void +test_crazy_rw(void) +{ + unsigned block_len = 512; + int num_blocks = 4; + int i; + + /* Multi block size read, single element, strange IOV makeup */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->u.bdev.iovcnt = 3; + g_bdev_io->u.bdev.num_blocks = num_blocks; + g_bdev_io->u.bdev.iovs[0].iov_len = 512; + g_bdev_io->u.bdev.iovs[0].iov_base = &test_crazy_rw; + g_bdev_io->u.bdev.iovs[1].iov_len = 1024; + g_bdev_io->u.bdev.iovs[1].iov_base = &test_crazy_rw + 512; + g_bdev_io->u.bdev.iovs[2].iov_len = 512; + g_bdev_io->u.bdev.iovs[2].iov_base = &test_crazy_rw + 512 + 1024; + + g_crypto_bdev.crypto_bdev.blocklen = block_len; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; + g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = num_blocks; + + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_io_ctx->cryop_cnt_remaining == num_blocks); + + for (i = 0; i < num_blocks; i++) { + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->buf_addr == &test_crazy_rw + (i * block_len)); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->data_len == block_len); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->next == NULL); + CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.length == block_len); + CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.offset == 0); + CU_ASSERT(*RTE_MBUF_DYNFIELD(g_test_crypto_ops[i]->sym->m_src, g_mbuf_offset, + uint64_t *) == (uint64_t)g_bdev_io); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src == g_test_crypto_ops[i]->sym->m_src); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_dst == NULL); + rte_pktmbuf_free(g_test_crypto_ops[i]->sym->m_src); + } + + /* Multi block size write, single element strange IOV makeup */ + num_blocks = 8; + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->u.bdev.iovcnt = 4; + g_bdev_io->u.bdev.num_blocks = num_blocks; + g_bdev_io->u.bdev.iovs[0].iov_len = 2048; + g_bdev_io->u.bdev.iovs[0].iov_base = &test_crazy_rw; + g_bdev_io->u.bdev.iovs[1].iov_len = 512; + g_bdev_io->u.bdev.iovs[1].iov_base = &test_crazy_rw + 2048; + g_bdev_io->u.bdev.iovs[2].iov_len = 512; + g_bdev_io->u.bdev.iovs[2].iov_base = &test_crazy_rw + 2048 + 512; + g_bdev_io->u.bdev.iovs[3].iov_len = 1024; + g_bdev_io->u.bdev.iovs[3].iov_base = &test_crazy_rw + 2048 + 512 + 512; + + g_crypto_bdev.crypto_bdev.blocklen = block_len; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; + g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = num_blocks; + + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_io_ctx->cryop_cnt_remaining == num_blocks); + + for (i = 0; i < num_blocks; i++) { + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->buf_addr == &test_crazy_rw + (i * block_len)); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->data_len == block_len); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->next == NULL); + CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.length == block_len); + CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.offset == 0); + CU_ASSERT(*RTE_MBUF_DYNFIELD(g_test_crypto_ops[i]->sym->m_src, g_mbuf_offset, + uint64_t *) == (uint64_t)g_bdev_io); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src == g_test_crypto_ops[i]->sym->m_src); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_dst == g_test_crypto_ops[i]->sym->m_dst); + rte_pktmbuf_free(g_test_crypto_ops[i]->sym->m_src); + rte_pktmbuf_free(g_test_crypto_ops[i]->sym->m_dst); + } +} + +static void +test_passthru(void) +{ + /* Make sure these follow our completion callback, test success & fail. */ + g_bdev_io->type = SPDK_BDEV_IO_TYPE_UNMAP; + MOCK_SET(spdk_bdev_unmap_blocks, 0); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + MOCK_SET(spdk_bdev_unmap_blocks, -1); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + MOCK_CLEAR(spdk_bdev_unmap_blocks); + + g_bdev_io->type = SPDK_BDEV_IO_TYPE_FLUSH; + MOCK_SET(spdk_bdev_flush_blocks, 0); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + MOCK_SET(spdk_bdev_flush_blocks, -1); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + MOCK_CLEAR(spdk_bdev_flush_blocks); + + /* We should never get a WZ command, we report that we don't support it. */ + g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE_ZEROES; + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); +} + +static void +test_reset(void) +{ + /* TODO: There are a few different ways to do this given that + * the code uses spdk_for_each_channel() to implement reset + * handling. Submitting w/o UT for this function for now and + * will follow up with something shortly. + */ +} + +static void +init_cleanup(void) +{ + if (g_crypto_op_mp) { + rte_mempool_free(g_crypto_op_mp); + g_crypto_op_mp = NULL; + } + if (g_mbuf_mp) { + rte_mempool_free(g_mbuf_mp); + g_mbuf_mp = NULL; + } + if (g_session_mp) { + rte_mempool_free(g_session_mp); + g_session_mp = NULL; + } + if (g_session_mp_priv != NULL) { + /* g_session_mp_priv may or may not be set depending on the DPDK version */ + rte_mempool_free(g_session_mp_priv); + g_session_mp_priv = NULL; + } +} + +static void +test_initdrivers(void) +{ + int rc; + static struct rte_mempool *orig_mbuf_mp; + static struct rte_mempool *orig_session_mp; + static struct rte_mempool *orig_session_mp_priv; + + /* These tests will alloc and free our g_mbuf_mp + * so save that off here and restore it after each test is over. + */ + orig_mbuf_mp = g_mbuf_mp; + orig_session_mp = g_session_mp; + orig_session_mp_priv = g_session_mp_priv; + + g_session_mp_priv = NULL; + g_session_mp = NULL; + g_mbuf_mp = NULL; + + /* No drivers available, not an error though */ + MOCK_SET(rte_cryptodev_count, 0); + rc = vbdev_crypto_init_crypto_drivers(); + CU_ASSERT(rc == 0); + CU_ASSERT(g_mbuf_mp == NULL); + CU_ASSERT(g_session_mp == NULL); + CU_ASSERT(g_session_mp_priv == NULL); + + /* Can't create session pool. */ + MOCK_SET(rte_cryptodev_count, 2); + MOCK_SET(spdk_mempool_create, NULL); + rc = vbdev_crypto_init_crypto_drivers(); + CU_ASSERT(rc == -ENOMEM); + CU_ASSERT(g_mbuf_mp == NULL); + CU_ASSERT(g_session_mp == NULL); + CU_ASSERT(g_session_mp_priv == NULL); + MOCK_CLEAR(spdk_mempool_create); + + /* Can't create op pool. */ + MOCK_SET(rte_crypto_op_pool_create, NULL); + rc = vbdev_crypto_init_crypto_drivers(); + CU_ASSERT(rc == -ENOMEM); + CU_ASSERT(g_mbuf_mp == NULL); + CU_ASSERT(g_session_mp == NULL); + CU_ASSERT(g_session_mp_priv == NULL); + MOCK_CLEAR(rte_crypto_op_pool_create); + + /* Check resources are not sufficient */ + MOCK_CLEARED_ASSERT(spdk_mempool_create); + rc = vbdev_crypto_init_crypto_drivers(); + CU_ASSERT(rc == -EINVAL); + + /* Test crypto dev configure failure. */ + MOCK_SET(rte_cryptodev_device_count_by_driver, 2); + MOCK_SET(rte_cryptodev_info_get, MOCK_INFO_GET_1QP_AESNI); + MOCK_SET(rte_cryptodev_configure, -1); + MOCK_CLEARED_ASSERT(spdk_mempool_create); + rc = vbdev_crypto_init_crypto_drivers(); + MOCK_SET(rte_cryptodev_configure, 0); + CU_ASSERT(g_mbuf_mp == NULL); + CU_ASSERT(g_session_mp == NULL); + CU_ASSERT(g_session_mp_priv == NULL); + CU_ASSERT(rc == -EINVAL); + + /* Test failure of qp setup. */ + MOCK_SET(rte_cryptodev_queue_pair_setup, -1); + MOCK_CLEARED_ASSERT(spdk_mempool_create); + rc = vbdev_crypto_init_crypto_drivers(); + CU_ASSERT(rc == -EINVAL); + CU_ASSERT(g_mbuf_mp == NULL); + CU_ASSERT(g_session_mp == NULL); + CU_ASSERT(g_session_mp_priv == NULL); + MOCK_SET(rte_cryptodev_queue_pair_setup, 0); + + /* Test failure of dev start. */ + MOCK_SET(rte_cryptodev_start, -1); + MOCK_CLEARED_ASSERT(spdk_mempool_create); + rc = vbdev_crypto_init_crypto_drivers(); + CU_ASSERT(rc == -EINVAL); + CU_ASSERT(g_mbuf_mp == NULL); + CU_ASSERT(g_session_mp == NULL); + CU_ASSERT(g_session_mp_priv == NULL); + MOCK_SET(rte_cryptodev_start, 0); + + /* Test bogus PMD */ + MOCK_CLEARED_ASSERT(spdk_mempool_create); + MOCK_SET(rte_cryptodev_info_get, MOCK_INFO_GET_1QP_BOGUS_PMD); + rc = vbdev_crypto_init_crypto_drivers(); + CU_ASSERT(g_mbuf_mp == NULL); + CU_ASSERT(g_session_mp == NULL); + CU_ASSERT(rc == -EINVAL); + + /* Test happy path QAT. */ + MOCK_CLEARED_ASSERT(spdk_mempool_create); + MOCK_SET(rte_cryptodev_info_get, MOCK_INFO_GET_1QP_QAT); + rc = vbdev_crypto_init_crypto_drivers(); + CU_ASSERT(g_mbuf_mp != NULL); + CU_ASSERT(g_session_mp != NULL); + init_cleanup(); + CU_ASSERT(rc == 0); + + /* Test happy path AESNI. */ + MOCK_CLEARED_ASSERT(spdk_mempool_create); + MOCK_SET(rte_cryptodev_info_get, MOCK_INFO_GET_1QP_AESNI); + rc = vbdev_crypto_init_crypto_drivers(); + CU_ASSERT(g_mbuf_offset == DPDK_DYNFIELD_OFFSET); + init_cleanup(); + CU_ASSERT(rc == 0); + + /* Test happy path MLX5. */ + MOCK_CLEARED_ASSERT(spdk_mempool_create); + MOCK_SET(rte_cryptodev_info_get, MOCK_INFO_GET_1QP_MLX5); + rc = vbdev_crypto_init_crypto_drivers(); + CU_ASSERT(g_mbuf_offset == DPDK_DYNFIELD_OFFSET); + init_cleanup(); + CU_ASSERT(rc == 0); + + /* Test failure of DPDK dev init. By now it is not longer an error + * situation for entire crypto framework. */ + MOCK_SET(rte_cryptodev_count, 2); + MOCK_SET(rte_cryptodev_device_count_by_driver, 2); + MOCK_SET(rte_vdev_init, -1); + MOCK_CLEARED_ASSERT(spdk_mempool_create); + MOCK_SET(rte_cryptodev_info_get, MOCK_INFO_GET_1QP_QAT); + rc = vbdev_crypto_init_crypto_drivers(); + CU_ASSERT(rc == 0); + CU_ASSERT(g_mbuf_mp != NULL); + CU_ASSERT(g_session_mp != NULL); +#if RTE_VERSION < RTE_VERSION_NUM(22, 11, 0, 0) + CU_ASSERT(g_session_mp_priv != NULL); +#endif + init_cleanup(); + MOCK_SET(rte_vdev_init, 0); + MOCK_CLEAR(rte_cryptodev_device_count_by_driver); + + /* restore our initial values. */ + g_mbuf_mp = orig_mbuf_mp; + g_session_mp = orig_session_mp; + g_session_mp_priv = orig_session_mp_priv; +} + +static void +test_crypto_op_complete(void) +{ + /* Make sure completion code respects failure. */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED; + g_completion_called = false; + _crypto_operation_complete(g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + CU_ASSERT(g_completion_called == true); + + /* Test read completion. */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; + g_completion_called = false; + _crypto_operation_complete(g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_completion_called == true); + + /* Test write completion success. */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; + g_completion_called = false; + MOCK_SET(spdk_bdev_writev_blocks, 0); + _crypto_operation_complete(g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_completion_called == true); + + /* Test write completion failed. */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; + g_completion_called = false; + MOCK_SET(spdk_bdev_writev_blocks, -1); + _crypto_operation_complete(g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + CU_ASSERT(g_completion_called == true); + + /* Test bogus type for this completion. */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_RESET; + g_completion_called = false; + _crypto_operation_complete(g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + CU_ASSERT(g_completion_called == true); +} + +static void +test_supported_io(void) +{ + void *ctx = NULL; + bool rc = true; + + /* Make sure we always report false to WZ, we need the bdev layer to + * send real 0's so we can encrypt/decrypt them. + */ + rc = vbdev_crypto_io_type_supported(ctx, SPDK_BDEV_IO_TYPE_WRITE_ZEROES); + CU_ASSERT(rc == false); +} + +static void +test_poller(void) +{ + int rc; + struct rte_mbuf *src_mbufs[2]; + struct vbdev_crypto_op *op_to_resubmit; + + /* test regular 1 op to dequeue and complete */ + g_dequeue_mock = g_enqueue_mock = 1; + rte_pktmbuf_alloc_bulk(g_mbuf_mp, src_mbufs, 1); + g_test_crypto_ops[0]->sym->m_src = src_mbufs[0]; + *RTE_MBUF_DYNFIELD(g_test_crypto_ops[0]->sym->m_src, g_mbuf_offset, + uint64_t *) = (uintptr_t)g_bdev_io; + g_test_crypto_ops[0]->sym->m_dst = NULL; + g_io_ctx->cryop_cnt_remaining = 1; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; + rc = crypto_dev_poller(g_crypto_ch); + CU_ASSERT(rc == 1); + + /* We have nothing dequeued but have some to resubmit */ + g_dequeue_mock = 0; + CU_ASSERT(TAILQ_EMPTY(&g_crypto_ch->queued_cry_ops) == true); + + /* add an op to the queued list. */ + g_resubmit_test = true; + op_to_resubmit = (struct vbdev_crypto_op *)((uint8_t *)g_test_crypto_ops[0] + QUEUED_OP_OFFSET); + op_to_resubmit->crypto_op = (void *)0xDEADBEEF; + op_to_resubmit->bdev_io = g_bdev_io; + TAILQ_INSERT_TAIL(&g_crypto_ch->queued_cry_ops, + op_to_resubmit, + link); + CU_ASSERT(TAILQ_EMPTY(&g_crypto_ch->queued_cry_ops) == false); + rc = crypto_dev_poller(g_crypto_ch); + g_resubmit_test = false; + CU_ASSERT(rc == 0); + CU_ASSERT(TAILQ_EMPTY(&g_crypto_ch->queued_cry_ops) == true); + + /* 2 to dequeue but 2nd one failed */ + g_dequeue_mock = g_enqueue_mock = 2; + g_io_ctx->cryop_cnt_remaining = 2; + rte_pktmbuf_alloc_bulk(g_mbuf_mp, src_mbufs, 2); + g_test_crypto_ops[0]->sym->m_src = src_mbufs[0]; + *RTE_MBUF_DYNFIELD(g_test_crypto_ops[0]->sym->m_src, g_mbuf_offset, + uint64_t *) = (uint64_t)g_bdev_io; + g_test_crypto_ops[0]->sym->m_dst = NULL; + g_test_crypto_ops[0]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + g_test_crypto_ops[1]->sym->m_src = src_mbufs[1]; + *RTE_MBUF_DYNFIELD(g_test_crypto_ops[1]->sym->m_src, g_mbuf_offset, + uint64_t *) = (uint64_t)g_bdev_io; + g_test_crypto_ops[1]->sym->m_dst = NULL; + g_test_crypto_ops[1]->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + rc = crypto_dev_poller(g_crypto_ch); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + CU_ASSERT(rc == 2); +} + +/* Helper function for test_assign_device_qp() */ +static void +_clear_device_qp_lists(void) +{ + struct device_qp *device_qp = NULL; + + while (!TAILQ_EMPTY(&g_device_qp_qat)) { + device_qp = TAILQ_FIRST(&g_device_qp_qat); + TAILQ_REMOVE(&g_device_qp_qat, device_qp, link); + free(device_qp); + + } + CU_ASSERT(TAILQ_EMPTY(&g_device_qp_qat) == true); + while (!TAILQ_EMPTY(&g_device_qp_aesni_mb)) { + device_qp = TAILQ_FIRST(&g_device_qp_aesni_mb); + TAILQ_REMOVE(&g_device_qp_aesni_mb, device_qp, link); + free(device_qp); + } + CU_ASSERT(TAILQ_EMPTY(&g_device_qp_aesni_mb) == true); + while (!TAILQ_EMPTY(&g_device_qp_mlx5)) { + device_qp = TAILQ_FIRST(&g_device_qp_mlx5); + TAILQ_REMOVE(&g_device_qp_mlx5, device_qp, link); + free(device_qp); + } + CU_ASSERT(TAILQ_EMPTY(&g_device_qp_mlx5) == true); +} + +/* Helper function for test_assign_device_qp() */ +static void +_check_expected_values(struct vbdev_crypto *crypto_bdev, struct device_qp *device_qp, + struct crypto_io_channel *crypto_ch, uint8_t expected_index, + uint8_t current_index) +{ + _assign_device_qp(&g_crypto_bdev, device_qp, g_crypto_ch); + CU_ASSERT(g_crypto_ch->device_qp->index == expected_index); + CU_ASSERT(g_next_qat_index == current_index); +} + +static void +test_assign_device_qp(void) +{ + struct device_qp *device_qp = NULL; + int i; + + /* start with a known state, clear the device/qp lists */ + _clear_device_qp_lists(); + + /* make sure that one AESNI_MB qp is found */ + device_qp = calloc(1, sizeof(struct device_qp)); + TAILQ_INSERT_TAIL(&g_device_qp_aesni_mb, device_qp, link); + g_crypto_ch->device_qp = NULL; + g_crypto_bdev.opts->drv_name = AESNI_MB; + _assign_device_qp(&g_crypto_bdev, device_qp, g_crypto_ch); + CU_ASSERT(g_crypto_ch->device_qp != NULL); + + /* QAT testing is more complex as the code under test load balances by + * assigning each subsequent device/qp to every QAT_VF_SPREAD modulo + * g_qat_total_qp. For the current latest QAT we'll have 48 virtual functions + * each with 2 qp so the "spread" between assignments is 32. + */ + g_qat_total_qp = 96; + for (i = 0; i < g_qat_total_qp; i++) { + device_qp = calloc(1, sizeof(struct device_qp)); + device_qp->index = i; + TAILQ_INSERT_TAIL(&g_device_qp_qat, device_qp, link); + } + g_crypto_ch->device_qp = NULL; + g_crypto_bdev.opts->drv_name = QAT; + + /* First assignment will assign to 0 and next at 32. */ + _check_expected_values(&g_crypto_bdev, device_qp, g_crypto_ch, + 0, QAT_VF_SPREAD); + + /* Second assignment will assign to 32 and next at 64. */ + _check_expected_values(&g_crypto_bdev, device_qp, g_crypto_ch, + QAT_VF_SPREAD, QAT_VF_SPREAD * 2); + + /* Third assignment will assign to 64 and next at 0. */ + _check_expected_values(&g_crypto_bdev, device_qp, g_crypto_ch, + QAT_VF_SPREAD * 2, 0); + + /* Fourth assignment will assign to 1 and next at 33. */ + _check_expected_values(&g_crypto_bdev, device_qp, g_crypto_ch, + 1, QAT_VF_SPREAD + 1); + + /* make sure that one MLX5 qp is found */ + device_qp = calloc(1, sizeof(struct device_qp)); + TAILQ_INSERT_TAIL(&g_device_qp_mlx5, device_qp, link); + g_crypto_ch->device_qp = NULL; + g_crypto_bdev.opts->drv_name = MLX5; + _assign_device_qp(&g_crypto_bdev, device_qp, g_crypto_ch); + CU_ASSERT(g_crypto_ch->device_qp == device_qp); + + _clear_device_qp_lists(); +} + +int +main(int argc, char **argv) +{ + CU_pSuite suite = NULL; + unsigned int num_failures; + + CU_set_error_action(CUEA_ABORT); + CU_initialize_registry(); + + suite = CU_add_suite("crypto", test_setup, test_cleanup); + CU_ADD_TEST(suite, test_error_paths); + CU_ADD_TEST(suite, test_simple_write); + CU_ADD_TEST(suite, test_simple_read); + CU_ADD_TEST(suite, test_large_rw); + CU_ADD_TEST(suite, test_dev_full); + CU_ADD_TEST(suite, test_crazy_rw); + CU_ADD_TEST(suite, test_passthru); + CU_ADD_TEST(suite, test_initdrivers); + CU_ADD_TEST(suite, test_crypto_op_complete); + CU_ADD_TEST(suite, test_supported_io); + CU_ADD_TEST(suite, test_reset); + CU_ADD_TEST(suite, test_poller); + CU_ADD_TEST(suite, test_assign_device_qp); + + CU_basic_set_mode(CU_BRM_VERBOSE); + CU_basic_run_tests(); + num_failures = CU_get_number_of_failures(); + CU_cleanup_registry(); + return num_failures; +}