diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bf499f9f..6991afaae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,12 @@ does not at this time confer any SPDK ABI compatibility claims. spdk_bdev_alias_del_all() was added to delete all alias from block device. +A new virtual bdev module has been added to perform at rest data encryption using the DPDK CryptoDev +Framework. The module initially uses a software AESNI CBC cipher with experimental support for the +Intel QAT hardware accelerator also currently implemented with support for CBC cipher. Future work +may include additional ciphers as well as consideration for authentication. NOTE: this module is +currently marked as experimental. Do not use in production. + ### Environment Abstraction Layer and Event Framework The size parameter of spdk_mem_map_translate is now a pointer. This allows the diff --git a/configure b/configure index 9c5e5ce81..2ce91919a 100755 --- a/configure +++ b/configure @@ -339,6 +339,7 @@ than or equal to 4.14 will see significantly reduced performance. fi if [[ "$CONFIG_CRYPTO" = "y" ]]; then + echo NOTE: Crypto is currently marked as experimental. Do not use in production. set +e nasm_ver=$(nasm -v | sed 's/[^0-9]*//g' | awk '{print substr ($0, 0, 5)}') ipsec="$(find /usr -name intel-ipsec-mb.h 2>/dev/null)" diff --git a/doc/bdev.md b/doc/bdev.md index f302ffd90..8efd4d114 100644 --- a/doc/bdev.md +++ b/doc/bdev.md @@ -91,6 +91,41 @@ To remove a block device representation use the delete_rbd_bdev command. `rpc.py delete_rbd_bdev Rbd0` +# Crypto Virtual Bdev Module {#bdev_config_crypto} + +The crypto virtual bdev module can be configured to provide at rest data encryption +for any underlying bdev. The module relies on the DPDK CryptoDev Framework to provide +all cryptographic functionality. The framework provides support for many different software +only cryptographic modules as well hardware assisted support for the Intel QAT board. The +framework also provides support for cipher, hash, authentication and AEAD functions. At this +time the SPDK virtual bdev module supports cipher only as follows: + +- AESN-NI Multi Buffer Crypto Poll Mode Driver: RTE_CRYPTO_CIPHER_AES128_CBC +- Intel(R) QuickAssist (QAT) Crypto Poll Mode Driver: RTE_CRYPTO_CIPHER_AES128_CBC +(Note: QAT is functional however is marked as experimental until the hardware has +been fully integrated with the SPDK CI system.) + +Support for other DPDK drivers and capabilities may be added programmatically. Existing +functionality is configured through a .conf file as shown here: + +[crypto]
+ \# CRY \ \ \ \
+ \# key size depends on cipher
+ \# supported PMD names: crypto_aesni_mb, crypto_qat
+ \# Note: QAT is experimental while test HW is being setup
+ CRY Malloc4 crypto_ram 0123456789123456 crypto_aesni_mb + +In order to support using the bdev block offset (LBA) as the initialization vector (IV), +the crypto module break up all I/O into crypto operations of a size equal to the block +size of the underlying bdev. For example, a 4K I/O to a bdev with a 512B block size, +would result in 8 cryptographic operations. + +For reads, the buffer provided to the crypto module will be used as the destination buffer +for unencrypted data. For writes, however, a temporary scratch buffer is used as the +destination buffer for encryption which is then passed on to the underlying bdev as the +write buffer. This is done to avoid encrypting the data in the original source buffer which +may cause problems in some use cases. + # GPT (GUID Partition Table) {#bdev_config_gpt} The GPT virtual bdev driver is enabled by default and does not require any configuration. diff --git a/dpdkbuild/Makefile b/dpdkbuild/Makefile index ef911776e..fa3351703 100644 --- a/dpdkbuild/Makefile +++ b/dpdkbuild/Makefile @@ -46,10 +46,10 @@ ifneq ($(wildcard $(SPDK_ROOT_DIR)/intel-ipsec-mb/README),) INTEL_IPSEC_MB_LIB=$(SPDK_ROOT_DIR)/intel-ipsec-mb/libIPSec_MB.a IPSEC_MB_CFLAGS = -fPIC CRYPTO_ENABLED = y -DPDK_OPTS += AESNI_MULTI_BUFFER_LIB_PATH=$(SPDK_ROOT_DIR)/intel-ipsec-mb DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_AESNI_MB=y DPDK_OPTS += CONFIG_RTE_LIBRTE_CRYPTODEV=y DPDK_OPTS += CONFIG_RTE_LIBRTE_REORDER=y +DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_QAT=n endif endif endif @@ -58,6 +58,7 @@ ifeq ($(CRYPTO_ENABLED),n) DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_AESNI_MB=n DPDK_OPTS += CONFIG_RTE_LIBRTE_CRYPTODEV=n DPDK_OPTS += CONFIG_RTE_LIBRTE_REORDER=n +DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_QAT=n endif ifeq ($(TARGET_MACHINE),aarch64) diff --git a/lib/bdev/Makefile b/lib/bdev/Makefile index d29f9a56f..183792c40 100644 --- a/lib/bdev/Makefile +++ b/lib/bdev/Makefile @@ -44,6 +44,10 @@ LIBNAME = bdev DIRS-y += error gpt lvol malloc null nvme passthru rpc split +ifeq ($(CONFIG_CRYPTO),y) +DIRS-y += crypto +endif + ifeq ($(OS),Linux) DIRS-y += aio DIRS-$(CONFIG_ISCSI_INITIATOR) += iscsi diff --git a/lib/bdev/crypto/Makefile b/lib/bdev/crypto/Makefile new file mode 100644 index 000000000..c3eb1b741 --- /dev/null +++ b/lib/bdev/crypto/Makefile @@ -0,0 +1,42 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +CFLAGS += $(ENV_CFLAGS) + +C_SRCS = vbdev_crypto.c vbdev_crypto_rpc.c +LIBNAME = vbdev_crypto + +include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk diff --git a/lib/bdev/crypto/vbdev_crypto.c b/lib/bdev/crypto/vbdev_crypto.c new file mode 100755 index 000000000..5d2e58445 --- /dev/null +++ b/lib/bdev/crypto/vbdev_crypto.c @@ -0,0 +1,1506 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUcryptoION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "vbdev_crypto.h" + +#include "spdk/env.h" +#include "spdk/conf.h" +#include "spdk/endian.h" +#include "spdk/io_channel.h" +#include "spdk/bdev_module.h" + +#include +#include +#include +#include +#include + +/* To add support for new device types, follow the examples of the following... + * Note that the string names are defined by the DPDK PMD in question so be + * sure to use the exact names. + */ +#define MAX_NUM_DRV_TYPES 2 +#define AESNI_MB "crypto_aesni_mb" +/* NOTE: QAT is experimental. It has been tested in development but should not + * be used in production until this comment is removed following the addition of + * QAT hardware to the CI test pool. + */ +#define QAT "crypto_qat" +const char *g_driver_names[MAX_NUM_DRV_TYPES] = { AESNI_MB, QAT }; + +/* Global list of available crypto devices. */ +struct vbdev_dev { + struct rte_cryptodev_info cdev_info; /* includes device friendly name */ + uint8_t cdev_id; /* identifier for the device */ + TAILQ_ENTRY(vbdev_dev) link; +}; +static TAILQ_HEAD(, vbdev_dev) g_vbdev_devs = TAILQ_HEAD_INITIALIZER(g_vbdev_devs); + +/* Global list and lock for unique device/queue pair combos */ +struct device_qp { + struct vbdev_dev *device; /* ptr to crypto device */ + uint8_t qp; /* queue pair for this node */ + bool in_use; /* whether this node is in use or not */ + TAILQ_ENTRY(device_qp) link; +}; +static TAILQ_HEAD(, device_qp) g_device_qp = TAILQ_HEAD_INITIALIZER(g_device_qp); +static pthread_mutex_t g_device_qp_lock = PTHREAD_MUTEX_INITIALIZER; + + +/* In order to limit the number of resources we need to do one crypto + * operation per LBA (we use LBA as IV), we tell the bdev layer that + * our max IO size is something reasonable. Units here are in bytes. + */ +#define CRYPTO_MAX_IO (64 * 1024) + +/* This controls how many ops will be dequeued from the crypto driver in one run + * of the poller. It is mainly a performance knob as it effectively determines how + * much work the poller has to do. However even that can vary between crypto drivers + * as the AESNI_MB driver for example does all the crypto work on dequeue whereas the + * QAT drvier just dequeues what has been completed already. + */ +#define MAX_DEQUEUE_BURST_SIZE 64 + +/* When enqueueing, we need to supply the crypto driver with an array of pointers to + * operation structs. As each of these can be max 512B, we can adjust the CRYPTO_MAX_IO + * value in conjunction with the the other defiens to make sure we're not using crazy amounts + * of memory. All of these numbers can and probably should be adjusted based on the + * workload. By default we'll use the worst case (smallest) block size for the + * minimum number of array entries. As an example, a CRYPTO_MAX_IO size of 64K with 512B + * blocks would give us an enqueue array size of 128. + */ +#define MAX_ENQUEUE_ARRAY_SIZE (CRYPTO_MAX_IO / 512) + +/* The number of MBUFS we need must be a power of two and to support other small IOs + * in addition to the limits mentioned above, we go to the next power of two. It is + * big number because it is one mempool for source and desitnation mbufs. It may + * need to be bigger to support multiple crypto drivers at once. + */ +#define NUM_MBUFS 32768 +#define POOL_CACHE_SIZE 256 +#define NUM_SESSIONS NUM_MBUFS +#define SESS_MEMPOOL_CACHE_SIZE 256 + +/* This is the max number of IOs we can supply to any crypto device QP at one time. + * It can vary between drivers. + */ +#define CRYPTO_QP_DESCRIPTORS 2048 + +/* Specific to AES_CBC. */ +#define AES_CBC_IV_LENGTH 16 +#define AES_CBC_KEY_LENGTH 16 + +/* Common for suported devices. */ +#define IV_OFFSET (sizeof(struct rte_crypto_op) + \ + sizeof(struct rte_crypto_sym_op)) + +static void _complete_internal_io(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg); +static void _complete_internal_read(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg); +static void _complete_internal_write(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg); +static void vbdev_crypto_examine(struct spdk_bdev *bdev); +static void vbdev_crypto_claim(struct spdk_bdev *bdev); + +/* list of crypto_bdev names and their base bdevs via configuration file. + * Used so we can parse the conf once at init and use this list in examine(). + */ +struct bdev_names { + char *vbdev_name; /* name of the vbdev to create */ + char *bdev_name; /* base bdev name */ + + /* Note, for dev/test we allow use of key in the config file, for production + * use, you must use an RPC to specify the key for security reasons. + */ + uint8_t *key; /* key per bdev */ + char *drv_name; /* name of the crypto device driver */ + TAILQ_ENTRY(bdev_names) link; +}; +static TAILQ_HEAD(, bdev_names) g_bdev_names = TAILQ_HEAD_INITIALIZER(g_bdev_names); + +/* List of virtual bdevs and associated info for each. We keep the device friendly name here even + * though its also in the device struct because we use it early on. + */ +struct vbdev_crypto { + struct spdk_bdev *base_bdev; /* the thing we're attaching to */ + struct spdk_bdev_desc *base_desc; /* its descriptor we get from open */ + struct spdk_bdev crypto_bdev; /* the crypto virtual bdev */ + uint8_t *key; /* key per bdev */ + char *drv_name; /* name of the crypto device driver */ + TAILQ_ENTRY(vbdev_crypto) link; +}; +static TAILQ_HEAD(, vbdev_crypto) g_vbdev_crypto = TAILQ_HEAD_INITIALIZER(g_vbdev_crypto); + +/* Shared mempools between all devices on this system */ +static struct spdk_mempool *g_session_mp = NULL; /* session mempool */ +static struct spdk_mempool *g_mbuf_mp = NULL; /* mbuf mempool */ +static struct rte_mempool *g_crypto_op_mp = NULL; /* crypto operations, must be rte* mempool */ + +/* The crypto vbdev channel struct. It is allocated and freed on my behalf by the io channel code. + * We store things in here that are needed on per thread basis like the base_channel for this thread, + * and the poller for this thread. + */ +struct crypto_io_channel { + struct spdk_io_channel *base_ch; /* IO channel of base device */ + struct spdk_poller *poller; /* completion poller */ + struct device_qp *device_qp; /* unique device/qp combination for this channel */ +}; + +/* This is the crypto per IO context that the bdev layer allocates for us opaquely and attaches to + * each IO for us. + */ +struct crypto_bdev_io { + int cryop_cnt_remaining; /* counter used when completing crypto ops */ + struct crypto_io_channel *crypto_ch; /* need to store for crypto completion handling */ + struct vbdev_crypto *crypto_bdev; /* the crypto node struct associated with this IO */ + enum rte_crypto_cipher_operation crypto_op; /* the crypto control struct */ + struct rte_crypto_sym_xform cipher_xform; /* crypto control struct for this IO */ + struct spdk_bdev_io *orig_io; /* the original IO */ + struct spdk_bdev_io *read_io; /* the read IO we issued */ + + /* Used for the single contigous buffer that serves as the crypto destination target for writes */ + uint64_t cry_num_blocks; /* num of blocks for the contiguous buffer */ + uint64_t cry_offset_blocks; /* block offset on media */ + struct iovec cry_iov; /* iov representing contig write buffer */ +}; + +/* This is called from the module's init function. We setup all crypto devices early on as we are unable + * to easily dynamically configure queue pairs after the drivers are up and running. So, here, we + * configure the max capabilities of each device and assign threads to queue pairs as channels are + * requested. + */ +static int +vbdev_crypto_init_crypto_drivers(void) +{ + uint8_t cdev_count; + uint8_t cdrv_id, cdev_id, i, j; + int rc = 0; + struct vbdev_dev *device = NULL; + struct device_qp *dev_qp = NULL; + unsigned int max_sess_size = 0, sess_size; + uint16_t num_lcores = rte_lcore_count(); + + /* Only the first call, via RPC or module init should init the crypto drivers. */ + if (g_session_mp != NULL) { + return 0; + } + + /* We always init AESNI_MB */ + rc = rte_vdev_init(AESNI_MB, NULL); + if (rc == 0) { + SPDK_NOTICELOG("created virtual PMD %s\n", AESNI_MB); + } else { + SPDK_ERRLOG("error creating virtual PMD %s\n", AESNI_MB); + return -EINVAL; + } + + /* If we have no crypto devices, there's no reason to continue. */ + cdev_count = rte_cryptodev_count(); + if (cdev_count == 0) { + return 0; + } + + /* + * Create global mempools, shared by all devices regardless of type. + */ + + /* First determine max session size, most pools are shared by all the devices, + * so we need to find the global max sessions size. + */ + for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { + sess_size = rte_cryptodev_sym_get_private_session_size(cdev_id); + if (sess_size > max_sess_size) { + max_sess_size = sess_size; + } + } + + g_session_mp = spdk_mempool_create("session_mp", NUM_SESSIONS * 2, max_sess_size, + SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, + SPDK_ENV_SOCKET_ID_ANY); + if (g_session_mp == NULL) { + SPDK_ERRLOG("Cannot create session pool max size 0x%x\n", max_sess_size); + return -ENOMEM; + } + + g_mbuf_mp = spdk_mempool_create("mbuf_mp", NUM_MBUFS, sizeof(struct rte_mbuf), + SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, + SPDK_ENV_SOCKET_ID_ANY); + if (g_mbuf_mp == NULL) { + SPDK_ERRLOG("Cannot create mbuf pool\n"); + rc = -ENOMEM; + goto error_create_mbuf; + } + + g_crypto_op_mp = rte_crypto_op_pool_create("op_mp", + RTE_CRYPTO_OP_TYPE_SYMMETRIC, + NUM_MBUFS, + POOL_CACHE_SIZE, + AES_CBC_IV_LENGTH, + rte_socket_id()); + if (g_crypto_op_mp == NULL) { + SPDK_ERRLOG("Cannot create op pool\n"); + rc = -ENOMEM; + goto error_create_op; + } + + /* + * Now lets configure each device. + */ + for (i = 0; i < cdev_count; i++) { + device = calloc(1, sizeof(struct vbdev_dev)); + if (!device) { + rc = -ENOMEM; + goto error_create_device; + } + + /* Get details about this device. */ + rte_cryptodev_info_get(i, &device->cdev_info); + cdrv_id = device->cdev_info.driver_id; + cdev_id = device->cdev_id = i; + + /* Before going any further, make sure we have enough resources for this + * device type to function. We need a unique queue pair per core accross each + * device type to remain lockless.... + */ + if ((rte_cryptodev_device_count_by_driver(cdrv_id) * + device->cdev_info.max_nb_queue_pairs) < num_lcores) { + SPDK_ERRLOG("Insufficient unique queue pairs available for %s\n", + device->cdev_info.driver_name); + SPDK_ERRLOG("Either add more crypto devices or decrease core count\n"); + rc = -EINVAL; + goto error_qp; + } + + /* Setup queue pairs. */ + struct rte_cryptodev_config conf = { + .nb_queue_pairs = device->cdev_info.max_nb_queue_pairs, + .socket_id = SPDK_ENV_SOCKET_ID_ANY + }; + + rc = rte_cryptodev_configure(cdev_id, &conf); + if (rc < 0) { + SPDK_ERRLOG("Failed to configure cryptodev %u", cdev_id); + rc = -EINVAL; + goto error_dev_config; + } + + struct rte_cryptodev_qp_conf qp_conf = { + .nb_descriptors = CRYPTO_QP_DESCRIPTORS + }; + + /* Pre-setup all pottential qpairs now and assign them in the channel + * callback. If we were to create them there, we'd have to stop the + * entire device affecting all other threads that might be using it + * even on other queue pairs. + */ + for (j = 0; j < device->cdev_info.max_nb_queue_pairs; j++) { + rc = rte_cryptodev_queue_pair_setup(cdev_id, j, &qp_conf, SOCKET_ID_ANY, + (struct rte_mempool *)g_session_mp); + + if (rc < 0) { + SPDK_ERRLOG("Failed to setup queue pair %u on " + "cryptodev %u", j, cdev_id); + rc = -EINVAL; + goto error_qp_setup; + } + } + + rc = rte_cryptodev_start(cdev_id); + if (rc < 0) { + SPDK_ERRLOG("Failed to start device %u: error %d\n", + cdev_id, rc); + rc = -EINVAL; + goto error_device_start; + } + + /* Add to our list of available crypto devices. */ + TAILQ_INSERT_TAIL(&g_vbdev_devs, device, link); + + /* Build up list of device/qp combinations */ + for (j = 0; j < device->cdev_info.max_nb_queue_pairs; j++) { + dev_qp = calloc(1, sizeof(struct device_qp)); + if (!dev_qp) { + rc = -ENOMEM; + goto error_create_devqp; + } + dev_qp->device = device; + dev_qp->qp = j; + dev_qp->in_use = false; + TAILQ_INSERT_TAIL(&g_device_qp, dev_qp, link); + } + } + return 0; + + /* Error cleanup paths. */ +error_create_devqp: + while ((dev_qp = TAILQ_FIRST(&g_device_qp))) { + TAILQ_REMOVE(&g_device_qp, dev_qp, link); + free(dev_qp); + } +error_device_start: +error_qp_setup: +error_dev_config: +error_qp: + free(device); +error_create_device: + rte_mempool_free(g_crypto_op_mp); +error_create_op: + spdk_mempool_free(g_mbuf_mp); +error_create_mbuf: + spdk_mempool_free(g_session_mp); + return rc; +} + +/* Following an encrypt or decrypt we need to then either write the encrypted data or finish + * the read on decrypted data. Do that here. + */ +static void +_crypto_operation_complete(struct spdk_bdev_io *bdev_io) +{ + struct vbdev_crypto *crypto_bdev = SPDK_CONTAINEROF(bdev_io->bdev, struct vbdev_crypto, + crypto_bdev); + struct crypto_bdev_io *io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; + struct crypto_io_channel *crypto_ch = io_ctx->crypto_ch; + struct spdk_bdev_io *free_me = io_ctx->read_io; + int rc = 0; + + if (bdev_io->internal.status != SPDK_BDEV_IO_STATUS_FAILED) { + if (bdev_io->type == SPDK_BDEV_IO_TYPE_READ) { + + /* Complete the original IO and then free the one that we created + * as a result of issuing an IO via submit_reqeust. + */ + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); + spdk_bdev_free_io(free_me); + + } else if (bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE) { + + /* Write the encrypted data. */ + rc = spdk_bdev_writev_blocks(crypto_bdev->base_desc, crypto_ch->base_ch, + &io_ctx->cry_iov, 1, io_ctx->cry_offset_blocks, + io_ctx->cry_num_blocks, _complete_internal_write, + bdev_io); + } else { + + /* Something really went haywire if this function got called with a type + * other than read or write. + */ + rc = -1; + } + } else { + /* If the poller found that one of the crypto ops had failed as part of this + * bdev_io it would have updated the internal status indicate failure. + */ + rc = -1; + } + + if (rc != 0) { + SPDK_ERRLOG("ERROR on crypto operation completion!\n"); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + } + +} + +/* This is the poller for the crypto device. It uses a single API to dequeue whatever is ready at + * the device. Then we need to decide if what we've got so far (including previous poller + * runs) totals up to one or more complete bdev_ios and if so continue with the bdev_io + * accordingly. This means either completing a read or issuing a new write. + */ +static int +crypto_dev_poller(void *args) +{ + struct crypto_io_channel *crypto_ch = args; + uint8_t cdev_id = crypto_ch->device_qp->device->cdev_id; + int i, num_dequeued_ops; + struct spdk_bdev_io *bdev_io = NULL; + struct crypto_bdev_io *io_ctx = NULL; + struct rte_crypto_op *dequeued_ops[MAX_DEQUEUE_BURST_SIZE]; + + /* Each run of the poller will get just what the device has available + * at the moment we call it, we don't check again after draining the + * first batch. + */ + num_dequeued_ops = rte_cryptodev_dequeue_burst(cdev_id, crypto_ch->device_qp->qp, + dequeued_ops, MAX_DEQUEUE_BURST_SIZE); + + /* Check if operation was processed successfully */ + for (i = 0; i < num_dequeued_ops; i++) { + + /* We don't know the order or association of the crypto ops wrt any + * partiular bdev_io so need to look at each and determine if it's + * the last one for it's bdev_io or not. + */ + bdev_io = (struct spdk_bdev_io *)dequeued_ops[i]->sym->m_src->userdata; + assert(bdev_io != NULL); + + if (dequeued_ops[i]->status != RTE_CRYPTO_OP_STATUS_SUCCESS) { + SPDK_ERRLOG("error with op %d status %u\n", i, + dequeued_ops[i]->status); + /* Update the bdev status to error, we'll still process the + * rest of the crypto ops for this bdev_io though so they + * aren't left hanging. + */ + bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED; + } + + io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; + assert(io_ctx->cryop_cnt_remaining > 0); + + /* return the associated src_mbufs */ + dequeued_ops[i]->sym->m_src->userdata = NULL; + spdk_mempool_put(g_mbuf_mp, dequeued_ops[i]->sym->m_src); + + /* For encryption, free the mbuf we used to encrypt, the data buffer + * will be freed on write completion. + */ + if (dequeued_ops[i]->sym->m_dst) { + spdk_mempool_put(g_mbuf_mp, dequeued_ops[i]->sym->m_dst); + } + + /* done encrypting, complete the bdev_io */ + if (--io_ctx->cryop_cnt_remaining == 0) { + + /* Complete the IO */ + _crypto_operation_complete(bdev_io); + + /* Return session */ + rte_cryptodev_sym_session_clear(cdev_id, dequeued_ops[i]->sym->session); + rte_cryptodev_sym_session_free(dequeued_ops[i]->sym->session); + } + + /* Free the operation */ + rte_crypto_op_free(dequeued_ops[i]); + } + return num_dequeued_ops; +} + +/* We're either encrypting on the way down or decrypting on the way back. */ +static int +_crypto_operation(struct spdk_bdev_io *bdev_io, enum rte_crypto_cipher_operation crypto_op) +{ + struct rte_cryptodev_sym_session *session; + uint16_t num_enqueued_ops = 0; + uint32_t cryop_cnt; + struct crypto_bdev_io *io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; + struct crypto_io_channel *crypto_ch = io_ctx->crypto_ch; + uint8_t cdev_id = crypto_ch->device_qp->device->cdev_id; + uint32_t crypto_len = io_ctx->crypto_bdev->crypto_bdev.blocklen; + uint64_t total_length = bdev_io->u.bdev.num_blocks * crypto_len; + int rc; + uint32_t enqueued = 0; + uint32_t iov_index = 0; + uint32_t allocated = 0; + uint8_t *current_iov = NULL; + uint64_t total_remaining = 0; + uint64_t current_iov_remaining = 0; + int completed = 0; + int crypto_index = 0; + uint32_t iov_offset = 0; + uint32_t en_offset = 0; + uint8_t *iv_ptr = NULL; + uint64_t op_block_offset, current_len; + struct rte_crypto_op *crypto_ops[MAX_ENQUEUE_ARRAY_SIZE]; + struct rte_mbuf *src_mbufs[MAX_ENQUEUE_ARRAY_SIZE]; + struct rte_mbuf *dst_mbufs[MAX_ENQUEUE_ARRAY_SIZE]; + int burst; + + assert((bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen) <= CRYPTO_MAX_IO); + + /* We fix the size of a crypto op to an LBA so we can use LBA as init vector (IV) + * for the cipher. + */ + cryop_cnt = total_length / crypto_len; + + /* Get the number of source mbufs that we need. These will always be 1:1 because we + * don't support chaining. The reason we don't is because of our decision to use + * LBA as IV, there can be no case where we'd need >1 mbuf per crypto op or the + * op would be > 1 LBA. + */ + rc = spdk_mempool_get_bulk(g_mbuf_mp, (void **)&src_mbufs[0], cryop_cnt); + if (rc) { + SPDK_ERRLOG("ERROR trying to get src_mbufs!\n"); + return -ENOMEM; + } + + /* Get the same amount but these buffers to describe the encrypted data location (dst). */ + if (crypto_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + rc = spdk_mempool_get_bulk(g_mbuf_mp, (void **)&dst_mbufs[0], cryop_cnt); + if (rc) { + SPDK_ERRLOG("ERROR trying to get dst_mbufs!\n"); + rc = -ENOMEM; + goto error_get_dst; + } + } + + /* Allocate crypto operations. */ + allocated = rte_crypto_op_bulk_alloc(g_crypto_op_mp, + RTE_CRYPTO_OP_TYPE_SYMMETRIC, + crypto_ops, cryop_cnt); + if (allocated < cryop_cnt) { + SPDK_ERRLOG("ERROR trying to get crypto ops!\n"); + rc = -ENOMEM; + goto error_get_ops; + } + + /* Get sessions. */ + session = rte_cryptodev_sym_session_create((struct rte_mempool *)g_session_mp); + if (NULL == session) { + SPDK_ERRLOG("ERROR trying to create crypto session!\n"); + rc = -EINVAL; + goto error_session_create; + } + + /* Init our session with the desired cipher options. */ + io_ctx->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + io_ctx->cipher_xform.cipher.key.data = io_ctx->crypto_bdev->key; + io_ctx->cipher_xform.cipher.op = io_ctx->crypto_op = crypto_op; + io_ctx->cipher_xform.cipher.iv.offset = IV_OFFSET; + io_ctx->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC; + io_ctx->cipher_xform.cipher.key.length = AES_CBC_KEY_LENGTH; + io_ctx->cipher_xform.cipher.iv.length = AES_CBC_IV_LENGTH; + + rc = rte_cryptodev_sym_session_init(cdev_id, session, + &io_ctx->cipher_xform, + (struct rte_mempool *)g_session_mp); + if (rc < 0) { + SPDK_ERRLOG("ERROR trying to init crypto session!\n"); + rc = -EINVAL; + goto error_session_init; + } + + /* For encryption, we need to prepare a single contiguous buffer as the encryption + * destination, we'll then pass that along for the write after encryption is done. + * This is done to avoiding encrypting the provided write buffer which may be + * undesirable in some use cases. + */ + + if (crypto_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + io_ctx->cry_iov.iov_len = total_length; + /* For now just allocate in the I/O path, not optimal but the current bdev API + * for getting a buffer from the pool won't work if the bdev_io passed in + * has a buffer, which ours always will. So, until we modify that API + * or better yet the current ZCOPY work lands, this is the best we can do. + */ + io_ctx->cry_iov.iov_base = spdk_dma_malloc(total_length, 0x10, NULL); + if (!io_ctx->cry_iov.iov_base) { + SPDK_ERRLOG("ERROR trying to allocate write buffer for encryption!\n"); + rc = -ENOMEM; + goto error_get_write_buffer; + } + io_ctx->cry_offset_blocks = bdev_io->u.bdev.offset_blocks; + io_ctx->cry_num_blocks = bdev_io->u.bdev.num_blocks; + } + + /* This value is used in the completion callback to determine when the bdev_io is + * complete. + */ + io_ctx->cryop_cnt_remaining = cryop_cnt; + + /* As we don't support chaining because of a decision to use LBA as IV, construction + * of crypto operaations is straightforward. We build both the op, the mbuf and the + * dst_mbuf in our local arrays by looping through the length of the bdev IO and + * picking off LBA sized blocks of memory from the IOVs as we walk through them. Each + * LBA sized chunck of memory will correspond 1:1 to a crypto operation and a single + * mbuf per crypto operation. + */ + total_remaining = total_length; + current_iov = bdev_io->u.bdev.iovs[iov_index].iov_base; + current_iov_remaining = bdev_io->u.bdev.iovs[iov_index].iov_len; + do { + /* Set the mbuf elements address and length. */ + src_mbufs[crypto_index]->buf_addr = current_iov; + src_mbufs[crypto_index]->buf_iova = spdk_vtophys((void *)current_iov); + current_len = spdk_min(total_remaining, crypto_len); + current_len = spdk_min(current_iov_remaining, current_len); + src_mbufs[crypto_index]->data_len = current_len; + + /* If we have an IOV that is not a block multiple, our use of LBA as IV + * doesn't work. This can be addressed in the future but only with crypto + * drivers that support chaining or some ugly internal double buffering + * to create LBA sized IOVs. + */ + if (current_len % crypto_len != 0) { + SPDK_ERRLOG("Fatal error, unsupported IOV makeup.\n"); + rc = -EINVAL; + goto error_iov_makeup; + } + + /* Subtract our running totals for the op in progress and the overall bdev io */ + total_remaining -= current_len; + current_iov_remaining -= current_len; + + /* Set the IV - we use the LBA of the crypto_op */ + iv_ptr = rte_crypto_op_ctod_offset(crypto_ops[crypto_index], uint8_t *, + IV_OFFSET); + memset(iv_ptr, 0, AES_CBC_IV_LENGTH); + op_block_offset = bdev_io->u.bdev.offset_blocks + (iov_offset / crypto_len); + rte_memcpy(iv_ptr, &op_block_offset, sizeof(uint64_t)); + + /* move our current IOV pointer accordingly and track the offset for IV calc */ + current_iov += current_len; + iov_offset += current_len; + src_mbufs[crypto_index]->next = NULL; + + /* Set the data to encrypt/decrypt length */ + crypto_ops[crypto_index]->sym->cipher.data.length = current_len; + crypto_ops[crypto_index]->sym->cipher.data.offset = 0; + + /* Store context in every mbuf as we don't know anything about completion order */ + src_mbufs[crypto_index]->userdata = bdev_io; + + /* link the mbuf to the crypto op. */ + crypto_ops[crypto_index]->sym->m_src = src_mbufs[crypto_index]; + if (crypto_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + crypto_ops[crypto_index]->sym->m_dst = src_mbufs[crypto_index]; + } else { + crypto_ops[crypto_index]->sym->m_dst = NULL; + } + + /* For encrypt, point the destination to a buffer we allocate and redirect the bdev_io + * that will be used to process the write on completion to the same buffer. Setting + * up the en_buffer is a little simpler as we know the destination buffer is single IOV. + */ + if (crypto_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + + /* Set the relevant destination en_mbuf elements. */ + dst_mbufs[crypto_index]->buf_addr = io_ctx->cry_iov.iov_base + en_offset; + dst_mbufs[crypto_index]->buf_iova = spdk_vtophys(dst_mbufs[crypto_index]->buf_addr); + dst_mbufs[crypto_index]->data_len = current_len; + crypto_ops[crypto_index]->sym->m_dst = dst_mbufs[crypto_index]; + en_offset += current_len; + dst_mbufs[crypto_index]->next = NULL; + } + + /* Attach the crypto session to the operation */ + rc = rte_crypto_op_attach_sym_session(crypto_ops[crypto_index], session); + if (rc) { + rc = -EINVAL; + goto error_attach_session; + } + + /* If we're done with this IOV, move to the next one. */ + if (current_iov_remaining == 0 && total_remaining > 0) { + iov_index++; + current_iov = bdev_io->u.bdev.iovs[iov_index].iov_base; + current_iov_remaining = bdev_io->u.bdev.iovs[iov_index].iov_len; + } + + /* move on to the next crypto operation */ + crypto_index++; + + } while (total_remaining > 0); + + /* Enqueue everything we've got but limit by the max number of descriptors we + * configured the crypto device for. + */ + do { + burst = spdk_min((cryop_cnt - enqueued), CRYPTO_QP_DESCRIPTORS); + num_enqueued_ops = rte_cryptodev_enqueue_burst(cdev_id, crypto_ch->device_qp->qp, + &crypto_ops[enqueued], + burst); + enqueued += num_enqueued_ops; + + /* Dequeue all inline if the device is full. We don't defer anything simply + * because of the complexity involved as we're building 1 or more crypto + * ops per IO. Dequeue will free up space for more enqueue. + */ + if (enqueued < cryop_cnt) { + + /* Dequeue everything, this may include ops that were already + * in the device before this submission.... + */ + do { + completed = crypto_dev_poller(crypto_ch); + } while (completed > 0); + } + } while (enqueued < cryop_cnt); + + return rc; + + /* Error cleanup paths. */ +error_attach_session: +error_iov_makeup: +error_get_write_buffer: +error_session_init: + rte_cryptodev_sym_session_clear(cdev_id, session); + rte_cryptodev_sym_session_free(session); +error_session_create: + rte_mempool_put_bulk(g_crypto_op_mp, (void **)crypto_ops, cryop_cnt); + allocated = 0; +error_get_ops: + if (crypto_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + spdk_mempool_put_bulk(g_mbuf_mp, (void **)&dst_mbufs[0], + cryop_cnt); + } + if (allocated > 0) { + rte_mempool_put_bulk(g_crypto_op_mp, (void **)crypto_ops, + allocated); + } +error_get_dst: + spdk_mempool_put_bulk(g_mbuf_mp, (void **)&src_mbufs[0], + cryop_cnt); + return rc; +} + +/* Completion callback for IO that were issued from this bdev other than read/write. + * They have their own for readability. + */ +static void +_complete_internal_io(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct spdk_bdev_io *orig_io = cb_arg; + int status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; + + spdk_bdev_io_complete(orig_io, status); + spdk_bdev_free_io(bdev_io); +} + +/* Completion callback for writes that were issued from this bdev. */ +static void +_complete_internal_write(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct spdk_bdev_io *orig_io = cb_arg; + int status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED; + struct crypto_bdev_io *orig_ctx = (struct crypto_bdev_io *)orig_io->driver_ctx; + + spdk_dma_free(orig_ctx->cry_iov.iov_base); + spdk_bdev_io_complete(orig_io, status); + spdk_bdev_free_io(bdev_io); +} + +/* Completion callback for reads that were issued from this bdev. */ +static void +_complete_internal_read(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct spdk_bdev_io *orig_io = cb_arg; + struct crypto_bdev_io *orig_ctx = (struct crypto_bdev_io *)orig_io->driver_ctx; + + if (success) { + + /* Save off this bdev_io so it can be freed after decryption. */ + orig_ctx->read_io = bdev_io; + + if (_crypto_operation(orig_io, RTE_CRYPTO_CIPHER_OP_DECRYPT)) { + SPDK_ERRLOG("ERROR decrypting"); + spdk_bdev_io_complete(orig_io, SPDK_BDEV_IO_STATUS_FAILED); + spdk_bdev_free_io(bdev_io); + } + } else { + SPDK_ERRLOG("ERROR on read prior to decrypting"); + spdk_bdev_io_complete(orig_io, SPDK_BDEV_IO_STATUS_FAILED); + spdk_bdev_free_io(bdev_io); + } +} + +/* Callback for getting a buf from the bdev pool in the event that the caller passed + * in NULL, we need to own the buffer so it doesn't get freed by another vbdev module + * beneath us before we're done with it. + */ +static void +crypto_read_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) +{ + struct vbdev_crypto *crypto_bdev = SPDK_CONTAINEROF(bdev_io->bdev, struct vbdev_crypto, + crypto_bdev); + struct crypto_io_channel *crypto_ch = spdk_io_channel_get_ctx(ch); + int rc; + + rc = spdk_bdev_readv_blocks(crypto_bdev->base_desc, crypto_ch->base_ch, bdev_io->u.bdev.iovs, + bdev_io->u.bdev.iovcnt, bdev_io->u.bdev.offset_blocks, + bdev_io->u.bdev.num_blocks, _complete_internal_read, + bdev_io); + if (rc != 0) { + SPDK_ERRLOG("ERROR on bdev_io submission!\n"); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + } +} + +/* Called when someone submits IO to this crypto vbdev. For IO's not relevant to crypto, + * we're simply passing it on here via SPDK IO calls which in turn allocate another bdev IO + * and call our cpl callback provided below along with the original bdev_io so that we can + * complete it once this IO completes. For crypto operations, we'll either encrypt it first + * (writes) then call back into bdev to submit it or we'll submit a read and then catch it + * on the way back for decryption. + */ +static void +vbdev_crypto_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) +{ + struct vbdev_crypto *crypto_bdev = SPDK_CONTAINEROF(bdev_io->bdev, struct vbdev_crypto, + crypto_bdev); + struct crypto_io_channel *crypto_ch = spdk_io_channel_get_ctx(ch); + struct crypto_bdev_io *io_ctx = (struct crypto_bdev_io *)bdev_io->driver_ctx; + int rc = 0; + + memset(io_ctx, 0, sizeof(struct crypto_bdev_io)); + io_ctx->crypto_bdev = crypto_bdev; + io_ctx->crypto_ch = crypto_ch; + io_ctx->orig_io = bdev_io; + + switch (bdev_io->type) { + case SPDK_BDEV_IO_TYPE_READ: + spdk_bdev_io_get_buf(bdev_io, crypto_read_get_buf_cb, + bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); + break; + case SPDK_BDEV_IO_TYPE_WRITE: + rc = _crypto_operation(bdev_io, RTE_CRYPTO_CIPHER_OP_ENCRYPT); + break; + case SPDK_BDEV_IO_TYPE_UNMAP: + rc = spdk_bdev_unmap_blocks(crypto_bdev->base_desc, crypto_ch->base_ch, + bdev_io->u.bdev.offset_blocks, + bdev_io->u.bdev.num_blocks, + _complete_internal_io, bdev_io); + break; + case SPDK_BDEV_IO_TYPE_FLUSH: + rc = spdk_bdev_flush_blocks(crypto_bdev->base_desc, crypto_ch->base_ch, + bdev_io->u.bdev.offset_blocks, + bdev_io->u.bdev.num_blocks, + _complete_internal_io, bdev_io); + break; + case SPDK_BDEV_IO_TYPE_RESET: + rc = spdk_bdev_reset(crypto_bdev->base_desc, crypto_ch->base_ch, + _complete_internal_io, bdev_io); + break; + case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: + default: + SPDK_ERRLOG("crypto: unknown I/O type %d\n", bdev_io->type); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + return; + } + + if (rc != 0) { + SPDK_ERRLOG("ERROR on bdev_io submission!\n"); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + } +} + +/* We'll just call the base bdev and let it answer except for WZ command which + * we always say we don't support so that the bdev layer will actually send us + * real writes that we can encrypt. + */ +static bool +vbdev_crypto_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) +{ + struct vbdev_crypto *crypto_bdev = (struct vbdev_crypto *)ctx; + + /* Force the bdev layer to issue actual writes of zeroes so we can + * encrypt them as regular writes. + */ + if (io_type == SPDK_BDEV_IO_TYPE_WRITE_ZEROES) { + return false; + } + return spdk_bdev_io_type_supported(crypto_bdev->base_bdev, io_type); +} + +/* Called after we've unregistered following a hot remove callback. + * Our finish entry point will be called next. + */ +static int +vbdev_crypto_destruct(void *ctx) +{ + struct vbdev_crypto *crypto_bdev = (struct vbdev_crypto *)ctx; + + /* Unclaim the underlying bdev. */ + spdk_bdev_module_release_bdev(crypto_bdev->base_bdev); + + /* Close the underlying bdev. */ + spdk_bdev_close(crypto_bdev->base_desc); + + /* Done with this crypto_bdev. */ + TAILQ_REMOVE(&g_vbdev_crypto, crypto_bdev, link); + free(crypto_bdev->drv_name); + free(crypto_bdev->key); + free(crypto_bdev->crypto_bdev.name); + free(crypto_bdev); + return 0; +} + +/* We supplied this as an entry point for upper layers who want to communicate to this + * bdev. This is how they get a channel. We are passed the same context we provided when + * we created our crypto vbdev in examine() which, for this bdev, is the address of one of + * our context nodes. From here we'll ask the SPDK channel code to fill out our channel + * struct and we'll keep it in our crypto node. + */ +static struct spdk_io_channel * +vbdev_crypto_get_io_channel(void *ctx) +{ + struct vbdev_crypto *crypto_bdev = (struct vbdev_crypto *)ctx; + + /* The IO channel code will allocate a channel for us which consists of + * the SPDK cahnnel structure plus the size of our crypto_io_channel struct + * that we passed in when we registered our IO device. It will then call + * our channel create callback to populate any elements that we need to + * update. + */ + return spdk_get_io_channel(crypto_bdev); +} + +/* This is the output for get_bdevs() for this vbdev */ +static int +vbdev_crypto_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) +{ + struct vbdev_crypto *crypto_bdev = (struct vbdev_crypto *)ctx; + + spdk_json_write_name(w, "crypto"); + spdk_json_write_object_begin(w); + spdk_json_write_name(w, "crypto_bdev_name"); + spdk_json_write_string(w, spdk_bdev_get_name(&crypto_bdev->crypto_bdev)); + spdk_json_write_name(w, "base_bdev_name"); + spdk_json_write_string(w, spdk_bdev_get_name(crypto_bdev->base_bdev)); + spdk_json_write_name(w, "crypto_pmd"); + spdk_json_write_string(w, crypto_bdev->drv_name); + spdk_json_write_name(w, "key"); + spdk_json_write_string(w, crypto_bdev->key); + spdk_json_write_object_end(w); + return 0; +} + +static int +vbdev_crypto_config_json(struct spdk_json_write_ctx *w) +{ + struct vbdev_crypto *crypto_bdev, *tmp; + + TAILQ_FOREACH_SAFE(crypto_bdev, &g_vbdev_crypto, link, tmp) { + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "method", "construct_crypto_bdev"); + spdk_json_write_named_object_begin(w, "params"); + spdk_json_write_named_string(w, "base_bdev_name", spdk_bdev_get_name(crypto_bdev->base_bdev)); + spdk_json_write_named_string(w, "crypto_bdev_name", spdk_bdev_get_name(&crypto_bdev->crypto_bdev)); + spdk_json_write_named_string(w, "crypto_pmd", crypto_bdev->drv_name); + spdk_json_write_named_string(w, "key", crypto_bdev->key); + spdk_json_write_object_end(w); + spdk_json_write_object_end(w); + } + return 0; +} + +/* We provide this callback for the SPDK channel code to create a channel using + * the channel struct we provided in our module get_io_channel() entry point. Here + * we get and save off an underlying base channel of the device below us so that + * we can communicate with the base bdev on a per channel basis. We also register the + * poller used to complete crypto operations from the device. + */ +static int +crypto_bdev_ch_create_cb(void *io_device, void *ctx_buf) +{ + struct crypto_io_channel *crypto_ch = ctx_buf; + struct vbdev_crypto *crypto_bdev = io_device; + struct device_qp *device_qp; + + crypto_ch->base_ch = spdk_bdev_get_io_channel(crypto_bdev->base_desc); + crypto_ch->poller = spdk_poller_register(crypto_dev_poller, crypto_ch, 0); + crypto_ch->device_qp = NULL; + + pthread_mutex_lock(&g_device_qp_lock); + TAILQ_FOREACH(device_qp, &g_device_qp, link) { + if ((strcmp(device_qp->device->cdev_info.driver_name, crypto_bdev->drv_name) == 0) && + (device_qp->in_use == false)) { + crypto_ch->device_qp = device_qp; + device_qp->in_use = true; + SPDK_NOTICELOG("Device queue pair assignment: ch %p device %p qpid %u %s\n", + crypto_ch, device_qp->device, crypto_ch->device_qp->qp, crypto_bdev->drv_name); + break; + } + } + pthread_mutex_unlock(&g_device_qp_lock); + assert(crypto_ch->device_qp); + return 0; +} + +/* We provide this callback for the SPDK channel code to destroy a channel + * created with our create callback. We just need to undo anything we did + * when we created. + */ +static void +crypto_bdev_ch_destroy_cb(void *io_device, void *ctx_buf) +{ + struct crypto_io_channel *crypto_ch = ctx_buf; + + pthread_mutex_lock(&g_device_qp_lock); + crypto_ch->device_qp->in_use = false; + pthread_mutex_unlock(&g_device_qp_lock); + + spdk_poller_unregister(&crypto_ch->poller); + spdk_put_io_channel(crypto_ch->base_ch); +} + +/* Create the association from the bdev and vbdev name and insert + * on the global list. */ +static int +vbdev_crypto_insert_name(const char *bdev_name, const char *vbdev_name, + const char *crypto_pmd, const char *key) +{ + struct bdev_names *name; + int rc; + + name = calloc(1, sizeof(struct bdev_names)); + if (!name) { + SPDK_ERRLOG("could not allocate bdev_names\n"); + return -ENOMEM; + } + + name->bdev_name = strdup(bdev_name); + if (!name->bdev_name) { + SPDK_ERRLOG("could not allocate name->bdev_name\n"); + rc = -ENOMEM; + goto error_alloc_bname; + } + + name->vbdev_name = strdup(vbdev_name); + if (!name->vbdev_name) { + SPDK_ERRLOG("could not allocate name->vbdev_name\n"); + rc = -ENOMEM; + goto error_alloc_vname; + } + + name->drv_name = strdup(crypto_pmd); + if (!name->drv_name) { + SPDK_ERRLOG("could not allocate name->drv_name\n"); + rc = -ENOMEM; + goto error_alloc_dname; + } + + name->key = strdup(key); + if (!name->key) { + SPDK_ERRLOG("could not allocate name->key\n"); + rc = -ENOMEM; + goto error_alloc_key; + } + if (strlen(name->key) != AES_CBC_KEY_LENGTH) { + SPDK_ERRLOG("invalid AES_CCB key length\n"); + rc = -EINVAL; + goto error_invalid_key; + } + + TAILQ_INSERT_TAIL(&g_bdev_names, name, link); + + return 0; + + /* Error cleanup paths. */ +error_invalid_key: +error_alloc_key: + free(name->drv_name); +error_alloc_dname: + free(name->vbdev_name); +error_alloc_vname: + free(name->bdev_name); +error_alloc_bname: + free(name); + return rc; +} + +/* RPC entry point for crypto creation. */ +int +create_crypto_disk(const char *bdev_name, const char *vbdev_name, + const char *crypto_pmd, const char *key) +{ + struct spdk_bdev *bdev = NULL; + struct vbdev_crypto *crypto_bdev, *tmp; + int rc = 0; + + bdev = spdk_bdev_get_by_name(bdev_name); + if (!bdev) { + return -1; + } + + rc = vbdev_crypto_insert_name(bdev_name, vbdev_name, crypto_pmd, key); + if (rc != 0) { + return rc; + } + + vbdev_crypto_claim(bdev); + + rc = vbdev_crypto_init_crypto_drivers(); + if (rc) { + SPDK_ERRLOG("Error setting up crypto devices\n"); + return rc; + } + + TAILQ_FOREACH_SAFE(crypto_bdev, &g_vbdev_crypto, link, tmp) { + if (strcmp(crypto_bdev->base_bdev->name, bdev->name) == 0) { + rc = spdk_vbdev_register(&crypto_bdev->crypto_bdev, + &crypto_bdev->base_bdev, 1); + if (rc) { + SPDK_ERRLOG("could not register crypto_bdev\n"); + spdk_bdev_close(crypto_bdev->base_desc); + TAILQ_REMOVE(&g_vbdev_crypto, crypto_bdev, link); + free(crypto_bdev->crypto_bdev.name); + free(crypto_bdev->key); + free(crypto_bdev); + } + break; + } + } + + return rc; +} + +/* Called at driver init time, parses config file to preapre for examine calls, + * also fully initializes the crypto drivers. + */ +static int +vbdev_crypto_init(void) +{ + struct spdk_conf_section *sp = NULL; + const char *conf_bdev_name = NULL; + const char *conf_vbdev_name = NULL; + const char *crypto_pmd = NULL; + bool found = false; + int i, j; + int rc = 0; + const char *key = NULL; + + sp = spdk_conf_find_section(NULL, "crypto"); + if (sp == NULL) { + return 0; + } + + for (i = 0; ; i++) { + + if (!spdk_conf_section_get_nval(sp, "CRY", i)) { + break; + } + + conf_bdev_name = spdk_conf_section_get_nmval(sp, "CRY", i, 0); + if (!conf_bdev_name) { + SPDK_ERRLOG("crypto configuration missing bdev name\n"); + return -EINVAL; + } + + conf_vbdev_name = spdk_conf_section_get_nmval(sp, "CRY", i, 1); + if (!conf_vbdev_name) { + SPDK_ERRLOG("crypto configuration missing crypto_bdev name\n"); + return -EINVAL; + } + + key = spdk_conf_section_get_nmval(sp, "CRY", i, 2); + if (!key) { + SPDK_ERRLOG("crypto configuration missing crypto_bdev key\n"); + return -EINVAL; + } + SPDK_NOTICELOG("WARNING: You are storing your key in a plain text file!!\n"); + + crypto_pmd = spdk_conf_section_get_nmval(sp, "CRY", i, 3); + if (!crypto_pmd) { + SPDK_ERRLOG("crypto configuration missing driver type\n"); + return -EINVAL; + } + + for (j = 0; j < MAX_NUM_DRV_TYPES ; j++) { + if (strcmp(crypto_pmd, g_driver_names[j]) == 0) { + found = true; + break; + } + } + if (!found) { + SPDK_ERRLOG("crypto configuration invalid PMD type\n"); + return -EINVAL; + } + + rc = vbdev_crypto_insert_name(conf_bdev_name, conf_vbdev_name, + crypto_pmd, key); + if (rc != 0) { + return rc; + } + } + + /* Fully configure both SW and HW drivers. */ + rc = vbdev_crypto_init_crypto_drivers(); + if (rc) { + SPDK_ERRLOG("Error setting up crypto devices\n"); + return rc; + } + + return rc; +} + +/* Called when the entire module is being torn down. */ +static void +vbdev_crypto_finish(void) +{ + struct bdev_names *name; + struct vbdev_dev *device; + struct device_qp *dev_qp; + + while ((name = TAILQ_FIRST(&g_bdev_names))) { + TAILQ_REMOVE(&g_bdev_names, name, link); + free(name->drv_name); + free(name->key); + free(name->bdev_name); + free(name->vbdev_name); + free(name); + } + + while ((device = TAILQ_FIRST(&g_vbdev_devs))) { + TAILQ_REMOVE(&g_vbdev_devs, device, link); + rte_cryptodev_stop(device->cdev_id); + free(device); + } + + while ((dev_qp = TAILQ_FIRST(&g_device_qp))) { + TAILQ_REMOVE(&g_device_qp, dev_qp, link); + free(dev_qp); + } +} + +/* During init we'll be asked how much memory we'd like passed to us + * in bev_io structures as context. Here's where we specify how + * much context we want per IO. + */ +static int +vbdev_crypto_get_ctx_size(void) +{ + return sizeof(struct crypto_bdev_io); +} + +/* Called when SPDK wants to save the current config of this vbdev module to + * a file. + */ +static void +vbdev_crypto_get_spdk_running_config(FILE *fp) +{ + struct bdev_names *names = NULL; + fprintf(fp, "\n[crypto]\n"); + TAILQ_FOREACH(names, &g_bdev_names, link) { + fprintf(fp, " crypto %s %s ", names->bdev_name, names->vbdev_name); + fprintf(fp, "\n"); + } + + fprintf(fp, "\n"); +} + +/* Called when the underlying base bdev goes away. */ +static void +vbdev_crypto_examine_hotremove_cb(void *ctx) +{ + struct vbdev_crypto *crypto_bdev, *tmp; + struct spdk_bdev *bdev_find = ctx; + + TAILQ_FOREACH_SAFE(crypto_bdev, &g_vbdev_crypto, link, tmp) { + if (bdev_find == crypto_bdev->base_bdev) { + spdk_bdev_unregister(&crypto_bdev->crypto_bdev, NULL, NULL); + } + } +} + +static void +vbdev_crypto_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) +{ + /* No config per bdev needed */ +} + +/* When we register our bdev this is how we specify our entry points. */ +static const struct spdk_bdev_fn_table vbdev_crypto_fn_table = { + .destruct = vbdev_crypto_destruct, + .submit_request = vbdev_crypto_submit_request, + .io_type_supported = vbdev_crypto_io_type_supported, + .get_io_channel = vbdev_crypto_get_io_channel, + .dump_info_json = vbdev_crypto_dump_info_json, + .write_config_json = vbdev_crypto_write_config_json +}; + +static struct spdk_bdev_module crypto_if = { + .name = "crypto", + .module_init = vbdev_crypto_init, + .config_text = vbdev_crypto_get_spdk_running_config, + .get_ctx_size = vbdev_crypto_get_ctx_size, + .examine_config = vbdev_crypto_examine, + .module_fini = vbdev_crypto_finish, + .config_json = vbdev_crypto_config_json +}; + +SPDK_BDEV_MODULE_REGISTER(&crypto_if) + +static void +vbdev_crypto_claim(struct spdk_bdev *bdev) +{ + struct bdev_names *name; + struct vbdev_crypto *vbdev; + int rc; + + /* Check our list of names from config versus this bdev and if + * there's a match, create the crypto_bdev & bdev accordingly. + */ + TAILQ_FOREACH(name, &g_bdev_names, link) { + if (strcmp(name->bdev_name, bdev->name) != 0) { + continue; + } + + SPDK_NOTICELOG("Match on %s\n", bdev->name); + vbdev = calloc(1, sizeof(struct vbdev_crypto)); + if (!vbdev) { + SPDK_ERRLOG("could not allocate crypto_bdev\n"); + break; + } + + /* The base bdev that we're attaching to. */ + vbdev->base_bdev = bdev; + vbdev->crypto_bdev.name = strdup(name->vbdev_name); + if (!vbdev->crypto_bdev.name) { + SPDK_ERRLOG("could not allocate crypto_bdev name\n"); + goto error_bdev_name; + } + + vbdev->key = strdup(name->key); + if (!vbdev->key) { + SPDK_ERRLOG("could not allocate crypto_bdev key\n"); + goto error_alloc_key; + } + + vbdev->drv_name = strdup(name->drv_name); + if (!vbdev->drv_name) { + SPDK_ERRLOG("could not allocate crypto_bdev drv_name\n"); + goto error_drv_name; + } + + vbdev->crypto_bdev.product_name = "crypto"; + vbdev->crypto_bdev.write_cache = bdev->write_cache; + vbdev->crypto_bdev.need_aligned_buffer = bdev->need_aligned_buffer; + /* Note: CRYPTO_MAX_IO is in units of bytes, optimal_io_boundary is + * in units of blocks. + */ + if (bdev->optimal_io_boundary > 0) { + vbdev->crypto_bdev.optimal_io_boundary = + spdk_min((CRYPTO_MAX_IO / bdev->blocklen), bdev->optimal_io_boundary); + } else { + vbdev->crypto_bdev.optimal_io_boundary = (CRYPTO_MAX_IO / bdev->blocklen); + } + vbdev->crypto_bdev.split_on_optimal_io_boundary = true; + vbdev->crypto_bdev.blocklen = bdev->blocklen; + vbdev->crypto_bdev.blockcnt = bdev->blockcnt; + + /* This is the context that is passed to us when the bdev + * layer calls in so we'll save our crypto_bdev node here. + */ + vbdev->crypto_bdev.ctxt = vbdev; + vbdev->crypto_bdev.fn_table = &vbdev_crypto_fn_table; + vbdev->crypto_bdev.module = &crypto_if; + TAILQ_INSERT_TAIL(&g_vbdev_crypto, vbdev, link); + + spdk_io_device_register(vbdev, crypto_bdev_ch_create_cb, crypto_bdev_ch_destroy_cb, + sizeof(struct crypto_io_channel), name->bdev_name); + + rc = spdk_bdev_open(bdev, true, vbdev_crypto_examine_hotremove_cb, + bdev, &vbdev->base_desc); + if (rc) { + SPDK_ERRLOG("could not open bdev %s\n", spdk_bdev_get_name(bdev)); + goto error_open; + } + + rc = spdk_bdev_module_claim_bdev(bdev, vbdev->base_desc, vbdev->crypto_bdev.module); + if (rc) { + SPDK_ERRLOG("could not claim bdev %s\n", spdk_bdev_get_name(bdev)); + goto error_claim; + } + + SPDK_NOTICELOG("registered crypto_bdev for: %s\n", name->vbdev_name); + } + + return; + + /* Error cleanup paths. */ +error_claim: + spdk_bdev_close(vbdev->base_desc); +error_open: + TAILQ_REMOVE(&g_vbdev_crypto, vbdev, link); + spdk_io_device_unregister(vbdev, NULL); +error_drv_name: + free(vbdev->key); +error_alloc_key: + free(vbdev->crypto_bdev.name); +error_bdev_name: + free(vbdev); +} + +/* RPC entry for deleting a crypto vbdev. */ +void +delete_crypto_disk(struct spdk_bdev *bdev, spdk_delete_crypto_complete cb_fn, + void *cb_arg) +{ + struct bdev_names *name; + + if (!bdev || bdev->module != &crypto_if) { + cb_fn(cb_arg, -ENODEV); + return; + } + + /* Remove the association (vbdev, bdev) from g_bdev_names. This is required so that the + * vbdev does not get re-created if the same bdev is constructed at some other time, + * unless the underlying bdev was hot-removed. + */ + TAILQ_FOREACH(name, &g_bdev_names, link) { + if (strcmp(name->vbdev_name, bdev->name) == 0) { + TAILQ_REMOVE(&g_bdev_names, name, link); + free(name->bdev_name); + free(name->vbdev_name); + free(name->drv_name); + free(name->key); + free(name); + break; + } + } + + spdk_bdev_unregister(bdev, cb_fn, cb_arg); +} + +/* Because we specified this function in our crypto bdev function table when we + * registered our crypto bdev, we'll get this call anytime a new bdev shows up. + * Here we need to decide if we care about it and if so what to do. We + * parsed the config file at init so we check the new bdev against the list + * we built up at that time and if the user configured us to attach to this + * bdev, here's where we do it. + */ +static void +vbdev_crypto_examine(struct spdk_bdev *bdev) +{ + struct vbdev_crypto *crypto_bdev, *tmp; + int rc; + + vbdev_crypto_claim(bdev); + + TAILQ_FOREACH_SAFE(crypto_bdev, &g_vbdev_crypto, link, tmp) { + if (strcmp(crypto_bdev->base_bdev->name, bdev->name) == 0) { + rc = spdk_vbdev_register(&crypto_bdev->crypto_bdev, + &crypto_bdev->base_bdev, 1); + if (rc) { + SPDK_ERRLOG("could not register crypto_bdev\n"); + spdk_bdev_close(crypto_bdev->base_desc); + TAILQ_REMOVE(&g_vbdev_crypto, crypto_bdev, link); + free(crypto_bdev->crypto_bdev.name); + free(crypto_bdev->key); + free(crypto_bdev); + } + break; + } + } + + spdk_bdev_module_examine_done(&crypto_if); +} + +SPDK_LOG_REGISTER_COMPONENT("vbdev_crypto", SPDK_LOG_VBDEV_crypto) diff --git a/lib/bdev/crypto/vbdev_crypto.h b/lib/bdev/crypto/vbdev_crypto.h new file mode 100755 index 000000000..c8ef8d16a --- /dev/null +++ b/lib/bdev/crypto/vbdev_crypto.h @@ -0,0 +1,66 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_VBDEV_CRYPTO_H +#define SPDK_VBDEV_CRYPTO_H + +#include "spdk/rpc.h" +#include "spdk/util.h" +#include "spdk/string.h" +#include "spdk_internal/log.h" + +#include "spdk/bdev.h" + +typedef void (*spdk_delete_crypto_complete)(void *cb_arg, int bdeverrno); + +/** + * Create new crypto bdev. + * + * \param bdev_name Bdev on which crypto vbdev will be created. + * \param bdev_name Vbdev name crypto_pmd key + * \return 0 on success, other on failure. + */ +int create_crypto_disk(const char *bdev_name, const char *vbdev_name, + const char *crypto_pmd, const char *key); + +/** + * Delete crypto bdev. + * + * \param bdev Pointer to crypto bdev. + * \param cb_fn Function to call after deletion. + * \param cb_arg Argument to pass to cb_fn. + */ +void delete_crypto_disk(struct spdk_bdev *bdev, spdk_delete_crypto_complete cb_fn, + void *cb_arg); + +#endif /* SPDK_VBDEV_CRYPTO_H */ diff --git a/lib/bdev/crypto/vbdev_crypto_rpc.c b/lib/bdev/crypto/vbdev_crypto_rpc.c new file mode 100755 index 000000000..024c886e1 --- /dev/null +++ b/lib/bdev/crypto/vbdev_crypto_rpc.c @@ -0,0 +1,163 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "vbdev_crypto.h" + +/* Structure to hold the parameters for this RPC method. */ +struct rpc_construct_crypto { + char *base_bdev_name; + char *crypto_bdev_name; + char *crypto_pmd; + char *key; +}; + +/* Free the allocated memory resource after the RPC handling. */ +static void +free_rpc_construct_crypto(struct rpc_construct_crypto *r) +{ + free(r->base_bdev_name); + free(r->crypto_bdev_name); + free(r->crypto_pmd); + free(r->key); +} + +/* Structure to decode the input parameters for this RPC method. */ +static const struct spdk_json_object_decoder rpc_construct_crypto_decoders[] = { + {"base_bdev_name", offsetof(struct rpc_construct_crypto, base_bdev_name), spdk_json_decode_string}, + {"crypto_bdev_name", offsetof(struct rpc_construct_crypto, crypto_bdev_name), spdk_json_decode_string}, + {"crypto_pmd", offsetof(struct rpc_construct_crypto, crypto_pmd), spdk_json_decode_string}, + {"key", offsetof(struct rpc_construct_crypto, key), spdk_json_decode_string}, +}; + +/* Decode the parameters for this RPC method and properly construct the crypto + * device. Error status returned in the failed cases. + */ +static void +spdk_rpc_construct_crypto_bdev(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_construct_crypto req = {NULL}; + struct spdk_json_write_ctx *w; + int rc; + + if (spdk_json_decode_object(params, rpc_construct_crypto_decoders, + SPDK_COUNTOF(rpc_construct_crypto_decoders), + &req)) { + SPDK_DEBUGLOG(SPDK_LOG_VBDEV_crypto, "spdk_json_decode_object failed\n"); + goto invalid; + } + + rc = create_crypto_disk(req.base_bdev_name, req.crypto_bdev_name, + req.crypto_pmd, req.key); + if (rc != 0) { + goto invalid; + } + + w = spdk_jsonrpc_begin_result(request); + if (w == NULL) { + free_rpc_construct_crypto(&req); + return; + } + + spdk_json_write_string(w, req.crypto_bdev_name); + spdk_jsonrpc_end_result(request, w); + free_rpc_construct_crypto(&req); + return; + +invalid: + free_rpc_construct_crypto(&req); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters"); +} +SPDK_RPC_REGISTER("construct_crypto_bdev", spdk_rpc_construct_crypto_bdev, SPDK_RPC_RUNTIME) + +struct rpc_delete_crypto { + char *name; +}; + +static void +free_rpc_delete_crypto(struct rpc_delete_crypto *req) +{ + free(req->name); +} + +static const struct spdk_json_object_decoder rpc_delete_crypto_decoders[] = { + {"name", offsetof(struct rpc_delete_crypto, name), spdk_json_decode_string}, +}; + +static void +_spdk_rpc_delete_crypto_bdev_cb(void *cb_arg, int bdeverrno) +{ + struct spdk_jsonrpc_request *request = cb_arg; + struct spdk_json_write_ctx *w; + + w = spdk_jsonrpc_begin_result(request); + if (w == NULL) { + return; + } + + spdk_json_write_bool(w, bdeverrno == 0); + spdk_jsonrpc_end_result(request, w); +} + +static void +spdk_rpc_delete_crypto_bdev(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_delete_crypto req = {NULL}; + struct spdk_bdev *bdev; + int rc; + + if (spdk_json_decode_object(params, rpc_delete_crypto_decoders, + SPDK_COUNTOF(rpc_delete_crypto_decoders), + &req)) { + rc = -EINVAL; + goto invalid; + } + + bdev = spdk_bdev_get_by_name(req.name); + if (bdev == NULL) { + rc = -ENODEV; + goto invalid; + } + + delete_crypto_disk(bdev, _spdk_rpc_delete_crypto_bdev_cb, request); + + free_rpc_delete_crypto(&req); + + return; + +invalid: + free_rpc_delete_crypto(&req); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, spdk_strerror(-rc)); +} +SPDK_RPC_REGISTER("delete_crypto_bdev", spdk_rpc_delete_crypto_bdev, SPDK_RPC_RUNTIME) diff --git a/lib/env_dpdk/env.mk b/lib/env_dpdk/env.mk index cacb43279..a9e2878ea 100644 --- a/lib/env_dpdk/env.mk +++ b/lib/env_dpdk/env.mk @@ -78,11 +78,18 @@ ifneq (, $(wildcard $(DPDK_ABS_DIR)/lib/librte_bus_pci.*)) DPDK_LIB_LIST += rte_bus_pci endif +ifeq ($(CONFIG_CRYPTO),y) +DPDK_LIB_LIST += rte_cryptodev rte_reorder rte_bus_vdev rte_pmd_aesni_mb rte_mbuf +endif + ifneq (, $(wildcard $(DPDK_ABS_DIR)/lib/librte_kvargs.*)) DPDK_LIB_LIST += rte_kvargs endif DPDK_LIB = $(DPDK_LIB_LIST:%=$(DPDK_ABS_DIR)/lib/lib%$(DPDK_LIB_EXT)) +ifeq ($(CONFIG_CRYPTO),y) +DPDK_LIB += $(SPDK_ROOT_DIR)/intel-ipsec-mb/libIPSec_MB.a +endif # SPDK memory registration requires experimental (deprecated) rte_memory API for DPDK 18.05 ENV_CFLAGS = $(DPDK_INC) -Wno-deprecated-declarations diff --git a/mk/spdk.modules.mk b/mk/spdk.modules.mk index f9826417f..07737164b 100644 --- a/mk/spdk.modules.mk +++ b/mk/spdk.modules.mk @@ -38,6 +38,10 @@ LVOL_MODULES_LIST += blob blob_bdev lvol BLOCKDEV_MODULES_LIST = $(LVOL_MODULES_LIST) BLOCKDEV_MODULES_LIST += bdev_malloc bdev_null bdev_nvme nvme vbdev_passthru vbdev_error vbdev_gpt vbdev_split +ifeq ($(CONFIG_CRYPTO),y) +BLOCKDEV_MODULES_LIST += vbdev_crypto +endif + ifeq ($(CONFIG_RDMA),y) BLOCKDEV_MODULES_DEPS += -libverbs -lrdmacm endif diff --git a/scripts/rpc.py b/scripts/rpc.py index 2d037e92c..f7061f43d 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -131,6 +131,30 @@ if __name__ == "__main__": p.add_argument('-c', '--bdev-io-cache-size', help='Maximum number of bdev_io structures cached per thread', type=int) p.set_defaults(func=set_bdev_options) + @call_cmd + def construct_crypto_bdev(args): + print(rpc.bdev.construct_crypto_bdev(args.client, + base_bdev_name=args.base_bdev_name, + crypto_bdev_name=args.crypto_bdev_name, + crypto_pmd=args.crypto_pmd, + key=args.key)) + p = subparsers.add_parser('construct_crypto_bdev', + help='Add a crypto vbdev') + p.add_argument('-b', '--base_bdev_name', help="Name of the base bdev") + p.add_argument('-c', '--crypto_bdev_name', help="Name of the crypto vbdev") + p.add_argument('-d', '--crypto_pmd', help="Name of the crypto device driver") + p.add_argument('-k', '--key', help="Key") + p.set_defaults(func=construct_crypto_bdev) + + @call_cmd + def delete_crypto_bdev(args): + rpc.bdev.delete_crypto_bdev(args.client, + name=args.name) + + p = subparsers.add_parser('delete_crypto_bdev', help='Delete a crypto disk') + p.add_argument('name', help='crypto bdev name') + p.set_defaults(func=delete_crypto_bdev) + @call_cmd def construct_malloc_bdev(args): num_blocks = (args.total_size * 1024 * 1024) // args.block_size @@ -139,7 +163,6 @@ if __name__ == "__main__": block_size=args.block_size, name=args.name, uuid=args.uuid)) - p = subparsers.add_parser('construct_malloc_bdev', help='Add a bdev with malloc backend') p.add_argument('-b', '--name', help="Name of the bdev") diff --git a/scripts/rpc/bdev.py b/scripts/rpc/bdev.py index 7d4b71096..6241fca3c 100755 --- a/scripts/rpc/bdev.py +++ b/scripts/rpc/bdev.py @@ -15,6 +15,33 @@ def set_bdev_options(client, bdev_io_pool_size=None, bdev_io_cache_size=None): return client.call('set_bdev_options', params) +def construct_crypto_bdev(client, base_bdev_name, crypto_bdev_name, crypto_pmd, key): + """Construct a crypto virtual block device. + + Args: + base_bdev_name: name of the underlying base bdev + crypto_bdev_name: name for the crypto vbdev + crypto_pmd: name of of the DPDK crypto driver to use + key: key + + Returns: + Name of created virtual block device. + """ + params = {'base_bdev_name': base_bdev_name, 'crypto_bdev_name': crypto_bdev_name, 'crypto_pmd': crypto_pmd, 'key': key} + + return client.call('construct_crypto_bdev', params) + + +def delete_crypto_bdev(client, name): + """Delete crypto virtual block device. + + Args: + bdev_name: name of crypto vbdev to delete + """ + params = {'name': name} + return client.call('delete_crypto_bdev', params) + + def construct_malloc_bdev(client, num_blocks, block_size, name=None, uuid=None): """Construct a malloc block device. diff --git a/test/bdev/bdev.conf.in b/test/bdev/bdev.conf.in index 22cfcca4d..bb1d733f8 100644 --- a/test/bdev/bdev.conf.in +++ b/test/bdev/bdev.conf.in @@ -2,8 +2,15 @@ # PT PT Malloc3 TestPT +[crypto] + # CRY + # key size depends on cipher + # supported PMD names: crypto_aesni_mb, crypto_qat + # Note: QAT is experimental while test HW is being setup + CRY Malloc6 crypto_ram 0123456789123456 crypto_aesni_mb + [Malloc] - NumberOfLuns 6 + NumberOfLuns 7 LunSizeInMB 64 [Split] diff --git a/test/common/autotest_common.sh b/test/common/autotest_common.sh index 6c08479c7..3bd5c677e 100755 --- a/test/common/autotest_common.sh +++ b/test/common/autotest_common.sh @@ -58,6 +58,7 @@ fi : ${SPDK_RUN_ASAN=1}; export SPDK_RUN_ASAN : ${SPDK_RUN_UBSAN=1}; export SPDK_RUN_UBSAN : ${SPDK_RUN_INSTALLED_DPDK=1}; export SPDK_RUN_INSTALLED_DPDK +: ${SPDK_TEST_CRYPTO=1}; export SPDK_TEST_CRYPTO if [ -z "$DEPENDENCY_DIR" ]; then export DEPENDENCY_DIR=/home/sys_sgsw @@ -86,6 +87,10 @@ fi # sure it gets built and run against a limited set of use cases for now. config_params+=' --with-raid' +if [ $SPDK_TEST_CRYPTO -eq 1 ]; then + config_params+=' --with-crypto' +fi + export UBSAN_OPTIONS='halt_on_error=1:print_stacktrace=1:abort_on_error=1' # On Linux systems, override the default HUGEMEM in scripts/setup.sh to diff --git a/test/nvme/nvme.sh b/test/nvme/nvme.sh index 9cdcb21e2..c52b926ed 100755 --- a/test/nvme/nvme.sh +++ b/test/nvme/nvme.sh @@ -87,7 +87,7 @@ if [ `uname` = Linux ]; then fi if [ `uname` = Linux ]; then - start_stub "-s 2048 -i 0 -m 0xF" + start_stub "-s 4096 -i 0 -m 0xF" trap "kill_stub; exit 1" SIGINT SIGTERM EXIT fi diff --git a/test/unit/lib/bdev/Makefile b/test/unit/lib/bdev/Makefile index 3721b5eea..61efba78a 100644 --- a/test/unit/lib/bdev/Makefile +++ b/test/unit/lib/bdev/Makefile @@ -36,6 +36,10 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk DIRS-y = bdev.c part.c scsi_nvme.c gpt vbdev_lvol.c mt bdev_raid.c +ifeq ($(CONFIG_CRYPTO),y) +DIRS-y += crypto.c +endif + DIRS-$(CONFIG_PMDK) += pmem .PHONY: all clean $(DIRS-y) diff --git a/test/unit/lib/bdev/crypto.c/.gitignore b/test/unit/lib/bdev/crypto.c/.gitignore new file mode 100644 index 000000000..b2777562d --- /dev/null +++ b/test/unit/lib/bdev/crypto.c/.gitignore @@ -0,0 +1 @@ +crypto_ut diff --git a/test/unit/lib/bdev/crypto.c/Makefile b/test/unit/lib/bdev/crypto.c/Makefile new file mode 100644 index 000000000..3241464be --- /dev/null +++ b/test/unit/lib/bdev/crypto.c/Makefile @@ -0,0 +1,41 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk +include $(SPDK_ROOT_DIR)/mk/spdk.app.mk + +TEST_FILE = crypto_ut.c +CFLAGS += $(ENV_CFLAGS) + +include $(SPDK_ROOT_DIR)/mk/spdk.unittest.mk diff --git a/test/unit/lib/bdev/crypto.c/crypto_ut.c b/test/unit/lib/bdev/crypto.c/crypto_ut.c new file mode 100755 index 000000000..f01aba19f --- /dev/null +++ b/test/unit/lib/bdev/crypto.c/crypto_ut.c @@ -0,0 +1,908 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk_cunit.h" + +#include "common/lib/test_env.c" +#include "spdk_internal/mock.h" +#include "unit/lib/json_mock.c" + +/* these rte_ headers are our local copies of the DPDK headers hacked to mock some functions + * included in them that can't be done with our mock library. + */ +#include "rte_crypto.h" +#include "rte_cryptodev.h" +DEFINE_STUB_V(rte_crypto_op_free, (struct rte_crypto_op *op)); +#include "bdev/crypto/vbdev_crypto.c" + +/* SPDK stubs */ +DEFINE_STUB(spdk_conf_find_section, struct spdk_conf_section *, + (struct spdk_conf *cp, const char *name), NULL); +DEFINE_STUB(spdk_conf_section_get_nval, char *, + (struct spdk_conf_section *sp, const char *key, int idx), NULL); +DEFINE_STUB(spdk_conf_section_get_nmval, char *, + (struct spdk_conf_section *sp, const char *key, int idx1, int idx2), NULL); + +DEFINE_STUB_V(spdk_bdev_module_list_add, (struct spdk_bdev_module *bdev_module)); +DEFINE_STUB_V(spdk_bdev_free_io, (struct spdk_bdev_io *g_bdev_io)); +DEFINE_STUB(spdk_bdev_io_type_supported, bool, (struct spdk_bdev *bdev, + enum spdk_bdev_io_type io_type), 0); +DEFINE_STUB_V(spdk_bdev_module_release_bdev, (struct spdk_bdev *bdev)); +DEFINE_STUB_V(spdk_bdev_close, (struct spdk_bdev_desc *desc)); +DEFINE_STUB(spdk_bdev_get_name, const char *, (const struct spdk_bdev *bdev), 0); +DEFINE_STUB(spdk_env_get_current_core, uint32_t, (void), 0); +DEFINE_STUB(spdk_bdev_get_io_channel, struct spdk_io_channel *, (struct spdk_bdev_desc *desc), 0); +DEFINE_STUB_V(spdk_bdev_unregister, (struct spdk_bdev *bdev, spdk_bdev_unregister_cb cb_fn, + void *cb_arg)); +DEFINE_STUB(spdk_bdev_open, int, (struct spdk_bdev *bdev, bool write, + spdk_bdev_remove_cb_t remove_cb, + void *remove_ctx, struct spdk_bdev_desc **_desc), 0); +DEFINE_STUB(spdk_bdev_module_claim_bdev, int, (struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, + struct spdk_bdev_module *module), 0); +DEFINE_STUB_V(spdk_bdev_module_examine_done, (struct spdk_bdev_module *module)); +DEFINE_STUB(spdk_vbdev_register, int, (struct spdk_bdev *vbdev, struct spdk_bdev **base_bdevs, + int base_bdev_count), 0); +DEFINE_STUB(spdk_bdev_get_by_name, struct spdk_bdev *, (const char *bdev_name), NULL); +DEFINE_STUB(spdk_env_get_socket_id, uint32_t, (uint32_t core), 0); + +/* DPDK stubs */ +DEFINE_STUB(rte_cryptodev_count, uint8_t, (void), 0); +DEFINE_STUB(rte_eal_get_configuration, struct rte_config *, (void), NULL); +DEFINE_STUB_V(rte_mempool_free, (struct rte_mempool *mp)); +DEFINE_STUB(rte_socket_id, unsigned, (void), 0); +DEFINE_STUB(rte_crypto_op_pool_create, struct rte_mempool *, + (const char *name, enum rte_crypto_op_type type, unsigned nb_elts, + unsigned cache_size, uint16_t priv_size, int socket_id), (struct rte_mempool *)1); +DEFINE_STUB(rte_cryptodev_device_count_by_driver, uint8_t, (uint8_t driver_id), 0); +DEFINE_STUB(rte_cryptodev_socket_id, int, (uint8_t dev_id), 0); +DEFINE_STUB(rte_cryptodev_configure, int, (uint8_t dev_id, struct rte_cryptodev_config *config), 0); +DEFINE_STUB(rte_cryptodev_queue_pair_setup, int, (uint8_t dev_id, uint16_t queue_pair_id, + const struct rte_cryptodev_qp_conf *qp_conf, + int socket_id, struct rte_mempool *session_pool), 0); +DEFINE_STUB(rte_cryptodev_start, int, (uint8_t dev_id), 0) +DEFINE_STUB_V(rte_cryptodev_stop, (uint8_t dev_id)); +DEFINE_STUB(rte_cryptodev_sym_session_create, struct rte_cryptodev_sym_session *, + (struct rte_mempool *mempool), (struct rte_cryptodev_sym_session *)1); +DEFINE_STUB(rte_cryptodev_sym_session_clear, int, (uint8_t dev_id, + struct rte_cryptodev_sym_session *sess), 0); +DEFINE_STUB(rte_cryptodev_sym_session_free, int, (struct rte_cryptodev_sym_session *sess), 0); +DEFINE_STUB(rte_cryptodev_sym_session_init, int, (uint8_t dev_id, + struct rte_cryptodev_sym_session *sess, + struct rte_crypto_sym_xform *xforms, struct rte_mempool *mempool), 0); +DEFINE_STUB(rte_vdev_init, int, (const char *name, const char *args), 0); +void __attribute__((noreturn)) __rte_panic(const char *funcname, const char *format, ...) +{ + abort(); +} +struct rte_mempool_ops_table rte_mempool_ops_table; +struct rte_cryptodev *rte_cryptodevs; +__thread unsigned per_lcore__lcore_id = 0; + +/* global vars and setup/cleanup functions used for all test functions */ +struct spdk_bdev_io *g_bdev_io; +struct crypto_bdev_io *g_io_ctx; +struct crypto_io_channel *g_crypto_ch; +struct spdk_io_channel *g_io_ch; +struct vbdev_dev g_device; +struct vbdev_crypto g_crypto_bdev; +struct rte_config *g_test_config; +struct device_qp g_dev_qp; + +#define MAX_TEST_BLOCKS 8192 +struct rte_crypto_op *g_test_crypto_ops[MAX_TEST_BLOCKS]; +struct rte_crypto_op *g_test_dequeued_ops[MAX_TEST_BLOCKS]; +struct rte_crypto_op *g_test_dev_full_ops[MAX_TEST_BLOCKS]; + +/* These globals are externs in our local rte_ header files so we can control + * specific functions for mocking. + */ +uint16_t g_dequeue_mock; +uint16_t g_enqueue_mock; +unsigned ut_rte_crypto_op_bulk_alloc; +int ut_rte_crypto_op_attach_sym_session = 0; + +int ut_rte_cryptodev_info_get = 0; +bool ut_rte_cryptodev_info_get_mocked = false; +void +rte_cryptodev_info_get(uint8_t dev_id, struct rte_cryptodev_info *dev_info) +{ + dev_info->max_nb_queue_pairs = ut_rte_cryptodev_info_get; +} + +unsigned int +rte_cryptodev_sym_get_private_session_size(uint8_t dev_id) +{ + return (unsigned int)dev_id; +} + +void +spdk_bdev_io_get_buf(struct spdk_bdev_io *bdev_io, spdk_bdev_io_get_buf_cb cb, uint64_t len) +{ + cb(g_io_ch, g_bdev_io); +} + +/* Mock these functions to call the callback and then return the value we require */ +int ut_spdk_bdev_readv_blocks = 0; +bool ut_spdk_bdev_readv_blocks_mocked = false; +int +spdk_bdev_readv_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + struct iovec *iov, int iovcnt, + uint64_t offset_blocks, uint64_t num_blocks, + spdk_bdev_io_completion_cb cb, void *cb_arg) +{ + cb(g_bdev_io, !ut_spdk_bdev_readv_blocks, cb_arg); + return ut_spdk_bdev_readv_blocks; +} + +int ut_spdk_bdev_writev_blocks = 0; +bool ut_spdk_bdev_writev_blocks_mocked = false; +int +spdk_bdev_writev_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + struct iovec *iov, int iovcnt, + uint64_t offset_blocks, uint64_t num_blocks, + spdk_bdev_io_completion_cb cb, void *cb_arg) +{ + cb(g_bdev_io, !ut_spdk_bdev_writev_blocks, cb_arg); + return ut_spdk_bdev_writev_blocks; +} + +int ut_spdk_bdev_unmap_blocks = 0; +bool ut_spdk_bdev_unmap_blocks_mocked = false; +int +spdk_bdev_unmap_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + uint64_t offset_blocks, uint64_t num_blocks, + spdk_bdev_io_completion_cb cb, void *cb_arg) +{ + cb(g_bdev_io, !ut_spdk_bdev_unmap_blocks, cb_arg); + return ut_spdk_bdev_unmap_blocks; +} + +int ut_spdk_bdev_flush_blocks = 0; +bool ut_spdk_bdev_flush_blocks_mocked = false; +int +spdk_bdev_flush_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + uint64_t offset_blocks, uint64_t num_blocks, spdk_bdev_io_completion_cb cb, + void *cb_arg) +{ + cb(g_bdev_io, !ut_spdk_bdev_flush_blocks, cb_arg); + return ut_spdk_bdev_flush_blocks; +} + +int ut_spdk_bdev_reset = 0; +bool ut_spdk_bdev_reset_mocked = false; +int +spdk_bdev_reset(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + spdk_bdev_io_completion_cb cb, void *cb_arg) +{ + cb(g_bdev_io, !ut_spdk_bdev_reset, cb_arg); + return ut_spdk_bdev_reset; +} + +bool g_completion_called = false; +void +spdk_bdev_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status status) +{ + bdev_io->internal.status = status; + g_completion_called = true; +} + +/* Used in testing device full condition */ +static inline uint16_t +rte_cryptodev_enqueue_burst(uint8_t dev_id, uint16_t qp_id, + struct rte_crypto_op **ops, uint16_t nb_ops) +{ + int i; + + CU_ASSERT(nb_ops > 0); + + for (i = 0; i < nb_ops; i++) { + /* Use this empty (til now) array of pointers to store + * enqueued operations for assertion in dev_full test. + */ + g_test_dev_full_ops[i] = *ops++; + } + + return g_enqueue_mock; +} + +/* This is pretty ugly but in order to complete an IO via the + * poller in the submit path, we need to first call to this func + * to return the dequeued value and also decrement it. On the subsequent + * call it needs to return 0 to indicate to the caller that there are + * no more IOs to drain. + */ +int g_test_overflow = 0; +static inline uint16_t +rte_cryptodev_dequeue_burst(uint8_t dev_id, uint16_t qp_id, + struct rte_crypto_op **ops, uint16_t nb_ops) +{ + CU_ASSERT(nb_ops > 0); + + /* A crypto device can be full on enqueue, the driver is designed to drain + * the device at the time by calling the poller until it's empty, then + * submitting the remaining crypto ops. + */ + if (g_test_overflow) { + if (g_dequeue_mock == 0) { + return 0; + } + *ops = g_test_crypto_ops[g_enqueue_mock]; + (*ops)->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + g_dequeue_mock -= 1; + } + return (g_dequeue_mock + 1); +} + +/* Instead of allocating real memory, assign the allocations to our + * test array for assertion in tests. + */ +static inline unsigned +rte_crypto_op_bulk_alloc(struct rte_mempool *mempool, + enum rte_crypto_op_type type, + struct rte_crypto_op **ops, uint16_t nb_ops) +{ + int i; + + for (i = 0; i < nb_ops; i++) { + *ops++ = g_test_crypto_ops[i]; + } + return ut_rte_crypto_op_bulk_alloc; +} + +static __rte_always_inline void +rte_mempool_put_bulk(struct rte_mempool *mp, void *const *obj_table, + unsigned int n) +{ + return; +} + +static inline void *rte_mempool_get_priv(struct rte_mempool *mp) +{ + return NULL; +} + + +static inline int +rte_crypto_op_attach_sym_session(struct rte_crypto_op *op, + struct rte_cryptodev_sym_session *sess) +{ + return ut_rte_crypto_op_attach_sym_session; +} + +/* Global setup for all tests that share a bunch of preparation... */ +static int +test_setup(void) +{ + int i; + + /* Prepare essential variables for test routines */ + g_bdev_io = calloc(1, sizeof(struct spdk_bdev_io) + sizeof(struct crypto_bdev_io)); + g_bdev_io->u.bdev.iovs = calloc(1, sizeof(struct iovec) * 128); + g_bdev_io->bdev = &g_crypto_bdev.crypto_bdev; + g_io_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct crypto_io_channel)); + g_crypto_ch = (struct crypto_io_channel *)((uint8_t *)g_io_ch + sizeof(struct spdk_io_channel)); + g_io_ctx = (struct crypto_bdev_io *)g_bdev_io->driver_ctx; + memset(&g_device, 0, sizeof(struct vbdev_dev)); + memset(&g_crypto_bdev, 0, sizeof(struct vbdev_crypto)); + g_dev_qp.device = &g_device; + g_io_ctx->crypto_ch = g_crypto_ch; + g_io_ctx->crypto_bdev = &g_crypto_bdev; + g_crypto_ch->device_qp = &g_dev_qp; + g_test_config = calloc(1, sizeof(struct rte_config)); + g_test_config->lcore_count = 1; + + /* Allocate a real mbuf pool so we can test error paths */ + g_mbuf_mp = spdk_mempool_create("mbuf_mp", NUM_MBUFS, sizeof(struct rte_mbuf), + SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, + SPDK_ENV_SOCKET_ID_ANY); + + /* Instead of allocating real rte mempools for these, it's easier and provides the + * same coverage just calloc them here. + */ + for (i = 0; i < MAX_TEST_BLOCKS; i++) { + g_test_crypto_ops[i] = calloc(1, sizeof(struct rte_crypto_op) + + sizeof(struct rte_crypto_sym_op)); + g_test_dequeued_ops[i] = calloc(1, sizeof(struct rte_crypto_op) + + sizeof(struct rte_crypto_sym_op)); + } + return 0; +} + +/* Global teardown for all tests */ +static int +test_cleanup(void) +{ + int i; + + free(g_test_config); + spdk_mempool_free(g_mbuf_mp); + for (i = 0; i < MAX_TEST_BLOCKS; i++) { + free(g_test_crypto_ops[i]); + free(g_test_dequeued_ops[i]); + } + free(g_bdev_io->u.bdev.iovs); + free(g_bdev_io); + free(g_io_ch); + return 0; +} + +static void +test_error_paths(void) +{ + /* Single element block size write, just to test error paths + * in vbdev_crypto_submit_request(). + */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->u.bdev.iovcnt = 1; + g_bdev_io->u.bdev.num_blocks = 1; + g_bdev_io->u.bdev.iovs[0].iov_len = 512; + g_crypto_bdev.crypto_bdev.blocklen = 512; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; + g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = 1; + + /* test failure of spdk_mempool_get_bulk() */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + MOCK_SET(spdk_mempool_get, NULL); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + + /* same thing but switch to reads to test error path in _crypto_complete_io() */ + g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + /* Now with the read_blocks failing */ + g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + MOCK_SET(spdk_bdev_readv_blocks, -1); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + MOCK_SET(spdk_bdev_readv_blocks, 0); + MOCK_CLEAR(spdk_mempool_get); + + /* test failure of rte_crypto_op_bulk_alloc() */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + ut_rte_crypto_op_bulk_alloc = 0; + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + ut_rte_crypto_op_bulk_alloc = 1; + + /* test failure of rte_cryptodev_sym_session_create() */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + MOCK_SET(rte_cryptodev_sym_session_create, NULL); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + MOCK_SET(rte_cryptodev_sym_session_create, (struct rte_cryptodev_sym_session *)1); + + /* test failure of rte_cryptodev_sym_session_init() */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + MOCK_SET(rte_cryptodev_sym_session_init, -1); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + MOCK_SET(rte_cryptodev_sym_session_init, 0); + + /* test failure of rte_crypto_op_attach_sym_session() */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + ut_rte_crypto_op_attach_sym_session = -1; + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + ut_rte_crypto_op_attach_sym_session = 0; +} + +static void +test_simple_write(void) +{ + /* Single element block size write */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->u.bdev.iovcnt = 1; + g_bdev_io->u.bdev.num_blocks = 1; + g_bdev_io->u.bdev.offset_blocks = 0; + g_bdev_io->u.bdev.iovs[0].iov_len = 512; + g_bdev_io->u.bdev.iovs[0].iov_base = &test_simple_write; + g_crypto_bdev.crypto_bdev.blocklen = 512; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; + g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = 1; + + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_io_ctx->cryop_cnt_remaining == 1); + CU_ASSERT(g_io_ctx->crypto_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT); + CU_ASSERT(g_io_ctx->cry_iov.iov_len == 512); + CU_ASSERT(g_io_ctx->cry_iov.iov_base != NULL); + CU_ASSERT(g_io_ctx->cry_offset_blocks == 0); + CU_ASSERT(g_io_ctx->cry_num_blocks == 1); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->buf_addr == &test_simple_write); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->data_len == 512); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->next == NULL); + CU_ASSERT(g_test_crypto_ops[0]->sym->cipher.data.length == 512); + CU_ASSERT(g_test_crypto_ops[0]->sym->cipher.data.offset == 0); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->userdata == g_bdev_io); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_dst->buf_addr != NULL); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_dst->data_len == 512); + + spdk_dma_free(g_io_ctx->cry_iov.iov_base); + spdk_mempool_put(g_mbuf_mp, g_test_crypto_ops[0]->sym->m_src); + spdk_mempool_put(g_mbuf_mp, g_test_crypto_ops[0]->sym->m_dst); +} + +static void +test_simple_read(void) +{ + /* Single element block size read */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->u.bdev.iovcnt = 1; + g_bdev_io->u.bdev.num_blocks = 1; + g_bdev_io->u.bdev.iovs[0].iov_len = 512; + g_bdev_io->u.bdev.iovs[0].iov_base = &test_simple_read; + g_crypto_bdev.crypto_bdev.blocklen = 512; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; + g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = 1; + + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_io_ctx->cryop_cnt_remaining == 1); + CU_ASSERT(g_io_ctx->crypto_op == RTE_CRYPTO_CIPHER_OP_DECRYPT); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->buf_addr == &test_simple_read); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->data_len == 512); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->next == NULL); + CU_ASSERT(g_test_crypto_ops[0]->sym->cipher.data.length == 512); + CU_ASSERT(g_test_crypto_ops[0]->sym->cipher.data.offset == 0); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_src->userdata == g_bdev_io); + CU_ASSERT(g_test_crypto_ops[0]->sym->m_dst == NULL); + + spdk_mempool_put(g_mbuf_mp, g_test_crypto_ops[0]->sym->m_src); +} + +static void +test_large_rw(void) +{ + unsigned block_len = 512; + unsigned num_blocks = CRYPTO_MAX_IO / block_len; + unsigned io_len = block_len * num_blocks; + unsigned i; + + /* Multi block size read, multi-element */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->u.bdev.iovcnt = 1; + g_bdev_io->u.bdev.num_blocks = num_blocks; + g_bdev_io->u.bdev.iovs[0].iov_len = io_len; + g_bdev_io->u.bdev.iovs[0].iov_base = &test_large_rw; + g_crypto_bdev.crypto_bdev.blocklen = block_len; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; + g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = num_blocks; + + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_io_ctx->cryop_cnt_remaining == (int)num_blocks); + CU_ASSERT(g_io_ctx->crypto_op == RTE_CRYPTO_CIPHER_OP_DECRYPT); + + for (i = 0; i < num_blocks; i++) { + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->buf_addr == &test_large_rw + (i * block_len)); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->data_len == block_len); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->next == NULL); + CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.length == block_len); + CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.offset == 0); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->userdata == g_bdev_io); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_dst == NULL); + spdk_mempool_put(g_mbuf_mp, g_test_crypto_ops[i]->sym->m_src); + } + + /* Multi block size write, multi-element */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->u.bdev.iovcnt = 1; + g_bdev_io->u.bdev.num_blocks = num_blocks; + g_bdev_io->u.bdev.iovs[0].iov_len = io_len; + g_bdev_io->u.bdev.iovs[0].iov_base = &test_large_rw; + g_crypto_bdev.crypto_bdev.blocklen = block_len; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; + g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = num_blocks; + + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_io_ctx->cryop_cnt_remaining == (int)num_blocks); + CU_ASSERT(g_io_ctx->crypto_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT); + + for (i = 0; i < num_blocks; i++) { + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->buf_addr == &test_large_rw + (i * block_len)); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->data_len == block_len); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->next == NULL); + CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.length == block_len); + CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.offset == 0); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->userdata == g_bdev_io); + CU_ASSERT(g_io_ctx->cry_iov.iov_len == io_len); + CU_ASSERT(g_io_ctx->cry_iov.iov_base != NULL); + CU_ASSERT(g_io_ctx->cry_offset_blocks == 0); + CU_ASSERT(g_io_ctx->cry_num_blocks == num_blocks); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_dst->buf_addr != NULL); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_dst->data_len == block_len); + spdk_mempool_put(g_mbuf_mp, g_test_crypto_ops[i]->sym->m_src); + spdk_mempool_put(g_mbuf_mp, g_test_crypto_ops[i]->sym->m_dst); + } + spdk_dma_free(g_io_ctx->cry_iov.iov_base); +} + +static void +test_dev_full(void) +{ + unsigned block_len = 512; + unsigned num_blocks = 2; + unsigned io_len = block_len * num_blocks; + unsigned i; + + g_test_overflow = 1; + + /* Multi block size read, multi-element */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->u.bdev.iovcnt = 1; + g_bdev_io->u.bdev.num_blocks = num_blocks; + g_bdev_io->u.bdev.iovs[0].iov_len = io_len; + g_bdev_io->u.bdev.iovs[0].iov_base = &test_dev_full; + g_crypto_bdev.crypto_bdev.blocklen = block_len; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; + g_enqueue_mock = g_dequeue_mock = 1; + ut_rte_crypto_op_bulk_alloc = num_blocks; + + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + + /* this test only completes one of the 2 IOs (in the drain path) */ + CU_ASSERT(g_io_ctx->cryop_cnt_remaining == 1); + CU_ASSERT(g_io_ctx->crypto_op == RTE_CRYPTO_CIPHER_OP_DECRYPT); + + for (i = 0; i < num_blocks; i++) { + /* One of the src_mbufs was freed because of the device full condition so + * we can't assert its value here. + */ + CU_ASSERT(g_test_dev_full_ops[i]->sym->cipher.data.length == block_len); + CU_ASSERT(g_test_dev_full_ops[i]->sym->cipher.data.offset == 0); + CU_ASSERT(g_test_dev_full_ops[i]->sym->m_src == g_test_dev_full_ops[i]->sym->m_src); + CU_ASSERT(g_test_dev_full_ops[i]->sym->m_dst == NULL); + } + + /* Only one of the 2 blocks in the test was freed on completion by design, so + * we need to free th other one here. + */ + spdk_mempool_put(g_mbuf_mp, g_test_crypto_ops[0]->sym->m_src); + g_test_overflow = 0; +} + +static void +test_crazy_rw(void) +{ + unsigned block_len = 512; + int num_blocks = 4; + int i; + + /* Multi block size read, single element, strange IOV makeup */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->u.bdev.iovcnt = 3; + g_bdev_io->u.bdev.num_blocks = num_blocks; + g_bdev_io->u.bdev.iovs[0].iov_len = 512; + g_bdev_io->u.bdev.iovs[0].iov_base = &test_crazy_rw; + g_bdev_io->u.bdev.iovs[1].iov_len = 1024; + g_bdev_io->u.bdev.iovs[1].iov_base = &test_crazy_rw + 512; + g_bdev_io->u.bdev.iovs[2].iov_len = 512; + g_bdev_io->u.bdev.iovs[2].iov_base = &test_crazy_rw + 512 + 1024; + + g_crypto_bdev.crypto_bdev.blocklen = block_len; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; + g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = num_blocks; + + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_io_ctx->cryop_cnt_remaining == num_blocks); + CU_ASSERT(g_io_ctx->crypto_op == RTE_CRYPTO_CIPHER_OP_DECRYPT); + + for (i = 0; i < num_blocks; i++) { + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->buf_addr == &test_crazy_rw + (i * block_len)); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->data_len == block_len); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->next == NULL); + CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.length == block_len); + CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.offset == 0); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->userdata == g_bdev_io); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src == g_test_crypto_ops[i]->sym->m_src); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_dst == NULL); + spdk_mempool_put(g_mbuf_mp, g_test_crypto_ops[i]->sym->m_src); + } + + /* Multi block size write, single element strange IOV makeup */ + num_blocks = 8; + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->u.bdev.iovcnt = 4; + g_bdev_io->u.bdev.num_blocks = num_blocks; + g_bdev_io->u.bdev.iovs[0].iov_len = 2048; + g_bdev_io->u.bdev.iovs[0].iov_base = &test_crazy_rw; + g_bdev_io->u.bdev.iovs[1].iov_len = 512; + g_bdev_io->u.bdev.iovs[1].iov_base = &test_crazy_rw + 2048; + g_bdev_io->u.bdev.iovs[2].iov_len = 512; + g_bdev_io->u.bdev.iovs[2].iov_base = &test_crazy_rw + 2048 + 512; + g_bdev_io->u.bdev.iovs[3].iov_len = 1024; + g_bdev_io->u.bdev.iovs[3].iov_base = &test_crazy_rw + 2048 + 512 + 512; + + g_crypto_bdev.crypto_bdev.blocklen = block_len; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; + g_enqueue_mock = g_dequeue_mock = ut_rte_crypto_op_bulk_alloc = num_blocks; + + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_io_ctx->cryop_cnt_remaining == num_blocks); + CU_ASSERT(g_io_ctx->crypto_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT); + + for (i = 0; i < num_blocks; i++) { + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->buf_addr == &test_crazy_rw + (i * block_len)); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->data_len == block_len); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->next == NULL); + CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.length == block_len); + CU_ASSERT(g_test_crypto_ops[i]->sym->cipher.data.offset == 0); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src->userdata == g_bdev_io); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_src == g_test_crypto_ops[i]->sym->m_src); + CU_ASSERT(g_test_crypto_ops[i]->sym->m_dst == g_test_crypto_ops[i]->sym->m_dst); + spdk_mempool_put(g_mbuf_mp, g_test_crypto_ops[i]->sym->m_src); + spdk_mempool_put(g_mbuf_mp, g_test_crypto_ops[i]->sym->m_dst); + } + spdk_dma_free(g_io_ctx->cry_iov.iov_base); +} + +static void +test_passthru(void) +{ + /* Make sure these follow our completion callback, test success & fail. */ + g_bdev_io->type = SPDK_BDEV_IO_TYPE_UNMAP; + MOCK_SET(spdk_bdev_unmap_blocks, 0); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + MOCK_SET(spdk_bdev_unmap_blocks, -1); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + MOCK_CLEAR(spdk_bdev_unmap_blocks); + + g_bdev_io->type = SPDK_BDEV_IO_TYPE_FLUSH; + MOCK_SET(spdk_bdev_flush_blocks, 0); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + MOCK_SET(spdk_bdev_flush_blocks, -1); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + MOCK_CLEAR(spdk_bdev_flush_blocks); + + g_bdev_io->type = SPDK_BDEV_IO_TYPE_RESET; + MOCK_SET(spdk_bdev_reset, 0); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + MOCK_SET(spdk_bdev_reset, -1); + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + MOCK_CLEAR(spdk_bdev_reset); + + /* We should never get a WZ command, we report that we don't support it. */ + g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE_ZEROES; + vbdev_crypto_submit_request(g_io_ch, g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); +} + +static void +test_initdrivers(void) +{ + int rc; + static struct spdk_mempool *orig_mbuf_mp; + static struct spdk_mempool *orig_session_mp; + + /* No drivers available, not an error though */ + MOCK_SET(rte_eal_get_configuration, g_test_config); + MOCK_SET(rte_cryptodev_count, 0); + rc = vbdev_crypto_init_crypto_drivers(); + CU_ASSERT(rc == 0); + + /* Test failure of DPDK dev init. */ + MOCK_SET(rte_cryptodev_count, 2); + MOCK_SET(rte_vdev_init, -1); + rc = vbdev_crypto_init_crypto_drivers(); + CU_ASSERT(rc == -EINVAL); + MOCK_SET(rte_vdev_init, 0); + + /* Can't create session pool. */ + MOCK_SET(spdk_mempool_create, NULL); + orig_mbuf_mp = g_mbuf_mp; + orig_session_mp = g_session_mp; + rc = vbdev_crypto_init_crypto_drivers(); + g_mbuf_mp = orig_mbuf_mp; + g_session_mp = orig_session_mp; + CU_ASSERT(rc == -ENOMEM); + MOCK_CLEAR(spdk_mempool_create); + + /* Can't create op pool. These tests will alloc and free our g_mbuf_mp + * so save that off here and restore it after each test is over. + */ + orig_mbuf_mp = g_mbuf_mp; + orig_session_mp = g_session_mp; + MOCK_SET(rte_crypto_op_pool_create, NULL); + rc = vbdev_crypto_init_crypto_drivers(); + g_mbuf_mp = orig_mbuf_mp; + g_session_mp = orig_session_mp; + CU_ASSERT(rc == -ENOMEM); + MOCK_SET(rte_crypto_op_pool_create, (struct rte_mempool *)1); + + /* Check resources are sufficient failure. */ + orig_mbuf_mp = g_mbuf_mp; + orig_session_mp = g_session_mp; + rc = vbdev_crypto_init_crypto_drivers(); + g_mbuf_mp = orig_mbuf_mp; + g_session_mp = orig_session_mp; + CU_ASSERT(rc == -EINVAL); + + /* Test crypto dev configure failure. */ + MOCK_SET(rte_cryptodev_device_count_by_driver, 2); + MOCK_SET(rte_cryptodev_info_get, 1); + MOCK_SET(rte_cryptodev_configure, -1); + orig_mbuf_mp = g_mbuf_mp; + orig_session_mp = g_session_mp; + rc = vbdev_crypto_init_crypto_drivers(); + g_mbuf_mp = orig_mbuf_mp; + g_session_mp = orig_session_mp; + MOCK_SET(rte_cryptodev_configure, 0); + CU_ASSERT(rc == -EINVAL); + + /* Test failure of qp setup. */ + MOCK_SET(rte_cryptodev_queue_pair_setup, -1); + orig_mbuf_mp = g_mbuf_mp; + orig_session_mp = g_session_mp; + rc = vbdev_crypto_init_crypto_drivers(); + g_mbuf_mp = orig_mbuf_mp; + g_session_mp = orig_session_mp; + CU_ASSERT(rc == -EINVAL); + MOCK_SET(rte_cryptodev_queue_pair_setup, 0); + + /* Test failure of dev start. */ + MOCK_SET(rte_cryptodev_start, -1); + orig_mbuf_mp = g_mbuf_mp; + orig_session_mp = g_session_mp; + rc = vbdev_crypto_init_crypto_drivers(); + g_mbuf_mp = orig_mbuf_mp; + g_session_mp = orig_session_mp; + CU_ASSERT(rc == -EINVAL); + MOCK_SET(rte_cryptodev_start, 0); + + /* Test happy path. */ + rc = vbdev_crypto_init_crypto_drivers(); + CU_ASSERT(rc == 0); +} + +static void +test_crypto_op_complete(void) +{ + /* Make sure completion code respects failure. */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED; + g_completion_called = false; + _crypto_operation_complete(g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + CU_ASSERT(g_completion_called == true); + + /* Test read completion. */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_READ; + g_completion_called = false; + _crypto_operation_complete(g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_completion_called == true); + + /* Test write completion success. */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; + g_completion_called = false; + MOCK_SET(spdk_bdev_writev_blocks, 0); + /* Code under test will free this, if not ASAN will complain. */ + g_io_ctx->cry_iov.iov_base = spdk_dma_malloc(16, 0x10, NULL); + _crypto_operation_complete(g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS); + CU_ASSERT(g_completion_called == true); + + /* Test write completion failed. */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE; + g_completion_called = false; + MOCK_SET(spdk_bdev_writev_blocks, -1); + /* Code under test will free this, if not ASAN will complain. */ + g_io_ctx->cry_iov.iov_base = spdk_dma_malloc(16, 0x10, NULL); + _crypto_operation_complete(g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + CU_ASSERT(g_completion_called == true); + + /* Test bogus type for this completion. */ + g_bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS; + g_bdev_io->type = SPDK_BDEV_IO_TYPE_RESET; + g_completion_called = false; + _crypto_operation_complete(g_bdev_io); + CU_ASSERT(g_bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FAILED); + CU_ASSERT(g_completion_called == true); +} + +static void +test_supported_io(void) +{ + void *ctx = NULL; + bool rc = true; + + /* Make sure we always report false to WZ, we need the bdev layer to + * send real 0's so we can encrypt/decrypt them. + */ + rc = vbdev_crypto_io_type_supported(ctx, SPDK_BDEV_IO_TYPE_WRITE_ZEROES); + CU_ASSERT(rc == false); +} + +int +main(int argc, char **argv) +{ + CU_pSuite suite = NULL; + unsigned int num_failures; + + if (CU_initialize_registry() != CUE_SUCCESS) { + return CU_get_error(); + } + + suite = CU_add_suite("crypto", test_setup, test_cleanup); + if (suite == NULL) { + CU_cleanup_registry(); + return CU_get_error(); + } + + if (CU_add_test(suite, "test_error_paths", + test_error_paths) == NULL || + CU_add_test(suite, "test_simple_write", + test_simple_write) == NULL || + CU_add_test(suite, "test_simple_read", + test_simple_read) == NULL || + CU_add_test(suite, "test_large_rw", + test_large_rw) == NULL || + CU_add_test(suite, "test_dev_full", + test_dev_full) == NULL || + CU_add_test(suite, "test_crazy_rw", + test_crazy_rw) == NULL || + CU_add_test(suite, "test_passthru", + test_passthru) == NULL || + CU_add_test(suite, "test_initdrivers", + test_initdrivers) == NULL || + CU_add_test(suite, "test_crypto_op_complete", + test_crypto_op_complete) == NULL || + CU_add_test(suite, "test_supported_io", + test_supported_io) == NULL + ) { + CU_cleanup_registry(); + return CU_get_error(); + } + + CU_basic_set_mode(CU_BRM_VERBOSE); + CU_basic_run_tests(); + num_failures = CU_get_number_of_failures(); + CU_cleanup_registry(); + return num_failures; +} diff --git a/test/unit/lib/bdev/crypto.c/rte_crypto.h b/test/unit/lib/bdev/crypto.c/rte_crypto.h new file mode 100644 index 000000000..a53a71dfe --- /dev/null +++ b/test/unit/lib/bdev/crypto.c/rte_crypto.h @@ -0,0 +1,95 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * Copyright(c) 2016 6WIND S.A. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUcryptoION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_CRYPTO_H_ +#define _RTE_CRYPTO_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* In order to mock some DPDK functions, we place headers here with the name name as the DPDK headers + * so these definitions wil be picked up. Only what's mocked is included. + */ + +#include "rte_mbuf.h" +#include "rte_mempool.h" +#include "rte_crypto_sym.h" + +enum rte_crypto_op_type { + RTE_CRYPTO_OP_TYPE_UNDEFINED, + RTE_CRYPTO_OP_TYPE_SYMMETRIC, +}; + +enum rte_crypto_op_status { + RTE_CRYPTO_OP_STATUS_SUCCESS, + RTE_CRYPTO_OP_STATUS_NOT_PROCESSED, + RTE_CRYPTO_OP_STATUS_AUTH_FAILED, + RTE_CRYPTO_OP_STATUS_INVALID_SESSION, + RTE_CRYPTO_OP_STATUS_INVALID_ARGS, + RTE_CRYPTO_OP_STATUS_ERROR, +}; + +struct rte_crypto_op { + uint8_t type; + uint8_t status; + uint8_t sess_type; + uint8_t reserved[5]; + struct rte_mempool *mempool; + rte_iova_t phys_addr; + __extension__ + union { + struct rte_crypto_sym_op sym[0]; + }; +}; + +extern struct rte_mempool * +rte_crypto_op_pool_create(const char *name, enum rte_crypto_op_type type, + unsigned nb_elts, unsigned cache_size, uint16_t priv_size, + int socket_id); + +static inline unsigned +rte_crypto_op_bulk_alloc(struct rte_mempool *mempool, + enum rte_crypto_op_type type, + struct rte_crypto_op **ops, uint16_t nb_ops); + +static inline int +rte_crypto_op_attach_sym_session(struct rte_crypto_op *op, + struct rte_cryptodev_sym_session *sess); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/test/unit/lib/bdev/crypto.c/rte_cryptodev.h b/test/unit/lib/bdev/crypto.c/rte_cryptodev.h new file mode 100644 index 000000000..ec72ffd7a --- /dev/null +++ b/test/unit/lib/bdev/crypto.c/rte_cryptodev.h @@ -0,0 +1,147 @@ +/*- + * + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. + * Copyright 2014 6WIND S.A. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_CRYPTODEV_H_ +#define _RTE_CRYPTODEV_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* In order to mock some DPDK functions, we place headers here with the name name as the DPDK headers + * so these definitions wil be picked up. Only what's mocked is included. + */ + +uint8_t dummy[16]; +#define rte_crypto_op_ctod_offset(c, t, o) &dummy[0] + +#define RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER (1ULL << 9) + +struct rte_cryptodev_info { + const char *driver_name; + uint8_t driver_id; + struct rte_pci_device *pci_dev; + uint64_t feature_flags; + const struct rte_cryptodev_capabilities *capabilities; + unsigned max_nb_queue_pairs; + struct { + unsigned max_nb_sessions; + unsigned int max_nb_sessions_per_qp; + } sym; +}; + +enum rte_cryptodev_event_type { + RTE_CRYPTODEV_EVENT_UNKNOWN, + RTE_CRYPTODEV_EVENT_ERROR, + RTE_CRYPTODEV_EVENT_MAX +}; + +struct rte_cryptodev_qp_conf { + uint32_t nb_descriptors; +}; + +struct rte_cryptodev_stats { + uint64_t enqueued_count; + uint64_t dequeued_count; + uint64_t enqueue_err_count; + uint64_t dequeue_err_count; +}; + +#define RTE_CRYPTODEV_NAME_MAX_LEN (64) + +extern uint8_t +rte_cryptodev_count(void); + +extern uint8_t +rte_cryptodev_device_count_by_driver(uint8_t driver_id); + +extern int +rte_cryptodev_socket_id(uint8_t dev_id); + +struct rte_cryptodev_config { + int socket_id; + uint16_t nb_queue_pairs; +}; + +extern int +rte_cryptodev_configure(uint8_t dev_id, struct rte_cryptodev_config *config); + +extern int +rte_cryptodev_start(uint8_t dev_id); + +extern void +rte_cryptodev_stop(uint8_t dev_id); + +extern int +rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id, + const struct rte_cryptodev_qp_conf *qp_conf, int socket_id, + struct rte_mempool *session_pool); + +extern void +rte_cryptodev_info_get(uint8_t dev_id, struct rte_cryptodev_info *dev_info); + +static inline uint16_t +rte_cryptodev_dequeue_burst(uint8_t dev_id, uint16_t qp_id, + struct rte_crypto_op **ops, uint16_t nb_ops); + +static inline uint16_t +rte_cryptodev_enqueue_burst(uint8_t dev_id, uint16_t qp_id, + struct rte_crypto_op **ops, uint16_t nb_ops); + +struct rte_cryptodev_sym_session { + __extension__ void *sess_private_data[0]; +}; + +struct rte_cryptodev_sym_session * +rte_cryptodev_sym_session_create(struct rte_mempool *mempool); + +int +rte_cryptodev_sym_session_free(struct rte_cryptodev_sym_session *sess); + +int +rte_cryptodev_sym_session_init(uint8_t dev_id, + struct rte_cryptodev_sym_session *sess, + struct rte_crypto_sym_xform *xforms, + struct rte_mempool *mempool); + +int +rte_cryptodev_sym_session_clear(uint8_t dev_id, + struct rte_cryptodev_sym_session *sess); + +unsigned int +rte_cryptodev_sym_get_private_session_size(uint8_t dev_id); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/test/unit/lib/bdev/crypto.c/rte_mbuf.h b/test/unit/lib/bdev/crypto.c/rte_mbuf.h new file mode 100644 index 000000000..4d69f4826 --- /dev/null +++ b/test/unit/lib/bdev/crypto.c/rte_mbuf.h @@ -0,0 +1,148 @@ +/*- + * + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. + * Copyright 2014 6WIND S.A. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_MBUF_H_ +#define _RTE_MBUF_H_ + +#include "rte_mempool.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* In order to mock some DPDK functions, we place headers here with the name name as the DPDK headers + * so these definitions wil be picked up. Only what's mocked is included. + */ + +__extension__ +typedef void *MARKER[0]; +__extension__ +typedef uint8_t MARKER8[0]; +__extension__ +typedef uint64_t MARKER64[0]; + +struct rte_mbuf { + MARKER cacheline0; + void *buf_addr; + RTE_STD_C11 + union { + rte_iova_t buf_iova; + rte_iova_t buf_physaddr; + } __rte_aligned(sizeof(rte_iova_t)); + MARKER64 rearm_data; + uint16_t data_off; + RTE_STD_C11 + union { + rte_atomic16_t refcnt_atomic; + uint16_t refcnt; + }; + uint16_t nb_segs; + uint16_t port; + uint64_t ol_flags; + MARKER rx_descriptor_fields1; + RTE_STD_C11 + union { + uint32_t packet_type; + struct { + uint32_t l2_type: 4; + uint32_t l3_type: 4; + uint32_t l4_type: 4; + uint32_t tun_type: 4; + RTE_STD_C11 + union { + uint8_t inner_esp_next_proto; + __extension__ + struct { + uint8_t inner_l2_type: 4; + uint8_t inner_l3_type: 4; + }; + }; + uint32_t inner_l4_type: 4; + }; + }; + uint32_t pkt_len; + uint16_t data_len; + uint16_t vlan_tci; + union { + uint32_t rss; + struct { + RTE_STD_C11 + union { + struct { + uint16_t hash; + uint16_t id; + }; + uint32_t lo; + }; + uint32_t hi; + } fdir; + struct { + uint32_t lo; + uint32_t hi; + } sched; + uint32_t usr; + } hash; + uint16_t vlan_tci_outer; + uint16_t buf_len; + uint64_t timestamp; + MARKER cacheline1 __rte_cache_min_aligned; + RTE_STD_C11 + union { + void *userdata; + uint64_t udata64; + }; + struct rte_mempool *pool; + struct rte_mbuf *next; + RTE_STD_C11 + union { + uint64_t tx_offload; + __extension__ + struct { + uint64_t l2_len: 7; + uint64_t l3_len: 9; + uint64_t l4_len: 8; + uint64_t tso_segsz: 16; + uint64_t outer_l3_len: 9; + uint64_t outer_l2_len: 7; + }; + }; + uint16_t priv_size; + uint16_t timesync; + uint32_t seqn; + +} __rte_cache_aligned; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/test/unit/lib/bdev/crypto.c/rte_mempool.h b/test/unit/lib/bdev/crypto.c/rte_mempool.h new file mode 100644 index 000000000..5750d30f2 --- /dev/null +++ b/test/unit/lib/bdev/crypto.c/rte_mempool.h @@ -0,0 +1,145 @@ +/*- + * + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. + * Copyright 2014 6WIND S.A. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_MEMPOOL_H_ +#define _RTE_MEMPOOL_H_ + +/** + * @file + * RTE Mempool. + * + * A memory pool is an allocator of fixed-size object. It is + * identified by its name, and uses a ring to store free objects. It + * provides some other optional services, like a per-core object + * cache, and an alignment helper to ensure that objects are padded + * to spread them equally on all RAM channels, ranks, and so on. + * + * Objects owned by a mempool should never be added in another + * mempool. When an object is freed using rte_mempool_put() or + * equivalent, the object data is not modified; the user can save some + * meta-data in the object data and retrieve them when allocating a + * new object. + * + * Note: the mempool implementation is not preemptible. An lcore must not be + * interrupted by another task that uses the same mempool (because it uses a + * ring which is not preemptible). Also, usual mempool functions like + * rte_mempool_get() or rte_mempool_put() are designed to be called from an EAL + * thread due to the internal per-lcore cache. Due to the lack of caching, + * rte_mempool_get() or rte_mempool_put() performance will suffer when called + * by non-EAL threads. Instead, non-EAL threads should call + * rte_mempool_generic_get() or rte_mempool_generic_put() with a user cache + * created with rte_mempool_cache_create(). + */ + +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* In order to mock some DPDK functions, we place headers here with the name name as the DPDK headers + * so these definitions wil be picked up. Only what's mocked is included. + */ + +STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr); +STAILQ_HEAD(rte_mempool_memhdr_list, rte_mempool_memhdr); +struct rte_mempool { + char name[RTE_MEMZONE_NAMESIZE]; + RTE_STD_C11 + union { + void *pool_data; + uint64_t pool_id; + }; + void *pool_config; + const struct rte_memzone *mz; + unsigned int flags; + int socket_id; + uint32_t size; + uint32_t cache_size; + uint32_t elt_size; + uint32_t header_size; + uint32_t trailer_size; + unsigned private_data_size; + int32_t ops_index; + struct rte_mempool_cache *local_cache; + uint32_t populated_size; + struct rte_mempool_objhdr_list elt_list; + uint32_t nb_mem_chunks; + struct rte_mempool_memhdr_list mem_list; +#ifdef RTE_LIBRTE_MEMPOOL_DEBUG + struct rte_mempool_debug_stats stats[RTE_MAX_LCORE]; +#endif +} __rte_cache_aligned; +#define RTE_MEMPOOL_OPS_NAMESIZE 32 +typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp); +typedef void (*rte_mempool_free_t)(struct rte_mempool *mp); +typedef int (*rte_mempool_enqueue_t)(struct rte_mempool *mp, + void *const *obj_table, unsigned int n); +typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp, + void **obj_table, unsigned int n); +typedef unsigned(*rte_mempool_get_count)(const struct rte_mempool *mp); +typedef int (*rte_mempool_get_capabilities_t)(const struct rte_mempool *mp, + unsigned int *flags); +typedef int (*rte_mempool_ops_register_memory_area_t) +(const struct rte_mempool *mp, char *vaddr, rte_iova_t iova, size_t len); +struct rte_mempool_ops { + char name[RTE_MEMPOOL_OPS_NAMESIZE]; + rte_mempool_alloc_t alloc; + rte_mempool_free_t free; + rte_mempool_enqueue_t enqueue; + rte_mempool_dequeue_t dequeue; + rte_mempool_get_count get_count; + rte_mempool_get_capabilities_t get_capabilities; + rte_mempool_ops_register_memory_area_t register_memory_area; +} __rte_cache_aligned; +#define RTE_MEMPOOL_MAX_OPS_IDX 16 +struct rte_mempool_ops_table { + rte_spinlock_t sl; + uint32_t num_ops; + struct rte_mempool_ops ops[RTE_MEMPOOL_MAX_OPS_IDX]; +} __rte_cache_aligned; +extern struct rte_mempool_ops_table rte_mempool_ops_table; +void +rte_mempool_free(struct rte_mempool *mp); +static __rte_always_inline void +rte_mempool_put_bulk(struct rte_mempool *mp, void *const *obj_table, + unsigned int n); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_MEMPOOL_H_ */ diff --git a/test/unit/unittest.sh b/test/unit/unittest.sh index b17443a1e..ede370305 100755 --- a/test/unit/unittest.sh +++ b/test/unit/unittest.sh @@ -54,6 +54,10 @@ $valgrind $testdir/lib/bdev/scsi_nvme.c/scsi_nvme_ut $valgrind $testdir/lib/bdev/gpt/gpt.c/gpt_ut $valgrind $testdir/lib/bdev/vbdev_lvol.c/vbdev_lvol_ut +if grep -q '#define SPDK_CONFIG_CRYPTO 1' $rootdir/config.h; then + $valgrind $testdir/lib/bdev/crypto.c/crypto_ut +fi + if grep -q '#define SPDK_CONFIG_PMDK 1' $rootdir/config.h; then $valgrind $testdir/lib/bdev/pmem/bdev_pmem_ut fi