ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

module/accel: Add mlx5 accel module

Browse Source 
The mlx5 accel module supports crypto operations.
Data buffer is split into `block_size` chunks and each
chunk is enrypted individually.
mlx5 library contains some utility functions that will
later be used by other libraries, this lib will be
exntended later.

Signed-off-by: Alexey Marchuk <alexeymar@nvidia.com>
Change-Id: Iacdd8caaade477277d5a95cfd53e9910e280a73b
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15420
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Paul Luse <paul.e.luse@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
This commit is contained in:
Alexey Marchuk 2022-10-10 10:53:49 +02:00 committed by Jim Harris
parent 13f97e6737
commit a1dfa7ec92
19 changed files with 1809 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -11,6 +11,9 @@ New functions `spdk_accel_submit_encrypt` and `spdk_accel_submit_decrypt` were a
New accel module `dpdk_cryptodev` has been added. It uses DPDK crypto PMD and support encrypt and
decrypt operations. New RPC `dpdk_cryptodev_scan_accel_module` has been added to enable this accel module.
New accel module `mlx5` was added. It implements crypto operations, enabled when SPDK is configured with
RDMA provider is mlx5_dv and crypto support.
### bdev
Added RPCs bdev_nvme_start_mdns_discovery, bdev_nvme_get_mdns_discovery_info and

16
configure vendored
View File

@ -874,12 +874,16 @@ than or equal to 4.14 will see significantly reduced performance.
fi
if [ "${CONFIG[RDMA_PROV]}" == "mlx5_dv" ]; then
if ! echo -e '#include <spdk/stdinc.h>\n' \
'#include <infiniband/mlx5dv.h>\n' \
'#include <rdma/rdma_cma.h>\n' \
'int main(void) { return rdma_establish(NULL) || ' \
'!!IBV_QP_INIT_ATTR_SEND_OPS_FLAGS || !!MLX5_OPCODE_RDMA_WRITE; }\n' \
| "${BUILD_CMD[@]}" -lmlx5 -I${rootdir}/include -c - 2> /dev/null; then
MLX5_DV_BUILD_BUILD_CMD="
#include <infiniband/mlx5dv.h>\n
#include <rdma/rdma_cma.h>\n
int main(void) { return rdma_establish(NULL) ||\n
!!IBV_QP_INIT_ATTR_SEND_OPS_FLAGS || !!MLX5_OPCODE_RDMA_WRITE"
if [ "${CONFIG[CRYPTO]}" = "y" ]; then
MLX5_DV_BUILD_BUILD_CMD+="|| !!MLX5DV_CRYPTO_ENGINES_CAP_AES_XTS_SINGLE_BLOCK"
fi
MLX5_DV_BUILD_BUILD_CMD+=";}"
if ! echo -e $MLX5_DV_BUILD_BUILD_CMD | "${BUILD_CMD[@]}" -lmlx5 -I${rootdir}/include -c -; then
echo "mlx5_dv provider is not supported"
exit 1
fi

38
doc/jsonrpc.md
View File

@ -450,6 +450,7 @@ Example response:
"dpdk_cryptodev_scan_accel_module",
"dpdk_cryptodev_set_driver",
"dpdk_cryptodev_get_driver",
"mlx5_scan_accel_module",
"bdev_virtio_attach_controller",
"bdev_virtio_scsi_get_devices",
"bdev_virtio_detach_controller",
@ -2132,6 +2133,43 @@ Example response:
}
~~~
### mlx5_scan_accel_module {#rpc_mlx5_scan_accel_module}
Enable mlx5 accel offload
#### Parameters
Name | Optional | Type | Description
----------------------- | -------- |--------| -----------
qp_size | Optional | number | qpair size
num_requests | Optional | number | Size of the shared requests pool
#### Example
Example request:
~~~json
{
"jsonrpc": "2.0",
"method": "mlx5_scan_accel_module",
"id": 1,
"params": {
"qp_size": 256,
"num_requests": 2047
}
}
~~~
Example response:
~~~json
{
"jsonrpc": "2.0",
"id": 1,
"result": true
}
~~~
## Block Device Abstraction Layer {#jsonrpc_components_bdev}
### bdev_set_options {#rpc_bdev_set_options}

69
include/spdk_internal/mlx5.h Normal file
View File

@ -0,0 +1,69 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.
* All rights reserved.
*/
#ifndef SPDK_MLX5_H
#define SPDK_MLX5_H
#include <infiniband/mlx5dv.h>
struct spdk_mlx5_crypto_dek;
struct spdk_mlx5_crypto_keytag;
struct spdk_mlx5_crypto_dek_create_attr {
/* Data Encryption Key in binary form */
char *dek;
/* Length of the dek */
size_t dek_len;
};
/**
* Return a NULL terminated array of devices which support crypto operation on Nvidia NICs
*
* \param dev_num The size of the array or 0
* \return Array of contexts. This array must be released with \b spdk_mlx5_crypto_devs_release
*/
struct ibv_context **spdk_mlx5_crypto_devs_get(int *dev_num);
/**
* Releases array of devices allocated by \b spdk_mlx5_crypto_devs_get
*
* \param rdma_devs Array of device to be released
*/
void spdk_mlx5_crypto_devs_release(struct ibv_context **rdma_devs);
/**
* Create a keytag which contains DEKs per each crypto device in the system
*
* \param attr Crypto attributes
* \param out Keytag
* \return 0 on success, negated errno of failure
*/
int spdk_mlx5_crypto_keytag_create(struct spdk_mlx5_crypto_dek_create_attr *attr,
struct spdk_mlx5_crypto_keytag **out);
/**
* Destroy a keytag created using \b spdk_mlx5_crypto_keytag_create
*
* \param keytag Keytag pointer
*/
void spdk_mlx5_crypto_keytag_destroy(struct spdk_mlx5_crypto_keytag *keytag);
/**
* Fills attributes used to register UMR with crypto operation
*
* \param attr_out Configured UMR attributes
* \param keytag Keytag with DEKs
* \param pd Protection Domain which is going to be used to register UMR. This function will find a DEK in \b keytag with the same PD
* \param block_size Logical block size
* \param iv Initialization vector or tweak. Usually that is logical block address
* \param encrypt_on_tx If set, memory data will be encrypted during TX and wire data will be decrypted during RX. If not set, memory data will be decrypted during TX and wire data will be encrypted during RX.
* \return 0 on success, negated errno on failure
*/
int spdk_mlx5_crypto_set_attr(struct mlx5dv_crypto_attr *attr_out,
struct spdk_mlx5_crypto_keytag *keytag, struct ibv_pd *pd,
uint32_t block_size, uint64_t iv, bool encrypt_on_tx);
#endif /* SPDK_MLX5_H */

3
lib/Makefile
View File

@ -22,6 +22,9 @@ DIRS-$(CONFIG_VBDEV_COMPRESS) += reduce
DIRS-$(CONFIG_RDMA) += rdma
DIRS-$(CONFIG_VFIO_USER) += vfu_tgt
ifeq ($(CONFIG_RDMA_PROV),mlx5_dv)
DIRS-y += mlx5
endif
# If CONFIG_ENV is pointing at a directory in lib, build it.
# Out-of-tree env implementations must be built separately by the user.
ENV_NAME := $(notdir $(CONFIG_ENV))

19
lib/mlx5/Makefile Normal file
View File

@ -0,0 +1,19 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
#
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
SO_VER := 1
SO_MINOR := 0
C_SRCS = mlx5_crypto.c
LIBNAME = mlx5
LOCAL_SYS_LIBS += -lmlx5 -libverbs -lrdmacm
SPDK_MAP_FILE = $(abspath $(CURDIR)/spdk_mlx5.map)
include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

345
lib/mlx5/mlx5_crypto.c Normal file
View File

@ -0,0 +1,345 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
*/
#include <rdma/rdma_cma.h>
#include <infiniband/verbs.h>
#include <infiniband/mlx5dv.h>
#include "spdk/stdinc.h"
#include "spdk/queue.h"
#include "spdk/log.h"
#include "spdk/likely.h"
#include "spdk/util.h"
#include "spdk_internal/mlx5.h"
#include "spdk_internal/rdma.h"
#define MLX5_VENDOR_ID_MELLANOX 0x2c9
/* Plaintext key sizes */
/* 64b keytag */
#define SPDK_MLX5_AES_XTS_KEYTAG_SIZE 8
/* key1_128b + key2_128b */
#define SPDK_MLX5_AES_XTS_128_DEK_BYTES 32
/* key1_256b + key2_256b */
#define SPDK_MLX5_AES_XTS_256_DEK_BYTES 64
/* key1_128b + key2_128b + 64b_keytag */
#define SPDK_MLX5_AES_XTS_128_DEK_BYTES_WITH_KEYTAG (SPDK_MLX5_AES_XTS_128_DEK_BYTES + SPDK_MLX5_AES_XTS_KEYTAG_SIZE)
/* key1_256b + key2_256b + 64b_keytag */
#define SPDK_MLX5_AES_XTS_256_DEK_BYTES_WITH_KEYTAG (SPDK_MLX5_AES_XTS_256_DEK_BYTES + SPDK_MLX5_AES_XTS_KEYTAG_SIZE)
struct spdk_mlx5_crypto_dek {
struct mlx5dv_dek *dek_obj;
struct ibv_pd *pd;
struct ibv_context *context;
};
struct spdk_mlx5_crypto_keytag {
struct spdk_mlx5_crypto_dek *deks;
uint32_t deks_num;
bool has_keytag;
char keytag[8];
};
struct ibv_context **
spdk_mlx5_crypto_devs_get(int *dev_num)
{
struct ibv_context **rdma_devs, **rdma_devs_out = NULL, *dev;
struct ibv_device_attr dev_attr;
struct mlx5dv_context dv_dev_attr;
int num_rdma_devs = 0, i, rc;
int num_crypto_devs = 0;
/* query all devices, save mlx5 with crypto support */
rdma_devs = rdma_get_devices(&num_rdma_devs);
if (!rdma_devs || !num_rdma_devs) {
*dev_num = 0;
return NULL;
}
rdma_devs_out = calloc(num_rdma_devs + 1, sizeof(*rdma_devs_out));
if (!rdma_devs_out) {
SPDK_ERRLOG("Memory allocation failed\n");
return NULL;
}
for (i = 0; i < num_rdma_devs; i++) {
dev = rdma_devs[i];
rc = ibv_query_device(dev, &dev_attr);
if (rc) {
SPDK_ERRLOG("Failed to query dev %s, skipping\n", dev->device->name);
continue;
}
if (dev_attr.vendor_id != MLX5_VENDOR_ID_MELLANOX) {
SPDK_DEBUGLOG(mlx5, "dev %s is not Mellanox device, skipping\n", dev->device->name);
continue;
}
memset(&dv_dev_attr, 0, sizeof(dv_dev_attr));
dv_dev_attr.comp_mask |= MLX5DV_CONTEXT_MASK_CRYPTO_OFFLOAD;
rc = mlx5dv_query_device(dev, &dv_dev_attr);
if (rc) {
SPDK_ERRLOG("Failed to query mlx5 dev %s, skipping\n", dev->device->name);
continue;
}
if (!(dv_dev_attr.crypto_caps.flags & MLX5DV_CRYPTO_CAPS_CRYPTO)) {
SPDK_DEBUGLOG(mlx5, "dev %s crypto engine doesn't support crypto, skipping\n", dev->device->name);
continue;
}
if (!(dv_dev_attr.crypto_caps.crypto_engines & (MLX5DV_CRYPTO_ENGINES_CAP_AES_XTS |
MLX5DV_CRYPTO_ENGINES_CAP_AES_XTS_SINGLE_BLOCK))) {
SPDK_DEBUGLOG(mlx5, "dev %s crypto engine doesn't support AES_XTS, skipping\n", dev->device->name);
continue;
}
if (dv_dev_attr.crypto_caps.wrapped_import_method &
MLX5DV_CRYPTO_WRAPPED_IMPORT_METHOD_CAP_AES_XTS) {
SPDK_WARNLOG("dev %s uses wrapped import method (0x%x) which is not supported by mlx5 accel module\n",
dev->device->name, dv_dev_attr.crypto_caps.wrapped_import_method);
continue;
}
SPDK_NOTICELOG("Crypto dev %s\n", dev->device->name);
rdma_devs_out[num_crypto_devs++] = dev;
}
if (!num_crypto_devs) {
SPDK_DEBUGLOG(mlx5, "Found no mlx5 crypto devices\n");
goto err_out;
}
rdma_free_devices(rdma_devs);
*dev_num = num_crypto_devs;
return rdma_devs_out;
err_out:
free(rdma_devs_out);
rdma_free_devices(rdma_devs);
*dev_num = 0;
return NULL;
}
void
spdk_mlx5_crypto_devs_release(struct ibv_context **rdma_devs)
{
if (rdma_devs) {
free(rdma_devs);
}
}
void
spdk_mlx5_crypto_keytag_destroy(struct spdk_mlx5_crypto_keytag *keytag)
{
struct spdk_mlx5_crypto_dek *dek;
uint32_t i;
if (!keytag) {
return;
}
for (i = 0; i < keytag->deks_num; i++) {
dek = &keytag->deks[i];
if (dek->dek_obj) {
mlx5dv_dek_destroy(dek->dek_obj);
}
if (dek->pd) {
spdk_rdma_put_pd(dek->pd);
}
}
spdk_memset_s(keytag->keytag, sizeof(keytag->keytag), 0, sizeof(keytag->keytag));
free(keytag->deks);
free(keytag);
}
int
spdk_mlx5_crypto_keytag_create(struct spdk_mlx5_crypto_dek_create_attr *attr,
struct spdk_mlx5_crypto_keytag **out)
{
struct spdk_mlx5_crypto_dek *dek;
struct spdk_mlx5_crypto_keytag *keytag;
struct ibv_context **devs;
struct mlx5dv_dek_init_attr init_attr = {};
struct mlx5dv_dek_attr query_attr;
int num_devs = 0, i, rc;
bool has_keytag;
if (!attr || !attr->dek) {
return -EINVAL;
}
switch (attr->dek_len) {
case SPDK_MLX5_AES_XTS_128_DEK_BYTES_WITH_KEYTAG:
init_attr.key_size = MLX5DV_CRYPTO_KEY_SIZE_128;
has_keytag = true;
SPDK_DEBUGLOG(mlx5, "128b AES_XTS with keytag\n");
break;
case SPDK_MLX5_AES_XTS_256_DEK_BYTES_WITH_KEYTAG:
init_attr.key_size = MLX5DV_CRYPTO_KEY_SIZE_256;
has_keytag = true;
SPDK_DEBUGLOG(mlx5, "256b AES_XTS with keytag\n");
break;
case SPDK_MLX5_AES_XTS_128_DEK_BYTES:
init_attr.key_size = MLX5DV_CRYPTO_KEY_SIZE_128;
has_keytag = false;
SPDK_DEBUGLOG(mlx5, "128b AES_XTS\n");
break;
case SPDK_MLX5_AES_XTS_256_DEK_BYTES:
init_attr.key_size = MLX5DV_CRYPTO_KEY_SIZE_256;
has_keytag = false;
SPDK_DEBUGLOG(mlx5, "256b AES_XTS\n");
break;
default:
SPDK_ERRLOG("Invalid key length %zu. The following keys are supported:\n"
"128b key + key2, %u bytes;\n"
"256b key + key2, %u bytes\n"
"128b key + key2 + keytag, %u bytes\n"
"256b lye + key2 + keytag, %u bytes\n",
attr->dek_len, SPDK_MLX5_AES_XTS_128_DEK_BYTES, MLX5DV_CRYPTO_KEY_SIZE_256,
SPDK_MLX5_AES_XTS_128_DEK_BYTES_WITH_KEYTAG, SPDK_MLX5_AES_XTS_256_DEK_BYTES_WITH_KEYTAG);
return -EINVAL;
}
devs = spdk_mlx5_crypto_devs_get(&num_devs);
if (!devs || !num_devs) {
SPDK_DEBUGLOG(mlx5, "No crypto devices found\n");
return -ENOTSUP;
}
keytag = calloc(1, sizeof(*keytag));
if (!keytag) {
SPDK_ERRLOG("Memory allocation failed\n");
spdk_mlx5_crypto_devs_release(devs);
return -ENOMEM;
}
keytag->deks = calloc(num_devs, sizeof(struct spdk_mlx5_crypto_dek));
if (!keytag->deks) {
SPDK_ERRLOG("Memory allocation failed\n");
spdk_mlx5_crypto_devs_release(devs);
free(keytag);
return -ENOMEM;
}
for (i = 0; i < num_devs; i++) {
keytag->deks_num++;
dek = &keytag->deks[i];
dek->pd = spdk_rdma_get_pd(devs[i]);
if (!dek->pd) {
SPDK_ERRLOG("Failed to get PD on device %s\n", devs[i]->device->name);
rc = -EINVAL;
goto err_out;
}
dek->context = devs[i];
init_attr.pd = dek->pd;
init_attr.has_keytag = has_keytag;
init_attr.key_purpose = MLX5DV_CRYPTO_KEY_PURPOSE_AES_XTS;
init_attr.comp_mask = MLX5DV_DEK_INIT_ATTR_CRYPTO_LOGIN;
init_attr.crypto_login = NULL;
memcpy(init_attr.key, attr->dek, attr->dek_len);
dek->dek_obj = mlx5dv_dek_create(dek->context, &init_attr);
spdk_memset_s(init_attr.key, sizeof(init_attr.key), 0, sizeof(init_attr.key));
if (!dek->dek_obj) {
SPDK_ERRLOG("mlx5dv_dek_create failed on dev %s, errno %d\n", dek->context->device->name, errno);
rc = -EINVAL;
goto err_out;
}
memset(&query_attr, 0, sizeof(query_attr));
rc = mlx5dv_dek_query(dek->dek_obj, &query_attr);
if (rc) {
SPDK_ERRLOG("Failed to query DEK on dev %s, rc %d\n", dek->context->device->name, rc);
goto err_out;
}
if (query_attr.state != MLX5DV_DEK_STATE_READY) {
SPDK_ERRLOG("DEK on dev %s state %d\n", dek->context->device->name, query_attr.state);
rc = -EINVAL;
goto err_out;
}
}
if (has_keytag) {
/* Save keytag, it will be used to configure crypto MKEY */
keytag->has_keytag = true;
memcpy(keytag->keytag, attr->dek + attr->dek_len - SPDK_MLX5_AES_XTS_KEYTAG_SIZE,
SPDK_MLX5_AES_XTS_KEYTAG_SIZE);
}
spdk_mlx5_crypto_devs_release(devs);
*out = keytag;
return 0;
err_out:
spdk_mlx5_crypto_keytag_destroy(keytag);
spdk_mlx5_crypto_devs_release(devs);
return rc;
}
static inline struct spdk_mlx5_crypto_dek *
mlx5_crypto_get_dek_by_pd(struct spdk_mlx5_crypto_keytag *keytag, struct ibv_pd *pd)
{
struct spdk_mlx5_crypto_dek *dek;
uint32_t i;
for (i = 0; i < keytag->deks_num; i++) {
dek = &keytag->deks[i];
if (dek->pd == pd) {
return dek;
}
}
return NULL;
}
int
spdk_mlx5_crypto_set_attr(struct mlx5dv_crypto_attr *attr_out,
struct spdk_mlx5_crypto_keytag *keytag, struct ibv_pd *pd,
uint32_t block_size, uint64_t iv, bool encrypt_on_tx)
{
struct spdk_mlx5_crypto_dek *dek;
enum mlx5dv_block_size bs;
dek = mlx5_crypto_get_dek_by_pd(keytag, pd);
if (spdk_unlikely(!dek)) {
SPDK_ERRLOG("No DEK for pd %p (dev %s)\n", pd, pd->context->device->name);
return -EINVAL;
}
switch (block_size) {
case 512:
bs = MLX5DV_BLOCK_SIZE_512;
break;
case 520:
bs = MLX5DV_BLOCK_SIZE_520;
break;
case 4048:
bs = MLX5DV_BLOCK_SIZE_4048;
break;
case 4096:
bs = MLX5DV_BLOCK_SIZE_4096;
break;
case 4160:
bs = MLX5DV_BLOCK_SIZE_4160;
break;
default:
SPDK_ERRLOG("Unsupported block size %u\n", block_size);
return -EINVAL;
}
memset(attr_out, 0, sizeof(*attr_out));
attr_out->dek = dek->dek_obj;
attr_out->crypto_standard = MLX5DV_CRYPTO_STANDARD_AES_XTS;
attr_out->data_unit_size = bs;
attr_out->encrypt_on_tx = encrypt_on_tx;
memcpy(attr_out->initial_tweak, &iv, sizeof(iv));
if (keytag->has_keytag) {
memcpy(attr_out->keytag, keytag->keytag, sizeof(keytag->keytag));
}
return 0;
}
SPDK_LOG_REGISTER_COMPONENT(mlx5)

11
lib/mlx5/spdk_mlx5.map Normal file
View File

@ -0,0 +1,11 @@
{
global:
spdk_mlx5_crypto_devs_get;
spdk_mlx5_crypto_devs_release;
spdk_mlx5_crypto_keytag_create;
spdk_mlx5_crypto_keytag_destroy;
spdk_mlx5_crypto_set_attr;
local: *;
};

7
mk/spdk.lib_deps.mk
View File

@ -70,6 +70,9 @@ DEPDIRS-nvmf := accel log sock util nvme thread $(JSON_LIBS) trace bdev
ifeq ($(CONFIG_RDMA),y)
DEPDIRS-nvmf += rdma
endif
ifeq ($(CONFIG_RDMA_PROV),mlx5_dv)
DEPDIRS-mlx5 = log rdma util
endif
DEPDIRS-scsi := log util thread $(JSON_LIBS) trace bdev
DEPDIRS-iscsi := log sock util conf thread $(JSON_LIBS) trace scsi
@ -100,6 +103,10 @@ DEPDIRS-accel_iaa := log idxd thread $(JSON_LIBS) accel trace
DEPDIRS-accel_dpdk_cryptodev := log thread $(JSON_LIBS) accel
DEPDIRS-accel_dpdk_compressdev := log thread $(JSON_LIBS) accel util
ifeq ($(CONFIG_RDMA_PROV),mlx5_dv)
DEPDIRS-accel_mlx5 := accel thread log mlx5 rdma util
endif
# module/env_dpdk
DEPDIRS-env_dpdk_rpc := log $(JSON_LIBS)

4
mk/spdk.modules.mk
View File

@ -108,6 +108,10 @@ ifeq ($(CONFIG_DPDK_COMPRESSDEV),y)
ACCEL_MODULES_LIST += accel_dpdk_compressdev
endif
ifeq ($(CONFIG_RDMA_PROV)|$(CONFIG_CRYPTO),mlx5_dv|y)
ACCEL_MODULES_LIST += accel_mlx5
endif
SCHEDULER_MODULES_LIST = scheduler_dynamic
ifeq (y,$(DPDK_POWER))
SCHEDULER_MODULES_LIST += env_dpdk scheduler_dpdk_governor scheduler_gscheduler

3
module/accel/Makefile
View File

@ -12,6 +12,9 @@ DIRS-$(CONFIG_DPDK_COMPRESSDEV) += dpdk_compressdev
DIRS-$(CONFIG_IDXD) += dsa
DIRS-$(CONFIG_IDXD) += iaa
DIRS-$(CONFIG_CRYPTO) += dpdk_cryptodev
ifeq ($(CONFIG_RDMA_PROV)|$(CONFIG_CRYPTO),mlx5_dv|y)
DIRS-y += mlx5
endif
.PHONY: all clean $(DIRS-y)

19
module/accel/mlx5/Makefile Normal file
View File

@ -0,0 +1,19 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES
# All rights reserved.
#
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
SO_VER := 1
SO_MINOR := 0
LIBNAME = accel_mlx5
C_SRCS = accel_mlx5.c accel_mlx5_rpc.c
SPDK_MAP_FILE = $(SPDK_ROOT_DIR)/mk/spdk_blank.map
LOCAL_SYS_LIBS += -libverbs -lmlx5
include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

1182
module/accel/mlx5/accel_mlx5.c Normal file
View File

File diff suppressed because it is too large Load Diff

15
module/accel/mlx5/accel_mlx5.h Normal file
View File

@ -0,0 +1,15 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
*/
#include "spdk/stdinc.h"
struct accel_mlx5_attr {
/* The number of entries in qp submission/receive queue */
uint16_t qp_size;
/* The number of requests in the global pool */
uint32_t num_requests;
};
void accel_mlx5_get_default_attr(struct accel_mlx5_attr *attr);
int accel_mlx5_enable(struct accel_mlx5_attr *attr);

43
module/accel/mlx5/accel_mlx5_rpc.c Normal file
View File

@ -0,0 +1,43 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
*/
#include "spdk/rpc.h"
#include "spdk/util.h"
#include "spdk/log.h"
#include "accel_mlx5.h"
static const struct spdk_json_object_decoder rpc_mlx5_module_decoder[] = {
{"qp_size", offsetof(struct accel_mlx5_attr, qp_size), spdk_json_decode_uint16, true},
{"num_requests", offsetof(struct accel_mlx5_attr, num_requests), spdk_json_decode_uint32, true},
};
static void
rpc_mlx5_scan_accel_module(struct spdk_jsonrpc_request *request,
const struct spdk_json_val *params)
{
struct accel_mlx5_attr attr;
int rc;
accel_mlx5_get_default_attr(&attr);
if (params != NULL) {
if (spdk_json_decode_object(params, rpc_mlx5_module_decoder,
SPDK_COUNTOF(rpc_mlx5_module_decoder),
&attr)) {
SPDK_ERRLOG("spdk_json_decode_object() failed\n");
spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_PARSE_ERROR,
"spdk_json_decode_object failed");
return;
}
}
rc = accel_mlx5_enable(&attr);
if (rc) {
spdk_jsonrpc_send_error_response_fmt(request, rc, "mlx5 scan failed with %d\n", rc);
} else {
spdk_jsonrpc_send_bool_response(request, true);
}
}
SPDK_RPC_REGISTER("mlx5_scan_accel_module", rpc_mlx5_scan_accel_module, SPDK_RPC_STARTUP)

1
python/spdk/rpc/__init__.py
View File

@ -33,6 +33,7 @@ from . import sock
from . import vfio_user
from . import iobuf
from . import dpdk_cryptodev
from . import mlx5
from . import client as rpc_client

21
python/spdk/rpc/mlx5.py Normal file
View File

@ -0,0 +1,21 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
from spdk.rpc.helpers import deprecated_alias
def mlx5_scan_accel_module(client, qp_size=None, num_requests=None):
"""Enable mlx5 accel module. Scans all mlx5 devices which can perform needed operations
Args:
qp_size: Qpair size. (optional)
num_requests: size of a global requests pool per mlx5 device (optional)
"""
params = {}
if qp_size is not None:
params['qp_size'] = qp_size
if num_requests is not None:
params['num_requests'] = num_requests
return client.call('mlx5_scan_accel_module', params)

11
scripts/rpc.py
View File

@ -2908,6 +2908,17 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
p = subparsers.add_parser('dpdk_cryptodev_get_driver', help='Get the DPDK cryptodev driver')
p.set_defaults(func=dpdk_cryptodev_get_driver)
# mlx5
def mlx5_scan_accel_module(args):
rpc.mlx5.mlx5_scan_accel_module(args.client,
qp_size=args.qp_size,
num_requests=args.num_requests)
p = subparsers.add_parser('mlx5_scan_accel_module', help='Enable mlx5 accel module.')
p.add_argument('-q', '--qp-size', type=int, help='QP size')
p.add_argument('-r', '--num-requests', type=int, help='Size of the shared requests pool')
p.set_defaults(func=mlx5_scan_accel_module)
# opal
def bdev_nvme_opal_init(args):
rpc.nvme.bdev_nvme_opal_init(args.client,

5
test/common/skipped_build_files.txt
View File

@ -56,3 +56,8 @@ module/bdev/daos/bdev_daos_rpc
# Not configured to test xNVMe bdev
module/bdev/xnvme/bdev_xnvme
module/bdev/xnvme/bdev_xnvme_rpc
# Not configured to test mlx5 accel module & crypto lib
module/accel/mlx5/accel_mlx5
module/accel/mlx5/accel_mlx5_rpc
lib/mlx5/mlx5_crypto