ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

module/accel: Add compressDev accel_module

Browse Source 
This is the port of the vbdev compress logic into the accel
framework.  It includes just one enhancement, to only fill each
mbuf in either src or dst array with max "window size" param to
avoid QAT errors. Note that DPDK ISAL PMD was not ported as we
have native ISAL compression in accel now.

Note: ISAL w/DPDK is still built w/this patch, that can't be
removed until the vbdev module moves to accel fw as it still
depends on DPDK ISAL PMD.

Follow-on patches will include addition C API for PMD selection,
this patch just gets equivalent functionality going.  Upcoming
patches will also convert the vbdev compress module to use the
accel framework instead of talking directly to compressdev.

More patches will also address comments on vbdev common code
that addressed here would make the review challenging.

This patch also fixes a bug in the ported code that needs to
be fixed here to pass CI.  Capability discovery was incorrect
causing all devices to appear to not support chained mbufs,
with the mbuf splitting code this is important to get right.

Signed-off-by: paul luse <paul.e.luse@intel.com>
Change-Id: I7f526404819b145ef26e40877122ba80a02fcf51
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15178
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@nvidia.com>
This commit is contained in:
paul luse 2022-10-28 14:56:55 -07:00 committed by Tomasz Zawadzki
parent 05e7a8a8bf
commit 976f8b0992
21 changed files with 579 additions and 1357 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CONFIG
View File

@ -130,6 +130,9 @@ CONFIG_PMDK_DIR=
# Build with xNVMe
CONFIG_XNVME=n
# Enable the dependencies for building the DPDK accel compress module
CONFIG_DPDK_COMPRESSDEV=n
# Enable the dependencies for building the compress vbdev, includes the reduce library
CONFIG_VBDEV_COMPRESS=n

9
configure vendored
View File

@ -76,6 +76,8 @@ function usage() {
echo " --without-pmdk No path required."
echo " --with-vbdev-compress Build vbdev compression module and dependencies."
echo " --without-vbdev-compress No path required."
echo " --with-dpdk-compressdev Build accel DPDK compression module and dependencies."
echo " --without-dpdk-compressdev No path required."
echo " --with-rbd Build Ceph RBD bdev module."
echo " --without-rbd No path required."
echo " --with-rdma[=DIR] Build RDMA transport for NVMf target and initiator."
@ -535,6 +537,12 @@ for i in "$@"; do
--without-vbdev-compress)
CONFIG[VBDEV_COMPRESS]=n
;;
--with-dpdk-compressdev)
CONFIG[DPDK_COMPRESSDEV]=y
;;
--without-dpdk-compressdev)
CONFIG[DPDK_COMPRESSDEV]=n
;;
--with-xnvme)
CONFIG[XNVME]=y
;;
@ -1192,6 +1200,7 @@ else
echo "so these features will be disabled."
CONFIG[CRYPTO]=n
CONFIG[VBDEV_COMPRESS]=n
CONFIG[DPDK_COMPRESSDEV]=n
fi
# ISA-L-crypto complements ISA-L functionality, it is only enabled together with ISA-L

37
doc/jsonrpc.md
View File

@ -1867,6 +1867,43 @@ Example response:
}
~~~
### compressdev_scan_accel_module {#rpc_compressdev_scan_accel_module}
Set config and enable compressdev accel module offload.
Select the DPDK polled mode driver (pmd) for the accel compress module,
0 = auto-select, 1= QAT only, 2 = mlx5_pci only.
#### Parameters
Name | Optional | Type | Description
----------------------- | -------- | ----------- | -----------
pmd | Required | int | pmd selection
#### Example
Example request:
~~~json
{
"params": {
"pmd": 1
},
"jsonrpc": "2.0",
"method": "compressdev_scan_accel_module",
"id": 1
}
~~~
Example response:
~~~json
{
"jsonrpc": "2.0",
"id": 1,
"result": true
}
~~~
### dsa_scan_accel_module {#rpc_dsa_scan_accel_module}
Set config and enable dsa accel module offload.

4
dpdkbuild/Makefile
View File

@ -40,7 +40,7 @@ DPDK_LIBS += kvargs telemetry
DPDK_LIBS += power timer ethdev net
# common crypto/compress drivers
ifeq ($(findstring y,$(CONFIG_CRYPTO)$(CONFIG_VBDEV_COMPRESS)),y)
ifeq ($(findstring y,$(CONFIG_DPDK_COMPRESSDEV)$(CONFIG_CRYPTO)$(CONFIG_VBDEV_COMPRESS)),y)
DPDK_DRIVERS += crypto/qat compress/qat common/qat
endif
@ -63,7 +63,7 @@ DPDK_LDFLAGS += -L$(IPSEC_MB_DIR)
endif
endif
ifeq ($(CONFIG_VBDEV_COMPRESS),y)
ifeq ($(findstring y,$(CONFIG_DPDK_COMPRESSDEV)$(CONFIG_VBDEV_COMPRESS)),y)
DPDK_DRIVERS += compress compress/isal
ifeq ($(CONFIG_VBDEV_COMPRESS_MLX5),y)
DPDK_DRIVERS += compress/mlx5

2
lib/env_dpdk/env.mk
View File

@ -80,7 +80,7 @@ endif
endif
endif
ifeq ($(CONFIG_VBDEV_COMPRESS),y)
ifeq ($(findstring y,$(CONFIG_DPDK_COMPRESSDEV)$(CONFIG_VBDEV_COMPRESS)),y)
DPDK_FRAMEWORK=y
ifneq (, $(wildcard $(DPDK_LIB_DIR)/librte_compress_isal.*))
DPDK_LIB_LIST += rte_compress_isal

1
mk/spdk.lib_deps.mk
View File

@ -98,6 +98,7 @@ DEPDIRS-accel_ioat := log ioat thread jsonrpc rpc accel
DEPDIRS-accel_dsa := log idxd thread $(JSON_LIBS) accel trace
DEPDIRS-accel_iaa := log idxd thread $(JSON_LIBS) accel trace
DEPDIRS-accel_dpdk_cryptodev := log thread $(JSON_LIBS) accel
DEPDIRS-accel_dpdk_compressdev := log thread $(JSON_LIBS) accel util
# module/env_dpdk
DEPDIRS-env_dpdk_rpc := log $(JSON_LIBS)

3
mk/spdk.modules.mk
View File

@ -104,6 +104,9 @@ endif
ifeq ($(CONFIG_CRYPTO),y)
ACCEL_MODULES_LIST += accel_dpdk_cryptodev
endif
ifeq ($(CONFIG_DPDK_COMPRESSDEV),y)
ACCEL_MODULES_LIST += accel_dpdk_compressdev
endif
SCHEDULER_MODULES_LIST = scheduler_dynamic
ifeq (y,$(DPDK_POWER))

2
module/accel/Makefile
View File

@ -8,7 +8,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
DIRS-y = ioat
DIRS-$(CONFIG_DPDK_COMPRESSDEV) += dpdk_compressdev
DIRS-$(CONFIG_IDXD) += dsa
DIRS-$(CONFIG_IDXD) += iaa
DIRS-$(CONFIG_CRYPTO) += dpdk_cryptodev

18
module/accel/dpdk_compressdev/Makefile Normal file
View File

@ -0,0 +1,18 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (C) 2022 Intel Corporation.
# All rights reserved.
#
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
SO_VER := 1
SO_MINOR := 0
CFLAGS += $(ENV_CFLAGS)
LIBNAME = accel_dpdk_compressdev
C_SRCS = accel_dpdk_compressdev.c accel_dpdk_compressdev_rpc.c
SPDK_MAP_FILE = $(SPDK_ROOT_DIR)/mk/spdk_blank.map
include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

1729
module/accel/dpdk_compressdev/accel_dpdk_compressdev.c
View File

File diff suppressed because it is too large Load Diff

16
module/accel/dpdk_compressdev/accel_dpdk_compressdev.h Normal file
View File

@ -0,0 +1,16 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright (C) 2022 Intel Corporation.
* All rights reserved.
*/
#include "spdk/stdinc.h"
enum compress_pmd {
COMPRESS_PMD_AUTO = 0,
COMPRESS_PMD_QAT_ONLY,
COMPRESS_PMD_MLX5_PCI_ONLY,
COMPRESS_PMD_MAX,
};
void accel_dpdk_compressdev_enable(void);
int accel_compressdev_enable_probe(enum compress_pmd *opts);

52
module/accel/dpdk_compressdev/accel_dpdk_compressdev_rpc.c Normal file
View File

@ -0,0 +1,52 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright (C) 2022 Intel Corporation.
* All rights reserved.
*/
#include "accel_dpdk_compressdev.h"
#include "spdk/rpc.h"
#include "spdk/util.h"
#include "spdk/string.h"
#include "spdk/log.h"
struct rpc_compressdev_scan_accel_module {
uint32_t pmd;
};
static const struct spdk_json_object_decoder rpc_compressdev_scan_accel_module_decoder[] = {
{"pmd", offsetof(struct rpc_compressdev_scan_accel_module, pmd), spdk_json_decode_uint32},
};
static void
rpc_compressdev_scan_accel_module(struct spdk_jsonrpc_request *request,
const struct spdk_json_val *params)
{
struct rpc_compressdev_scan_accel_module req;
int rc = 0;
if (spdk_json_decode_object(params, rpc_compressdev_scan_accel_module_decoder,
SPDK_COUNTOF(rpc_compressdev_scan_accel_module_decoder),
&req)) {
SPDK_ERRLOG("spdk_json_decode_object failed\n");
spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_PARSE_ERROR,
"spdk_json_decode_object failed");
return;
}
if (req.pmd >= COMPRESS_PMD_MAX) {
spdk_jsonrpc_send_error_response_fmt(request, -EINVAL,
"PMD value %d should be less than %d", req.pmd, COMPRESS_PMD_MAX);
return;
}
rc = accel_compressdev_enable_probe(&req.pmd);
if (rc) {
spdk_jsonrpc_send_error_response(request, rc, spdk_strerror(-rc));
return;
}
accel_dpdk_compressdev_enable();
spdk_jsonrpc_send_bool_response(request, true);
}
SPDK_RPC_REGISTER("compressdev_scan_accel_module", rpc_compressdev_scan_accel_module,
SPDK_RPC_STARTUP)

6
module/bdev/compress/vbdev_compress.c
View File

@ -353,6 +353,8 @@ vbdev_init_compress_drivers(void)
return -EINVAL;
}
/* TODO: make these global pools per thread but do in a follow-up patch to make
* it easier to review against the old compressdev code */
g_mbuf_mp = rte_pktmbuf_pool_create("comp_mbuf_mp", NUM_MBUFS, POOL_CACHE_SIZE,
sizeof(struct rte_mbuf), 0, rte_socket_id());
if (g_mbuf_mp == NULL) {
@ -1401,6 +1403,10 @@ comp_bdev_ch_create_cb(void *io_device, void *ctx_buf)
struct vbdev_compress *comp_bdev = io_device;
struct comp_device_qp *device_qp;
/* TODO look into associating the device_qp with the channel vs the thread,
* doing in next patch to make this one easier to review against code taken
* from the vbdev module */
/* Now set the reduce channel if it's not already set. */
pthread_mutex_lock(&comp_bdev->reduce_lock);
if (comp_bdev->ch_count == 0) {

1
python/spdk/rpc/__init__.py
View File

@ -12,6 +12,7 @@ from . import accel
from . import app
from . import bdev
from . import blobfs
from . import compressdev
from . import env_dpdk
from . import dsa
from . import iaa

1
python/spdk/rpc/accel.py
View File

@ -1,6 +1,7 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (C) 2022 Intel Corporation.
# All rights reserved.
#
from spdk.rpc.helpers import deprecated_alias

14
python/spdk/rpc/compressdev.py Normal file
View File

@ -0,0 +1,14 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (C) 2022 Intel Corporation.
# All rights reserved.
#
def compressdev_scan_accel_module(client, pmd):
"""Scan and enable compressdev module and set pmd option.
Args:
pmd: 0 = auto-select, 1 = QAT, 2 = mlx5_pci
"""
params = {'pmd': pmd}
return client.call('compressdev_scan_accel_module', params)

8
scripts/rpc.py
View File

@ -2840,6 +2840,14 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
p = subparsers.add_parser('ioat_scan_accel_module', help='Enable IOAT accel module offload.')
p.set_defaults(func=ioat_scan_accel_module)
# dpdk compressdev
def compressdev_scan_accel_module(args):
rpc.compressdev.compressdev_scan_accel_module(args.client, pmd=args.pmd)
p = subparsers.add_parser('compressdev_scan_accel_module', help='Scan and enable compressdev module and set pmd option.')
p.add_argument('-p', '--pmd', type=int, help='0 = auto-select, 1= QAT only, 2 = mlx5_pci only')
p.set_defaults(func=compressdev_scan_accel_module)
# dsa
def dsa_scan_accel_module(args):
rpc.dsa.dsa_scan_accel_module(args.client, config_kernel_mode=args.config_kernel_mode)

9
test/accel/accel.sh
View File

@ -28,6 +28,15 @@ if [[ $CONFIG_ISAL == y ]]; then
run_test "accel_decomp_mthread" $SPDK_EXAMPLE_DIR/accel_perf -t 1 -w decompress -l $testdir/bib -y -T 2
run_test "accel_deomp_full_mthread" $SPDK_EXAMPLE_DIR/accel_perf -t 1 -w decompress -l $testdir/bib -y -o 0 -T 2
fi
if [[ $CONFIG_DPDK_COMPRESSDEV == y ]]; then
run_test "accel_cdev_comp" $SPDK_EXAMPLE_DIR/accel_perf -t 1 -w compress -l $testdir/bib -c $testdir/dpdk.json
run_test "accel_cdev_decomp" $SPDK_EXAMPLE_DIR/accel_perf -t 1 -w decompress -l $testdir/bib -y -c $testdir/dpdk.json
run_test "accel_cdev_decmop_full" $SPDK_EXAMPLE_DIR/accel_perf -t 1 -w decompress -l $testdir/bib -y -o 0 -c $testdir/dpdk.json
run_test "accel_cdev_decomp_mcore" $SPDK_EXAMPLE_DIR/accel_perf -t 1 -w decompress -l $testdir/bib -y -m 0xf -c $testdir/dpdk.json
run_test "accel_cdev_decomp_full_mcore" $SPDK_EXAMPLE_DIR/accel_perf -t 1 -w decompress -l $testdir/bib -y -o 0 -m 0xf -c $testdir/dpdk.json
run_test "accel_cdev_decomp_mthread" $SPDK_EXAMPLE_DIR/accel_perf -t 1 -w decompress -l $testdir/bib -y -T 2 -c $testdir/dpdk.json
run_test "accel_cdev_deomp_full_mthread" $SPDK_EXAMPLE_DIR/accel_perf -t 1 -w decompress -l $testdir/bib -y -o 0 -T 2 -c $testdir/dpdk.json
fi
trap 'killprocess $spdk_tgt_pid; exit 1' ERR

15
test/accel/dpdk.json Normal file
View File

@ -0,0 +1,15 @@
{
"subsystems": [
{
"subsystem": "accel",
"config": [
{
"method": "compressdev_scan_accel_module",
"params": {
"pmd": 0
}
}
]
}
]
}

2
test/common/autotest_common.sh
View File

@ -444,7 +444,7 @@ function get_config_params() {
if [ -f /usr/include/libpmem.h ] && [ $SPDK_TEST_VBDEV_COMPRESS -eq 1 ]; then
if ge "$(nasm --version | awk '{print $3}')" 2.14 && [[ $SPDK_TEST_ISAL -eq 1 ]]; then
config_params+=' --with-vbdev-compress'
config_params+=' --with-vbdev-compress --with-dpdk-compressdev'
fi
fi

4
test/common/skipped_build_files.txt
View File

@ -56,7 +56,3 @@ module/bdev/daos/bdev_daos_rpc
# Not configured to test xNVMe bdev
module/bdev/xnvme/bdev_xnvme
module/bdev/xnvme/bdev_xnvme_rpc
# Temporarily include this in the skipped files. Next patches will remove
# this and start building it.
module/accel/dpdk_compressdev/accel_dpdk_compressdev