From 69c448a30e228d4b010a769e961aad17f8dd5af5 Mon Sep 17 00:00:00 2001 From: Artur Paszkiewicz Date: Mon, 28 Mar 2022 14:17:55 +0200 Subject: [PATCH] lib/util: add ISA-L accelerated xor generation Signed-off-by: Artur Paszkiewicz Change-Id: I3ef9dadb4c68e92760c8426f0fffb7b249829e2b Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12080 Tested-by: SPDK CI Jenkins Community-CI: Mellanox Build Bot Reviewed-by: Ben Walker Reviewed-by: Tomasz Zawadzki --- CHANGELOG.md | 3 + include/spdk/xor.h | 42 +++++++++ lib/util/Makefile | 2 +- lib/util/spdk_util.map | 4 + lib/util/xor.c | 145 ++++++++++++++++++++++++++++++ test/unit/lib/util/Makefile | 2 +- test/unit/lib/util/xor.c/Makefile | 10 +++ test/unit/lib/util/xor.c/xor_ut.c | 121 +++++++++++++++++++++++++ test/unit/unittest.sh | 1 + 9 files changed, 328 insertions(+), 2 deletions(-) create mode 100644 include/spdk/xor.h create mode 100644 lib/util/xor.c create mode 100644 test/unit/lib/util/xor.c/Makefile create mode 100644 test/unit/lib/util/xor.c/xor_ut.c diff --git a/CHANGELOG.md b/CHANGELOG.md index ba613d56a..2f148ed38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -108,6 +108,9 @@ For now we are using hard-coded PSK and only support TLS 1.3. Added new functions: `spdk_hexlify` and `spdk_unhexlify`. +A new API `spdk_xor_gen` was added to generate XOR from multiple source buffers. It is going to be +used by `raid5f` for calculating parity. + ### virtio virtio-vhost-user no longer tries to support dynamic memory allocation. The vhost target does diff --git a/include/spdk/xor.h b/include/spdk/xor.h new file mode 100644 index 000000000..4bbe4aa5d --- /dev/null +++ b/include/spdk/xor.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) Intel Corporation. + * All rights reserved. + */ + +/** + * \file + * XOR utility functions + */ + +#ifndef SPDK_XOR_H +#define SPDK_XOR_H + +#include "spdk/stdinc.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Generate XOR from multiple source buffers. + * + * \param dest Destination buffer. + * \param sources Array of source buffers. + * \param n Number of source buffers in the array. + * \param len Length of each buffer in bytes. + * \return 0 on success, negative error code otherwise. + */ +int spdk_xor_gen(void *dest, void **sources, uint32_t n, uint32_t len); + +/** + * Get the optimal buffer alignment for XOR functions. + * + * \return The alignment in bytes. + */ +size_t spdk_xor_get_optimal_alignment(void); + +#ifdef __cplusplus +} +#endif + +#endif /* SPDK_XOR_H */ diff --git a/lib/util/Makefile b/lib/util/Makefile index f3f39968a..fe30f7219 100644 --- a/lib/util/Makefile +++ b/lib/util/Makefile @@ -11,7 +11,7 @@ SO_MINOR := 1 C_SRCS = base64.c bit_array.c cpuset.c crc16.c crc32.c crc32c.c crc32_ieee.c \ dif.c fd.c file.c hexlify.c iov.c math.c pipe.c strerror_tls.c string.c uuid.c \ - fd_group.c zipf.c + fd_group.c xor.c zipf.c LIBNAME = util LOCAL_SYS_LIBS = -luuid diff --git a/lib/util/spdk_util.map b/lib/util/spdk_util.map index b6db4ebf0..41f94089d 100644 --- a/lib/util/spdk_util.map +++ b/lib/util/spdk_util.map @@ -153,6 +153,10 @@ spdk_fd_group_event_modify; spdk_fd_group_get_fd; + # public functions in xor.h + spdk_xor_gen; + spdk_xor_get_optimal_alignment; + # public functions in zipf.h spdk_zipf_create; spdk_zipf_free; diff --git a/lib/util/xor.c b/lib/util/xor.c new file mode 100644 index 000000000..d5ec7795a --- /dev/null +++ b/lib/util/xor.c @@ -0,0 +1,145 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) Intel Corporation. + * All rights reserved. + */ + +#include "spdk/xor.h" +#include "spdk/config.h" +#include "spdk/assert.h" +#include "spdk/util.h" + +/* maximum number of source buffers */ +#define SPDK_XOR_MAX_SRC 256 + +static inline bool +is_aligned(void *ptr, size_t alignment) +{ + uintptr_t p = (uintptr_t)ptr; + + return p == SPDK_ALIGN_FLOOR(p, alignment); +} + +static bool +buffers_aligned(void *dest, void **sources, uint32_t n, size_t alignment) +{ + uint32_t i; + + for (i = 0; i < n; i++) { + if (!is_aligned(sources[i], alignment)) { + return false; + } + } + + return is_aligned(dest, alignment); +} + +static void +xor_gen_unaligned(void *dest, void **sources, uint32_t n, uint32_t len) +{ + uint32_t i, j; + + for (i = 0; i < len; i++) { + uint8_t b = 0; + + for (j = 0; j < n; j++) { + b ^= ((uint8_t *)sources[j])[i]; + } + ((uint8_t *)dest)[i] = b; + } +} + +static void +xor_gen_basic(void *dest, void **sources, uint32_t n, uint32_t len) +{ + uint32_t shift; + uint32_t len_div, len_rem; + uint32_t i, j; + + if (!buffers_aligned(dest, sources, n, sizeof(uint64_t))) { + xor_gen_unaligned(dest, sources, n, len); + return; + } + + shift = spdk_u32log2(sizeof(uint64_t)); + len_div = len >> shift; + len_rem = len_div << shift; + + for (i = 0; i < len_div; i++) { + uint64_t w = 0; + + for (j = 0; j < n; j++) { + w ^= ((uint64_t *)sources[j])[i]; + } + ((uint64_t *)dest)[i] = w; + } + + if (len_rem < len) { + void *sources2[SPDK_XOR_MAX_SRC]; + + for (j = 0; j < n; j++) { + sources2[j] = sources[j] + len_rem; + } + + xor_gen_unaligned(dest + len_rem, sources2, n, len - len_rem); + } +} + +#ifdef SPDK_CONFIG_ISAL +#include "isa-l/include/raid.h" + +#define SPDK_XOR_BUF_ALIGN 32 + +static int +do_xor_gen(void *dest, void **sources, uint32_t n, uint32_t len) +{ + if (buffers_aligned(dest, sources, n, SPDK_XOR_BUF_ALIGN)) { + void *buffers[SPDK_XOR_MAX_SRC + 1]; + + if (n >= INT_MAX) { + return -EINVAL; + } + + memcpy(buffers, sources, n * sizeof(buffers[0])); + buffers[n] = dest; + + if (xor_gen(n + 1, len, buffers)) { + return -EINVAL; + } + } else { + xor_gen_basic(dest, sources, n, len); + } + + return 0; +} + +#else + +#define SPDK_XOR_BUF_ALIGN sizeof(uint64_t) + +static inline int +do_xor_gen(void *dest, void **sources, uint32_t n, uint32_t len) +{ + xor_gen_basic(dest, sources, n, len); + return 0; +} + +#endif + +int +spdk_xor_gen(void *dest, void **sources, uint32_t n, uint32_t len) +{ + if (n < 2 || n > SPDK_XOR_MAX_SRC) { + return -EINVAL; + } + + return do_xor_gen(dest, sources, n, len); +} + +size_t +spdk_xor_get_optimal_alignment(void) +{ + return SPDK_XOR_BUF_ALIGN; +} + +SPDK_STATIC_ASSERT(SPDK_XOR_BUF_ALIGN > 0 && !(SPDK_XOR_BUF_ALIGN & (SPDK_XOR_BUF_ALIGN - 1)), + "Must be power of 2"); diff --git a/test/unit/lib/util/Makefile b/test/unit/lib/util/Makefile index e95d58cd9..4304791d8 100644 --- a/test/unit/lib/util/Makefile +++ b/test/unit/lib/util/Makefile @@ -7,7 +7,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk DIRS-y = base64.c bit_array.c cpuset.c crc16.c crc32_ieee.c crc32c.c dif.c \ - iov.c math.c pipe.c string.c + iov.c math.c pipe.c string.c xor.c .PHONY: all clean $(DIRS-y) diff --git a/test/unit/lib/util/xor.c/Makefile b/test/unit/lib/util/xor.c/Makefile new file mode 100644 index 000000000..0758a3b38 --- /dev/null +++ b/test/unit/lib/util/xor.c/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) Intel Corporation. +# All rights reserved. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../../..) + +TEST_FILE = xor_ut.c + +include $(SPDK_ROOT_DIR)/mk/spdk.unittest.mk diff --git a/test/unit/lib/util/xor.c/xor_ut.c b/test/unit/lib/util/xor.c/xor_ut.c new file mode 100644 index 000000000..72361ee59 --- /dev/null +++ b/test/unit/lib/util/xor.c/xor_ut.c @@ -0,0 +1,121 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) Intel Corporation. + * All rights reserved. + */ + +#include "spdk/stdinc.h" + +#include "spdk_cunit.h" + +#include "util/xor.c" +#include "common/lib/test_env.c" + +#define BUF_COUNT 8 +#define SRC_BUF_COUNT (BUF_COUNT - 1) +#define BUF_SIZE 4096 + +static void +test_xor_gen(void) +{ + void *bufs[BUF_COUNT]; + void *bufs2[SRC_BUF_COUNT]; + uint8_t *ref, *dest; + int ret; + size_t i, j; + uint32_t *tmp; + + /* alloc and fill the buffers with a pattern */ + for (i = 0; i < BUF_COUNT; i++) { + ret = posix_memalign(&bufs[i], spdk_xor_get_optimal_alignment(), BUF_SIZE); + SPDK_CU_ASSERT_FATAL(ret == 0); + + tmp = bufs[i]; + for (j = 0; j < BUF_SIZE / sizeof(*tmp); j++) { + tmp[j] = (i << 16) + j; + } + } + dest = bufs[SRC_BUF_COUNT]; + + /* prepare the reference buffer */ + ref = malloc(BUF_SIZE); + SPDK_CU_ASSERT_FATAL(ref != NULL); + + memset(ref, 0, BUF_SIZE); + for (i = 0; i < SRC_BUF_COUNT; i++) { + for (j = 0; j < BUF_SIZE; j++) { + ref[j] ^= ((uint8_t *)bufs[i])[j]; + } + } + + /* generate xor, compare the dest and reference buffers */ + ret = spdk_xor_gen(dest, bufs, SRC_BUF_COUNT, BUF_SIZE); + CU_ASSERT(ret == 0); + ret = memcmp(ref, dest, BUF_SIZE); + CU_ASSERT(ret == 0); + + /* len not multiple of alignment */ + memset(dest, 0xba, BUF_SIZE); + ret = spdk_xor_gen(dest, bufs, SRC_BUF_COUNT, BUF_SIZE - 1); + CU_ASSERT(ret == 0); + ret = memcmp(ref, dest, BUF_SIZE - 1); + CU_ASSERT(ret == 0); + + /* unaligned buffer */ + memcpy(bufs2, bufs, sizeof(bufs2)); + bufs2[1] += 1; + bufs2[2] += 2; + bufs2[3] += 3; + + memset(ref, 0, BUF_SIZE); + for (i = 0; i < SRC_BUF_COUNT; i++) { + for (j = 0; j < BUF_SIZE - SRC_BUF_COUNT; j++) { + ref[j] ^= ((uint8_t *)bufs2[i])[j]; + } + } + + memset(dest, 0xba, BUF_SIZE); + ret = spdk_xor_gen(dest, bufs2, SRC_BUF_COUNT, BUF_SIZE - SRC_BUF_COUNT); + CU_ASSERT(ret == 0); + ret = memcmp(ref, dest, BUF_SIZE - SRC_BUF_COUNT); + CU_ASSERT(ret == 0); + + /* xoring a buffer with itself should result in all zeros */ + memset(ref, 0, BUF_SIZE); + bufs2[0] = bufs[0]; + bufs2[1] = bufs[0]; + dest = bufs[0]; + + ret = spdk_xor_gen(dest, bufs2, 2, BUF_SIZE); + CU_ASSERT(ret == 0); + ret = memcmp(ref, dest, BUF_SIZE); + CU_ASSERT(ret == 0); + + /* cleanup */ + for (i = 0; i < BUF_COUNT; i++) { + free(bufs[i]); + } + free(ref); +} + +int +main(int argc, char **argv) +{ + CU_pSuite suite = NULL; + unsigned int num_failures; + + CU_set_error_action(CUEA_ABORT); + CU_initialize_registry(); + + suite = CU_add_suite("xor", NULL, NULL); + + CU_ADD_TEST(suite, test_xor_gen); + + CU_basic_set_mode(CU_BRM_VERBOSE); + + CU_basic_run_tests(); + + num_failures = CU_get_number_of_failures(); + CU_cleanup_registry(); + + return num_failures; +} diff --git a/test/unit/unittest.sh b/test/unit/unittest.sh index 4e175bcb5..21a3f08f8 100755 --- a/test/unit/unittest.sh +++ b/test/unit/unittest.sh @@ -133,6 +133,7 @@ function unittest_util() { $valgrind $testdir/lib/util/iov.c/iov_ut $valgrind $testdir/lib/util/math.c/math_ut $valgrind $testdir/lib/util/pipe.c/pipe_ut + $valgrind $testdir/lib/util/xor.c/xor_ut } function unittest_init() {