lib/util: add ISA-L accelerated xor generation

Signed-off-by: Artur Paszkiewicz <artur.paszkiewicz@intel.com>
Change-Id: I3ef9dadb4c68e92760c8426f0fffb7b249829e2b
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12080
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>

CHANGELOG.md

@@ -108,6 +108,9 @@ For now we are using hard-coded PSK and only support TLS 1.3.
 
 Added new functions: `spdk_hexlify` and `spdk_unhexlify`.
 
+A new API `spdk_xor_gen` was added to generate XOR from multiple source buffers. It is going to be
+used by `raid5f` for calculating parity.
+
 ### virtio
 
 virtio-vhost-user no longer tries to support dynamic memory allocation.  The vhost target does

include/spdk/xor.h

@@ -0,0 +1,42 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ */
+
+/**
+ * \file
+ * XOR utility functions
+ */
+
+#ifndef SPDK_XOR_H
+#define SPDK_XOR_H
+
+#include "spdk/stdinc.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Generate XOR from multiple source buffers.
+ *
+ * \param dest Destination buffer.
+ * \param sources Array of source buffers.
+ * \param n Number of source buffers in the array.
+ * \param len Length of each buffer in bytes.
+ * \return 0 on success, negative error code otherwise.
+ */
+int spdk_xor_gen(void *dest, void **sources, uint32_t n, uint32_t len);
+
+/**
+ * Get the optimal buffer alignment for XOR functions.
+ *
+ * \return The alignment in bytes.
+ */
+size_t spdk_xor_get_optimal_alignment(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SPDK_XOR_H */

lib/util/Makefile

@@ -11,7 +11,7 @@ SO_MINOR := 1
 
 C_SRCS = base64.c bit_array.c cpuset.c crc16.c crc32.c crc32c.c crc32_ieee.c \
 	 dif.c fd.c file.c hexlify.c iov.c math.c pipe.c strerror_tls.c string.c uuid.c \
-	 fd_group.c zipf.c
+	 fd_group.c xor.c zipf.c
 LIBNAME = util
 LOCAL_SYS_LIBS = -luuid
 

lib/util/spdk_util.map

@@ -153,6 +153,10 @@
 	spdk_fd_group_event_modify;
 	spdk_fd_group_get_fd;
 
+	# public functions in xor.h
+	spdk_xor_gen;
+	spdk_xor_get_optimal_alignment;
+
 	# public functions in zipf.h
 	spdk_zipf_create;
 	spdk_zipf_free;

lib/util/xor.c

@@ -0,0 +1,145 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ */
+
+#include "spdk/xor.h"
+#include "spdk/config.h"
+#include "spdk/assert.h"
+#include "spdk/util.h"
+
+/* maximum number of source buffers */
+#define SPDK_XOR_MAX_SRC	256
+
+static inline bool
+is_aligned(void *ptr, size_t alignment)
+{
+	uintptr_t p = (uintptr_t)ptr;
+
+	return p == SPDK_ALIGN_FLOOR(p, alignment);
+}
+
+static bool
+buffers_aligned(void *dest, void **sources, uint32_t n, size_t alignment)
+{
+	uint32_t i;
+
+	for (i = 0; i < n; i++) {
+		if (!is_aligned(sources[i], alignment)) {
+			return false;
+		}
+	}
+
+	return is_aligned(dest, alignment);
+}
+
+static void
+xor_gen_unaligned(void *dest, void **sources, uint32_t n, uint32_t len)
+{
+	uint32_t i, j;
+
+	for (i = 0; i < len; i++) {
+		uint8_t b = 0;
+
+		for (j = 0; j < n; j++) {
+			b ^= ((uint8_t *)sources[j])[i];
+		}
+		((uint8_t *)dest)[i] = b;
+	}
+}
+
+static void
+xor_gen_basic(void *dest, void **sources, uint32_t n, uint32_t len)
+{
+	uint32_t shift;
+	uint32_t len_div, len_rem;
+	uint32_t i, j;
+
+	if (!buffers_aligned(dest, sources, n, sizeof(uint64_t))) {
+		xor_gen_unaligned(dest, sources, n, len);
+		return;
+	}
+
+	shift = spdk_u32log2(sizeof(uint64_t));
+	len_div = len >> shift;
+	len_rem = len_div << shift;
+
+	for (i = 0; i < len_div; i++) {
+		uint64_t w = 0;
+
+		for (j = 0; j < n; j++) {
+			w ^= ((uint64_t *)sources[j])[i];
+		}
+		((uint64_t *)dest)[i] = w;
+	}
+
+	if (len_rem < len) {
+		void *sources2[SPDK_XOR_MAX_SRC];
+
+		for (j = 0; j < n; j++) {
+			sources2[j] = sources[j] + len_rem;
+		}
+
+		xor_gen_unaligned(dest + len_rem, sources2, n, len - len_rem);
+	}
+}
+
+#ifdef SPDK_CONFIG_ISAL
+#include "isa-l/include/raid.h"
+
+#define SPDK_XOR_BUF_ALIGN 32
+
+static int
+do_xor_gen(void *dest, void **sources, uint32_t n, uint32_t len)
+{
+	if (buffers_aligned(dest, sources, n, SPDK_XOR_BUF_ALIGN)) {
+		void *buffers[SPDK_XOR_MAX_SRC + 1];
+
+		if (n >= INT_MAX) {
+			return -EINVAL;
+		}
+
+		memcpy(buffers, sources, n * sizeof(buffers[0]));
+		buffers[n] = dest;
+
+		if (xor_gen(n + 1, len, buffers)) {
+			return -EINVAL;
+		}
+	} else {
+		xor_gen_basic(dest, sources, n, len);
+	}
+
+	return 0;
+}
+
+#else
+
+#define SPDK_XOR_BUF_ALIGN sizeof(uint64_t)
+
+static inline int
+do_xor_gen(void *dest, void **sources, uint32_t n, uint32_t len)
+{
+	xor_gen_basic(dest, sources, n, len);
+	return 0;
+}
+
+#endif
+
+int
+spdk_xor_gen(void *dest, void **sources, uint32_t n, uint32_t len)
+{
+	if (n < 2 || n > SPDK_XOR_MAX_SRC) {
+		return -EINVAL;
+	}
+
+	return do_xor_gen(dest, sources, n, len);
+}
+
+size_t
+spdk_xor_get_optimal_alignment(void)
+{
+	return SPDK_XOR_BUF_ALIGN;
+}
+
+SPDK_STATIC_ASSERT(SPDK_XOR_BUF_ALIGN > 0 && !(SPDK_XOR_BUF_ALIGN & (SPDK_XOR_BUF_ALIGN - 1)),
+		   "Must be power of 2");

test/unit/lib/util/Makefile

@@ -7,7 +7,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
 DIRS-y = base64.c bit_array.c cpuset.c crc16.c crc32_ieee.c crc32c.c dif.c \
-	 iov.c math.c pipe.c string.c
+	 iov.c math.c pipe.c string.c xor.c
 
 .PHONY: all clean $(DIRS-y)
 

test/unit/lib/util/xor.c/Makefile

@@ -0,0 +1,10 @@
+#  SPDX-License-Identifier: BSD-3-Clause
+#  Copyright (c) Intel Corporation.
+#  All rights reserved.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../../..)
+
+TEST_FILE = xor_ut.c
+
+include $(SPDK_ROOT_DIR)/mk/spdk.unittest.mk

test/unit/lib/util/xor.c/xor_ut.c

@@ -0,0 +1,121 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk_cunit.h"
+
+#include "util/xor.c"
+#include "common/lib/test_env.c"
+
+#define BUF_COUNT 8
+#define SRC_BUF_COUNT (BUF_COUNT - 1)
+#define BUF_SIZE 4096
+
+static void
+test_xor_gen(void)
+{
+	void *bufs[BUF_COUNT];
+	void *bufs2[SRC_BUF_COUNT];
+	uint8_t *ref, *dest;
+	int ret;
+	size_t i, j;
+	uint32_t *tmp;
+
+	/* alloc and fill the buffers with a pattern */
+	for (i = 0; i < BUF_COUNT; i++) {
+		ret = posix_memalign(&bufs[i], spdk_xor_get_optimal_alignment(), BUF_SIZE);
+		SPDK_CU_ASSERT_FATAL(ret == 0);
+
+		tmp = bufs[i];
+		for (j = 0; j < BUF_SIZE / sizeof(*tmp); j++) {
+			tmp[j] = (i << 16) + j;
+		}
+	}
+	dest = bufs[SRC_BUF_COUNT];
+
+	/* prepare the reference buffer */
+	ref = malloc(BUF_SIZE);
+	SPDK_CU_ASSERT_FATAL(ref != NULL);
+
+	memset(ref, 0, BUF_SIZE);
+	for (i = 0; i < SRC_BUF_COUNT; i++) {
+		for (j = 0; j < BUF_SIZE; j++) {
+			ref[j] ^= ((uint8_t *)bufs[i])[j];
+		}
+	}
+
+	/* generate xor, compare the dest and reference buffers */
+	ret = spdk_xor_gen(dest, bufs, SRC_BUF_COUNT, BUF_SIZE);
+	CU_ASSERT(ret == 0);
+	ret = memcmp(ref, dest, BUF_SIZE);
+	CU_ASSERT(ret == 0);
+
+	/* len not multiple of alignment */
+	memset(dest, 0xba, BUF_SIZE);
+	ret = spdk_xor_gen(dest, bufs, SRC_BUF_COUNT, BUF_SIZE - 1);
+	CU_ASSERT(ret == 0);
+	ret = memcmp(ref, dest, BUF_SIZE - 1);
+	CU_ASSERT(ret == 0);
+
+	/* unaligned buffer */
+	memcpy(bufs2, bufs, sizeof(bufs2));
+	bufs2[1] += 1;
+	bufs2[2] += 2;
+	bufs2[3] += 3;
+
+	memset(ref, 0, BUF_SIZE);
+	for (i = 0; i < SRC_BUF_COUNT; i++) {
+		for (j = 0; j < BUF_SIZE - SRC_BUF_COUNT; j++) {
+			ref[j] ^= ((uint8_t *)bufs2[i])[j];
+		}
+	}
+
+	memset(dest, 0xba, BUF_SIZE);
+	ret = spdk_xor_gen(dest, bufs2, SRC_BUF_COUNT, BUF_SIZE - SRC_BUF_COUNT);
+	CU_ASSERT(ret == 0);
+	ret = memcmp(ref, dest, BUF_SIZE - SRC_BUF_COUNT);
+	CU_ASSERT(ret == 0);
+
+	/* xoring a buffer with itself should result in all zeros */
+	memset(ref, 0, BUF_SIZE);
+	bufs2[0] = bufs[0];
+	bufs2[1] = bufs[0];
+	dest = bufs[0];
+
+	ret = spdk_xor_gen(dest, bufs2, 2, BUF_SIZE);
+	CU_ASSERT(ret == 0);
+	ret = memcmp(ref, dest, BUF_SIZE);
+	CU_ASSERT(ret == 0);
+
+	/* cleanup */
+	for (i = 0; i < BUF_COUNT; i++) {
+		free(bufs[i]);
+	}
+	free(ref);
+}
+
+int
+main(int argc, char **argv)
+{
+	CU_pSuite	suite = NULL;
+	unsigned int	num_failures;
+
+	CU_set_error_action(CUEA_ABORT);
+	CU_initialize_registry();
+
+	suite = CU_add_suite("xor", NULL, NULL);
+
+	CU_ADD_TEST(suite, test_xor_gen);
+
+	CU_basic_set_mode(CU_BRM_VERBOSE);
+
+	CU_basic_run_tests();
+
+	num_failures = CU_get_number_of_failures();
+	CU_cleanup_registry();
+
+	return num_failures;
+}

test/unit/unittest.sh

@@ -133,6 +133,7 @@ function unittest_util() {
 	$valgrind $testdir/lib/util/iov.c/iov_ut
 	$valgrind $testdir/lib/util/math.c/math_ut
 	$valgrind $testdir/lib/util/pipe.c/pipe_ut
+	$valgrind $testdir/lib/util/xor.c/xor_ut
 }
 
 function unittest_init() {