FTL: Add ftl mempools

Optimized for single thread utilization

Signed-off-by: Kozlowski Mateusz <mateusz.kozlowski@intel.com>
Signed-off-by: Artur Paszkiewicz <artur.paszkiewicz@intel.com>
Change-Id: I56602a3d85e0cd47256c8f3e5d7a3f0ed4e38743
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13303
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

lib/ftl/Makefile

@@ -20,7 +20,7 @@ FTL_SUBDIRS := mngt utils
 C_SRCS = ftl_core.c ftl_init.c ftl_layout.c ftl_debug.c
 C_SRCS += mngt/ftl_mngt.c mngt/ftl_mngt_bdev.c mngt/ftl_mngt_shutdown.c mngt/ftl_mngt_startup.c
 C_SRCS += mngt/ftl_mngt_md.c mngt/ftl_mngt_misc.c
-C_SRCS += utils/ftl_conf.c utils/ftl_md.c
+C_SRCS += utils/ftl_conf.c utils/ftl_md.c utils/ftl_mempool.c
 
 SPDK_MAP_FILE = $(abspath $(CURDIR)/spdk_ftl.map)
 

lib/ftl/ftl_core.c

@@ -16,6 +16,7 @@
 #include "ftl_debug.h"
 #include "ftl_internal.h"
 #include "mngt/ftl_mngt.h"
+#include "utils/ftl_mempool.h"
 
 static int
 ftl_shutdown_complete(struct spdk_ftl_dev *dev)

lib/ftl/ftl_utils.h

@@ -7,6 +7,7 @@
 #define FTL_FTL_UTILS_H
 
 #include "utils/ftl_defs.h"
+#include "utils/ftl_mempool.h"
 #include "utils/ftl_conf.h"
 #include "utils/ftl_md.h"
 

lib/ftl/utils/ftl_mempool.c

@@ -0,0 +1,147 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/queue.h"
+#include "spdk/util.h"
+#include "spdk/env.h"
+#include "spdk/likely.h"
+
+#include "ftl_mempool.h"
+
+struct ftl_mempool_element {
+	SLIST_ENTRY(ftl_mempool_element) entry;
+};
+
+struct ftl_mempool {
+	SLIST_HEAD(, ftl_mempool_element) list;
+	size_t element_size;
+	void *buffer;
+	size_t buffer_size;
+	size_t count;
+	size_t alignment;
+	int socket_id;
+};
+
+
+static size_t
+element_size_aligned(size_t size, size_t alignment)
+{
+	if (!alignment) {
+		return size;
+	}
+
+	if (size % alignment) {
+		return (size / alignment + 1) * alignment;
+	}
+
+	return size;
+}
+
+static inline bool
+is_element_valid(struct ftl_mempool *mpool, void *element)
+{
+	if (element < mpool->buffer) {
+		return false;
+	}
+
+	if (element + mpool->element_size > mpool->buffer + mpool->buffer_size) {
+		return false;
+	}
+
+	if (!mpool->alignment) {
+		return true;
+	}
+
+	if ((size_t)element % mpool->alignment) {
+		return false;
+	}
+
+	if ((element - mpool->buffer) % mpool->element_size != 0) {
+		return false;
+	}
+
+	return true;
+}
+
+struct ftl_mempool *ftl_mempool_create(size_t count, size_t size,
+				       size_t alignment, int socket_id)
+{
+	struct ftl_mempool *mp;
+	void *buffer;
+	size_t i;
+
+	assert(count > 0);
+	assert(size > 0);
+
+	if (!spdk_u64_is_pow2(alignment)) {
+		return NULL;
+	}
+
+	mp = calloc(1, sizeof(*mp));
+	if (!mp) {
+		return NULL;
+	}
+
+	size = spdk_max(size, sizeof(struct ftl_mempool_element));
+
+	mp->count = count;
+	mp->element_size = element_size_aligned(size, alignment);
+	mp->alignment = alignment;
+	mp->socket_id = socket_id;
+	SLIST_INIT(&mp->list);
+
+	mp->buffer_size = mp->element_size * mp->count;
+	mp->buffer = spdk_dma_malloc_socket(mp->buffer_size, mp->alignment,
+					    NULL, socket_id);
+	if (!mp->buffer) {
+		free(mp);
+		return NULL;
+	}
+
+	buffer = mp->buffer;
+	for (i = 0; i < count; i++, buffer += mp->element_size) {
+		struct ftl_mempool_element *el = buffer;
+		assert(is_element_valid(mp, el));
+		SLIST_INSERT_HEAD(&mp->list, el, entry);
+	}
+
+	return mp;
+}
+
+void
+ftl_mempool_destroy(struct ftl_mempool *mpool)
+{
+	if (!mpool) {
+		return;
+	}
+
+	spdk_dma_free(mpool->buffer);
+	free(mpool);
+}
+
+void *
+ftl_mempool_get(struct ftl_mempool *mpool)
+{
+	struct ftl_mempool_element *el;
+
+	if (spdk_unlikely(SLIST_EMPTY(&mpool->list))) {
+		return NULL;
+	}
+
+	el = SLIST_FIRST(&mpool->list);
+	SLIST_REMOVE_HEAD(&mpool->list, entry);
+
+	return el;
+}
+
+void
+ftl_mempool_put(struct ftl_mempool *mpool, void *element)
+{
+	struct ftl_mempool_element *el = element;
+
+	assert(is_element_valid(mpool, element));
+	SLIST_INSERT_HEAD(&mpool->list, el, entry);
+}

lib/ftl/utils/ftl_mempool.h

@@ -0,0 +1,53 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ */
+
+#ifndef FTL_MEMPOOL_H
+#define FTL_MEMPOOL_H
+
+#include "spdk/stdinc.h"
+
+/* TODO: Consider porting this mempool to general SPDK utils */
+
+/**
+ * @brief Creates custom FTL memory pool using DMA kind memory
+ *
+ * The pool is being initialized.
+ *
+ * @param count Count of element in the memory pool
+ * @param size Size of elements in the memory pool
+ * @param alignment Memory alignment of element in the memory pool
+ * @param socket_id It is the socket identifier in the case of NUMA. The value
+ * can be *SOCKET_ID_ANY* if there is no NUMA constraint for the reserved zone.
+ *
+ * @return Pointer to the memory pool
+ */
+struct ftl_mempool *ftl_mempool_create(size_t count, size_t size,
+				       size_t alignment, int socket_id);
+
+/**
+ * @brief Destroys the FTL memory pool
+
+ * @param mpool The memory pool to be destroyed
+ */
+void ftl_mempool_destroy(struct ftl_mempool *mpool);
+
+/**
+ * @brief Gets (allocates) an element from the memory pool
+ *
+ * @param mpool The memory pool
+ *
+ * @return Element from memory pool. If memory pool empty it returns NULL.
+ */
+void *ftl_mempool_get(struct ftl_mempool *mpool);
+
+/**
+ * @brief Puts (releases) the element to the memory pool
+ *
+ * @param mpool The memory pool
+ * @param element The element to be released
+ */
+void ftl_mempool_put(struct ftl_mempool *mpool, void *element);
+
+#endif /* FTL_MEMPOOL_H */