ftl: bitmap on external memory

Main use case is to allow for keeping it in shared memory, to
speed up the recovery time after application crash.

Signed-off-by: Artur Paszkiewicz <artur.paszkiewicz@intel.com>
Signed-off-by: Kozlowski Mateusz <mateusz.kozlowski@intel.com>
Change-Id: I36b6b8331cd6483c5bd202e5f9103c351d705da8
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13345
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

lib/ftl/Makefile

@@ -26,7 +26,7 @@ C_SRCS += ftl_nv_cache.c ftl_band.c ftl_band_ops.c ftl_writer.c ftl_rq.c ftl_rel
 C_SRCS += mngt/ftl_mngt.c mngt/ftl_mngt_bdev.c mngt/ftl_mngt_shutdown.c mngt/ftl_mngt_startup.c
 C_SRCS += mngt/ftl_mngt_md.c mngt/ftl_mngt_misc.c mngt/ftl_mngt_ioch.c mngt/ftl_mngt_l2p.c
 C_SRCS += mngt/ftl_mngt_band.c
-C_SRCS += utils/ftl_conf.c utils/ftl_md.c utils/ftl_mempool.c
+C_SRCS += utils/ftl_conf.c utils/ftl_md.c utils/ftl_mempool.c utils/ftl_bitmap.c
 
 SPDK_MAP_FILE = $(abspath $(CURDIR)/spdk_ftl.map)
 

lib/ftl/utils/ftl_bitmap.c

@@ -0,0 +1,186 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ */
+
+#include "spdk/log.h"
+#include "spdk/util.h"
+
+#include "ftl_bitmap.h"
+#include "ftl_internal.h"
+
+typedef unsigned long bitmap_word;
+
+const size_t ftl_bitmap_buffer_alignment = sizeof(bitmap_word);
+
+#define FTL_BITMAP_WORD_SHIFT	spdk_u32log2(sizeof(bitmap_word) * 8)
+#define FTL_BITMAP_WORD_MASK	(~(~0UL << FTL_BITMAP_WORD_SHIFT))
+
+uint64_t
+ftl_bitmap_bits_to_size(uint64_t bits)
+{
+	uint64_t size;
+
+	if (bits < ftl_bitmap_buffer_alignment) {
+		bits = ftl_bitmap_buffer_alignment;
+	}
+
+	size = spdk_divide_round_up(bits, 8);
+	size = spdk_divide_round_up(size, ftl_bitmap_buffer_alignment) * ftl_bitmap_buffer_alignment;
+
+	return size;
+}
+
+uint64_t
+ftl_bitmap_bits_to_blocks(uint64_t bits)
+{
+	uint64_t size = ftl_bitmap_bits_to_size(bits);
+
+	return spdk_divide_round_up(size, FTL_BLOCK_SIZE);
+}
+
+struct ftl_bitmap {
+	bitmap_word *buf;
+	size_t size;
+};
+
+struct ftl_bitmap *ftl_bitmap_create(void *buf, size_t size)
+{
+	struct ftl_bitmap *bitmap;
+
+	if ((uintptr_t)buf % ftl_bitmap_buffer_alignment) {
+		SPDK_ERRLOG("Buffer for bitmap must be aligned to %lu bytes\n",
+			    ftl_bitmap_buffer_alignment);
+		return NULL;
+	}
+
+	if (size % ftl_bitmap_buffer_alignment) {
+		SPDK_ERRLOG("Size of buffer for bitmap must be divisible by %lu bytes\n",
+			    ftl_bitmap_buffer_alignment);
+		return NULL;
+	}
+
+	bitmap = calloc(1, sizeof(*bitmap));
+	if (!bitmap) {
+		return NULL;
+	}
+
+	bitmap->buf = buf;
+	bitmap->size = size / sizeof(bitmap_word);
+
+	return bitmap;
+}
+
+void
+ftl_bitmap_destroy(struct ftl_bitmap *bitmap)
+{
+	free(bitmap);
+}
+
+static inline void
+locate_bit(const struct ftl_bitmap *bitmap, uint64_t bit,
+	   bitmap_word **word_out, uint8_t *word_bit_idx_out)
+{
+	size_t word_idx = bit >> FTL_BITMAP_WORD_SHIFT;
+
+	assert(word_idx < bitmap->size);
+
+	*word_bit_idx_out = bit & FTL_BITMAP_WORD_MASK;
+	*word_out = &bitmap->buf[word_idx];
+}
+
+bool
+ftl_bitmap_get(const struct ftl_bitmap *bitmap, uint64_t bit)
+{
+	bitmap_word *word;
+	uint8_t word_bit_idx;
+
+	locate_bit(bitmap, bit, &word, &word_bit_idx);
+
+	return *word & (1UL << word_bit_idx);
+}
+
+void
+ftl_bitmap_set(struct ftl_bitmap *bitmap, uint64_t bit)
+{
+	bitmap_word *word;
+	uint8_t word_bit_idx;
+
+	locate_bit(bitmap, bit, &word, &word_bit_idx);
+
+	*word |= (1UL << word_bit_idx);
+}
+
+void
+ftl_bitmap_clear(struct ftl_bitmap *bitmap, uint64_t bit)
+{
+	bitmap_word *word;
+	uint8_t word_bit_idx;
+
+	locate_bit(bitmap, bit, &word, &word_bit_idx);
+
+	*word &= ~(1UL << word_bit_idx);
+}
+
+static uint64_t
+ftl_bitmap_find_first(struct ftl_bitmap *bitmap, uint64_t start_bit,
+		      uint64_t end_bit, bool value)
+{
+	bitmap_word skip = (value ? 0 : ~0UL);
+	bitmap_word word;
+	size_t i, end;
+	uint64_t ret;
+
+	assert(start_bit <= end_bit);
+
+	i = start_bit >> FTL_BITMAP_WORD_SHIFT;
+	assert(i < bitmap->size);
+
+	word = (bitmap->buf[i] ^ skip) & (~0UL << (start_bit & FTL_BITMAP_WORD_MASK));
+	if (word != 0) {
+		goto found;
+	}
+
+	end = spdk_min((end_bit >> FTL_BITMAP_WORD_SHIFT) + 1, bitmap->size);
+	for (i = i + 1; i < end; i++) {
+		word = bitmap->buf[i] ^ skip;
+		if (word != 0) {
+			goto found;
+		}
+	}
+
+	return UINT64_MAX;
+found:
+	ret = (i << FTL_BITMAP_WORD_SHIFT) + __builtin_ctzl(word);
+	if (ret > end_bit) {
+		return UINT64_MAX;
+	}
+	return ret;
+}
+
+uint64_t
+ftl_bitmap_find_first_set(struct ftl_bitmap *bitmap, uint64_t start_bit, uint64_t end_bit)
+{
+	return ftl_bitmap_find_first(bitmap, start_bit, end_bit, true);
+}
+
+uint64_t
+ftl_bitmap_find_first_clear(struct ftl_bitmap *bitmap, uint64_t start_bit,
+			    uint64_t end_bit)
+{
+	return ftl_bitmap_find_first(bitmap, start_bit, end_bit, false);
+}
+
+uint64_t
+ftl_bitmap_count_set(struct ftl_bitmap *bitmap)
+{
+	size_t i;
+	bitmap_word *word = bitmap->buf;
+	uint64_t count = 0;
+
+	for (i = 0; i < bitmap->size; i++, word++) {
+		count += __builtin_popcountl(*word);
+	}
+
+	return count;
+}

lib/ftl/utils/ftl_bitmap.h

@@ -0,0 +1,111 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ */
+
+#ifndef FTL_BITMAP_H_
+#define FTL_BITMAP_H_
+
+#include "spdk/stdinc.h"
+
+struct ftl_bitmap;
+
+/**
+ * @brief The required alignment for buffer used for bitmap
+ */
+extern const size_t ftl_bitmap_buffer_alignment;
+
+/**
+ * @brief Converts number of bits to bitmap size need to create it
+ *
+ * @param bits Number of bits
+ *
+ * @return Size needed to create bitmap which will hold space for specified number of bits
+ */
+uint64_t ftl_bitmap_bits_to_size(uint64_t bits);
+
+/**
+ * @brief Converts number of bits to blocks
+ *
+ * @param bits Number of bits
+ *
+ * @return Number of blocks needed to create bitmap which will hold space for specified number of bits
+ */
+uint64_t ftl_bitmap_bits_to_blocks(uint64_t bits);
+
+/**
+ * @brief Creates a bitmap object using a preallocated buffer
+ *
+ * @param buf The buffer
+ * @param size Size of the buffer
+ *
+ * @return On success - pointer to the allocated bitmap object, otherwise NULL
+ */
+struct ftl_bitmap *ftl_bitmap_create(void *buf, size_t size);
+
+/**
+ * @brief Destroys the bitmap object
+ *
+ * @param bitmap The bitmap
+ */
+void ftl_bitmap_destroy(struct ftl_bitmap *bitmap);
+
+/**
+ * @brief Gets the value of the specified bit
+ *
+ * @param bitmap The bitmap
+ * @param bit Index of the bit
+ *
+ * @return True if bit is set, otherwise false
+ */
+bool ftl_bitmap_get(const struct ftl_bitmap *bitmap, uint64_t bit);
+
+/**
+ * @brief Sets the specified bit
+ *
+ * @param bitmap The bitmap
+ * @param bit Index of the bit
+ */
+void ftl_bitmap_set(struct ftl_bitmap *bitmap, uint64_t bit);
+
+/**
+ * @brief Clears the specified bit
+ *
+ * @param bitmap The bitmap
+ * @param bit Index of the bit
+ */
+void ftl_bitmap_clear(struct ftl_bitmap *bitmap, uint64_t bit);
+
+/**
+ * @brief Finds the first set bit
+ *
+ * @param bitmap The bitmap
+ * @param start_bit Index of the bit from which to begin searching
+ * @param end_bit Index of the bit up to which to search
+ *
+ * @return Index of the first set bit or UINT64_MAX if none found
+ */
+uint64_t ftl_bitmap_find_first_set(struct ftl_bitmap *bitmap, uint64_t start_bit, uint64_t end_bit);
+
+/**
+ * @brief Finds the first clear bit
+ *
+ * @param bitmap The bitmap
+ * @param start_bit Index of the bit from which to begin searching
+ * @param end_bit Index of the bit up to which to search
+ *
+ * @return Index of the first clear bit or UINT64_MAX if none found
+ */
+uint64_t ftl_bitmap_find_first_clear(struct ftl_bitmap *bitmap, uint64_t start_bit,
+				     uint64_t end_bit);
+
+/**
+ * @brief Iterates over and counts set bits
+ *
+ * @param bitmap The bitmap
+ *
+ * @return Count of sets bits
+ */
+uint64_t ftl_bitmap_count_set(struct ftl_bitmap *bitmap);
+
+#endif /* FTL_BITMAP_H_ */

test/unit/lib/ftl/Makefile

@@ -7,7 +7,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
 DIRS-y = ftl_l2p ftl_band.c ftl_io.c
-DIRS-y += ftl_mempool.c ftl_mngt
+DIRS-y += ftl_bitmap.c ftl_mempool.c ftl_mngt
 
 .PHONY: all clean $(DIRS-y)
 

test/unit/lib/ftl/ftl_bitmap.c/.gitignore

@@ -0,0 +1 @@
+ftl_bitmap_ut

test/unit/lib/ftl/ftl_bitmap.c/Makefile

@@ -0,0 +1,12 @@
+#  SPDX-License-Identifier: BSD-3-Clause
+#  Copyright (c) Intel Corporation.
+#  All rights reserved.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../../..)
+
+TEST_FILE = ftl_bitmap_ut.c
+
+include $(SPDK_ROOT_DIR)/mk/spdk.unittest.mk
+
+CFLAGS += -I$(SPDK_ROOT_DIR)/lib/ftl

test/unit/lib/ftl/ftl_bitmap.c/ftl_bitmap_ut.c

@@ -0,0 +1,241 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk_cunit.h"
+
+#include "ftl/utils/ftl_bitmap.c"
+
+#define BITMAP_SIZE	64
+#define BITMAP_CAPACITY	(BITMAP_SIZE * 8)
+
+#define TEST_BIT(bi, bbi) \
+{ .byte_idx = (bi), .byte_bit_idx = (bbi), .bit_idx = (((bi) * 8) + (bbi)) }
+
+static struct {
+	size_t byte_idx;
+	uint8_t byte_bit_idx;
+	uint64_t bit_idx;
+} g_test_bits[] = {
+	TEST_BIT(0, 0),
+	TEST_BIT(0, 1),
+	TEST_BIT(0, 2),
+	TEST_BIT(1, 3),
+	TEST_BIT(2, 4),
+	TEST_BIT(3, 5),
+	TEST_BIT(15, 7),
+	TEST_BIT(42, 6),
+	TEST_BIT(BITMAP_SIZE - 1, 0),
+	TEST_BIT(BITMAP_SIZE - 1, 7),
+};
+static const size_t g_test_bits_count = sizeof(g_test_bits) / sizeof(*g_test_bits);
+
+static unsigned long g_buf[BITMAP_SIZE / sizeof(unsigned long)];
+static struct ftl_bitmap *g_bitmap;
+
+static uint64_t
+count_set_bits(const struct ftl_bitmap *bitmap)
+{
+	uint64_t n = 0;
+	uint64_t i;
+
+	for (i = 0; i < BITMAP_CAPACITY; i++) {
+		if (ftl_bitmap_get(bitmap, i)) {
+			n++;
+		}
+	}
+
+	return n;
+}
+
+static void
+test_ftl_bitmap_create(void)
+{
+	struct ftl_bitmap *ret;
+
+	/* unaligned buffer */
+	ret = ftl_bitmap_create(((uint8_t *)g_buf) + 1, BITMAP_SIZE);
+	CU_ASSERT_EQUAL(ret, NULL);
+
+	/* wrong size */
+	ret = ftl_bitmap_create(g_buf, BITMAP_SIZE - 1);
+	CU_ASSERT_EQUAL(ret, NULL);
+}
+
+static void
+test_ftl_bitmap_get(void)
+{
+	uint8_t *buf = (uint8_t *)g_buf;
+	size_t i;
+
+	memset(g_buf, 0, BITMAP_SIZE);
+
+	for (i = 0; i < g_test_bits_count; i++) {
+		buf[g_test_bits[i].byte_idx] += (1 << g_test_bits[i].byte_bit_idx);
+	}
+
+	CU_ASSERT_EQUAL(count_set_bits(g_bitmap), g_test_bits_count);
+
+	for (i = 0; i < g_test_bits_count; i++) {
+		CU_ASSERT_TRUE(ftl_bitmap_get(g_bitmap, g_test_bits[i].bit_idx));
+	}
+}
+
+static void
+test_ftl_bitmap_set(void)
+{
+	size_t i;
+
+	memset(g_buf, 0, BITMAP_SIZE);
+
+	for (i = 0; i < g_test_bits_count; i++) {
+		ftl_bitmap_set(g_bitmap, g_test_bits[i].bit_idx);
+	}
+
+	CU_ASSERT_EQUAL(count_set_bits(g_bitmap), g_test_bits_count);
+
+	for (i = 0; i < g_test_bits_count; i++) {
+		CU_ASSERT_TRUE(ftl_bitmap_get(g_bitmap, g_test_bits[i].bit_idx));
+	}
+}
+
+static void
+test_ftl_bitmap_clear(void)
+{
+	size_t i;
+
+	memset(g_buf, 0xff, BITMAP_SIZE);
+
+	for (i = 0; i < g_test_bits_count; i++) {
+		ftl_bitmap_clear(g_bitmap, g_test_bits[i].bit_idx);
+	}
+
+	CU_ASSERT_EQUAL(count_set_bits(g_bitmap), BITMAP_CAPACITY - g_test_bits_count);
+
+	for (i = 0; i < g_test_bits_count; i++) {
+		CU_ASSERT_FALSE(ftl_bitmap_get(g_bitmap, g_test_bits[i].bit_idx));
+	}
+}
+
+static void
+test_ftl_bitmap_find_first_set(void)
+{
+	size_t i;
+	uint64_t bit;
+
+	memset(g_buf, 0, BITMAP_SIZE);
+
+	CU_ASSERT_EQUAL(ftl_bitmap_find_first_set(g_bitmap, 0, UINT64_MAX), UINT64_MAX);
+
+	for (i = 1; i <= g_test_bits_count; i++) {
+		bit = g_test_bits[g_test_bits_count - i].bit_idx;
+
+		ftl_bitmap_set(g_bitmap, bit);
+
+		CU_ASSERT_EQUAL(ftl_bitmap_find_first_set(g_bitmap, 0, UINT64_MAX), bit);
+		CU_ASSERT_EQUAL(ftl_bitmap_find_first_set(g_bitmap, 0, bit), bit);
+		if (bit > 0) {
+			CU_ASSERT_EQUAL(ftl_bitmap_find_first_set(g_bitmap, 0, bit - 1), UINT64_MAX);
+		}
+	}
+
+	for (i = 0; i < g_test_bits_count; i++) {
+		bit = g_test_bits[i].bit_idx;
+
+		CU_ASSERT_EQUAL(ftl_bitmap_find_first_set(g_bitmap, bit, UINT64_MAX), bit);
+		CU_ASSERT_EQUAL(ftl_bitmap_find_first_set(g_bitmap, bit, bit), bit);
+	}
+}
+
+static void
+test_ftl_bitmap_find_first_clear(void)
+{
+	size_t i;
+	uint64_t bit;
+
+	memset(g_buf, 0xff, BITMAP_SIZE);
+
+	CU_ASSERT_EQUAL(ftl_bitmap_find_first_clear(g_bitmap, 0, UINT64_MAX), UINT64_MAX);
+
+	for (i = 1; i <= g_test_bits_count; i++) {
+		bit = g_test_bits[g_test_bits_count - i].bit_idx;
+
+		ftl_bitmap_clear(g_bitmap, bit);
+
+		CU_ASSERT_EQUAL(ftl_bitmap_find_first_clear(g_bitmap, 0, UINT64_MAX), bit);
+		CU_ASSERT_EQUAL(ftl_bitmap_find_first_clear(g_bitmap, 0, bit), bit);
+		if (bit > 0) {
+			CU_ASSERT_EQUAL(ftl_bitmap_find_first_clear(g_bitmap, 0, bit - 1), UINT64_MAX);
+		}
+	}
+
+	for (i = 0; i < g_test_bits_count; i++) {
+		bit = g_test_bits[i].bit_idx;
+
+		CU_ASSERT_EQUAL(ftl_bitmap_find_first_clear(g_bitmap, bit, UINT64_MAX), bit);
+		CU_ASSERT_EQUAL(ftl_bitmap_find_first_clear(g_bitmap, bit, bit), bit);
+	}
+}
+
+static void
+test_ftl_bitmap_count_set(void)
+{
+	size_t i;
+
+	memset(g_buf, 0, BITMAP_SIZE);
+
+	for (i = 0; i < g_test_bits_count; i++) {
+		ftl_bitmap_set(g_bitmap, g_test_bits[i].bit_idx);
+	}
+
+	CU_ASSERT_EQUAL(g_test_bits_count, ftl_bitmap_count_set(g_bitmap));
+	CU_ASSERT_EQUAL(count_set_bits(g_bitmap), ftl_bitmap_count_set(g_bitmap));
+}
+
+static int
+test_setup(void)
+{
+	g_bitmap = ftl_bitmap_create(g_buf, BITMAP_SIZE);
+	if (!g_bitmap) {
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int
+test_cleanup(void)
+{
+	free(g_bitmap);
+	g_bitmap = NULL;
+	return 0;
+}
+
+int
+main(int argc, char **argv)
+{
+	CU_pSuite suite = NULL;
+	unsigned int num_failures;
+
+	CU_set_error_action(CUEA_ABORT);
+	CU_initialize_registry();
+
+	suite = CU_add_suite("ftl_bitmap", test_setup, test_cleanup);
+	CU_ADD_TEST(suite, test_ftl_bitmap_create);
+	CU_ADD_TEST(suite, test_ftl_bitmap_get);
+	CU_ADD_TEST(suite, test_ftl_bitmap_set);
+	CU_ADD_TEST(suite, test_ftl_bitmap_clear);
+	CU_ADD_TEST(suite, test_ftl_bitmap_find_first_set);
+	CU_ADD_TEST(suite, test_ftl_bitmap_find_first_clear);
+	CU_ADD_TEST(suite, test_ftl_bitmap_count_set);
+
+	CU_basic_set_mode(CU_BRM_VERBOSE);
+	CU_basic_run_tests();
+	num_failures = CU_get_number_of_failures();
+	CU_cleanup_registry();
+
+	return num_failures;
+}

test/unit/unittest.sh

@@ -46,6 +46,7 @@ function unittest_event() {
 
 function unittest_ftl() {
 	$valgrind $testdir/lib/ftl/ftl_band.c/ftl_band_ut
+	$valgrind $testdir/lib/ftl/ftl_bitmap.c/ftl_bitmap_ut
 	$valgrind $testdir/lib/ftl/ftl_io.c/ftl_io_ut
 	$valgrind $testdir/lib/ftl/ftl_mngt/ftl_mngt_ut
 	$valgrind $testdir/lib/ftl/ftl_mempool.c/ftl_mempool_ut