From f1b079b49f877c9965c6f595cbac2277b18cf62c Mon Sep 17 00:00:00 2001 From: Artur Paszkiewicz Date: Fri, 10 Jun 2022 10:28:16 +0200 Subject: [PATCH] ftl: bitmap on external memory Main use case is to allow for keeping it in shared memory, to speed up the recovery time after application crash. Signed-off-by: Artur Paszkiewicz Signed-off-by: Kozlowski Mateusz Change-Id: I36b6b8331cd6483c5bd202e5f9103c351d705da8 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13345 Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Ben Walker --- lib/ftl/Makefile | 2 +- lib/ftl/utils/ftl_bitmap.c | 186 ++++++++++++++ lib/ftl/utils/ftl_bitmap.h | 111 ++++++++ test/unit/lib/ftl/Makefile | 2 +- test/unit/lib/ftl/ftl_bitmap.c/.gitignore | 1 + test/unit/lib/ftl/ftl_bitmap.c/Makefile | 12 + .../unit/lib/ftl/ftl_bitmap.c/ftl_bitmap_ut.c | 241 ++++++++++++++++++ test/unit/unittest.sh | 1 + 8 files changed, 554 insertions(+), 2 deletions(-) create mode 100644 lib/ftl/utils/ftl_bitmap.c create mode 100644 lib/ftl/utils/ftl_bitmap.h create mode 100644 test/unit/lib/ftl/ftl_bitmap.c/.gitignore create mode 100644 test/unit/lib/ftl/ftl_bitmap.c/Makefile create mode 100644 test/unit/lib/ftl/ftl_bitmap.c/ftl_bitmap_ut.c diff --git a/lib/ftl/Makefile b/lib/ftl/Makefile index 96215bdd4..81eddbd51 100644 --- a/lib/ftl/Makefile +++ b/lib/ftl/Makefile @@ -26,7 +26,7 @@ C_SRCS += ftl_nv_cache.c ftl_band.c ftl_band_ops.c ftl_writer.c ftl_rq.c ftl_rel C_SRCS += mngt/ftl_mngt.c mngt/ftl_mngt_bdev.c mngt/ftl_mngt_shutdown.c mngt/ftl_mngt_startup.c C_SRCS += mngt/ftl_mngt_md.c mngt/ftl_mngt_misc.c mngt/ftl_mngt_ioch.c mngt/ftl_mngt_l2p.c C_SRCS += mngt/ftl_mngt_band.c -C_SRCS += utils/ftl_conf.c utils/ftl_md.c utils/ftl_mempool.c +C_SRCS += utils/ftl_conf.c utils/ftl_md.c utils/ftl_mempool.c utils/ftl_bitmap.c SPDK_MAP_FILE = $(abspath $(CURDIR)/spdk_ftl.map) diff --git a/lib/ftl/utils/ftl_bitmap.c b/lib/ftl/utils/ftl_bitmap.c new file mode 100644 index 000000000..d52cca91d --- /dev/null +++ b/lib/ftl/utils/ftl_bitmap.c @@ -0,0 +1,186 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) Intel Corporation. + * All rights reserved. + */ + +#include "spdk/log.h" +#include "spdk/util.h" + +#include "ftl_bitmap.h" +#include "ftl_internal.h" + +typedef unsigned long bitmap_word; + +const size_t ftl_bitmap_buffer_alignment = sizeof(bitmap_word); + +#define FTL_BITMAP_WORD_SHIFT spdk_u32log2(sizeof(bitmap_word) * 8) +#define FTL_BITMAP_WORD_MASK (~(~0UL << FTL_BITMAP_WORD_SHIFT)) + +uint64_t +ftl_bitmap_bits_to_size(uint64_t bits) +{ + uint64_t size; + + if (bits < ftl_bitmap_buffer_alignment) { + bits = ftl_bitmap_buffer_alignment; + } + + size = spdk_divide_round_up(bits, 8); + size = spdk_divide_round_up(size, ftl_bitmap_buffer_alignment) * ftl_bitmap_buffer_alignment; + + return size; +} + +uint64_t +ftl_bitmap_bits_to_blocks(uint64_t bits) +{ + uint64_t size = ftl_bitmap_bits_to_size(bits); + + return spdk_divide_round_up(size, FTL_BLOCK_SIZE); +} + +struct ftl_bitmap { + bitmap_word *buf; + size_t size; +}; + +struct ftl_bitmap *ftl_bitmap_create(void *buf, size_t size) +{ + struct ftl_bitmap *bitmap; + + if ((uintptr_t)buf % ftl_bitmap_buffer_alignment) { + SPDK_ERRLOG("Buffer for bitmap must be aligned to %lu bytes\n", + ftl_bitmap_buffer_alignment); + return NULL; + } + + if (size % ftl_bitmap_buffer_alignment) { + SPDK_ERRLOG("Size of buffer for bitmap must be divisible by %lu bytes\n", + ftl_bitmap_buffer_alignment); + return NULL; + } + + bitmap = calloc(1, sizeof(*bitmap)); + if (!bitmap) { + return NULL; + } + + bitmap->buf = buf; + bitmap->size = size / sizeof(bitmap_word); + + return bitmap; +} + +void +ftl_bitmap_destroy(struct ftl_bitmap *bitmap) +{ + free(bitmap); +} + +static inline void +locate_bit(const struct ftl_bitmap *bitmap, uint64_t bit, + bitmap_word **word_out, uint8_t *word_bit_idx_out) +{ + size_t word_idx = bit >> FTL_BITMAP_WORD_SHIFT; + + assert(word_idx < bitmap->size); + + *word_bit_idx_out = bit & FTL_BITMAP_WORD_MASK; + *word_out = &bitmap->buf[word_idx]; +} + +bool +ftl_bitmap_get(const struct ftl_bitmap *bitmap, uint64_t bit) +{ + bitmap_word *word; + uint8_t word_bit_idx; + + locate_bit(bitmap, bit, &word, &word_bit_idx); + + return *word & (1UL << word_bit_idx); +} + +void +ftl_bitmap_set(struct ftl_bitmap *bitmap, uint64_t bit) +{ + bitmap_word *word; + uint8_t word_bit_idx; + + locate_bit(bitmap, bit, &word, &word_bit_idx); + + *word |= (1UL << word_bit_idx); +} + +void +ftl_bitmap_clear(struct ftl_bitmap *bitmap, uint64_t bit) +{ + bitmap_word *word; + uint8_t word_bit_idx; + + locate_bit(bitmap, bit, &word, &word_bit_idx); + + *word &= ~(1UL << word_bit_idx); +} + +static uint64_t +ftl_bitmap_find_first(struct ftl_bitmap *bitmap, uint64_t start_bit, + uint64_t end_bit, bool value) +{ + bitmap_word skip = (value ? 0 : ~0UL); + bitmap_word word; + size_t i, end; + uint64_t ret; + + assert(start_bit <= end_bit); + + i = start_bit >> FTL_BITMAP_WORD_SHIFT; + assert(i < bitmap->size); + + word = (bitmap->buf[i] ^ skip) & (~0UL << (start_bit & FTL_BITMAP_WORD_MASK)); + if (word != 0) { + goto found; + } + + end = spdk_min((end_bit >> FTL_BITMAP_WORD_SHIFT) + 1, bitmap->size); + for (i = i + 1; i < end; i++) { + word = bitmap->buf[i] ^ skip; + if (word != 0) { + goto found; + } + } + + return UINT64_MAX; +found: + ret = (i << FTL_BITMAP_WORD_SHIFT) + __builtin_ctzl(word); + if (ret > end_bit) { + return UINT64_MAX; + } + return ret; +} + +uint64_t +ftl_bitmap_find_first_set(struct ftl_bitmap *bitmap, uint64_t start_bit, uint64_t end_bit) +{ + return ftl_bitmap_find_first(bitmap, start_bit, end_bit, true); +} + +uint64_t +ftl_bitmap_find_first_clear(struct ftl_bitmap *bitmap, uint64_t start_bit, + uint64_t end_bit) +{ + return ftl_bitmap_find_first(bitmap, start_bit, end_bit, false); +} + +uint64_t +ftl_bitmap_count_set(struct ftl_bitmap *bitmap) +{ + size_t i; + bitmap_word *word = bitmap->buf; + uint64_t count = 0; + + for (i = 0; i < bitmap->size; i++, word++) { + count += __builtin_popcountl(*word); + } + + return count; +} diff --git a/lib/ftl/utils/ftl_bitmap.h b/lib/ftl/utils/ftl_bitmap.h new file mode 100644 index 000000000..98fe6d199 --- /dev/null +++ b/lib/ftl/utils/ftl_bitmap.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) Intel Corporation. + * All rights reserved. + */ + +#ifndef FTL_BITMAP_H_ +#define FTL_BITMAP_H_ + +#include "spdk/stdinc.h" + +struct ftl_bitmap; + +/** + * @brief The required alignment for buffer used for bitmap + */ +extern const size_t ftl_bitmap_buffer_alignment; + +/** + * @brief Converts number of bits to bitmap size need to create it + * + * @param bits Number of bits + * + * @return Size needed to create bitmap which will hold space for specified number of bits + */ +uint64_t ftl_bitmap_bits_to_size(uint64_t bits); + +/** + * @brief Converts number of bits to blocks + * + * @param bits Number of bits + * + * @return Number of blocks needed to create bitmap which will hold space for specified number of bits + */ +uint64_t ftl_bitmap_bits_to_blocks(uint64_t bits); + +/** + * @brief Creates a bitmap object using a preallocated buffer + * + * @param buf The buffer + * @param size Size of the buffer + * + * @return On success - pointer to the allocated bitmap object, otherwise NULL + */ +struct ftl_bitmap *ftl_bitmap_create(void *buf, size_t size); + +/** + * @brief Destroys the bitmap object + * + * @param bitmap The bitmap + */ +void ftl_bitmap_destroy(struct ftl_bitmap *bitmap); + +/** + * @brief Gets the value of the specified bit + * + * @param bitmap The bitmap + * @param bit Index of the bit + * + * @return True if bit is set, otherwise false + */ +bool ftl_bitmap_get(const struct ftl_bitmap *bitmap, uint64_t bit); + +/** + * @brief Sets the specified bit + * + * @param bitmap The bitmap + * @param bit Index of the bit + */ +void ftl_bitmap_set(struct ftl_bitmap *bitmap, uint64_t bit); + +/** + * @brief Clears the specified bit + * + * @param bitmap The bitmap + * @param bit Index of the bit + */ +void ftl_bitmap_clear(struct ftl_bitmap *bitmap, uint64_t bit); + +/** + * @brief Finds the first set bit + * + * @param bitmap The bitmap + * @param start_bit Index of the bit from which to begin searching + * @param end_bit Index of the bit up to which to search + * + * @return Index of the first set bit or UINT64_MAX if none found + */ +uint64_t ftl_bitmap_find_first_set(struct ftl_bitmap *bitmap, uint64_t start_bit, uint64_t end_bit); + +/** + * @brief Finds the first clear bit + * + * @param bitmap The bitmap + * @param start_bit Index of the bit from which to begin searching + * @param end_bit Index of the bit up to which to search + * + * @return Index of the first clear bit or UINT64_MAX if none found + */ +uint64_t ftl_bitmap_find_first_clear(struct ftl_bitmap *bitmap, uint64_t start_bit, + uint64_t end_bit); + +/** + * @brief Iterates over and counts set bits + * + * @param bitmap The bitmap + * + * @return Count of sets bits + */ +uint64_t ftl_bitmap_count_set(struct ftl_bitmap *bitmap); + +#endif /* FTL_BITMAP_H_ */ diff --git a/test/unit/lib/ftl/Makefile b/test/unit/lib/ftl/Makefile index 4cd33c5c9..fbe0ca20c 100644 --- a/test/unit/lib/ftl/Makefile +++ b/test/unit/lib/ftl/Makefile @@ -7,7 +7,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk DIRS-y = ftl_l2p ftl_band.c ftl_io.c -DIRS-y += ftl_mempool.c ftl_mngt +DIRS-y += ftl_bitmap.c ftl_mempool.c ftl_mngt .PHONY: all clean $(DIRS-y) diff --git a/test/unit/lib/ftl/ftl_bitmap.c/.gitignore b/test/unit/lib/ftl/ftl_bitmap.c/.gitignore new file mode 100644 index 000000000..873cf41a0 --- /dev/null +++ b/test/unit/lib/ftl/ftl_bitmap.c/.gitignore @@ -0,0 +1 @@ +ftl_bitmap_ut diff --git a/test/unit/lib/ftl/ftl_bitmap.c/Makefile b/test/unit/lib/ftl/ftl_bitmap.c/Makefile new file mode 100644 index 000000000..66e0d9cee --- /dev/null +++ b/test/unit/lib/ftl/ftl_bitmap.c/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) Intel Corporation. +# All rights reserved. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../../..) + +TEST_FILE = ftl_bitmap_ut.c + +include $(SPDK_ROOT_DIR)/mk/spdk.unittest.mk + +CFLAGS += -I$(SPDK_ROOT_DIR)/lib/ftl diff --git a/test/unit/lib/ftl/ftl_bitmap.c/ftl_bitmap_ut.c b/test/unit/lib/ftl/ftl_bitmap.c/ftl_bitmap_ut.c new file mode 100644 index 000000000..a744bb53d --- /dev/null +++ b/test/unit/lib/ftl/ftl_bitmap.c/ftl_bitmap_ut.c @@ -0,0 +1,241 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) Intel Corporation. + * All rights reserved. + */ + +#include "spdk/stdinc.h" + +#include "spdk_cunit.h" + +#include "ftl/utils/ftl_bitmap.c" + +#define BITMAP_SIZE 64 +#define BITMAP_CAPACITY (BITMAP_SIZE * 8) + +#define TEST_BIT(bi, bbi) \ +{ .byte_idx = (bi), .byte_bit_idx = (bbi), .bit_idx = (((bi) * 8) + (bbi)) } + +static struct { + size_t byte_idx; + uint8_t byte_bit_idx; + uint64_t bit_idx; +} g_test_bits[] = { + TEST_BIT(0, 0), + TEST_BIT(0, 1), + TEST_BIT(0, 2), + TEST_BIT(1, 3), + TEST_BIT(2, 4), + TEST_BIT(3, 5), + TEST_BIT(15, 7), + TEST_BIT(42, 6), + TEST_BIT(BITMAP_SIZE - 1, 0), + TEST_BIT(BITMAP_SIZE - 1, 7), +}; +static const size_t g_test_bits_count = sizeof(g_test_bits) / sizeof(*g_test_bits); + +static unsigned long g_buf[BITMAP_SIZE / sizeof(unsigned long)]; +static struct ftl_bitmap *g_bitmap; + +static uint64_t +count_set_bits(const struct ftl_bitmap *bitmap) +{ + uint64_t n = 0; + uint64_t i; + + for (i = 0; i < BITMAP_CAPACITY; i++) { + if (ftl_bitmap_get(bitmap, i)) { + n++; + } + } + + return n; +} + +static void +test_ftl_bitmap_create(void) +{ + struct ftl_bitmap *ret; + + /* unaligned buffer */ + ret = ftl_bitmap_create(((uint8_t *)g_buf) + 1, BITMAP_SIZE); + CU_ASSERT_EQUAL(ret, NULL); + + /* wrong size */ + ret = ftl_bitmap_create(g_buf, BITMAP_SIZE - 1); + CU_ASSERT_EQUAL(ret, NULL); +} + +static void +test_ftl_bitmap_get(void) +{ + uint8_t *buf = (uint8_t *)g_buf; + size_t i; + + memset(g_buf, 0, BITMAP_SIZE); + + for (i = 0; i < g_test_bits_count; i++) { + buf[g_test_bits[i].byte_idx] += (1 << g_test_bits[i].byte_bit_idx); + } + + CU_ASSERT_EQUAL(count_set_bits(g_bitmap), g_test_bits_count); + + for (i = 0; i < g_test_bits_count; i++) { + CU_ASSERT_TRUE(ftl_bitmap_get(g_bitmap, g_test_bits[i].bit_idx)); + } +} + +static void +test_ftl_bitmap_set(void) +{ + size_t i; + + memset(g_buf, 0, BITMAP_SIZE); + + for (i = 0; i < g_test_bits_count; i++) { + ftl_bitmap_set(g_bitmap, g_test_bits[i].bit_idx); + } + + CU_ASSERT_EQUAL(count_set_bits(g_bitmap), g_test_bits_count); + + for (i = 0; i < g_test_bits_count; i++) { + CU_ASSERT_TRUE(ftl_bitmap_get(g_bitmap, g_test_bits[i].bit_idx)); + } +} + +static void +test_ftl_bitmap_clear(void) +{ + size_t i; + + memset(g_buf, 0xff, BITMAP_SIZE); + + for (i = 0; i < g_test_bits_count; i++) { + ftl_bitmap_clear(g_bitmap, g_test_bits[i].bit_idx); + } + + CU_ASSERT_EQUAL(count_set_bits(g_bitmap), BITMAP_CAPACITY - g_test_bits_count); + + for (i = 0; i < g_test_bits_count; i++) { + CU_ASSERT_FALSE(ftl_bitmap_get(g_bitmap, g_test_bits[i].bit_idx)); + } +} + +static void +test_ftl_bitmap_find_first_set(void) +{ + size_t i; + uint64_t bit; + + memset(g_buf, 0, BITMAP_SIZE); + + CU_ASSERT_EQUAL(ftl_bitmap_find_first_set(g_bitmap, 0, UINT64_MAX), UINT64_MAX); + + for (i = 1; i <= g_test_bits_count; i++) { + bit = g_test_bits[g_test_bits_count - i].bit_idx; + + ftl_bitmap_set(g_bitmap, bit); + + CU_ASSERT_EQUAL(ftl_bitmap_find_first_set(g_bitmap, 0, UINT64_MAX), bit); + CU_ASSERT_EQUAL(ftl_bitmap_find_first_set(g_bitmap, 0, bit), bit); + if (bit > 0) { + CU_ASSERT_EQUAL(ftl_bitmap_find_first_set(g_bitmap, 0, bit - 1), UINT64_MAX); + } + } + + for (i = 0; i < g_test_bits_count; i++) { + bit = g_test_bits[i].bit_idx; + + CU_ASSERT_EQUAL(ftl_bitmap_find_first_set(g_bitmap, bit, UINT64_MAX), bit); + CU_ASSERT_EQUAL(ftl_bitmap_find_first_set(g_bitmap, bit, bit), bit); + } +} + +static void +test_ftl_bitmap_find_first_clear(void) +{ + size_t i; + uint64_t bit; + + memset(g_buf, 0xff, BITMAP_SIZE); + + CU_ASSERT_EQUAL(ftl_bitmap_find_first_clear(g_bitmap, 0, UINT64_MAX), UINT64_MAX); + + for (i = 1; i <= g_test_bits_count; i++) { + bit = g_test_bits[g_test_bits_count - i].bit_idx; + + ftl_bitmap_clear(g_bitmap, bit); + + CU_ASSERT_EQUAL(ftl_bitmap_find_first_clear(g_bitmap, 0, UINT64_MAX), bit); + CU_ASSERT_EQUAL(ftl_bitmap_find_first_clear(g_bitmap, 0, bit), bit); + if (bit > 0) { + CU_ASSERT_EQUAL(ftl_bitmap_find_first_clear(g_bitmap, 0, bit - 1), UINT64_MAX); + } + } + + for (i = 0; i < g_test_bits_count; i++) { + bit = g_test_bits[i].bit_idx; + + CU_ASSERT_EQUAL(ftl_bitmap_find_first_clear(g_bitmap, bit, UINT64_MAX), bit); + CU_ASSERT_EQUAL(ftl_bitmap_find_first_clear(g_bitmap, bit, bit), bit); + } +} + +static void +test_ftl_bitmap_count_set(void) +{ + size_t i; + + memset(g_buf, 0, BITMAP_SIZE); + + for (i = 0; i < g_test_bits_count; i++) { + ftl_bitmap_set(g_bitmap, g_test_bits[i].bit_idx); + } + + CU_ASSERT_EQUAL(g_test_bits_count, ftl_bitmap_count_set(g_bitmap)); + CU_ASSERT_EQUAL(count_set_bits(g_bitmap), ftl_bitmap_count_set(g_bitmap)); +} + +static int +test_setup(void) +{ + g_bitmap = ftl_bitmap_create(g_buf, BITMAP_SIZE); + if (!g_bitmap) { + return -ENOMEM; + } + + return 0; +} + +static int +test_cleanup(void) +{ + free(g_bitmap); + g_bitmap = NULL; + return 0; +} + +int +main(int argc, char **argv) +{ + CU_pSuite suite = NULL; + unsigned int num_failures; + + CU_set_error_action(CUEA_ABORT); + CU_initialize_registry(); + + suite = CU_add_suite("ftl_bitmap", test_setup, test_cleanup); + CU_ADD_TEST(suite, test_ftl_bitmap_create); + CU_ADD_TEST(suite, test_ftl_bitmap_get); + CU_ADD_TEST(suite, test_ftl_bitmap_set); + CU_ADD_TEST(suite, test_ftl_bitmap_clear); + CU_ADD_TEST(suite, test_ftl_bitmap_find_first_set); + CU_ADD_TEST(suite, test_ftl_bitmap_find_first_clear); + CU_ADD_TEST(suite, test_ftl_bitmap_count_set); + + CU_basic_set_mode(CU_BRM_VERBOSE); + CU_basic_run_tests(); + num_failures = CU_get_number_of_failures(); + CU_cleanup_registry(); + + return num_failures; +} diff --git a/test/unit/unittest.sh b/test/unit/unittest.sh index e8c8c70cf..e72b956db 100755 --- a/test/unit/unittest.sh +++ b/test/unit/unittest.sh @@ -46,6 +46,7 @@ function unittest_event() { function unittest_ftl() { $valgrind $testdir/lib/ftl/ftl_band.c/ftl_band_ut + $valgrind $testdir/lib/ftl/ftl_bitmap.c/ftl_bitmap_ut $valgrind $testdir/lib/ftl/ftl_io.c/ftl_io_ut $valgrind $testdir/lib/ftl/ftl_mngt/ftl_mngt_ut $valgrind $testdir/lib/ftl/ftl_mempool.c/ftl_mempool_ut