diff --git a/lib/ftl/mngt/ftl_mngt_md.c b/lib/ftl/mngt/ftl_mngt_md.c index ffbd836b0..cabe345be 100644 --- a/lib/ftl/mngt/ftl_mngt_md.c +++ b/lib/ftl/mngt/ftl_mngt_md.c @@ -50,6 +50,7 @@ ftl_mngt_init_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) struct ftl_layout *layout = &dev->layout; struct ftl_layout_region *region = layout->region; uint64_t i; + int md_flags; for (i = 0; i < FTL_LAYOUT_REGION_TYPE_MAX; i++, region++) { if (layout->md[i]) { @@ -61,8 +62,11 @@ ftl_mngt_init_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) */ continue; } + md_flags = is_buffer_needed(i) ? + FTL_MD_CREATE_SHM | FTL_MD_CREATE_SHM_NEW : + FTL_MD_CREATE_NO_MEM; layout->md[i] = ftl_md_create(dev, region->current.blocks, region->vss_blksz, region->name, - !is_buffer_needed(i), region); + md_flags, region); if (NULL == layout->md[i]) { ftl_mngt_fail_step(mngt); return; @@ -245,6 +249,7 @@ ftl_mngt_superblock_init(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt struct ftl_layout *layout = &dev->layout; struct ftl_layout_region *region = &layout->region[FTL_LAYOUT_REGION_TYPE_SB]; char uuid[SPDK_UUID_STRING_LEN]; + int md_create_flags = FTL_MD_CREATE_SHM | FTL_MD_CREATE_SHM_NEW; /* Must generate UUID before MD create on SHM for the SB */ if (dev->conf.mode & SPDK_FTL_MODE_CREATE) { @@ -261,7 +266,8 @@ ftl_mngt_superblock_init(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt /* Allocate md buf */ layout->md[FTL_LAYOUT_REGION_TYPE_SB] = ftl_md_create(dev, region->current.blocks, - region->vss_blksz, region->name, false, region); + region->vss_blksz, region->name, + md_create_flags, region); if (NULL == layout->md[FTL_LAYOUT_REGION_TYPE_SB]) { ftl_mngt_fail_step(mngt); return; @@ -273,7 +279,7 @@ ftl_mngt_superblock_init(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt /* Setup superblock mirror to QLC */ region = &layout->region[FTL_LAYOUT_REGION_TYPE_SB_BASE]; layout->md[FTL_LAYOUT_REGION_TYPE_SB_BASE] = ftl_md_create(dev, region->current.blocks, - region->vss_blksz, NULL, false, region); + region->vss_blksz, NULL, FTL_MD_CREATE_HEAP, region); if (NULL == layout->md[FTL_LAYOUT_REGION_TYPE_SB_BASE]) { ftl_mngt_fail_step(mngt); return; diff --git a/lib/ftl/utils/ftl_md.c b/lib/ftl/utils/ftl_md.c index 335c3f78c..b29d396f4 100644 --- a/lib/ftl/utils/ftl_md.c +++ b/lib/ftl/utils/ftl_md.c @@ -61,8 +61,194 @@ xfer_size(struct ftl_md *md) return ftl_md_xfer_blocks(md->dev) * FTL_BLOCK_SIZE; } +static void +ftl_md_create_heap(struct ftl_md *md, uint64_t vss_blksz) +{ + md->shm_fd = -1; + md->vss_data = NULL; + md->data = calloc(md->data_blocks, FTL_BLOCK_SIZE + vss_blksz); + + if (md->data && vss_blksz) { + md->vss_data = ((char *)md->data) + md->data_blocks * FTL_BLOCK_SIZE; + } +} + +static void +ftl_md_destroy_heap(struct ftl_md *md) +{ + if (md->data) { + free(md->data); + md->data = NULL; + md->vss_data = NULL; + } +} + +static int +ftl_wrapper_open(const char *name, int of, mode_t m) +{ + return open(name, of, m); +} + +static void +ftl_md_setup_obj(struct ftl_md *md, int flags, + const char *name) +{ + char uuid_str[SPDK_UUID_STRING_LEN]; + const char *fmt; + + if (!(flags & FTL_MD_CREATE_SHM)) { + assert(false); + return; + } + + /* TODO: temporary, define a proper hugetlbfs mountpoint */ + fmt = "/dev/hugepages/ftl_%s_%s"; + md->shm_mmap_flags = MAP_SHARED; + md->shm_open = ftl_wrapper_open; + md->shm_unlink = unlink; + + if (name == NULL || + spdk_uuid_fmt_lower(uuid_str, SPDK_UUID_STRING_LEN, &md->dev->conf.uuid) || + snprintf(md->name, sizeof(md->name) / sizeof(md->name[0]), + fmt, uuid_str, name) <= 0) { + md->name[0] = 0; + } +} + +static void +ftl_md_create_shm(struct ftl_md *md, uint64_t vss_blksz, int flags) +{ + struct stat shm_stat; + size_t vss_blk_offs; + void *shm_ptr; + int open_flags = O_RDWR; + mode_t open_mode = S_IRUSR | S_IWUSR; + + assert(md->shm_open && md->shm_unlink); + md->data = NULL; + md->vss_data = NULL; + md->shm_sz = 0; + + /* Must have an object name */ + if (md->name[0] == 0) { + assert(false); + return; + } + + /* If specified, unlink before create a new SHM object */ + if (flags & FTL_MD_CREATE_SHM_NEW) { + if (md->shm_unlink(md->name) < 0 && errno != ENOENT) { + return; + } + open_flags += O_CREAT | O_TRUNC; + } + + /* Open existing or create a new SHM object, then query its props */ + md->shm_fd = md->shm_open(md->name, open_flags, open_mode); + if (md->shm_fd < 0 || fstat(md->shm_fd, &shm_stat) < 0) { + goto err_shm; + } + + /* Verify open mode hasn't changed */ + if ((shm_stat.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) != open_mode) { + goto err_shm; + } + + /* Round up the SHM obj size to the nearest blk size (i.e. page size) */ + md->shm_sz = spdk_divide_round_up(md->data_blocks * FTL_BLOCK_SIZE, shm_stat.st_blksize); + + /* Add some blks for VSS metadata */ + vss_blk_offs = md->shm_sz; + + if (vss_blksz) { + md->shm_sz += spdk_divide_round_up(md->data_blocks * vss_blksz, + shm_stat.st_blksize); + } + + /* Total SHM obj size */ + md->shm_sz *= shm_stat.st_blksize; + + /* Set or check the object size - zero init`d in case of set (FTL_MD_CREATE_SHM_NEW) */ + if ((shm_stat.st_size == 0 && (ftruncate(md->shm_fd, md->shm_sz) < 0 || + (flags & FTL_MD_CREATE_SHM_NEW) == 0)) + || (shm_stat.st_size > 0 && (size_t)shm_stat.st_size != md->shm_sz)) { + goto err_shm; + } + + /* Create a virtual memory mapping for the object */ + shm_ptr = mmap(NULL, md->shm_sz, PROT_READ | PROT_WRITE, md->shm_mmap_flags, + md->shm_fd, 0); + if (shm_ptr == MAP_FAILED) { + goto err_shm; + } + + md->data = shm_ptr; + if (vss_blksz) { + md->vss_data = ((char *)shm_ptr) + vss_blk_offs * shm_stat.st_blksize; + } + + /* Lock the pages in memory (i.e. prevent the pages to be paged out) */ + if (mlock(md->data, md->shm_sz) < 0) { + goto err_map; + } + + if (spdk_mem_register(md->data, md->shm_sz)) { + goto err_mlock; + } + md->mem_reg = true; + + return; + + /* Cleanup upon fault */ +err_mlock: + munlock(md->data, md->shm_sz); + +err_map: + munmap(md->data, md->shm_sz); + md->data = NULL; + md->vss_data = NULL; + md->shm_sz = 0; + +err_shm: + if (md->shm_fd >= 0) { + close(md->shm_fd); + md->shm_unlink(md->name); + md->shm_fd = -1; + } +} + +static void +ftl_md_destroy_shm(struct ftl_md *md) +{ + if (!md->data) { + return; + } + + assert(md->shm_sz > 0); + if (md->mem_reg) { + spdk_mem_unregister(md->data, md->shm_sz); + md->mem_reg = false; + } + + /* Unlock the pages in memory */ + munlock(md->data, md->shm_sz); + + /* Remove the virtual memory mapping for the object */ + munmap(md->data, md->shm_sz); + + /* Close SHM object fd */ + close(md->shm_fd); + + md->data = NULL; + md->vss_data = NULL; + + /* Otherwise destroy/unlink the object */ + assert(md->name[0] != 0 && md->shm_unlink != NULL); + md->shm_unlink(md->name); +} + struct ftl_md *ftl_md_create(struct spdk_ftl_dev *dev, uint64_t blocks, - uint64_t vss_blksz, const char *name, bool no_mem, + uint64_t vss_blksz, const char *name, int flags, const struct ftl_layout_region *region) { struct ftl_md *md; @@ -75,20 +261,19 @@ struct ftl_md *ftl_md_create(struct spdk_ftl_dev *dev, uint64_t blocks, md->data_blocks = blocks; md->mirror_enabled = true; - if (!no_mem) { - size_t buf_size = md->data_blocks * (FTL_BLOCK_SIZE + vss_blksz); - int ret; + if (flags != FTL_MD_CREATE_NO_MEM) { + if (flags & FTL_MD_CREATE_SHM) { + ftl_md_setup_obj(md, flags, name); + ftl_md_create_shm(md, vss_blksz, flags); + } else { + assert((flags & FTL_MD_CREATE_HEAP) == FTL_MD_CREATE_HEAP); + ftl_md_create_heap(md, vss_blksz); + } - ret = posix_memalign((void **)&md->data, FTL_BLOCK_SIZE, buf_size); - if (ret) { + if (!md->data) { free(md); return NULL; } - memset(md->data, 0, buf_size); - - if (vss_blksz) { - md->vss_data = ((char *)md->data) + md->data_blocks * FTL_BLOCK_SIZE; - } } if (region) { @@ -114,6 +299,22 @@ err: return NULL; } +int +ftl_md_unlink(struct spdk_ftl_dev *dev, const char *name, int flags) +{ + struct ftl_md md = { 0 }; + + if (0 == (flags & FTL_MD_CREATE_SHM)) { + /* Unlink can be called for shared memory only */ + return -EINVAL; + } + + md.dev = dev; + ftl_md_setup_obj(&md, flags, name); + + return md.shm_unlink(md.name); +} + void ftl_md_destroy(struct ftl_md *md) { @@ -136,10 +337,10 @@ ftl_md_free_buf(struct ftl_md *md) return; } - if (md->data) { - free(md->data); - md->data = NULL; - md->vss_data = NULL; + if (md->shm_fd < 0) { + ftl_md_destroy_heap(md); + } else { + ftl_md_destroy_shm(md); } } diff --git a/lib/ftl/utils/ftl_md.h b/lib/ftl/utils/ftl_md.h index 7d5a77c20..697821a8f 100644 --- a/lib/ftl/utils/ftl_md.h +++ b/lib/ftl/utils/ftl_md.h @@ -20,6 +20,9 @@ enum ftl_md_ops { FTL_MD_OP_CLEAR, }; +typedef int (*shm_open_t)(const char *, int, mode_t); +typedef int (*shm_unlink_t)(const char *); + /* FTL metadata container which allows to store/restore/recover */ struct ftl_md { /* Context of owner (Caller of restore/persist/clear operation) */ @@ -64,6 +67,27 @@ struct ftl_md { struct spdk_bdev_io_wait_entry bdev_io_wait; } io; + /* SHM object file descriptor or -1 if heap alloc */ + int shm_fd; + + /* Object name */ + char name[NAME_MAX + 1]; + + /* mmap flags for the SHM object */ + int shm_mmap_flags; + + /* Total size of SHM object (data + md) */ + size_t shm_sz; + + /* open() for the SHM object */ + shm_open_t shm_open; + + /* unlink() for the SHM object */ + shm_unlink_t shm_unlink; + + /* Memory was registered to SPDK */ + bool mem_reg; + /* Metadata primary object */ struct ftl_md *mirror; @@ -104,6 +128,23 @@ union ftl_md_vss { SPDK_STATIC_ASSERT(sizeof(union ftl_md_vss) == FTL_MD_VSS_SZ, "Invalid md vss size"); +/** + * FTL metadata creation flags + */ +enum ftl_md_create_flags { + /** FTL metadata will be created without memory allocation */ + FTL_MD_CREATE_NO_MEM = 0x0, + + /** FTL metadata data buf will be allocated in SHM */ + FTL_MD_CREATE_SHM = 0x1, + + /** Always create a new SHM obj, i.e. issue shm_unlink() before shm_open(), only valid with FTL_MD_CREATE_SHM */ + FTL_MD_CREATE_SHM_NEW = 0x2, + + /** FTL metadata will be created on heap */ + FTL_MD_CREATE_HEAP = 0x4, +}; + /** * @brief Creates FTL metadata * @@ -111,7 +152,7 @@ SPDK_STATIC_ASSERT(sizeof(union ftl_md_vss) == FTL_MD_VSS_SZ, "Invalid md vss si * @param blocks Size of buffer in FTL block size unit * @param vss_blksz Size of VSS MD * @param name Name of the object being created - * @param no_mem If true metadata will be created without memory allocation + * @param flags Bit flags of ftl_md_create_flags type * @param region Region associated with FTL metadata * * @note if buffer is NULL, the buffer will be allocated internally by the object @@ -119,9 +160,21 @@ SPDK_STATIC_ASSERT(sizeof(union ftl_md_vss) == FTL_MD_VSS_SZ, "Invalid md vss si * @return FTL metadata */ struct ftl_md *ftl_md_create(struct spdk_ftl_dev *dev, uint64_t blocks, - uint64_t vss_blksz, const char *name, bool no_mem, + uint64_t vss_blksz, const char *name, int flags, const struct ftl_layout_region *region); +/** + * @brief Unlinks metadata object from FS + * @param dev The FTL device + * @param name Name of the object being unlinked + * @param flags Bit flag describing the MD object + * + * @note Unlink is possible only for objects created with FTL_MD_CREATE_SHM flag + * + * @return Operation result + */ +int ftl_md_unlink(struct spdk_ftl_dev *dev, const char *name, int flags); + /** * @brief Destroys metadata * diff --git a/test/ftl/bdevperf.sh b/test/ftl/bdevperf.sh index 22893a07d..6fdbe8da0 100755 --- a/test/ftl/bdevperf.sh +++ b/test/ftl/bdevperf.sh @@ -32,3 +32,5 @@ for ((i = 0; i < ${#tests[@]}; i++)); do trap - SIGINT SIGTERM EXIT timing_exit "${tests[$i]}" done + +remove_shm diff --git a/test/ftl/common.sh b/test/ftl/common.sh index 612e68368..5b68c3c58 100644 --- a/test/ftl/common.sh +++ b/test/ftl/common.sh @@ -46,3 +46,13 @@ function create_base_bdev() { $rootdir/scripts/rpc.py bdev_lvol_create ${base_bdev}p0 $size -t -u $lvs fi } + +# Remove not needed files from shared memory +function remove_shm() { + echo Remove shared memory files + rm -f rm -f /dev/shm/ftl* + rm -f rm -f /dev/hugepages/ftl* + rm -f rm -f /dev/shm/spdk* + rm -f rm -f /dev/shm/iscsi + rm -f rm -f /dev/hugepages/spdk* +} diff --git a/test/ftl/fio.sh b/test/ftl/fio.sh index 8924e4be0..18655b849 100755 --- a/test/ftl/fio.sh +++ b/test/ftl/fio.sh @@ -66,3 +66,4 @@ for test in ${tests}; do done rm -f $FTL_JSON_CONF +remove_shm diff --git a/test/ftl/ftl.sh b/test/ftl/ftl.sh index 4cb745515..85575078c 100755 --- a/test/ftl/ftl.sh +++ b/test/ftl/ftl.sh @@ -22,6 +22,7 @@ function at_ftl_exit() { # restore original driver $rootdir/scripts/setup.sh reset + remove_shm } trap 'at_ftl_exit' SIGINT SIGTERM EXIT