diff --git a/CHANGELOG.md b/CHANGELOG.md index ae7807eac..f001af635 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,10 @@ Function spdk_blobfs_bdev_detect is added to detect whether blobfs exists on the Function spdk_blobfs_bdev_create is added to create a blobfs on the given block device. +Function spdk_blobfs_bdev_mount is added to mount a blobfs on the given block device to +a host path by FUSE. Then, a new thread is created dedicatedly for one mountpoint to handle +FUSE request by blobfs API. + ### build Option to build FUSE components into blobfs_bdev module for mounting a blobfs filesystem. diff --git a/include/spdk/blobfs_bdev.h b/include/spdk/blobfs_bdev.h index 4632c1601..e915b18f8 100644 --- a/include/spdk/blobfs_bdev.h +++ b/include/spdk/blobfs_bdev.h @@ -40,6 +40,7 @@ #include "spdk/stdinc.h" #include "spdk/bdev.h" +#include "spdk/config.h" #ifdef __cplusplus extern "C" { @@ -74,6 +75,22 @@ void spdk_blobfs_bdev_detect(const char *bdev_name, void spdk_blobfs_bdev_create(const char *bdev_name, uint32_t cluster_sz, spdk_blobfs_bdev_op_complete cb_fn, void *cb_arg); +#ifdef SPDK_CONFIG_FUSE +/** + * Mount a blobfs on given device to a host path by FUSE + * + * A new thread is created dedicatedly for one mountpoint to handle FUSE request + * by blobfs API. + * + * \param bdev_name Name of block device. + * \param mountpoint Host path to mount blobfs. + * \param cb_fn Called when mount operation is complete. fserrno is -EILSEQ if no blobfs exists. + * \param cb_arg Argument passed to function cb_fn. + */ +void spdk_blobfs_bdev_mount(const char *bdev_name, const char *mountpoint, + spdk_blobfs_bdev_op_complete cb_fn, void *cb_arg); +#endif + #ifdef __cplusplus } #endif diff --git a/module/blobfs/bdev/Makefile b/module/blobfs/bdev/Makefile index ec4539885..247c99fb5 100644 --- a/module/blobfs/bdev/Makefile +++ b/module/blobfs/bdev/Makefile @@ -35,6 +35,12 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk C_SRCS = blobfs_bdev.c blobfs_bdev_rpc.c + +# libfuse3 is required by blobfs_fuse.c +ifeq ($(CONFIG_FUSE),y) +C_SRCS += blobfs_fuse.c +endif + LIBNAME = blobfs_bdev include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk diff --git a/module/blobfs/bdev/blobfs_bdev.c b/module/blobfs/bdev/blobfs_bdev.c index 3eb248766..6e1f1c79e 100644 --- a/module/blobfs/bdev/blobfs_bdev.c +++ b/module/blobfs/bdev/blobfs_bdev.c @@ -45,6 +45,8 @@ #include "spdk_internal/log.h" +#include "blobfs_fuse.h" + /* Dummy bdev module used to to claim bdevs. */ static struct spdk_bdev_module blobfs_bdev_module = { .name = "blobfs", @@ -61,8 +63,21 @@ struct blobfs_bdev_operation_ctx { const char *bdev_name; struct spdk_filesystem *fs; + /* If cb_fn is already called in other function, not _blobfs_bdev_unload_cb. + * cb_fn should be set NULL after its being called, in order to avoid repeated + * calling in _blobfs_bdev_unload_cb. + */ spdk_blobfs_bdev_op_complete cb_fn; void *cb_arg; + + /* Variables for mount operation */ + const char *mountpoint; + struct spdk_thread *fs_loading_thread; + + /* Used in bdev_event_cb to do some proper operations on blobfs_fuse for + * asynchronous event of the backend bdev. + */ + struct spdk_blobfs_fuse *bfuse; }; static void @@ -74,7 +89,10 @@ _blobfs_bdev_unload_cb(void *_ctx, int fserrno) SPDK_ERRLOG("Failed to unload blobfs on bdev %s: errno %d\n", ctx->bdev_name, fserrno); } - ctx->cb_fn(ctx->cb_arg, fserrno); + if (ctx->cb_fn) { + ctx->cb_fn(ctx->cb_arg, fserrno); + } + free(ctx); } @@ -209,3 +227,132 @@ invalid: cb_fn(cb_arg, rc); } + +#ifdef SPDK_CONFIG_FUSE + +static void +blobfs_bdev_unmount(void *arg) +{ + struct blobfs_bdev_operation_ctx *ctx = arg; + + /* Keep blobfs unloaded in a same spdk thread with spdk_fs_load */ + spdk_thread_send_msg(ctx->fs_loading_thread, blobfs_bdev_unload, ctx); +} + +static void +_blobfs_bdev_mount_fuse_start(void *_ctx) +{ + struct blobfs_bdev_operation_ctx *ctx = _ctx; + spdk_blobfs_bdev_op_complete cb_fn = ctx->cb_fn; + int rc; + + /* Since function of ctx->cb_fn will be called in this function, set + * ctx->cb_fn to be NULL, in order to avoid repeated calling in unload_cb. + */ + ctx->cb_fn = NULL; + + rc = spdk_blobfs_fuse_start(ctx->bdev_name, ctx->mountpoint, ctx->fs, + blobfs_bdev_unmount, ctx, &ctx->bfuse); + if (rc != 0) { + SPDK_ERRLOG("Failed to mount blobfs on bdev %s to %s\n", ctx->bdev_name, ctx->mountpoint); + + /* Return failure state back */ + cb_fn(ctx->cb_arg, rc); + + blobfs_bdev_unmount(ctx); + + return; + } + + cb_fn(ctx->cb_arg, 0); +} + +static void +_blobfs_bdev_mount_load_cb(void *_ctx, struct spdk_filesystem *fs, int fserrno) +{ + struct blobfs_bdev_operation_ctx *ctx = _ctx; + + if (fserrno) { + SPDK_ERRLOG("Failed to load blobfs on bdev %s: errno %d\n", ctx->bdev_name, fserrno); + + ctx->cb_fn(ctx->cb_arg, fserrno); + free(ctx); + return; + } + + ctx->fs = fs; + ctx->fs_loading_thread = spdk_get_thread(); + + spdk_thread_send_msg(spdk_get_thread(), _blobfs_bdev_mount_fuse_start, ctx); +} + +static void +blobfs_bdev_fuse_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, + void *event_ctx) +{ + struct blobfs_bdev_operation_ctx *ctx = event_ctx; + + SPDK_WARNLOG("Async event(%d) is triggered in bdev %s\n", type, spdk_bdev_get_name(bdev)); + + if (type == SPDK_BDEV_EVENT_REMOVE) { + spdk_blobfs_fuse_stop(ctx->bfuse); + } +} + +void +spdk_blobfs_bdev_mount(const char *bdev_name, const char *mountpoint, + spdk_blobfs_bdev_op_complete cb_fn, void *cb_arg) +{ + struct blobfs_bdev_operation_ctx *ctx; + struct spdk_bs_dev *bs_dev; + struct spdk_bdev_desc *desc; + int rc; + + ctx = calloc(1, sizeof(*ctx)); + if (ctx == NULL) { + SPDK_ERRLOG("Failed to allocate ctx.\n"); + cb_fn(cb_arg, -ENOMEM); + + return; + } + + ctx->bdev_name = bdev_name; + ctx->mountpoint = mountpoint; + ctx->cb_fn = cb_fn; + ctx->cb_arg = cb_arg; + + rc = spdk_bdev_open_ext(bdev_name, true, blobfs_bdev_fuse_event_cb, ctx, &desc); + if (rc != 0) { + SPDK_INFOLOG(SPDK_LOG_BLOBFS, "Failed to open bdev(%s): %s\n", ctx->bdev_name, spdk_strerror(rc)); + + goto invalid; + } + + bs_dev = spdk_bdev_create_bs_dev_from_desc(desc); + if (bs_dev == NULL) { + SPDK_INFOLOG(SPDK_LOG_BLOBFS, "Failed to create a blobstore block device from bdev desc"); + rc = -ENOMEM; + spdk_bdev_close(desc); + + goto invalid; + } + + rc = spdk_bs_bdev_claim(bs_dev, &blobfs_bdev_module); + if (rc != 0) { + SPDK_INFOLOG(SPDK_LOG_BLOBFS, "Blobfs base bdev already claimed by another bdev\n"); + bs_dev->destroy(bs_dev); + + goto invalid; + } + + spdk_fs_load(bs_dev, spdk_blobfs_fuse_send_request, _blobfs_bdev_mount_load_cb, ctx); + + return; + +invalid: + free(ctx); + + cb_fn(cb_arg, rc); +} + +#endif diff --git a/module/blobfs/bdev/blobfs_fuse.c b/module/blobfs/bdev/blobfs_fuse.c new file mode 100644 index 000000000..ffe1f0564 --- /dev/null +++ b/module/blobfs/bdev/blobfs_fuse.c @@ -0,0 +1,358 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" + +#include "spdk/log.h" +#include "spdk/env.h" +#include "spdk/event.h" +#include "spdk/thread.h" +#include "spdk/string.h" +#include "spdk/blobfs.h" + +#include "blobfs_fuse.h" + +#define FUSE_USE_VERSION 30 +#include "fuse3/fuse.h" +#include "fuse3/fuse_lowlevel.h" + +struct spdk_blobfs_fuse { + char *bdev_name; + char *mountpoint; + struct spdk_fs_thread_ctx *channel; + struct spdk_filesystem *fs; + + struct fuse *fuse_handle; + pthread_t fuse_tid; + + blobfs_fuse_unmount_cb cb_fn; + void *cb_arg; +}; + +/* Each thread serves one blobfs */ +static __thread struct spdk_blobfs_fuse *thd_bfuse; + +static void +blobfs_fuse_free(struct spdk_blobfs_fuse *bfuse) +{ + if (bfuse == NULL) { + return; + } + + free(bfuse->bdev_name); + free(bfuse->mountpoint); + free(bfuse); +} + +static void +__call_fn(void *arg1, void *arg2) +{ + fs_request_fn fn; + + fn = (fs_request_fn)arg1; + fn(arg2); +} + +void +spdk_blobfs_fuse_send_request(fs_request_fn fn, void *arg) +{ + struct spdk_event *event; + + event = spdk_event_allocate(0, __call_fn, (void *)fn, arg); + spdk_event_call(event); +} + +static int +spdk_fuse_getattr(const char *path, struct stat *stbuf, struct fuse_file_info *fi) +{ + struct spdk_file_stat stat; + int rc; + + if (!strcmp(path, "/")) { + stbuf->st_mode = S_IFDIR | 0755; + stbuf->st_nlink = 2; + return 0; + } + + rc = spdk_fs_file_stat(thd_bfuse->fs, thd_bfuse->channel, path, &stat); + if (rc == 0) { + stbuf->st_mode = S_IFREG | 0644; + stbuf->st_nlink = 1; + stbuf->st_size = stat.size; + } + + return rc; +} + +static int +spdk_fuse_readdir(const char *path, void *buf, fuse_fill_dir_t filler, + off_t offset, struct fuse_file_info *fi, + enum fuse_readdir_flags flags) +{ + struct spdk_file *file; + const char *filename; + spdk_fs_iter iter; + + filler(buf, ".", NULL, 0, 0); + filler(buf, "..", NULL, 0, 0); + + iter = spdk_fs_iter_first(thd_bfuse->fs); + while (iter != NULL) { + file = spdk_fs_iter_get_file(iter); + iter = spdk_fs_iter_next(iter); + filename = spdk_file_get_name(file); + filler(buf, &filename[1], NULL, 0, 0); + } + + return 0; +} + +static int +spdk_fuse_mknod(const char *path, mode_t mode, dev_t rdev) +{ + return spdk_fs_create_file(thd_bfuse->fs, thd_bfuse->channel, path); +} + +static int +spdk_fuse_unlink(const char *path) +{ + return spdk_fs_delete_file(thd_bfuse->fs, thd_bfuse->channel, path); +} + +static int +spdk_fuse_truncate(const char *path, off_t size, struct fuse_file_info *fi) +{ + struct spdk_file *file; + int rc; + + rc = spdk_fs_open_file(thd_bfuse->fs, thd_bfuse->channel, path, 0, &file); + if (rc != 0) { + return -rc; + } + + rc = spdk_file_truncate(file, thd_bfuse->channel, size); + if (rc != 0) { + return -rc; + } + + spdk_file_close(file, thd_bfuse->channel); + + return 0; +} + +static int +spdk_fuse_utimens(const char *path, const struct timespec tv[2], struct fuse_file_info *fi) +{ + return 0; +} + +static int +spdk_fuse_open(const char *path, struct fuse_file_info *info) +{ + struct spdk_file *file; + int rc; + + rc = spdk_fs_open_file(thd_bfuse->fs, thd_bfuse->channel, path, 0, &file); + if (rc != 0) { + return -rc; + } + + info->fh = (uintptr_t)file; + return 0; +} + +static int +spdk_fuse_release(const char *path, struct fuse_file_info *info) +{ + struct spdk_file *file = (struct spdk_file *)info->fh; + + return spdk_file_close(file, thd_bfuse->channel); +} + +static int +spdk_fuse_read(const char *path, char *buf, size_t len, off_t offset, struct fuse_file_info *info) +{ + struct spdk_file *file = (struct spdk_file *)info->fh; + + return spdk_file_read(file, thd_bfuse->channel, buf, offset, len); +} + +static int +spdk_fuse_write(const char *path, const char *buf, size_t len, off_t offset, + struct fuse_file_info *info) +{ + struct spdk_file *file = (struct spdk_file *)info->fh; + int rc; + + rc = spdk_file_write(file, thd_bfuse->channel, (void *)buf, offset, len); + if (rc == 0) { + return len; + } else { + return rc; + } +} + +static int +spdk_fuse_flush(const char *path, struct fuse_file_info *info) +{ + return 0; +} + +static int +spdk_fuse_fsync(const char *path, int datasync, struct fuse_file_info *info) +{ + return 0; +} + +static int +spdk_fuse_rename(const char *old_path, const char *new_path, unsigned int flags) +{ + return spdk_fs_rename_file(thd_bfuse->fs, thd_bfuse->channel, old_path, new_path); +} + +static struct fuse_operations spdk_fuse_oper = { + .getattr = spdk_fuse_getattr, + .readdir = spdk_fuse_readdir, + .mknod = spdk_fuse_mknod, + .unlink = spdk_fuse_unlink, + .truncate = spdk_fuse_truncate, + .utimens = spdk_fuse_utimens, + .open = spdk_fuse_open, + .release = spdk_fuse_release, + .read = spdk_fuse_read, + .write = spdk_fuse_write, + .flush = spdk_fuse_flush, + .fsync = spdk_fuse_fsync, + .rename = spdk_fuse_rename, +}; + +static void * +fuse_loop_new_thread(void *arg) +{ + struct spdk_blobfs_fuse *bfuse = arg; + + spdk_unaffinitize_thread(); + + thd_bfuse = bfuse; + SPDK_NOTICELOG("Start to loop blobfs on bdev %s mounted at %s\n", bfuse->bdev_name, + bfuse->mountpoint); + + bfuse->channel = spdk_fs_alloc_thread_ctx(bfuse->fs); + + fuse_loop(bfuse->fuse_handle); + fuse_unmount(bfuse->fuse_handle); + fuse_destroy(bfuse->fuse_handle); + SPDK_NOTICELOG("Blobfs on bdev %s unmounted from %s\n", bfuse->bdev_name, bfuse->mountpoint); + + spdk_fs_free_thread_ctx(bfuse->channel); + + bfuse->cb_fn(bfuse->cb_arg); + + blobfs_fuse_free(bfuse); + + pthread_exit(NULL); +} + +int +spdk_blobfs_fuse_start(const char *bdev_name, const char *mountpoint, struct spdk_filesystem *fs, + blobfs_fuse_unmount_cb cb_fn, void *cb_arg, struct spdk_blobfs_fuse **_bfuse) +{ + /* Set argv[1] as bdev_name in order to show bdev_name as the mounting source */ + char *argv[1] = {(char *)bdev_name}; + struct fuse_args args = FUSE_ARGS_INIT(1, argv); + struct fuse_cmdline_opts opts = {}; + struct fuse *fuse_handle; + struct spdk_blobfs_fuse *bfuse; + pthread_t tid; + int rc; + + bfuse = (struct spdk_blobfs_fuse *)calloc(1, sizeof(*bfuse)); + if (bfuse == NULL) { + return -ENOMEM; + } + + rc = fuse_parse_cmdline(&args, &opts); + assert(rc == 0); + + bfuse->bdev_name = strdup(bdev_name); + bfuse->mountpoint = strdup(mountpoint); + bfuse->fs = fs; + bfuse->cb_fn = cb_fn; + bfuse->cb_arg = cb_arg; + + fuse_handle = fuse_new(&args, &spdk_fuse_oper, sizeof(spdk_fuse_oper), NULL); + fuse_opt_free_args(&args); + if (fuse_handle == NULL) { + SPDK_ERRLOG("could not create fuse handle!\n"); + rc = -1; + goto err; + } + bfuse->fuse_handle = fuse_handle; + + rc = fuse_mount(bfuse->fuse_handle, bfuse->mountpoint); + if (rc != 0) { + SPDK_ERRLOG("could not mount fuse handle\n"); + rc = -1; + goto err; + } + + rc = pthread_create(&tid, NULL, fuse_loop_new_thread, bfuse); + if (rc != 0) { + SPDK_ERRLOG("could not create thread: %s\n", spdk_strerror(rc)); + rc = -rc; + goto err; + } + bfuse->fuse_tid = tid; + + rc = pthread_detach(tid); + if (rc != 0) { + SPDK_ERRLOG("could not detach thread for fuse loop thread: %s\n", spdk_strerror(rc)); + rc = -rc; + goto err; + } + + *_bfuse = bfuse; + return 0; + +err: + blobfs_fuse_free(bfuse); + + return rc; +} + +void +spdk_blobfs_fuse_stop(struct spdk_blobfs_fuse *bfuse) +{ + fuse_session_exit(fuse_get_session(bfuse->fuse_handle)); + pthread_kill(bfuse->fuse_tid, SIGINT); +} diff --git a/module/blobfs/bdev/blobfs_fuse.h b/module/blobfs/bdev/blobfs_fuse.h new file mode 100644 index 000000000..f3950db02 --- /dev/null +++ b/module/blobfs/bdev/blobfs_fuse.h @@ -0,0 +1,52 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_BLOBFS_FUSE_H +#define SPDK_BLOBFS_FUSE_H + +#include "spdk/stdinc.h" +#include "spdk/blobfs.h" + +struct spdk_blobfs_fuse; + +void spdk_blobfs_fuse_send_request(fs_request_fn fn, void *arg); + +typedef void (*blobfs_fuse_unmount_cb)(void *arg); + +int spdk_blobfs_fuse_start(const char *bdev_name, const char *mountpoint, + struct spdk_filesystem *fs, blobfs_fuse_unmount_cb cb_fn, + void *cb_arg, struct spdk_blobfs_fuse **bfuse); + +void spdk_blobfs_fuse_stop(struct spdk_blobfs_fuse *bfuse); + +#endif /* SPDK_BLOBFS_FUSE_H */ diff --git a/test/blobfs/Makefile b/test/blobfs/Makefile index 1a9dfefac..d4275544b 100644 --- a/test/blobfs/Makefile +++ b/test/blobfs/Makefile @@ -35,11 +35,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk DIRS-y = mkfs - -# TODO: do not check a hardcoded path here -ifneq (,$(wildcard /usr/local/include/fuse3)) -DIRS-y += fuse -endif +DIRS-$(CONFIG_FUSE) += fuse .PHONY: all clean $(DIRS-y) diff --git a/test/blobfs/blobfs.sh b/test/blobfs/blobfs.sh index 0c6942f20..975d77b62 100755 --- a/test/blobfs/blobfs.sh +++ b/test/blobfs/blobfs.sh @@ -13,6 +13,7 @@ rpc_py="$rootdir/scripts/rpc.py -s $rpc_server" tmp_file=/tmp/blobfs_file conf_file=/tmp/blobfs.conf bdevname=BlobfsBdev +mount_dir=/tmp/spdk_tmp_mount source $rootdir/test/common/autotest_common.sh @@ -21,6 +22,7 @@ function on_error_exit() { killprocess $blobfs_pid fi + rm -rf $mount_dir rm -f $tmp_file rm -f $conf_file print_backtrace @@ -73,6 +75,33 @@ function blobfs_create_test() { killprocess $blobfs_pid } +function blobfs_fuse_test() { + if [ ! -d /usr/include/fuse3 ] && [ ! -d /usr/local/include/fuse3 ]; then + echo "libfuse3 is not installed which is required to this test." + return 0 + fi + + # mount blobfs on test dir + $rootdir/test/blobfs/fuse/fuse ${conf_file} ${bdevname} $mount_dir & + blobfs_pid=$! + echo "Process blobfs pid: $blobfs_pid" + + # Currently blobfs fuse APP doesn't support specific path of RPC sock. + # So directly use default sock path. + waitforlisten $blobfs_pid /var/tmp/spdk.sock + + # check mount status + mount || grep $mount_dir + + # basic file operations in mount dir + dd if=/dev/urandom of=${mount_dir}/rand_file bs=4k count=32 + md5sum ${mount_dir}/rand_file + rm ${mount_dir}/rand_file + + umount ${mount_dir} + killprocess $blobfs_pid +} + timing_enter blobfs trap 'on_error_exit;' ERR @@ -89,6 +118,12 @@ dd if=/dev/zero of=${tmp_file} bs=4k count=1M blobfs_create_test +# Create dir for FUSE mount +mkdir -p $mount_dir +blobfs_fuse_test + + +rm -rf $mount_dir rm -f $tmp_file report_test_completion "blobfs" diff --git a/test/blobfs/fuse/Makefile b/test/blobfs/fuse/Makefile index be9ea35c2..aa23cedb4 100644 --- a/test/blobfs/fuse/Makefile +++ b/test/blobfs/fuse/Makefile @@ -42,8 +42,9 @@ C_SRCS := fuse.c SPDK_LIB_LIST = $(ALL_MODULES_LIST) SPDK_LIB_LIST += event_bdev event_copy event_vmd SPDK_LIB_LIST += bdev copy event thread util conf trace \ - log jsonrpc json rpc sock notify + log jsonrpc json rpc sock notify blobfs_bdev +# libfuse3 is required internally by blobfs_bdev LIBS+= -L/usr/local/lib -lfuse3 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk diff --git a/test/blobfs/fuse/fuse.c b/test/blobfs/fuse/fuse.c index 3401aabb2..8afbe5740 100644 --- a/test/blobfs/fuse/fuse.c +++ b/test/blobfs/fuse/fuse.c @@ -42,281 +42,44 @@ #include "spdk/event.h" #include "spdk/thread.h" #include "spdk/blob_bdev.h" +#include "spdk/blobfs_bdev.h" #include "spdk/log.h" +#include "spdk/string.h" -struct fuse *g_fuse; char *g_bdev_name; char *g_mountpoint; -pthread_t g_fuse_thread; -struct spdk_bs_dev *g_bs_dev; -struct spdk_filesystem *g_fs; -struct spdk_fs_thread_ctx *g_channel; -struct spdk_file *g_file; -int g_fserrno; int g_fuse_argc = 0; char **g_fuse_argv = NULL; static void -__call_fn(void *arg1, void *arg2) +fuse_run_cb(void *cb_arg, int fserrno) { - fs_request_fn fn; + if (fserrno) { + printf("Failed to mount filesystem on bdev %s to path %s: %s", + g_bdev_name, g_mountpoint, spdk_strerror(fserrno)); - fn = (fs_request_fn)arg1; - fn(arg2); -} - -static void -__send_request(fs_request_fn fn, void *arg) -{ - struct spdk_event *event; - - event = spdk_event_allocate(0, __call_fn, (void *)fn, arg); - spdk_event_call(event); -} - -static int -spdk_fuse_getattr(const char *path, struct stat *stbuf, struct fuse_file_info *fi) -{ - struct spdk_file_stat stat; - int rc; - - if (!strcmp(path, "/")) { - stbuf->st_mode = S_IFDIR | 0755; - stbuf->st_nlink = 2; - return 0; - } - - rc = spdk_fs_file_stat(g_fs, g_channel, path, &stat); - if (rc == 0) { - stbuf->st_mode = S_IFREG | 0644; - stbuf->st_nlink = 1; - stbuf->st_size = stat.size; - } - - return rc; -} - -static int -spdk_fuse_readdir(const char *path, void *buf, fuse_fill_dir_t filler, - off_t offset, struct fuse_file_info *fi, - enum fuse_readdir_flags flags) -{ - struct spdk_file *file; - const char *filename; - spdk_fs_iter iter; - - filler(buf, ".", NULL, 0, 0); - filler(buf, "..", NULL, 0, 0); - - iter = spdk_fs_iter_first(g_fs); - while (iter != NULL) { - file = spdk_fs_iter_get_file(iter); - iter = spdk_fs_iter_next(iter); - filename = spdk_file_get_name(file); - filler(buf, &filename[1], NULL, 0, 0); - } - - return 0; -} - -static int -spdk_fuse_mknod(const char *path, mode_t mode, dev_t rdev) -{ - return spdk_fs_create_file(g_fs, g_channel, path); -} - -static int -spdk_fuse_unlink(const char *path) -{ - return spdk_fs_delete_file(g_fs, g_channel, path); -} - -static int -spdk_fuse_truncate(const char *path, off_t size, struct fuse_file_info *fi) -{ - struct spdk_file *file; - int rc; - - rc = spdk_fs_open_file(g_fs, g_channel, path, 0, &file); - if (rc != 0) { - return -rc; - } - - rc = spdk_file_truncate(file, g_channel, size); - if (rc != 0) { - return -rc; - } - - spdk_file_close(file, g_channel); - - return 0; -} - -static int -spdk_fuse_utimens(const char *path, const struct timespec tv[2], struct fuse_file_info *fi) -{ - return 0; -} - -static int -spdk_fuse_open(const char *path, struct fuse_file_info *info) -{ - struct spdk_file *file; - int rc; - - rc = spdk_fs_open_file(g_fs, g_channel, path, 0, &file); - if (rc != 0) { - return -rc; - } - - info->fh = (uintptr_t)file; - return 0; -} - -static int -spdk_fuse_release(const char *path, struct fuse_file_info *info) -{ - struct spdk_file *file = (struct spdk_file *)info->fh; - - return spdk_file_close(file, g_channel); -} - -static int -spdk_fuse_read(const char *path, char *buf, size_t len, off_t offset, struct fuse_file_info *info) -{ - struct spdk_file *file = (struct spdk_file *)info->fh; - - return spdk_file_read(file, g_channel, buf, offset, len); -} - -static int -spdk_fuse_write(const char *path, const char *buf, size_t len, off_t offset, - struct fuse_file_info *info) -{ - struct spdk_file *file = (struct spdk_file *)info->fh; - int rc; - - rc = spdk_file_write(file, g_channel, (void *)buf, offset, len); - if (rc == 0) { - return len; - } else { - return rc; - } -} - -static int -spdk_fuse_flush(const char *path, struct fuse_file_info *info) -{ - return 0; -} - -static int -spdk_fuse_fsync(const char *path, int datasync, struct fuse_file_info *info) -{ - return 0; -} - -static int -spdk_fuse_rename(const char *old_path, const char *new_path, unsigned int flags) -{ - return spdk_fs_rename_file(g_fs, g_channel, old_path, new_path); -} - -static struct fuse_operations spdk_fuse_oper = { - .getattr = spdk_fuse_getattr, - .readdir = spdk_fuse_readdir, - .mknod = spdk_fuse_mknod, - .unlink = spdk_fuse_unlink, - .truncate = spdk_fuse_truncate, - .utimens = spdk_fuse_utimens, - .open = spdk_fuse_open, - .release = spdk_fuse_release, - .read = spdk_fuse_read, - .write = spdk_fuse_write, - .flush = spdk_fuse_flush, - .fsync = spdk_fuse_fsync, - .rename = spdk_fuse_rename, -}; - -static void -construct_targets(void) -{ - struct spdk_bdev *bdev; - - bdev = spdk_bdev_get_by_name(g_bdev_name); - if (bdev == NULL) { - SPDK_ERRLOG("bdev %s not found\n", g_bdev_name); - exit(1); - } - - g_bs_dev = spdk_bdev_create_bs_dev(bdev, NULL, NULL); - - printf("Mounting BlobFS on bdev %s\n", spdk_bdev_get_name(bdev)); -} - -static void -start_fuse_fn(void *arg1, void *arg2) -{ - struct fuse_args args = FUSE_ARGS_INIT(g_fuse_argc, g_fuse_argv); - int rc; - struct fuse_cmdline_opts opts = {}; - - g_fuse_thread = pthread_self(); - rc = fuse_parse_cmdline(&args, &opts); - if (rc != 0) { - spdk_app_stop(-1); - fuse_opt_free_args(&args); - return; - } - g_fuse = fuse_new(&args, &spdk_fuse_oper, sizeof(spdk_fuse_oper), NULL); - fuse_opt_free_args(&args); - - rc = fuse_mount(g_fuse, g_mountpoint); - if (rc != 0) { - spdk_app_stop(-1); + spdk_app_stop(0); return; } - fuse_daemonize(true /* true = run in foreground */); - - fuse_loop(g_fuse); - - fuse_unmount(g_fuse); - fuse_destroy(g_fuse); -} - -static void -init_cb(void *ctx, struct spdk_filesystem *fs, int fserrno) -{ - struct spdk_event *event; - - g_fs = fs; - g_channel = spdk_fs_alloc_thread_ctx(g_fs); - event = spdk_event_allocate(1, start_fuse_fn, NULL, NULL); - spdk_event_call(event); + printf("done.\n"); } static void spdk_fuse_run(void *arg1) { - construct_targets(); - spdk_fs_load(g_bs_dev, __send_request, init_cb, NULL); -} + printf("Mounting filesystem on bdev %s to path %s...", + g_bdev_name, g_mountpoint); + fflush(stdout); -static void -shutdown_cb(void *ctx, int fserrno) -{ - fuse_session_exit(fuse_get_session(g_fuse)); - pthread_kill(g_fuse_thread, SIGINT); - spdk_fs_free_thread_ctx(g_channel); - spdk_app_stop(0); + spdk_blobfs_bdev_mount(g_bdev_name, g_mountpoint, fuse_run_cb, NULL); } static void spdk_fuse_shutdown(void) { - spdk_fs_unload(g_fs, shutdown_cb, NULL); + spdk_app_stop(0); } int main(int argc, char **argv) @@ -337,6 +100,8 @@ int main(int argc, char **argv) g_bdev_name = argv[2]; g_mountpoint = argv[3]; + + /* TODO: mount blobfs with extra FUSE options. */ g_fuse_argc = argc - 2; g_fuse_argv = &argv[2]; diff --git a/test/unit/lib/blobfs/blobfs_bdev.c/blobfs_bdev_ut.c b/test/unit/lib/blobfs/blobfs_bdev.c/blobfs_bdev_ut.c index 31a6c0e4b..530c859b0 100644 --- a/test/unit/lib/blobfs/blobfs_bdev.c/blobfs_bdev_ut.c +++ b/test/unit/lib/blobfs/blobfs_bdev.c/blobfs_bdev_ut.c @@ -44,6 +44,7 @@ bool g_bdev_create_bs_dev_from_desc_fail = false; bool g_fs_load_fail = false; bool g_fs_unload_fail = false; bool g_bs_bdev_claim_fail = false; +bool g_blobfs_fuse_start_fail = false; const char *g_bdev_name = "ut_bdev"; @@ -128,6 +129,17 @@ spdk_bs_bdev_claim(struct spdk_bs_dev *bs_dev, struct spdk_bdev_module *module) return 0; } +int +spdk_blobfs_fuse_start(const char *bdev_name, const char *mountpoint, struct spdk_filesystem *fs, + blobfs_fuse_unmount_cb cb_fn, void *cb_arg, struct spdk_blobfs_fuse **_bfuse) +{ + if (g_blobfs_fuse_start_fail == true) { + return -1; + } + + return 0; +} + void spdk_bdev_close(struct spdk_bdev_desc *desc) { @@ -158,6 +170,16 @@ spdk_fs_opts_init(struct spdk_blobfs_opts *opts) { } +void +spdk_blobfs_fuse_send_request(fs_request_fn fn, void *arg) +{ +} + +void +spdk_blobfs_fuse_stop(struct spdk_blobfs_fuse *bfuse) +{ +} + static void blobfs_bdev_op_complete(void *cb_arg, int fserrno) { @@ -245,6 +267,53 @@ spdk_blobfs_bdev_create_test(void) CU_ASSERT(g_fserrno == 0); } +static void +spdk_blobfs_bdev_mount_test(void) +{ +#ifdef SPDK_CONFIG_FUSE + const char *mountpoint = "/mnt"; + + /* spdk_bdev_open_ext() fails */ + g_bdev_open_ext_fail = true; + spdk_blobfs_bdev_mount(g_bdev_name, mountpoint, blobfs_bdev_op_complete, NULL); + CU_ASSERT(g_fserrno != 0); + + g_bdev_open_ext_fail = false; + + /* spdk_bdev_create_bs_dev_from_desc() fails */ + g_bdev_create_bs_dev_from_desc_fail = true; + spdk_blobfs_bdev_mount(g_bdev_name, mountpoint, blobfs_bdev_op_complete, NULL); + CU_ASSERT(g_fserrno != 0); + + g_bdev_create_bs_dev_from_desc_fail = false; + + /* spdk_bs_bdev_claim() fails */ + g_bs_bdev_claim_fail = true; + spdk_blobfs_bdev_mount(g_bdev_name, mountpoint, blobfs_bdev_op_complete, NULL); + CU_ASSERT(g_fserrno != 0); + + g_bs_bdev_claim_fail = false; + + /* spdk_fs_load() fails */ + g_fs_load_fail = true; + spdk_blobfs_bdev_mount(g_bdev_name, mountpoint, blobfs_bdev_op_complete, NULL); + CU_ASSERT(g_fserrno != 0); + + g_fs_load_fail = false; + + /* spdk_blobfs_fuse_start() fails */ + g_blobfs_fuse_start_fail = true; + spdk_blobfs_bdev_mount(g_bdev_name, mountpoint, blobfs_bdev_op_complete, NULL); + CU_ASSERT(g_fserrno != 0); + + g_blobfs_fuse_start_fail = false; + + /* no fail */ + spdk_blobfs_bdev_mount(g_bdev_name, mountpoint, blobfs_bdev_op_complete, NULL); + CU_ASSERT(g_fserrno == 0); +#endif +} + int main(int argc, char **argv) { CU_pSuite suite = NULL; @@ -262,7 +331,8 @@ int main(int argc, char **argv) if ( CU_add_test(suite, "spdk_blobfs_bdev_detect_test", spdk_blobfs_bdev_detect_test) == NULL || - CU_add_test(suite, "spdk_blobfs_bdev_create_test", spdk_blobfs_bdev_create_test) == NULL + CU_add_test(suite, "spdk_blobfs_bdev_create_test", spdk_blobfs_bdev_create_test) == NULL || + CU_add_test(suite, "spdk_blobfs_bdev_mount_test", spdk_blobfs_bdev_mount_test) == NULL ) { CU_cleanup_registry(); return CU_get_error();