From 413d25b0362384f3343c3df11776d55320b79bdd Mon Sep 17 00:00:00 2001 From: Tomasz Zawadzki Date: Wed, 4 Aug 2021 05:31:27 -0400 Subject: [PATCH] bdev_module: add async fini_start() fini_start() is called for each bdev module before iterating over all unclaimed bdevs to unregister them. This allows bdev modules to behave differently during each such unregister. Ex. unloading lvol store when all lvol bdevs on it are unregistered. Another use of this callback is to unclaim all bdevs that can be at that point. Especially ones that will not receive callback due to no bdev registered. Ex. offline raid bdev, when some underlying bdevs are missing. fini_start() being synchronous does not help in cases where to release claim on the bdev, an asynchronous operation is required. Ex. lvol store with no bdevs present, requires async lvs unload to be called. This patch adds async_fini_start flag for the bdev modules, to be used when async fini_start is required. When done, bdev module has to call spdk_bdev_module_finish_start_done(). Signed-off-by: Tomasz Zawadzki Change-Id: I63438b325d4cc53fd236bf9ff143abf6bdd81c49 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9094 Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Aleksey Marchuk --- CHANGELOG.md | 3 ++ include/spdk/bdev_module.h | 16 +++++++++++ lib/bdev/bdev.c | 57 ++++++++++++++++++++++++++++++++------ lib/bdev/spdk_bdev.map | 1 + module/blob/bdev/Makefile | 2 +- 5 files changed, 69 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bca4f5032..507b90183 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ These functions accept `spdk_bdev_ext_io_opts` structure with extended IO reques options, e.g. DMA memory domain which describes data that may belong to another memory domain and can't be accessed directly. +Added `async_fini_start` to allow bdev modules to complete the `fini_start` asynchronously, +with new `spdk_bdev_module_fini_start_done` API. + ### dma A new library, lib/dma, has been added. This library provides the necessary infrastructure for diff --git a/include/spdk/bdev_module.h b/include/spdk/bdev_module.h index 1f8d78da6..2e276b148 100644 --- a/include/spdk/bdev_module.h +++ b/include/spdk/bdev_module.h @@ -151,6 +151,13 @@ struct spdk_bdev_module { */ bool async_fini; + /** + * Denotes if the fini_start function may complete asynchronously. + * If set to true finishing has to be explicitly completed by calling + * spdk_bdev_module_fini_start_done(). + */ + bool async_fini_start; + /** * Fields that are used by the internal bdev subsystem. Bdev modules * must not read or write to these fields. @@ -832,6 +839,15 @@ void spdk_bdev_module_init_done(struct spdk_bdev_module *module); */ void spdk_bdev_module_finish_done(void); +/** + * Indicate that the module fini start has completed. + * + * To be called in response to the fini_start, only if async_fini_start is set. + * May be called during fini_start or asynchronously. + * + */ +void spdk_bdev_module_fini_start_done(void); + /** * Add alias to block device names list. * Aliases can be add only to registered bdev. diff --git a/lib/bdev/bdev.c b/lib/bdev/bdev.c index 2d804ca39..7ef4d6af4 100644 --- a/lib/bdev/bdev.c +++ b/lib/bdev/bdev.c @@ -1619,11 +1619,56 @@ bdev_finish_unregister_bdevs_iter(void *cb_arg, int bdeverrno) } } +static void +bdev_module_fini_start_iter(void *arg) +{ + struct spdk_bdev_module *bdev_module; + + if (!g_resume_bdev_module) { + bdev_module = TAILQ_LAST(&g_bdev_mgr.bdev_modules, bdev_module_list); + } else { + bdev_module = TAILQ_PREV(g_resume_bdev_module, bdev_module_list, internal.tailq); + } + + while (bdev_module) { + if (bdev_module->async_fini_start) { + /* Save our place so we can resume later. We must + * save the variable here, before calling fini_start() + * below, because in some cases the module may immediately + * call spdk_bdev_module_fini_start_done() and re-enter + * this function to continue iterating. */ + g_resume_bdev_module = bdev_module; + } + + if (bdev_module->fini_start) { + bdev_module->fini_start(); + } + + if (bdev_module->async_fini_start) { + return; + } + + bdev_module = TAILQ_PREV(bdev_module, bdev_module_list, internal.tailq); + } + + g_resume_bdev_module = NULL; + + bdev_finish_unregister_bdevs_iter(NULL, 0); +} + +void +spdk_bdev_module_fini_start_done(void) +{ + if (spdk_get_thread() != g_fini_thread) { + spdk_thread_send_msg(g_fini_thread, bdev_module_fini_start_iter, NULL); + } else { + bdev_module_fini_start_iter(NULL); + } +} + void spdk_bdev_finish(spdk_bdev_fini_cb cb_fn, void *cb_arg) { - struct spdk_bdev_module *m; - assert(cb_fn != NULL); g_fini_thread = spdk_get_thread(); @@ -1631,13 +1676,7 @@ spdk_bdev_finish(spdk_bdev_fini_cb cb_fn, void *cb_arg) g_fini_cb_fn = cb_fn; g_fini_cb_arg = cb_arg; - TAILQ_FOREACH(m, &g_bdev_mgr.bdev_modules, internal.tailq) { - if (m->fini_start) { - m->fini_start(); - } - } - - bdev_finish_unregister_bdevs_iter(NULL, 0); + bdev_module_fini_start_iter(NULL); } struct spdk_bdev_io * diff --git a/lib/bdev/spdk_bdev.map b/lib/bdev/spdk_bdev.map index 8862da661..8cebb8c07 100644 --- a/lib/bdev/spdk_bdev.map +++ b/lib/bdev/spdk_bdev.map @@ -105,6 +105,7 @@ spdk_bdev_module_examine_done; spdk_bdev_module_init_done; spdk_bdev_module_finish_done; + spdk_bdev_module_fini_start_done; spdk_bdev_module_claim_bdev; spdk_bdev_module_release_bdev; spdk_bdev_alias_add; diff --git a/module/blob/bdev/Makefile b/module/blob/bdev/Makefile index 842060219..53a0b0248 100644 --- a/module/blob/bdev/Makefile +++ b/module/blob/bdev/Makefile @@ -34,7 +34,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk -SO_VER := 5 +SO_VER := 6 SO_MINOR := 0 C_SRCS = blob_bdev.c