From 8bf4c089f6460e6b5aa6ee6db92d55bce7dfbd37 Mon Sep 17 00:00:00 2001 From: Jim Harris Date: Fri, 7 Jul 2017 09:14:33 -0700 Subject: [PATCH] bdev: initialize vbdev modules first This allows vbdev modules to be ready for examine calls for (physical) bdev modules when the latter initialize. This requires the following modifications to existing bdev modules: 1) error and split now search their config sections at examine time (instead of init) to see if the bdev should be consumed by the vbdev module 2) gpt is simplified considerably - it no longer needs to save bdevs to examine when gpt initializes later 3) nvme must register its io device before registering the bdev, since vbdevs may immediately start trying to send I/O to the new nvme bdev Signed-off-by: Jim Harris Change-Id: I8fe5686092ffb15fc8bdbc068b09add229d9da6c Reviewed-on: https://review.gerrithub.io/368598 Tested-by: SPDK Automated Test System Reviewed-by: Daniel Verkamp Reviewed-by: Ziye Yang Reviewed-by: Ben Walker --- lib/bdev/bdev.c | 6 ++-- lib/bdev/error/vbdev_error.c | 37 ++++++++++------------ lib/bdev/gpt/vbdev_gpt.c | 58 +++++++++-------------------------- lib/bdev/nvme/blockdev_nvme.c | 4 +-- lib/bdev/split/vbdev_split.c | 46 ++++++++++++--------------- 5 files changed, 54 insertions(+), 97 deletions(-) diff --git a/lib/bdev/bdev.c b/lib/bdev/bdev.c index cf3961615..d44ed3284 100644 --- a/lib/bdev/bdev.c +++ b/lib/bdev/bdev.c @@ -416,7 +416,7 @@ spdk_bdev_module_init_next(int rc) if (g_next_bdev_module) { g_next_bdev_module->module_init(); } else { - spdk_vbdev_module_init_next(0); + spdk_bdev_module_init_complete(rc); } } @@ -439,7 +439,7 @@ spdk_vbdev_module_init_next(int rc) if (g_next_vbdev_module) { g_next_vbdev_module->module_init(); } else { - spdk_bdev_module_init_complete(rc);; + spdk_bdev_module_init_next(0); } } @@ -526,7 +526,7 @@ spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg, sizeof(struct spdk_bdev_mgmt_channel)); end: - spdk_bdev_module_init_next(rc); + spdk_vbdev_module_init_next(rc); } int diff --git a/lib/bdev/error/vbdev_error.c b/lib/bdev/error/vbdev_error.c index 7ca12947d..00ededbdf 100644 --- a/lib/bdev/error/vbdev_error.c +++ b/lib/bdev/error/vbdev_error.c @@ -283,15 +283,21 @@ cleanup: static void vbdev_error_init(void) +{ + spdk_vbdev_module_init_next(0); +} + +static void +vbdev_error_examine(struct spdk_bdev *bdev) { struct spdk_conf_section *sp; const char *base_bdev_name; - int i, rc = 0; - struct spdk_bdev *base_bdev; + int i; sp = spdk_conf_find_section(NULL, "BdevError"); if (sp == NULL) { - goto end; + spdk_vbdev_module_examine_done(SPDK_GET_BDEV_MODULE(error)); + return; } for (i = 0; ; i++) { @@ -302,25 +308,20 @@ vbdev_error_init(void) base_bdev_name = spdk_conf_section_get_nmval(sp, "BdevError", i, 0); if (!base_bdev_name) { SPDK_ERRLOG("ErrorInjection configuration missing blockdev name\n"); - rc = -1; - goto end; + break; } - base_bdev = spdk_bdev_get_by_name(base_bdev_name); - if (!base_bdev) { - SPDK_ERRLOG("Could not find ErrorInjection bdev %s\n", base_bdev_name); - rc = -1; - goto end; + if (strcmp(base_bdev_name, bdev->name) != 0) { + continue; } - if (spdk_vbdev_error_create(base_bdev)) { - rc = -1; - goto end; + if (spdk_vbdev_error_create(bdev)) { + SPDK_ERRLOG("could not create error vbdev for bdev %s\n", bdev->name); + break; } } -end: - spdk_vbdev_module_init_next(rc); + spdk_vbdev_module_examine_done(SPDK_GET_BDEV_MODULE(error)); } static void @@ -333,11 +334,5 @@ vbdev_error_fini(void) } } -static void -vbdev_error_examine(struct spdk_bdev *bdev) -{ - spdk_vbdev_module_examine_done(SPDK_GET_BDEV_MODULE(error)); -} - SPDK_VBDEV_MODULE_REGISTER(error, vbdev_error_init, vbdev_error_fini, NULL, NULL, vbdev_error_examine) diff --git a/lib/bdev/gpt/vbdev_gpt.c b/lib/bdev/gpt/vbdev_gpt.c index ddc44efb9..f37f971d4 100644 --- a/lib/bdev/gpt/vbdev_gpt.c +++ b/lib/bdev/gpt/vbdev_gpt.c @@ -76,8 +76,6 @@ static TAILQ_HEAD(, gpt_partition_disk) g_gpt_partition_disks = TAILQ_HEAD_INITI g_gpt_partition_disks); static TAILQ_HEAD(, spdk_bdev) g_bdevs = TAILQ_HEAD_INITIALIZER(g_bdevs); -static int g_gpt_base_num; -static bool g_gpt_init_done; static bool g_gpt_disabled; static void @@ -407,7 +405,6 @@ static void spdk_gpt_bdev_complete(struct spdk_bdev_io *bdev_io, bool status, void *arg) { struct spdk_gpt_bdev *gpt_bdev = (struct spdk_gpt_bdev *)arg; - static int bdev_init_num = 0; int rc; /* free the ch and also close the bdev_desc */ @@ -416,7 +413,6 @@ spdk_gpt_bdev_complete(struct spdk_bdev_io *bdev_io, bool status, void *arg) spdk_bdev_close(gpt_bdev->bdev_desc); gpt_bdev->bdev_desc = NULL; - bdev_init_num++; if (status != SPDK_BDEV_IO_STATUS_SUCCESS) { SPDK_ERRLOG("Gpt: bdev=%s io error status=%d\n", spdk_bdev_get_name(gpt_bdev->bdev), status); @@ -436,19 +432,17 @@ spdk_gpt_bdev_complete(struct spdk_bdev_io *bdev_io, bool status, void *arg) } end: + /* + * Notify the generic bdev layer that the actions related to the original examine + * callback are now completed. + */ + spdk_vbdev_module_examine_done(SPDK_GET_BDEV_MODULE(gpt)); + spdk_bdev_free_io(bdev_io); if (gpt_bdev->ref == 0) { /* If no gpt_partition_disk instances were created, free the base context */ spdk_gpt_bdev_free(gpt_bdev); } - - if (!g_gpt_init_done) { - /* Call next vbdev module init after the last gpt creation */ - if (bdev_init_num == g_gpt_base_num) { - g_gpt_init_done = true; - spdk_vbdev_module_init_next(0); - } - } } static int @@ -477,36 +471,14 @@ vbdev_gpt_read_gpt(struct spdk_bdev *bdev) static void vbdev_gpt_init(void) { - struct spdk_bdev *base_bdev, *tmp; - int rc = 0; struct spdk_conf_section *sp = spdk_conf_find_section(NULL, "Gpt"); if (sp && spdk_conf_section_get_boolval(sp, "Disable", false)) { /* Disable Gpt probe */ g_gpt_disabled = true; - goto end; } - TAILQ_FOREACH_SAFE(base_bdev, &g_bdevs, link, tmp) { - TAILQ_REMOVE(&g_bdevs, base_bdev, link); - rc = vbdev_gpt_read_gpt(base_bdev); - if (rc) { - SPDK_ERRLOG("Failed to read info from bdev %s\n", - spdk_bdev_get_name(base_bdev)); - continue; - } - g_gpt_base_num++; - - } - - if (!g_gpt_base_num) { - g_gpt_init_done = true; - } -end: - /* if no gpt bdev num is counted, just call vbdev_module_init_next */ - if (!g_gpt_base_num) { - spdk_vbdev_module_init_next(rc); - } + spdk_vbdev_module_init_next(0); } static void @@ -522,19 +494,17 @@ vbdev_gpt_fini(void) static void vbdev_gpt_examine(struct spdk_bdev *bdev) { - /* - * TODO: this will get fixed in a later patch which refactors - * the GPT init/register code. For now, nothing is operating - * on the examination counts so just immediately indicate - * that the examination is done. - */ - spdk_vbdev_module_examine_done(SPDK_GET_BDEV_MODULE(gpt)); + int rc; + if (g_gpt_disabled) { + spdk_vbdev_module_examine_done(SPDK_GET_BDEV_MODULE(gpt)); return; } - if (!g_gpt_init_done) { - TAILQ_INSERT_TAIL(&g_bdevs, bdev, link); + rc = vbdev_gpt_read_gpt(bdev); + if (rc) { + spdk_vbdev_module_examine_done(SPDK_GET_BDEV_MODULE(gpt)); + SPDK_ERRLOG("Failed to read info from bdev %s\n", spdk_bdev_get_name(bdev)); } } diff --git a/lib/bdev/nvme/blockdev_nvme.c b/lib/bdev/nvme/blockdev_nvme.c index 07162d783..c39d49358 100644 --- a/lib/bdev/nvme/blockdev_nvme.c +++ b/lib/bdev/nvme/blockdev_nvme.c @@ -745,13 +745,13 @@ attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, nvme_ctrlr->trid = *trid; nvme_ctrlr->name = name; + spdk_io_device_register(ctrlr, bdev_nvme_create_cb, bdev_nvme_destroy_cb, + sizeof(struct nvme_io_channel)); nvme_ctrlr_create_bdevs(nvme_ctrlr); spdk_bdev_poller_start(&nvme_ctrlr->adminq_timer_poller, bdev_nvme_poll_adminq, ctrlr, spdk_env_get_current_core(), g_nvme_adminq_poll_timeout_us); - spdk_io_device_register(ctrlr, bdev_nvme_create_cb, bdev_nvme_destroy_cb, - sizeof(struct nvme_io_channel)); TAILQ_INSERT_TAIL(&g_nvme_ctrlrs, nvme_ctrlr, tailq); if (g_action_on_timeout != TIMEOUT_ACTION_NONE) { diff --git a/lib/bdev/split/vbdev_split.c b/lib/bdev/split/vbdev_split.c index 07024d841..c85662c55 100644 --- a/lib/bdev/split/vbdev_split.c +++ b/lib/bdev/split/vbdev_split.c @@ -337,17 +337,23 @@ cleanup: static void vbdev_split_init(void) +{ + spdk_vbdev_module_init_next(0); +} + +static void +vbdev_split_examine(struct spdk_bdev *bdev) { struct spdk_conf_section *sp; const char *base_bdev_name; const char *split_count_str; const char *split_size_str; - int i, split_count, split_size, rc = 0; - struct spdk_bdev *base_bdev; + int i, split_count, split_size; sp = spdk_conf_find_section(NULL, "Split"); if (sp == NULL) { - goto end; + spdk_vbdev_module_examine_done(SPDK_GET_BDEV_MODULE(split)); + return; } for (i = 0; ; i++) { @@ -358,29 +364,23 @@ vbdev_split_init(void) base_bdev_name = spdk_conf_section_get_nmval(sp, "Split", i, 0); if (!base_bdev_name) { SPDK_ERRLOG("Split configuration missing blockdev name\n"); - rc = -1; - goto end; + break; } - base_bdev = spdk_bdev_get_by_name(base_bdev_name); - if (!base_bdev) { - SPDK_ERRLOG("Could not find Split bdev %s\n", base_bdev_name); - rc = -1; - goto end; + if (strcmp(base_bdev_name, bdev->name) != 0) { + continue; } split_count_str = spdk_conf_section_get_nmval(sp, "Split", i, 1); if (!split_count_str) { SPDK_ERRLOG("Split configuration missing split count\n"); - rc = -1; - goto end; + break; } split_count = atoi(split_count_str); if (split_count < 1) { SPDK_ERRLOG("Invalid Split count %d\n", split_count); - rc = -1; - goto end; + break; } /* Optional split size in MB */ @@ -390,19 +390,17 @@ vbdev_split_init(void) split_size = atoi(split_size_str); if (split_size <= 0) { SPDK_ERRLOG("Invalid Split size %d\n", split_size); - rc = -1; - goto end; + break; } } - if (vbdev_split_create(base_bdev, split_count, split_size)) { - rc = -1; - goto end; + if (vbdev_split_create(bdev, split_count, split_size)) { + SPDK_ERRLOG("could not split bdev %s\n", bdev->name); + break; } } -end: - spdk_vbdev_module_init_next(rc); + spdk_vbdev_module_examine_done(SPDK_GET_BDEV_MODULE(split)); } static void @@ -425,12 +423,6 @@ vbdev_split_get_ctx_size(void) return sizeof(struct spdk_io_channel *); } -static void -vbdev_split_examine(struct spdk_bdev *bdev) -{ - spdk_vbdev_module_examine_done(SPDK_GET_BDEV_MODULE(split)); -} - SPDK_VBDEV_MODULE_REGISTER(split, vbdev_split_init, vbdev_split_fini, NULL, vbdev_split_get_ctx_size, vbdev_split_examine) SPDK_LOG_REGISTER_TRACE_FLAG("vbdev_split", SPDK_TRACE_VBDEV_SPLIT)