From 250acc1792748679e7e4a6d7c8f8e7913bf85226 Mon Sep 17 00:00:00 2001 From: Mike Gerdts Date: Fri, 14 Oct 2022 11:18:45 -0500 Subject: [PATCH] lvol: hotplug of missing esnaps This introduces spdk_lvs_notify_hotplug() to trigger the lvstore to call the appropriate lvstore's esnap_bs_dev_create() callback for each esnap clone lvol that is missing the device identified by esnap_id. Change-Id: I0e2eb26375c62043b0f895197b24d6e056905aa2 Signed-off-by: Mike Gerdts Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/16428 Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Ben Walker --- include/spdk_internal/lvolstore.h | 1 + lib/lvol/lvol.c | 128 ++++++++++++++++++++++++++++ lib/lvol/spdk_lvol.map | 1 + test/unit/lib/lvol/lvol.c/lvol_ut.c | 2 + 4 files changed, 132 insertions(+) diff --git a/include/spdk_internal/lvolstore.h b/include/spdk_internal/lvolstore.h index 7fc7b81d6..02a6ad742 100644 --- a/include/spdk_internal/lvolstore.h +++ b/include/spdk_internal/lvolstore.h @@ -120,5 +120,6 @@ void spdk_lvol_set_read_only(struct spdk_lvol *lvol, spdk_lvol_op_complete cb_fn int spdk_lvs_esnap_missing_add(struct spdk_lvol_store *lvs, struct spdk_lvol *lvol, const void *esnap_id, uint32_t id_len); void spdk_lvs_esnap_missing_remove(struct spdk_lvol *lvol); +bool spdk_lvs_notify_hotplug(const void *esnap_id, uint32_t id_len); #endif /* SPDK_INTERNAL_LVOLSTORE_H */ diff --git a/lib/lvol/lvol.c b/lib/lvol/lvol.c index a059698ee..017a60e48 100644 --- a/lib/lvol/lvol.c +++ b/lib/lvol/lvol.c @@ -1835,6 +1835,15 @@ lvs_esnap_bs_dev_create(void *bs_ctx, void *blob_ctx, struct spdk_blob *blob, * Missing external snapshots are tracked in a per-lvolstore tree, lvs->degraded_lvol_sets_tree. * Each tree node (struct spdk_lvs_degraded_lvol_set) contains a tailq of lvols that are missing * that particular external snapshot. + * + * When a potential missing snapshot becomes available, spdk_lvs_notify_hotplug() may be called to + * notify this library that it is available. It will then iterate through the active lvolstores and + * search each lvs->degraded_lvol_sets_tree for a set of degraded lvols that are missing an external + * snapshot matching the id passed in the notification. The lvols in the tailq on each matching tree + * node are then asked to create an external snapshot bs_dev using the esnap_bs_dev_create() + * callback that the consumer registered with the lvolstore. If lvs->esnap_bs_dev_create() returns + * 0, the lvol is removed from the spdk_lvs_degraded_lvol_set's lvol tailq. When this tailq becomes + * empty, the degraded lvol set node for this missing external snapshot is removed. */ static int lvs_esnap_name_cmp(struct spdk_lvs_degraded_lvol_set *m1, struct spdk_lvs_degraded_lvol_set *m2) @@ -1935,3 +1944,122 @@ spdk_lvs_esnap_missing_remove(struct spdk_lvol *lvol) free((char *)degraded_set->esnap_id); free(degraded_set); } + +static void +lvs_esnap_hotplug_done(void *cb_arg, int bserrno) +{ + struct spdk_lvol *lvol = cb_arg; + struct spdk_lvol_store *lvs = lvol->lvol_store; + + if (bserrno != 0) { + SPDK_ERRLOG("lvol %s/%s: failed to hotplug blob_bdev due to error %d\n", + lvs->name, lvol->name, bserrno); + } +} + +static void +lvs_esnap_degraded_hotplug(struct spdk_lvs_degraded_lvol_set *degraded_set) +{ + struct spdk_lvol_store *lvs = degraded_set->lvol_store; + struct spdk_lvol *lvol, *tmp, *last_missing; + struct spdk_bs_dev *bs_dev; + const void *esnap_id = degraded_set->esnap_id; + uint32_t id_len = degraded_set->id_len; + int rc; + + assert(lvs->thread == spdk_get_thread()); + + /* + * When lvs->esnap_bs_bdev_create() tries to load an external snapshot, it can encounter + * errors that lead it to calling spdk_lvs_esnap_missing_add(). This function needs to be + * sure that such modifications do not lead to degraded_set->lvols tailqs or references + * to memory that this function will free. + * + * While this function is running, no other thread can add items to degraded_set->lvols. If + * the list is mutated, it must have been done by this function or something in its call + * graph running on this thread. + */ + + /* Remember the last lvol on the list. Iteration will stop once it has been processed. */ + last_missing = TAILQ_LAST(°raded_set->lvols, degraded_lvols); + + TAILQ_FOREACH_SAFE(lvol, °raded_set->lvols, degraded_link, tmp) { + /* + * Remove the lvol from the tailq so that tailq corruption is avoided if + * lvs->esnap_bs_dev_create() calls spdk_lvs_esnap_missing_add(lvol). + */ + TAILQ_REMOVE(°raded_set->lvols, lvol, degraded_link); + lvol->degraded_set = NULL; + + bs_dev = NULL; + rc = lvs->esnap_bs_dev_create(lvs, lvol, lvol->blob, esnap_id, id_len, &bs_dev); + if (rc != 0) { + SPDK_ERRLOG("lvol %s: failed to create esnap bs_dev: error %d\n", + lvol->unique_id, rc); + lvol->degraded_set = degraded_set; + TAILQ_INSERT_TAIL(°raded_set->lvols, lvol, degraded_link); + } else { + spdk_blob_set_esnap_bs_dev(lvol->blob, bs_dev, + lvs_esnap_hotplug_done, lvol); + } + + if (lvol == last_missing) { + /* + * Anything after last_missing was added due to some problem encountered + * while trying to create the esnap bs_dev. + */ + break; + } + } + + if (TAILQ_EMPTY(°raded_set->lvols)) { + RB_REMOVE(degraded_lvol_sets_tree, &lvs->degraded_lvol_sets_tree, degraded_set); + free((void *)degraded_set->esnap_id); + free(degraded_set); + } +} + +/* + * Notify each lvstore created on this thread that is missing a bdev by the specified name or uuid + * that the bdev now exists. + */ +bool +spdk_lvs_notify_hotplug(const void *esnap_id, uint32_t id_len) +{ + struct spdk_lvs_degraded_lvol_set *found; + struct spdk_lvs_degraded_lvol_set find = { 0 }; + struct spdk_lvol_store *lvs; + struct spdk_thread *thread = spdk_get_thread(); + bool ret = false; + + find.esnap_id = esnap_id; + find.id_len = id_len; + + pthread_mutex_lock(&g_lvol_stores_mutex); + TAILQ_FOREACH(lvs, &g_lvol_stores, link) { + if (thread != lvs->thread) { + /* + * It is expected that this is called from vbdev_lvol's examine_config() + * callback. The lvstore was likely loaded do a creation happening as a + * result of an RPC call or opening of an existing lvstore via + * examine_disk() callback. RPC calls, examine_disk(), and examine_config() + * should all be happening only on the app thread. The "wrong thread" + * condition will only happen when an application is doing something weird. + */ + SPDK_NOTICELOG("Discarded examine for lvstore %s: wrong thread\n", + lvs->name); + continue; + } + + found = RB_FIND(degraded_lvol_sets_tree, &lvs->degraded_lvol_sets_tree, &find); + if (found == NULL) { + continue; + } + + ret = true; + lvs_esnap_degraded_hotplug(found); + } + pthread_mutex_unlock(&g_lvol_stores_mutex); + + return ret; +} diff --git a/lib/lvol/spdk_lvol.map b/lib/lvol/spdk_lvol.map index 45c323462..57de75c81 100644 --- a/lib/lvol/spdk_lvol.map +++ b/lib/lvol/spdk_lvol.map @@ -28,6 +28,7 @@ spdk_lvol_set_read_only; spdk_lvs_esnap_missing_add; spdk_lvs_esnap_missing_remove; + spdk_lvs_notify_hotplug; local: *; }; diff --git a/test/unit/lib/lvol/lvol.c/lvol_ut.c b/test/unit/lib/lvol/lvol.c/lvol_ut.c index d166b23bb..88caa4841 100644 --- a/test/unit/lib/lvol/lvol.c/lvol_ut.c +++ b/test/unit/lib/lvol/lvol.c/lvol_ut.c @@ -35,6 +35,8 @@ DEFINE_STUB(spdk_bdev_create_bs_dev_ro, int, (const char *bdev_name, spdk_bdev_event_cb_t event_cb, void *event_ctx, struct spdk_bs_dev **bs_dev), -ENOTSUP); DEFINE_STUB(spdk_blob_is_esnap_clone, bool, (const struct spdk_blob *blob), false); +DEFINE_STUB_V(spdk_blob_set_esnap_bs_dev, (struct spdk_blob *blob, struct spdk_bs_dev *back_bs_dev, + spdk_blob_op_complete cb_fn, void *cb_arg)); const char *uuid = "828d9766-ae50-11e7-bd8d-001e67edf350";