vbdev_lvol: load esnaps via examine_config

This introduces an examine_config callback that triggers hotplug of
missing esnap devices.

Signed-off-by: Mike Gerdts <mgerdts@nvidia.com>
Change-Id: I5ced2ff26bfd393d2df4fd4718700be30eb48063
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/16626
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

include/spdk_internal/lvolstore.h

@@ -120,6 +120,7 @@ void spdk_lvol_set_read_only(struct spdk_lvol *lvol, spdk_lvol_op_complete cb_fn
 int spdk_lvs_esnap_missing_add(struct spdk_lvol_store *lvs, struct spdk_lvol *lvol,
 			       const void *esnap_id, uint32_t id_len);
 void spdk_lvs_esnap_missing_remove(struct spdk_lvol *lvol);
-bool spdk_lvs_notify_hotplug(const void *esnap_id, uint32_t id_len);
+bool spdk_lvs_notify_hotplug(const void *esnap_id, uint32_t id_len,
+			     spdk_lvol_op_with_handle_complete cb_fn, void *cb_arg);
 
 #endif /* SPDK_INTERNAL_LVOLSTORE_H */

lib/lvol/lvol.c

@@ -1945,26 +1945,37 @@ spdk_lvs_esnap_missing_remove(struct spdk_lvol *lvol)
 	free(degraded_set);
 }
 
+struct lvs_esnap_hotplug_req {
+	struct spdk_lvol			*lvol;
+	spdk_lvol_op_with_handle_complete	cb_fn;
+	void					*cb_arg;
+};
+
 static void
 lvs_esnap_hotplug_done(void *cb_arg, int bserrno)
 {
-	struct spdk_lvol	*lvol = cb_arg;
+	struct lvs_esnap_hotplug_req *req = cb_arg;
+	struct spdk_lvol	*lvol = req->lvol;
 	struct spdk_lvol_store	*lvs = lvol->lvol_store;
 
 	if (bserrno != 0) {
 		SPDK_ERRLOG("lvol %s/%s: failed to hotplug blob_bdev due to error %d\n",
 			    lvs->name, lvol->name, bserrno);
 	}
+	req->cb_fn(req->cb_arg, lvol, bserrno);
+	free(req);
 }
 
 static void
-lvs_esnap_degraded_hotplug(struct spdk_lvs_degraded_lvol_set *degraded_set)
+lvs_esnap_degraded_hotplug(struct spdk_lvs_degraded_lvol_set *degraded_set,
+			   spdk_lvol_op_with_handle_complete cb_fn, void *cb_arg)
 {
 	struct spdk_lvol_store	*lvs = degraded_set->lvol_store;
 	struct spdk_lvol	*lvol, *tmp, *last_missing;
 	struct spdk_bs_dev	*bs_dev;
 	const void		*esnap_id = degraded_set->esnap_id;
 	uint32_t		id_len = degraded_set->id_len;
+	struct lvs_esnap_hotplug_req *req;
 	int			rc;
 
 	assert(lvs->thread == spdk_get_thread());
@@ -1984,6 +1995,17 @@ lvs_esnap_degraded_hotplug(struct spdk_lvs_degraded_lvol_set *degraded_set)
 	last_missing = TAILQ_LAST(&degraded_set->lvols, degraded_lvols);
 
 	TAILQ_FOREACH_SAFE(lvol, &degraded_set->lvols, degraded_link, tmp) {
+		req = calloc(1, sizeof(*req));
+		if (req == NULL) {
+			SPDK_ERRLOG("lvol %s: failed to create esnap bs_dev: out of memory\n",
+				    lvol->unique_id);
+			cb_fn(cb_arg, lvol, -ENOMEM);
+			/* The next one likely won't succeed either, but keep going so that all the
+			 * failed hotplugs are logged.
+			 */
+			goto next;
+		}
+
 		/*
 		 * Remove the lvol from the tailq so that tailq corruption is avoided if
 		 * lvs->esnap_bs_dev_create() calls spdk_lvs_esnap_missing_add(lvol).
@@ -1998,11 +2020,17 @@ lvs_esnap_degraded_hotplug(struct spdk_lvs_degraded_lvol_set *degraded_set)
 				    lvol->unique_id, rc);
 			lvol->degraded_set = degraded_set;
 			TAILQ_INSERT_TAIL(&degraded_set->lvols, lvol, degraded_link);
-		} else {
+			cb_fn(cb_arg, lvol, rc);
-			spdk_blob_set_esnap_bs_dev(lvol->blob, bs_dev,
+			free(req);
-						   lvs_esnap_hotplug_done, lvol);
+			goto next;
 		}
 
+		req->lvol = lvol;
+		req->cb_fn = cb_fn;
+		req->cb_arg = cb_arg;
+		spdk_blob_set_esnap_bs_dev(lvol->blob, bs_dev, lvs_esnap_hotplug_done, req);
+
+next:
 		if (lvol == last_missing) {
 			/*
 			 * Anything after last_missing was added due to some problem encountered
@@ -2024,7 +2052,8 @@ lvs_esnap_degraded_hotplug(struct spdk_lvs_degraded_lvol_set *degraded_set)
  * that the bdev now exists.
  */
 bool
-spdk_lvs_notify_hotplug(const void *esnap_id, uint32_t id_len)
+spdk_lvs_notify_hotplug(const void *esnap_id, uint32_t id_len,
+			spdk_lvol_op_with_handle_complete cb_fn, void *cb_arg)
 {
 	struct spdk_lvs_degraded_lvol_set *found;
 	struct spdk_lvs_degraded_lvol_set find = { 0 };
@@ -2057,7 +2086,7 @@ spdk_lvs_notify_hotplug(const void *esnap_id, uint32_t id_len)
 		}
 
 		ret = true;
-		lvs_esnap_degraded_hotplug(found);
+		lvs_esnap_degraded_hotplug(found, cb_fn, cb_arg);
 	}
 	pthread_mutex_unlock(&g_lvol_stores_mutex);
 

module/bdev/lvol/vbdev_lvol.c

@@ -24,7 +24,8 @@ static TAILQ_HEAD(, lvol_store_bdev) g_spdk_lvol_pairs = TAILQ_HEAD_INITIALIZER(
 static int vbdev_lvs_init(void);
 static void vbdev_lvs_fini_start(void);
 static int vbdev_lvs_get_ctx_size(void);
-static void vbdev_lvs_examine(struct spdk_bdev *bdev);
+static void vbdev_lvs_examine_config(struct spdk_bdev *bdev);
+static void vbdev_lvs_examine_disk(struct spdk_bdev *bdev);
 static bool g_shutdown_started = false;
 
 struct spdk_bdev_module g_lvol_if = {
@@ -32,7 +33,8 @@ struct spdk_bdev_module g_lvol_if = {
 	.module_init = vbdev_lvs_init,
 	.fini_start = vbdev_lvs_fini_start,
 	.async_fini_start = true,
-	.examine_disk = vbdev_lvs_examine,
+	.examine_config = vbdev_lvs_examine_config,
+	.examine_disk = vbdev_lvs_examine_disk,
 	.get_ctx_size = vbdev_lvs_get_ctx_size,
 
 };
@@ -1458,6 +1460,52 @@ end:
 	}
 }
 
+/* Walks a tree of clones that are no longer degraded to create bdevs. */
+static int
+create_esnap_clone_lvol_disks(void *ctx, struct spdk_lvol *lvol)
+{
+	struct spdk_bdev *bdev = ctx;
+	int rc;
+
+	rc = _create_lvol_disk(lvol, false);
+	if (rc != 0) {
+		SPDK_ERRLOG("lvol %s: failed to create bdev after esnap hotplug of %s: %d\n",
+			    lvol->unique_id, spdk_bdev_get_name(bdev), rc);
+		/* Do not prevent creation of other clones in case of one failure. */
+		return 0;
+	}
+
+	return spdk_lvol_iter_immediate_clones(lvol, create_esnap_clone_lvol_disks, ctx);
+}
+
+static void
+vbdev_lvs_hotplug(void *ctx, struct spdk_lvol *lvol, int lvolerrno)
+{
+	struct spdk_bdev *esnap_clone_bdev = ctx;
+
+	if (lvolerrno != 0) {
+		SPDK_ERRLOG("lvol %s: during examine of bdev %s: not creating clone bdev due to "
+			    "error %d\n", lvol->unique_id, spdk_bdev_get_name(esnap_clone_bdev),
+			    lvolerrno);
+		return;
+	}
+	create_esnap_clone_lvol_disks(esnap_clone_bdev, lvol);
+}
+
+static void
+vbdev_lvs_examine_config(struct spdk_bdev *bdev)
+{
+	char uuid_str[SPDK_UUID_STRING_LEN];
+
+	spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), &bdev->uuid);
+
+	if (spdk_lvs_notify_hotplug(uuid_str, sizeof(uuid_str), vbdev_lvs_hotplug, bdev)) {
+		SPDK_INFOLOG(vbdev_lvol, "bdev %s: claimed by one ore more esnap clones\n",
+			     uuid_str);
+	}
+	spdk_bdev_module_examine_done(&g_lvol_if);
+}
+
 static void
 _vbdev_lvs_examine_cb(void *arg, struct spdk_lvol_store *lvol_store, int lvserrno)
 {
@@ -1572,7 +1620,7 @@ vbdev_lvs_load(struct spdk_bs_dev *bs_dev, spdk_lvs_op_with_handle_complete cb_f
 }
 
 static void
-vbdev_lvs_examine(struct spdk_bdev *bdev)
+vbdev_lvs_examine_disk(struct spdk_bdev *bdev)
 {
 	struct spdk_lvs_req *req;
 

test/lvol/Makefile

@@ -5,7 +5,12 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-DIRS-y = esnap
+DIRS-y =
+
+ifeq ($(OS),Linux)
+# Tests in this directory mostly depend upon aio bdevs, which are not widely supported.
+DIRS-y += esnap
+endif
 
 .PHONY: all clean $(DIRS-y)
 

test/lvol/esnap/Makefile

@@ -2,9 +2,15 @@
 #  Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
 TEST_FILE = esnap.c
 
-SPDK_LIB_LIST = accel bdev blob blob_bdev dma init lvol notify
+SPDK_LIB_LIST = accel bdev blob blob_bdev dma init notify
+
+ifeq ($(OS),Linux)
+SPDK_LIB_LIST += bdev_aio
+endif
 
 include $(SPDK_ROOT_DIR)/mk/spdk.unittest.mk
+include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk

test/lvol/esnap/esnap.c

@@ -2,6 +2,7 @@
  *   Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  */
 
+#include "spdk/stdinc.h"
 #include "spdk_cunit.h"
 #include "spdk/string.h"
 #include "spdk/init.h"
@@ -9,10 +10,13 @@
 #include "common/lib/ut_multithread.c"
 
 #include "bdev/bdev.c"
+#include "lvol/lvol.c"
 #include "bdev/malloc/bdev_malloc.c"
 #include "bdev/lvol/vbdev_lvol.c"
 #include "accel/accel_sw.c"
+#include "bdev/part.c"
 #include "blob/blobstore.h"
+#include "bdev/aio/bdev_aio.h"
 
 #include "unit/lib/json_mock.c"
 
@@ -23,6 +27,51 @@ DEFINE_STUB(pmem_is_pmem, int, (const void *addr, size_t len), 0);
 DEFINE_STUB(pmem_memset_persist, void *, (void *pmemdest, int c, size_t len), NULL);
 #endif
 
+char g_testdir[PATH_MAX];
+
+static void
+set_testdir(const char *path)
+{
+	char *tmp;
+
+	tmp = realpath(path, NULL);
+	snprintf(g_testdir, sizeof(g_testdir), "%s", tmp ? dirname(tmp) : ".");
+	free(tmp);
+}
+
+static int
+make_test_file(size_t size, char *path, size_t len, const char *name)
+{
+	int fd;
+	int rc;
+
+	CU_ASSERT(len <= INT32_MAX);
+	if (snprintf(path, len, "%s/%s", g_testdir, name) >= (int)len) {
+		return -ENAMETOOLONG;
+	}
+	fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0600);
+	if (fd < 0) {
+		return -errno;
+	}
+	rc = ftruncate(fd, size);
+	if (rc != 0) {
+		rc = -errno;
+		unlink(path);
+	}
+	close(fd);
+	return rc;
+}
+
+static void
+unregister_cb(void *ctx, int bdeverrno)
+{
+	int *rc = ctx;
+
+	if (rc != NULL) {
+		*rc = bdeverrno;
+	}
+}
+
 struct op_with_handle_data {
 	union {
 		struct spdk_lvol_store *lvs;
@@ -40,6 +89,15 @@ clear_owh(struct op_with_handle_data *owh)
 	return owh;
 }
 
+/* spdk_poll_threads() doesn't have visibility into uncompleted aio operations. */
+static void
+poll_error_updated(int *error)
+{
+	while (*error == 0xbad) {
+		poll_threads();
+	}
+}
+
 static void
 lvs_op_with_handle_cb(void *cb_arg, struct spdk_lvol_store *lvs, int lvserrno)
 {
@@ -188,9 +246,9 @@ esnap_clone_io(void)
 
 	/* Create lvstore */
 	rc = vbdev_lvs_create("bs_malloc", "lvs1", cluster_size, 0, 0,
-			      lvs_op_with_handle_cb, &owh_data);
+			      lvs_op_with_handle_cb, clear_owh(&owh_data));
 	SPDK_CU_ASSERT_FATAL(rc == 0);
-	poll_threads();
+	poll_error_updated(&owh_data.lvserrno);
 	SPDK_CU_ASSERT_FATAL(owh_data.lvserrno == 0);
 	SPDK_CU_ASSERT_FATAL(owh_data.u.lvs != NULL);
 	lvs = owh_data.u.lvs;
@@ -218,7 +276,7 @@ esnap_clone_io(void)
 	/* Create esnap clone */
 	vbdev_lvol_create_bdev_clone(esnap_uuid, lvs, "clone1",
 				     lvol_op_with_handle_cb, clear_owh(&owh_data));
-	poll_threads();
+	poll_error_updated(&owh_data.lvserrno);
 	SPDK_CU_ASSERT_FATAL(owh_data.lvserrno == 0);
 	SPDK_CU_ASSERT_FATAL(owh_data.u.lvol != NULL);
 
@@ -279,6 +337,114 @@ esnap_clone_io(void)
 	poll_threads();
 }
 
+static void
+esnap_wait_for_examine(void *ctx)
+{
+	int *flag = ctx;
+
+	*flag = 0;
+}
+
+static void
+esnap_hotplug(void)
+{
+	const char *uuid_esnap = "22218fb6-6743-483d-88b1-de643dc7c0bc";
+	struct malloc_bdev_opts malloc_opts = { 0 };
+	const uint32_t bs_size_bytes = 10 * 1024 * 1024;
+	const uint32_t bs_block_size = 4096;
+	const uint32_t cluster_size = 32 * 1024;
+	const uint32_t esnap_size_bytes = 2 * cluster_size;
+	struct op_with_handle_data owh_data = { 0 };
+	struct spdk_bdev *malloc_bdev = NULL, *bdev;
+	struct spdk_lvol_store *lvs;
+	struct spdk_lvol *lvol;
+	char aiopath[PATH_MAX];
+	int rc, rc2;
+
+	g_bdev_opts.bdev_auto_examine = true;
+
+	/* Create aio device to hold the lvstore. */
+	rc = make_test_file(bs_size_bytes, aiopath, sizeof(aiopath), "esnap_hotplug.aio");
+	SPDK_CU_ASSERT_FATAL(rc == 0);
+	rc = create_aio_bdev("aio1", aiopath, bs_block_size, false);
+	SPDK_CU_ASSERT_FATAL(rc == 0);
+	poll_threads();
+
+	rc = vbdev_lvs_create("aio1", "lvs1", cluster_size, 0, 0,
+			      lvs_op_with_handle_cb, clear_owh(&owh_data));
+	SPDK_CU_ASSERT_FATAL(rc == 0);
+	poll_error_updated(&owh_data.lvserrno);
+	SPDK_CU_ASSERT_FATAL(owh_data.lvserrno == 0);
+	SPDK_CU_ASSERT_FATAL(owh_data.u.lvs != NULL);
+	lvs = owh_data.u.lvs;
+
+	/* Create esnap device */
+	spdk_uuid_parse(&malloc_opts.uuid, uuid_esnap);
+	malloc_opts.name = "esnap_malloc";
+	malloc_opts.num_blocks = esnap_size_bytes / bs_block_size;
+	malloc_opts.block_size = bs_block_size;
+	rc = create_malloc_disk(&malloc_bdev, &malloc_opts);
+	SPDK_CU_ASSERT_FATAL(rc == 0);
+
+	/* Create esnap clone */
+	vbdev_lvol_create_bdev_clone(uuid_esnap, lvs, "clone1",
+				     lvol_op_with_handle_cb, clear_owh(&owh_data));
+	poll_error_updated(&owh_data.lvserrno);
+	SPDK_CU_ASSERT_FATAL(owh_data.lvserrno == 0);
+	SPDK_CU_ASSERT_FATAL(owh_data.u.lvol != NULL);
+
+	/* Verify that lvol bdev exists */
+	bdev = spdk_bdev_get_by_name("lvs1/clone1");
+	SPDK_CU_ASSERT_FATAL(bdev != NULL);
+
+	/* Unload the lvstore and verify the bdev is gone. */
+	rc = rc2 = 0xbad;
+	bdev_aio_delete("aio1", unregister_cb, &rc);
+	CU_ASSERT(spdk_bdev_get_by_name(uuid_esnap) != NULL)
+	delete_malloc_disk(malloc_bdev->name, unregister_cb, &rc2);
+	malloc_bdev = NULL;
+	poll_error_updated(&rc);
+	poll_error_updated(&rc2);
+	SPDK_CU_ASSERT_FATAL(rc == 0);
+	SPDK_CU_ASSERT_FATAL(rc2 == 0);
+	SPDK_CU_ASSERT_FATAL(spdk_bdev_get_by_name("lvs1/clone1") == NULL);
+	SPDK_CU_ASSERT_FATAL(spdk_bdev_get_by_name(uuid_esnap) == NULL);
+
+	/* Trigger the reload of the lvstore */
+	rc = create_aio_bdev("aio1", aiopath, bs_block_size, false);
+	SPDK_CU_ASSERT_FATAL(rc == 0);
+	rc = 0xbad;
+	spdk_bdev_wait_for_examine(esnap_wait_for_examine, &rc);
+	poll_error_updated(&rc);
+
+	/* Verify the lvol is loaded without creating a bdev. */
+	lvol = spdk_lvol_get_by_names("lvs1", "clone1");
+	CU_ASSERT(spdk_bdev_get_by_name("lvs1/clone1") == NULL)
+	SPDK_CU_ASSERT_FATAL(lvol != NULL);
+	SPDK_CU_ASSERT_FATAL(lvol->degraded_set != NULL);
+
+	/* Create the esnap device and verify that the bdev is created. */
+	rc = create_malloc_disk(&malloc_bdev, &malloc_opts);
+	SPDK_CU_ASSERT_FATAL(rc == 0);
+	poll_threads();
+	CU_ASSERT(malloc_bdev != NULL);
+	CU_ASSERT(lvol->degraded_set == NULL);
+	CU_ASSERT(spdk_bdev_get_by_name("lvs1/clone1") != NULL);
+
+	/* Clean up */
+	rc = rc2 = 0xbad;
+	bdev_aio_delete("aio1", unregister_cb, &rc);
+	poll_error_updated(&rc);
+	CU_ASSERT(rc == 0);
+	if (malloc_bdev != NULL) {
+		delete_malloc_disk(malloc_bdev->name, unregister_cb, &rc2);
+		poll_threads();
+		CU_ASSERT(rc2 == 0);
+	}
+	rc = unlink(aiopath);
+	CU_ASSERT(rc == 0);
+}
+
 static void
 bdev_init_cb(void *arg, int rc)
 {
@@ -303,12 +469,15 @@ main(int argc, char **argv)
 	unsigned int	num_failures;
 	int rc;
 
+	set_testdir(argv[0]);
+
 	CU_set_error_action(CUEA_ABORT);
 	CU_initialize_registry();
 
 	suite = CU_add_suite("esnap_io", NULL, NULL);
 
 	CU_ADD_TEST(suite, esnap_clone_io);
+	CU_ADD_TEST(suite, esnap_hotplug);
 
 	allocate_threads(2);
 	set_thread(0);

test/lvol/external_snapshot.sh

@@ -196,7 +196,26 @@ function test_esnap_reload_missing() {
 	NOT rpc_cmd bdev_get_bdevs -b lvs_test/snap
 	NOT rpc_cmd bdev_get_bdevs -b "$snap_uuid"
 
+	# Create the esnap bdev and verify the degraded bdevs become not degraded.
+	esnap_dev=$(rpc_cmd bdev_malloc_create -u "$esnap_uuid" "$esnap_size_mb" "$block_size")
+	rpc_cmd bdev_wait_for_examine
+	lvols=$(rpc_cmd bdev_lvol_get_lvols)
+	[[ "$(jq -r '.[] | select(.name == "eclone").is_esnap_clone' <<< "$lvols")" == "true" ]]
+	[[ "$(jq -r '.[] | select(.name == "eclone").is_degraded' <<< "$lvols")" == "false" ]]
+	[[ "$(jq -r '.[] | select(.name == "clone").is_clone' <<< "$lvols")" == "true" ]]
+	[[ "$(jq -r '.[] | select(.name == "clone").is_degraded' <<< "$lvols")" == "false" ]]
+	[[ "$(jq -r '.[] | select(.name == "snap").is_clone' <<< "$lvols")" == "true" ]]
+	[[ "$(jq -r '.[] | select(.name == "snap").is_snapshot' <<< "$lvols")" == "true" ]]
+	[[ "$(jq -r '.[] | select(.name == "snap").is_degraded' <<< "$lvols")" == "false" ]]
+	rpc_cmd bdev_get_bdevs -b lvs_test/eclone
+	rpc_cmd bdev_get_bdevs -b "$eclone_uuid"
+	rpc_cmd bdev_get_bdevs -b lvs_test/clone
+	rpc_cmd bdev_get_bdevs -b "$clone_uuid"
+	rpc_cmd bdev_get_bdevs -b lvs_test/snap
+	rpc_cmd bdev_get_bdevs -b "$snap_uuid"
+
 	rpc_cmd bdev_aio_delete "$aio_bdev"
+	rpc_cmd bdev_malloc_delete "$esnap_uuid"
 }
 
 function log_jq_out() {
@@ -342,6 +361,52 @@ function test_esnap_clones() {
 	rpc_cmd bdev_malloc_delete "$esnap_dev"
 }
 
+function test_esnap_late_arrival() {
+	local bs_dev esnap_dev
+	local block_size=512
+	local esnap_size_mb=1
+	local lvs_cluster_size=$((16 * 1024))
+	local lvs_uuid esnap_uuid eclone_uuid snap_uuid clone_uuid uuid
+	local aio_bdev=test_esnap_reload_aio0
+	local lvols
+
+	# Create the lvstore on an aio device. Can't use malloc because we need to remove
+	# the device and re-add it to trigger an lvstore unload and then load.
+	rm -f $testdir/aio_bdev_0
+	truncate -s "${AIO_SIZE_MB}M" $testdir/aio_bdev_0
+	bs_dev=$(rpc_cmd bdev_aio_create "$testdir/aio_bdev_0" "$aio_bdev" "$block_size")
+	lvs_uuid=$(rpc_cmd bdev_lvol_create_lvstore -c "$lvs_cluster_size" "$bs_dev" lvs_test)
+
+	# Create a bdev that will be the external snapshot
+	esnap_uuid=e4b40d8b-f623-416d-8234-baf5a4c83cbd
+	esnap_dev=$(rpc_cmd bdev_malloc_create -u "$esnap_uuid" "$esnap_size_mb" "$block_size")
+	eclone_uuid=$(rpc_cmd bdev_lvol_clone_bdev "$esnap_uuid" lvs_test "eclone1")
+
+	# Unload the lvstore
+	rpc_cmd bdev_aio_delete "$aio_bdev"
+	NOT rpc_cmd bdev_lvol_get_lvstores -l lvs_test
+
+	# Delete the external snapshot device then reload the lvstore.
+	rpc_cmd bdev_malloc_delete "$esnap_dev"
+	bs_dev=$(rpc_cmd bdev_aio_create "$testdir/aio_bdev_0" "$aio_bdev" "$block_size")
+	lvs_uuid=$(rpc_cmd bdev_lvol_get_lvstores -l lvs_test)
+
+	# Verify that the esnap clone exists but does not have the esnap loaded.
+	NOT rpc_cmd bdev_get_bdevs -b "$esnap_uuid"
+	NOT rpc_cmd bdev_get_bdevs -b "$eclone_uuid"
+	lvols=$(rpc_cmd bdev_lvol_get_lvols)
+	[[ "$(jq -r '.[] | select(.uuid == "'$eclone_uuid'").is_esnap_clone' <<< "$lvols")" == "true" ]]
+	[[ "$(jq -r '.[] | select(.uuid == "'$eclone_uuid'").is_degraded' <<< "$lvols")" == "true" ]]
+
+	# Create the esnap device and verify that the esnap clone finds it.
+	esnap_dev=$(rpc_cmd bdev_malloc_create -u "$esnap_uuid" "$esnap_size_mb" "$block_size")
+	rpc_cmd bdev_wait_for_examine
+	verify_esnap_clone "$eclone_uuid" "$esnap_uuid"
+
+	rpc_cmd bdev_aio_delete "$aio_bdev"
+	rpc_cmd bdev_malloc_delete "$esnap_dev"
+}
+
 $SPDK_BIN_DIR/spdk_tgt &
 spdk_pid=$!
 trap 'killprocess "$spdk_pid"; rm -f "$testdir/aio_bdev_0"; exit 1' SIGINT SIGTERM SIGPIPE EXIT
@@ -351,6 +416,7 @@ modprobe nbd
 run_test "test_esnap_reload" test_esnap_reload
 run_test "test_esnap_reload" test_esnap_reload_missing
 run_test "test_esnap_clones" test_esnap_clones
+run_test "test_esnap_late_arrival" test_esnap_late_arrival
 
 trap - SIGINT SIGTERM SIGPIPE EXIT
 killprocess $spdk_pid

test/unit/lib/bdev/vbdev_lvol.c/vbdev_lvol_ut.c

@@ -36,6 +36,7 @@ bool g_examine_done = false;
 bool g_bdev_alias_already_exists = false;
 bool g_lvs_with_name_already_exists = false;
 bool g_ext_api_called;
+bool g_bdev_is_missing = false;
 
 DEFINE_STUB_V(spdk_bdev_module_fini_start_done, (void));
 DEFINE_STUB(spdk_bdev_get_memory_domains, int, (struct spdk_bdev *bdev,
@@ -876,6 +877,21 @@ spdk_lvol_create_clone(struct spdk_lvol *lvol, const char *clone_name,
 	cb_fn(cb_arg, clone, 0);
 }
 
+bool
+spdk_lvs_notify_hotplug(const void *esnap_id, uint32_t id_len,
+			spdk_lvol_op_with_handle_complete cb_fn, void *cb_arg)
+{
+	struct spdk_uuid uuid = { 0 };
+	char uuid_str[SPDK_UUID_STRING_LEN] = "bad";
+
+	CU_ASSERT(id_len == SPDK_UUID_STRING_LEN);
+	CU_ASSERT(spdk_uuid_parse(&uuid, esnap_id) == 0);
+	CU_ASSERT(spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), &uuid) == 0);
+	CU_ASSERT(strcmp(esnap_id, uuid_str) == 0);
+
+	return g_bdev_is_missing;
+}
+
 static void
 lvol_store_op_complete(void *cb_arg, int lvserrno)
 {
@@ -1127,6 +1143,23 @@ ut_lvol_hotremove(void)
 
 }
 
+static void
+ut_lvol_examine_config(void)
+{
+	/* No esnap clone needs the bdev. */
+	g_bdev_is_missing = false;
+	g_examine_done = false;
+	vbdev_lvs_examine_config(&g_bdev);
+	CU_ASSERT(g_examine_done);
+
+	g_bdev_is_missing = true;
+	g_examine_done = false;
+	vbdev_lvs_examine_config(&g_bdev);
+	CU_ASSERT(g_examine_done);
+
+	g_examine_done = false;
+}
+
 static void
 ut_lvs_examine_check(bool success)
 {
@@ -1153,18 +1186,18 @@ ut_lvs_examine_check(bool success)
 }
 
 static void
-ut_lvol_examine(void)
+ut_lvol_examine_disk(void)
 {
 	/* Examine unsuccessfully - bdev already opened */
 	g_lvserrno = -1;
 	lvol_already_opened = true;
-	vbdev_lvs_examine(&g_bdev);
+	vbdev_lvs_examine_disk(&g_bdev);
 	ut_lvs_examine_check(false);
 
 	/* Examine unsuccessfully - fail on lvol store */
 	g_lvserrno = -1;
 	lvol_already_opened = false;
-	vbdev_lvs_examine(&g_bdev);
+	vbdev_lvs_examine_disk(&g_bdev);
 	ut_lvs_examine_check(false);
 
 	/* Examine successfully
@@ -1175,7 +1208,7 @@ ut_lvol_examine(void)
 	g_num_lvols = 1;
 	lvol_already_opened = false;
 	g_registered_bdevs = 0;
-	vbdev_lvs_examine(&g_bdev);
+	vbdev_lvs_examine_disk(&g_bdev);
 	ut_lvs_examine_check(true);
 	CU_ASSERT(g_registered_bdevs == 0);
 	CU_ASSERT(TAILQ_EMPTY(&g_lvol_store->lvols));
@@ -1188,7 +1221,7 @@ ut_lvol_examine(void)
 	g_lvolerrno = 0;
 	g_registered_bdevs = 0;
 	lvol_already_opened = false;
-	vbdev_lvs_examine(&g_bdev);
+	vbdev_lvs_examine_disk(&g_bdev);
 	ut_lvs_examine_check(true);
 	CU_ASSERT(g_registered_bdevs != 0);
 	SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&g_lvol_store->lvols));
@@ -1199,7 +1232,7 @@ ut_lvol_examine(void)
 	g_num_lvols = 4;
 	g_registered_bdevs = 0;
 	lvol_already_opened = false;
-	vbdev_lvs_examine(&g_bdev);
+	vbdev_lvs_examine_disk(&g_bdev);
 	ut_lvs_examine_check(true);
 	CU_ASSERT(g_registered_bdevs == g_num_lvols);
 	SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&g_lvol_store->lvols));
@@ -1211,7 +1244,7 @@ ut_lvol_examine(void)
 	g_lvol_open_enomem = 2;
 	g_registered_bdevs = 0;
 	lvol_already_opened = false;
-	vbdev_lvs_examine(&g_bdev);
+	vbdev_lvs_examine_disk(&g_bdev);
 	ut_lvs_examine_check(true);
 	CU_ASSERT(g_registered_bdevs == g_num_lvols);
 	SPDK_CU_ASSERT_FATAL(!TAILQ_EMPTY(&g_lvol_store->lvols));
@@ -1917,7 +1950,8 @@ main(int argc, char **argv)
 	CU_ADD_TEST(suite, ut_vbdev_lvol_io_type_supported);
 	CU_ADD_TEST(suite, ut_lvol_read_write);
 	CU_ADD_TEST(suite, ut_vbdev_lvol_submit_request);
-	CU_ADD_TEST(suite, ut_lvol_examine);
+	CU_ADD_TEST(suite, ut_lvol_examine_config);
+	CU_ADD_TEST(suite, ut_lvol_examine_disk);
 	CU_ADD_TEST(suite, ut_lvol_rename);
 	CU_ADD_TEST(suite, ut_bdev_finish);
 	CU_ADD_TEST(suite, ut_lvs_rename);

test/unit/lib/lvol/lvol.c/lvol_ut.c

@@ -2964,7 +2964,8 @@ lvol_esnap_hotplug_scenario(struct hotplug_lvol *hotplug_lvols,
 
 	/* Perform hotplug */
 	for (m_esnap = degraded_lvol_sets_tree; m_esnap->esnap_id != NULL; m_esnap++) {
-		spdk_lvs_notify_hotplug(m_esnap->esnap_id, strlen(m_esnap->esnap_id) + 1);
+		spdk_lvs_notify_hotplug(m_esnap->esnap_id, strlen(m_esnap->esnap_id) + 1,
+					lvol_op_with_handle_complete, NULL);
 	}
 
 	/* Verify lvol->degraded_set and back_bs_dev */