vbdev_lvol: create esnap blobstore device

Register an spdk_bs_esnap_dev_create callback when initializing or
loading an lvstore. This is the first of several commits required to add
support enable lvol bdevs to support external snapshots and esnap
clones.

Signed-off-by: Mike Gerdts <mgerdts@nvidia.com>
Change-Id: I35c4e61fdbe5b93d65b9374e0ad91cb7fb94d1f4
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/16589
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

module/bdev/lvol/vbdev_lvol.c

@@ -1,7 +1,7 @@
 /*   SPDX-License-Identifier: BSD-3-Clause
  *   Copyright (C) 2017 Intel Corporation.
  *   All rights reserved.
- *   Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ *   Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  */
 
 #include "spdk/blob_bdev.h"
@@ -26,7 +26,7 @@ static int vbdev_lvs_get_ctx_size(void);
 static void vbdev_lvs_examine(struct spdk_bdev *bdev);
 static bool g_shutdown_started = false;
 
-static struct spdk_bdev_module g_lvol_if = {
+struct spdk_bdev_module g_lvol_if = {
 	.name = "lvol",
 	.module_init = vbdev_lvs_init,
 	.fini_start = vbdev_lvs_fini_start,
@@ -240,6 +240,7 @@ vbdev_lvs_create(const char *base_bdev_name, const char *name, uint32_t cluster_
 		return -EINVAL;
 	}
 	snprintf(opts.name, sizeof(opts.name), "%s", name);
+	opts.esnap_bs_dev_create = vbdev_lvol_esnap_dev_create;
 
 	lvs_req = calloc(1, sizeof(*lvs_req));
 	if (!lvs_req) {
@@ -1489,6 +1490,16 @@ vbdev_lvs_examine_done(void *arg, int lvserrno)
 	free(req);
 }
 
+static void
+vbdev_lvs_load(struct spdk_bs_dev *bs_dev, spdk_lvs_op_with_handle_complete cb_fn, void *cb_arg)
+{
+	struct spdk_lvs_opts lvs_opts;
+
+	spdk_lvs_opts_init(&lvs_opts);
+	lvs_opts.esnap_bs_dev_create = vbdev_lvol_esnap_dev_create;
+	spdk_lvs_load_ext(bs_dev, &lvs_opts, cb_fn, cb_arg);
+}
+
 static void
 vbdev_lvs_examine(struct spdk_bdev *bdev)
 {
@@ -1511,7 +1522,7 @@ vbdev_lvs_examine(struct spdk_bdev *bdev)
 	req->cb_fn = vbdev_lvs_examine_done;
 	req->cb_arg = req;
 
-	_vbdev_lvs_examine(bdev, req, spdk_lvs_load);
+	_vbdev_lvs_examine(bdev, req, vbdev_lvs_load);
 }
 
 struct spdk_lvol *
@@ -1615,4 +1626,71 @@ vbdev_lvs_grow(struct spdk_lvol_store *lvs,
 	}
 }
 
+/* Begin external snapshot support */
+
+static void
+vbdev_lvol_esnap_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev,
+			       void *event_ctx)
+{
+	SPDK_NOTICELOG("bdev name (%s) received unsupported event type %d\n",
+		       spdk_bdev_get_name(bdev), type);
+}
+
+int
+vbdev_lvol_esnap_dev_create(void *bs_ctx, void *blob_ctx, struct spdk_blob *blob,
+			    const void *esnap_id, uint32_t id_len,
+			    struct spdk_bs_dev **_bs_dev)
+{
+	struct spdk_lvol	*lvol = blob_ctx;
+	struct spdk_bs_dev	*bs_dev = NULL;
+	struct spdk_uuid	uuid;
+	int			rc;
+	char			uuid_str[SPDK_UUID_STRING_LEN] = { 0 };
+
+	if (esnap_id == NULL) {
+		SPDK_ERRLOG("lvol %s: NULL esnap ID\n", lvol->unique_id);
+		return -EINVAL;
+	}
+
+	/* Guard against arbitrary names and unterminated UUID strings */
+	if (id_len != SPDK_UUID_STRING_LEN) {
+		SPDK_ERRLOG("lvol %s: Invalid esnap ID length (%u)\n", lvol->unique_id, id_len);
+		return -EINVAL;
+	}
+
+	if (spdk_uuid_parse(&uuid, esnap_id)) {
+		SPDK_ERRLOG("lvol %s: Invalid esnap ID: not a UUID\n", lvol->unique_id);
+		return -EINVAL;
+	}
+
+	/* Format the UUID the same as it is in the bdev names tree. */
+	spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), &uuid);
+	if (strcmp(uuid_str, esnap_id) != 0) {
+		SPDK_WARNLOG("lvol %s: esnap_id '%*s' does not match parsed uuid '%s'\n",
+			     lvol->unique_id, SPDK_UUID_STRING_LEN, (const char *)esnap_id,
+			     uuid_str);
+		assert(false);
+	}
+
+	rc = spdk_bdev_create_bs_dev(uuid_str, false, NULL, 0,
+				     vbdev_lvol_esnap_bdev_event_cb, NULL, &bs_dev);
+	if (rc != 0) {
+		SPDK_ERRLOG("lvol %s: failed to create bs_dev from bdev '%s': %d\n",
+			    lvol->unique_id, uuid_str, rc);
+		return rc;
+	}
+	rc = spdk_bs_bdev_claim(bs_dev, &g_lvol_if);
+	if (rc != 0) {
+		SPDK_ERRLOG("lvol %s: unable to claim esnap bdev '%s': %d\n",
+			    lvol->unique_id, uuid_str, rc);
+		bs_dev->destroy(bs_dev);
+		return rc;
+	}
+
+	*_bs_dev = bs_dev;
+	return 0;
+}
+
+/* End external snapshot support */
+
 SPDK_LOG_REGISTER_COMPONENT(vbdev_lvol)

module/bdev/lvol/vbdev_lvol.h

@@ -1,6 +1,7 @@
 /*   SPDX-License-Identifier: BSD-3-Clause
  *   Copyright (C) 2017 Intel Corporation.
  *   All rights reserved.
+ *   Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  */
 
 #ifndef SPDK_VBDEV_LVOL_H
@@ -116,4 +117,8 @@ struct spdk_lvol *vbdev_lvol_get_from_bdev(struct spdk_bdev *bdev);
 void vbdev_lvs_grow(struct spdk_lvol_store *lvs,
 		    spdk_lvs_op_complete cb_fn, void *cb_arg);
 
+int vbdev_lvol_esnap_dev_create(void *bs_ctx, void *blob_ctx, struct spdk_blob *blob,
+				const void *esnap_id, uint32_t id_len,
+				struct spdk_bs_dev **_bs_dev);
+
 #endif /* SPDK_VBDEV_LVOL_H */

test/unit/lib/bdev/vbdev_lvol.c/vbdev_lvol_ut.c

@@ -45,6 +45,15 @@ DEFINE_STUB(spdk_blob_is_esnap_clone, bool, (const struct spdk_blob *blob), fals
 DEFINE_STUB(spdk_lvol_iter_immediate_clones, int,
 	    (struct spdk_lvol *lvol, spdk_lvol_iter_cb cb_fn, void *cb_arg), -ENOTSUP);
 
+struct spdk_blob {
+	uint64_t	id;
+	char		name[32];
+};
+
+struct spdk_blob_store {
+	spdk_bs_esnap_dev_create esnap_bs_dev_create;
+};
+
 const struct spdk_bdev_aliases_list *
 spdk_bdev_get_aliases(const struct spdk_bdev *bdev)
 {
@@ -125,6 +134,42 @@ spdk_bdev_destruct_done(struct spdk_bdev *bdev, int bdeverrno)
 	bdev->internal.unregister_cb(bdev->internal.unregister_ctx, bdeverrno);
 }
 
+struct ut_bs_dev {
+	struct spdk_bs_dev bs_dev;
+	struct spdk_bdev *bdev;
+};
+
+static void
+ut_bs_dev_destroy(struct spdk_bs_dev *bs_dev)
+{
+	struct ut_bs_dev *ut_bs_dev = SPDK_CONTAINEROF(bs_dev, struct ut_bs_dev, bs_dev);
+
+	free(ut_bs_dev);
+}
+
+int
+spdk_bdev_create_bs_dev(const char *bdev_name, bool write,
+			struct spdk_bdev_bs_dev_opts *opts, size_t opts_size,
+			spdk_bdev_event_cb_t event_cb, void *event_ctx,
+			struct spdk_bs_dev **bs_dev)
+{
+	struct spdk_bdev *bdev;
+	struct ut_bs_dev *ut_bs_dev;
+
+	bdev = spdk_bdev_get_by_name(bdev_name);
+	if (bdev == NULL) {
+		return -ENODEV;
+	}
+
+	ut_bs_dev = calloc(1, sizeof(*ut_bs_dev));
+	SPDK_CU_ASSERT_FATAL(ut_bs_dev != NULL);
+	ut_bs_dev->bs_dev.destroy = ut_bs_dev_destroy;
+	ut_bs_dev->bdev = bdev;
+	*bs_dev = &ut_bs_dev->bs_dev;
+
+	return 0;
+}
+
 void
 spdk_lvs_grow(struct spdk_bs_dev *bs_dev, spdk_lvs_op_with_handle_complete cb_fn, void *cb_arg)
 {
@@ -259,8 +304,8 @@ spdk_blob_is_thin_provisioned(struct spdk_blob *blob)
 
 static struct spdk_lvol *_lvol_create(struct spdk_lvol_store *lvs);
 
-void
+static void
-spdk_lvs_load(struct spdk_bs_dev *dev,
+lvs_load(struct spdk_bs_dev *dev, const struct spdk_lvs_opts *lvs_opts,
 	 spdk_lvs_op_with_handle_complete cb_fn, void *cb_arg)
 {
 	struct spdk_lvol_store *lvs = NULL;
@@ -279,6 +324,9 @@ spdk_lvs_load(struct spdk_bs_dev *dev,
 
 	lvs = calloc(1, sizeof(*lvs));
 	SPDK_CU_ASSERT_FATAL(lvs != NULL);
+	lvs->blobstore = calloc(1, sizeof(*lvs->blobstore));
+	lvs->blobstore->esnap_bs_dev_create = lvs_opts->esnap_bs_dev_create;
+	SPDK_CU_ASSERT_FATAL(lvs->blobstore != NULL);
 	TAILQ_INIT(&lvs->lvols);
 	TAILQ_INIT(&lvs->pending_lvols);
 	TAILQ_INIT(&lvs->retry_open_lvols);
@@ -292,11 +340,25 @@ spdk_lvs_load(struct spdk_bs_dev *dev,
 	cb_fn(cb_arg, lvs, lvserrno);
 }
 
+void
+spdk_lvs_load(struct spdk_bs_dev *dev,
+	      spdk_lvs_op_with_handle_complete cb_fn, void *cb_arg)
+{
+	lvs_load(dev, NULL, cb_fn, cb_arg);
+}
+
+void
+spdk_lvs_load_ext(struct spdk_bs_dev *bs_dev, const struct spdk_lvs_opts *lvs_opts,
+		  spdk_lvs_op_with_handle_complete cb_fn, void *cb_arg)
+{
+	lvs_load(bs_dev, lvs_opts, cb_fn, cb_arg);
+}
+
 int
 spdk_bs_bdev_claim(struct spdk_bs_dev *bs_dev, struct spdk_bdev_module *module)
 {
 	if (lvol_already_opened == true) {
-		return -1;
+		return -EPERM;
 	}
 
 	lvol_already_opened = true;
@@ -423,6 +485,7 @@ spdk_lvs_unload(struct spdk_lvol_store *lvs, spdk_lvs_op_complete cb_fn, void *c
 	g_lvol_store = NULL;
 
 	lvs->bs_dev->destroy(lvs->bs_dev);
+	free(lvs->blobstore);
 	free(lvs);
 
 	if (cb_fn != NULL) {
@@ -455,6 +518,7 @@ spdk_lvs_destroy(struct spdk_lvol_store *lvs, spdk_lvs_op_complete cb_fn,
 	g_lvol_store = NULL;
 
 	lvs->bs_dev->destroy(lvs->bs_dev);
+	free(lvs->blobstore);
 	free(lvs);
 
 	if (cb_fn != NULL) {
@@ -492,10 +556,22 @@ spdk_bs_get_cluster_size(struct spdk_blob_store *bs)
 struct spdk_bdev *
 spdk_bdev_get_by_name(const char *bdev_name)
 {
+	struct spdk_uuid uuid;
+	int rc;
+
+	if (g_base_bdev == NULL) {
+		return NULL;
+	}
+
 	if (!strcmp(g_base_bdev->name, bdev_name)) {
 		return g_base_bdev;
 	}
 
+	rc = spdk_uuid_parse(&uuid, bdev_name);
+	if (rc == 0 && spdk_uuid_compare(&uuid, &g_base_bdev->uuid) == 0) {
+		return g_base_bdev;
+	}
+
 	return NULL;
 }
 
@@ -660,7 +736,7 @@ spdk_bdev_module_list_add(struct spdk_bdev_module *bdev_module)
 const char *
 spdk_bdev_get_name(const struct spdk_bdev *bdev)
 {
-	return "test";
+	return bdev->name;
 }
 
 int
@@ -997,6 +1073,8 @@ ut_lvs_examine_check(bool success)
 		SPDK_CU_ASSERT_FATAL(lvs_bdev != NULL);
 		g_lvol_store = lvs_bdev->lvs;
 		SPDK_CU_ASSERT_FATAL(g_lvol_store != NULL);
+		SPDK_CU_ASSERT_FATAL(g_lvol_store->blobstore != NULL);
+		CU_ASSERT(g_lvol_store->blobstore->esnap_bs_dev_create != NULL);
 		CU_ASSERT(g_lvol_store->bs_dev != NULL);
 		CU_ASSERT(g_lvol_store->lvols_opened == spdk_min(g_num_lvols, g_registered_bdevs));
 	} else {
@@ -1615,6 +1693,69 @@ ut_lvol_seek(void)
 	free(g_lvol);
 }
 
+static void
+ut_esnap_dev_create(void)
+{
+	struct spdk_lvol_store lvs = { 0 };
+	struct spdk_lvol lvol = { 0 };
+	struct spdk_blob blob = { 0 };
+	struct spdk_bdev bdev = { 0 };
+	const char uuid_str[SPDK_UUID_STRING_LEN] = "a27fd8fe-d4b9-431e-a044-271016228ce4";
+	char bad_uuid_str[SPDK_UUID_STRING_LEN] = "a27fd8fe-d4b9-431e-a044-271016228ce4";
+	char *unterminated;
+	size_t len;
+	struct spdk_bs_dev *bs_dev = NULL;
+	int rc;
+
+	bdev.name = "bdev0";
+	spdk_uuid_parse(&bdev.uuid, uuid_str);
+
+	/* NULL esnap_id */
+	rc = vbdev_lvol_esnap_dev_create(&lvs, &lvol, &blob, NULL, 0, &bs_dev);
+	CU_ASSERT(rc == -EINVAL);
+	CU_ASSERT(bs_dev == NULL);
+
+	/* Unterminated UUID: asan should catch reads past end of allocated buffer. */
+	len = strlen(uuid_str);
+	unterminated = calloc(1, len);
+	SPDK_CU_ASSERT_FATAL(unterminated != NULL);
+	memcpy(unterminated, uuid_str, len);
+	rc = vbdev_lvol_esnap_dev_create(&lvs, &lvol, &blob, unterminated, len, &bs_dev);
+	CU_ASSERT(rc == -EINVAL);
+	CU_ASSERT(bs_dev == NULL);
+
+	/* Invaid UUID but the right length is invalid */
+	bad_uuid_str[2] = 'z';
+	rc = vbdev_lvol_esnap_dev_create(&lvs, &lvol, &blob, bad_uuid_str, sizeof(uuid_str),
+					 &bs_dev);
+	CU_ASSERT(rc == -EINVAL);
+	CU_ASSERT(bs_dev == NULL);
+
+	/* Bdev not found */
+	g_base_bdev = NULL;
+	rc = vbdev_lvol_esnap_dev_create(&lvs, &lvol, &blob, uuid_str, sizeof(uuid_str), &bs_dev);
+	CU_ASSERT(rc == -ENODEV);
+	CU_ASSERT(bs_dev == NULL);
+
+	/* Cannot get a claim */
+	g_base_bdev = &bdev;
+	lvol_already_opened = true;
+	rc = vbdev_lvol_esnap_dev_create(&lvs, &lvol, &blob, uuid_str, sizeof(uuid_str), &bs_dev);
+	CU_ASSERT(rc == -EPERM);
+	CU_ASSERT(bs_dev == NULL);
+
+	/* Happy path */
+	lvol_already_opened = false;
+	rc = vbdev_lvol_esnap_dev_create(&lvs, &lvol, &blob, uuid_str, sizeof(uuid_str), &bs_dev);
+	CU_ASSERT(rc == 0);
+	SPDK_CU_ASSERT_FATAL(bs_dev != NULL);
+	bs_dev->destroy(bs_dev);
+
+	g_base_bdev = NULL;
+	lvol_already_opened = false;
+	free(unterminated);
+}
+
 int
 main(int argc, char **argv)
 {
@@ -1644,6 +1785,7 @@ main(int argc, char **argv)
 	CU_ADD_TEST(suite, ut_bdev_finish);
 	CU_ADD_TEST(suite, ut_lvs_rename);
 	CU_ADD_TEST(suite, ut_lvol_seek);
+	CU_ADD_TEST(suite, ut_esnap_dev_create);
 
 	CU_basic_set_mode(CU_BRM_VERBOSE);
 	CU_basic_run_tests();