module/raid: write initial superblock

When creating the raid_bdev with enabled superblock option, write the
superblock to the base bdevs before bringing the array online.

Change-Id: I24659202ef3bbe6c87ca8603d514bd81660c9b41
Signed-off-by: Artur Paszkiewicz <artur.paszkiewicz@intel.com>

doc/bdev.md

@@ -536,10 +536,11 @@ Example commands
 ## RAID {#bdev_ug_raid}
 
 RAID virtual bdev module provides functionality to combine any SPDK bdevs into
-one RAID bdev. Currently SPDK supports only RAID 0. RAID functionality does not
+one RAID bdev. Currently SPDK supports only RAID 0. RAID metadata may be stored
-store on-disk metadata on the member disks, so user must recreate the RAID
+on member disks if enabled when creating the RAID bdev, so user does not have to
-volume when restarting application. User may specify member disks to create RAID
+recreate the RAID volume when restarting application. It is not enabled by
-volume event if they do not exists yet - as the member disks are registered at
+default for backward compatibility. User may specify member disks to create
+RAID volume even if they do not exist yet - as the member disks are registered at
 a later time, the RAID module will claim them and will surface the RAID volume
 after all of the member disks are available. It is allowed to use disks of
 different sizes - the smallest disk size will be the amount of space used on

module/bdev/raid/Makefile

@@ -10,7 +10,7 @@ SO_VER := 5
 SO_MINOR := 0
 
 CFLAGS += -I$(SPDK_ROOT_DIR)/lib/bdev/
-C_SRCS = bdev_raid.c bdev_raid_rpc.c raid0.c raid1.c concat.c
+C_SRCS = bdev_raid.c bdev_raid_rpc.c bdev_raid_sb.c raid0.c raid1.c concat.c
 
 ifeq ($(CONFIG_RAID5F),y)
 C_SRCS += raid5f.c

module/bdev/raid/bdev_raid.c

@@ -196,6 +196,7 @@ raid_bdev_cleanup(struct raid_bdev *raid_bdev)
 static void
 raid_bdev_free(struct raid_bdev *raid_bdev)
 {
+	spdk_dma_free(raid_bdev->sb);
 	pthread_mutex_destroy(&raid_bdev->mutex);
 	free(raid_bdev->base_bdev_info);
 	free(raid_bdev->bdev.name);
@@ -215,8 +216,7 @@ raid_bdev_cleanup_and_free(struct raid_bdev *raid_bdev)
  * params:
  * base_info - raid base bdev info
  * returns:
- * 0 - success
+ * none
- * non zero - failure
  */
 static void
 raid_bdev_free_base_bdev_resource(struct raid_base_bdev_info *base_info)
@@ -237,6 +237,8 @@ raid_bdev_free_base_bdev_resource(struct raid_base_bdev_info *base_info)
 	spdk_bdev_close(base_info->desc);
 	base_info->desc = NULL;
 	base_info->bdev = NULL;
+	spdk_put_io_channel(base_info->app_thread_ch);
+	base_info->app_thread_ch = NULL;
 
 	assert(raid_bdev->num_base_bdevs_discovered);
 	raid_bdev->num_base_bdevs_discovered--;
@@ -626,7 +628,7 @@ raid_bdev_write_info_json(struct raid_bdev *raid_bdev, struct spdk_json_write_ct
 	spdk_json_write_named_uint32(w, "strip_size_kb", raid_bdev->strip_size_kb);
 	spdk_json_write_named_string(w, "state", raid_bdev_state_to_str(raid_bdev->state));
 	spdk_json_write_named_string(w, "raid_level", raid_bdev_level_to_str(raid_bdev->level));
-	spdk_json_write_named_bool(w, "superblock", raid_bdev->superblock_enabled);
+	spdk_json_write_named_bool(w, "superblock", raid_bdev->sb != NULL);
 	spdk_json_write_named_uint32(w, "num_base_bdevs", raid_bdev->num_base_bdevs);
 	spdk_json_write_named_uint32(w, "num_base_bdevs_discovered", raid_bdev->num_base_bdevs_discovered);
 	spdk_json_write_name(w, "base_bdevs_list");
@@ -691,7 +693,7 @@ raid_bdev_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *
 	spdk_json_write_named_string(w, "name", bdev->name);
 	spdk_json_write_named_uint32(w, "strip_size_kb", raid_bdev->strip_size_kb);
 	spdk_json_write_named_string(w, "raid_level", raid_bdev_level_to_str(raid_bdev->level));
-	spdk_json_write_named_bool(w, "superblock", raid_bdev->superblock_enabled);
+	spdk_json_write_named_bool(w, "superblock", raid_bdev->sb != NULL);
 
 	spdk_json_write_named_array_begin(w, "base_bdevs");
 	RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
@@ -953,7 +955,7 @@ raid_bdev_init(void)
 
 /*
  * brief:
- * raid_bdev_create allocates raid bdev based on passed configuration
+ * _raid_bdev_create allocates raid bdev based on passed configuration
  * params:
  * name - name for raid bdev
  * strip_size - strip size in KB
@@ -964,10 +966,10 @@ raid_bdev_init(void)
  * 0 - success
  * non zero - failure
  */
-int
+static int
-raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs,
+_raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs,
-		 enum raid_level level, struct raid_bdev **raid_bdev_out,
+		  enum raid_level level, struct raid_bdev **raid_bdev_out,
-		 const struct spdk_uuid *uuid, bool superblock)
+		  const struct spdk_uuid *uuid, bool superblock)
 {
 	struct raid_bdev *raid_bdev;
 	struct spdk_bdev *raid_bdev_gen;
@@ -976,6 +978,11 @@ raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs,
 	uint8_t min_operational;
 	int rc;
 
+	if (strnlen(name, RAID_BDEV_SB_NAME_SIZE) == RAID_BDEV_SB_NAME_SIZE) {
+		SPDK_ERRLOG("Raid bdev name '%s' exceeds %d characters\n", name, RAID_BDEV_SB_NAME_SIZE - 1);
+		return -EINVAL;
+	}
+
 	if (raid_bdev_find_by_name(name) != NULL) {
 		SPDK_ERRLOG("Duplicate raid bdev name found: %s\n", name);
 		return -EEXIST;
@@ -1068,9 +1075,17 @@ raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs,
 	raid_bdev->state = RAID_BDEV_STATE_CONFIGURING;
 	raid_bdev->level = level;
 	raid_bdev->min_base_bdevs_operational = min_operational;
-	raid_bdev->superblock_enabled = superblock;
 	TAILQ_INIT(&raid_bdev->suspend_ctx);
 
+	if (superblock) {
+		raid_bdev->sb = spdk_dma_zmalloc(RAID_BDEV_SB_MAX_LENGTH, 0x1000, NULL);
+		if (!raid_bdev->sb) {
+			SPDK_ERRLOG("Failed to allocate raid bdev sb buffer\n");
+			raid_bdev_free(raid_bdev);
+			return -ENOMEM;
+		}
+	}
+
 	raid_bdev_gen = &raid_bdev->bdev;
 
 	raid_bdev_gen->name = strdup(name);
@@ -1097,6 +1112,28 @@ raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs,
 	return 0;
 }
 
+int
+raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs,
+		 enum raid_level level, struct raid_bdev **raid_bdev_out,
+		 const struct spdk_uuid *uuid, bool superblock)
+{
+	struct raid_bdev *raid_bdev;
+	int rc;
+
+	rc = _raid_bdev_create(name, strip_size, num_base_bdevs, level, &raid_bdev, uuid, superblock);
+	if (rc != 0) {
+		return rc;
+	}
+
+	if (superblock) {
+		spdk_uuid_generate(&raid_bdev->bdev.uuid);
+	}
+
+	*raid_bdev_out = raid_bdev;
+
+	return 0;
+}
+
 /*
  * brief:
  * Check underlying block devices against support for metadata. Do not configure
@@ -1138,6 +1175,140 @@ raid_bdev_configure_md(struct raid_bdev *raid_bdev)
 	return 0;
 }
 
+typedef void (*raid_bdev_write_superblock_cb)(bool success, struct raid_bdev *raid_bdev);
+
+struct raid_bdev_write_superblock_ctx {
+	bool success;
+	uint8_t remaining;
+	raid_bdev_write_superblock_cb cb;
+};
+
+static void
+raid_bdev_write_superblock_base_bdev_cb(int status, void *_ctx)
+{
+	struct raid_base_bdev_info *base_info = _ctx;
+	struct raid_bdev_write_superblock_ctx *ctx = base_info->raid_bdev->sb_write_ctx;
+
+	if (status != 0) {
+		SPDK_ERRLOG("Failed to save superblock on bdev %s: %s\n",
+			    base_info->name, spdk_strerror(-status));
+		ctx->success = false;
+	}
+
+	if (--ctx->remaining == 0) {
+		if (ctx->cb) {
+			ctx->cb(ctx->success, base_info->raid_bdev);
+		}
+		free(ctx);
+	}
+}
+
+static int
+raid_bdev_write_superblock(struct raid_bdev *raid_bdev, raid_bdev_write_superblock_cb cb)
+{
+	struct raid_base_bdev_info *base_info;
+	struct raid_bdev_write_superblock_ctx *ctx;
+	int rc = -ENODEV;
+
+	assert(spdk_get_thread() == spdk_thread_get_app_thread());
+	assert(raid_bdev->sb != NULL);
+
+	ctx = malloc(sizeof(*ctx));
+	if (!ctx) {
+		return -ENOMEM;
+	}
+
+	ctx->success = true;
+	ctx->remaining = raid_bdev->num_base_bdevs;
+	ctx->cb = cb;
+
+	raid_bdev->sb_write_ctx = ctx;
+
+	raid_bdev->sb->seq_number++;
+	raid_bdev_sb_update_crc(raid_bdev->sb);
+
+	RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
+		rc = raid_bdev_save_base_bdev_superblock(base_info->desc, base_info->app_thread_ch, raid_bdev->sb,
+				raid_bdev_write_superblock_base_bdev_cb, base_info);
+		if (rc != 0) {
+			raid_bdev_write_superblock_base_bdev_cb(rc, base_info);
+		}
+	}
+
+	return rc;
+}
+
+static void
+raid_bdev_init_superblock(struct raid_bdev *raid_bdev)
+{
+	struct raid_bdev_superblock *sb = raid_bdev->sb;
+	struct raid_base_bdev_info *base_info;
+	struct raid_bdev_sb_base_bdev *sb_base_bdev;
+
+	memset(sb, 0, RAID_BDEV_SB_MAX_LENGTH);
+
+	memcpy(&sb->signature, RAID_BDEV_SB_SIG, sizeof(sb->signature));
+	sb->version.major = RAID_BDEV_SB_VERSION_MAJOR;
+	sb->version.minor = RAID_BDEV_SB_VERSION_MINOR;
+	spdk_uuid_copy(&sb->uuid, &raid_bdev->bdev.uuid);
+	snprintf(sb->name, RAID_BDEV_SB_NAME_SIZE, "%s", raid_bdev->bdev.name);
+	sb->raid_size = raid_bdev->bdev.blockcnt;
+	sb->block_size = raid_bdev->bdev.blocklen;
+	sb->level = raid_bdev->level;
+	sb->strip_size = raid_bdev->strip_size;
+	/* TODO: sb->state */
+	sb->num_base_bdevs = sb->base_bdevs_size = raid_bdev->num_base_bdevs;
+	sb->length = sizeof(*sb) + sizeof(*sb_base_bdev) * sb->base_bdevs_size;
+
+	sb_base_bdev = &sb->base_bdevs[0];
+	RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
+		spdk_uuid_copy(&sb_base_bdev->uuid, spdk_bdev_get_uuid(base_info->bdev));
+		sb_base_bdev->data_offset = base_info->data_offset;
+		sb_base_bdev->data_size = base_info->data_size;
+		sb_base_bdev->state = RAID_SB_BASE_BDEV_CONFIGURED;
+		sb_base_bdev->slot = sb_base_bdev - sb->base_bdevs;
+		sb_base_bdev++;
+	}
+}
+
+static void
+raid_bdev_configure_cont(struct raid_bdev *raid_bdev)
+{
+	struct spdk_bdev *raid_bdev_gen = &raid_bdev->bdev;
+	int rc;
+
+	raid_bdev->state = RAID_BDEV_STATE_ONLINE;
+	SPDK_DEBUGLOG(bdev_raid, "io device register %p\n", raid_bdev);
+	SPDK_DEBUGLOG(bdev_raid, "blockcnt %" PRIu64 ", blocklen %u\n",
+		      raid_bdev_gen->blockcnt, raid_bdev_gen->blocklen);
+	spdk_io_device_register(raid_bdev, raid_bdev_create_cb, raid_bdev_destroy_cb,
+				sizeof(struct raid_bdev_io_channel),
+				raid_bdev->bdev.name);
+	rc = spdk_bdev_register(raid_bdev_gen);
+	if (rc != 0) {
+		SPDK_ERRLOG("Unable to register raid bdev and stay at configuring state\n");
+		if (raid_bdev->module->stop != NULL) {
+			raid_bdev->module->stop(raid_bdev);
+		}
+		spdk_io_device_unregister(raid_bdev, NULL);
+		raid_bdev->state = RAID_BDEV_STATE_CONFIGURING;
+		return;
+	}
+	SPDK_DEBUGLOG(bdev_raid, "raid bdev generic %p\n", raid_bdev_gen);
+	SPDK_DEBUGLOG(bdev_raid, "raid bdev is created with name %s, raid_bdev %p\n",
+		      raid_bdev_gen->name, raid_bdev);
+}
+
+static void
+raid_bdev_configure_write_sb_cb(bool success, struct raid_bdev *raid_bdev)
+{
+	if (success) {
+		raid_bdev_configure_cont(raid_bdev);
+	} else {
+		SPDK_ERRLOG("Failed to write raid bdev '%s' superblock\n", raid_bdev->bdev.name);
+	}
+}
+
 /*
  * brief:
  * If raid bdev config is complete, then only register the raid bdev to
@@ -1153,7 +1324,6 @@ static int
 raid_bdev_configure(struct raid_bdev *raid_bdev)
 {
 	uint32_t blocklen = 0;
-	struct spdk_bdev *raid_bdev_gen;
 	struct raid_base_bdev_info *base_info;
 	int rc = 0;
 
@@ -1186,9 +1356,7 @@ raid_bdev_configure(struct raid_bdev *raid_bdev)
 	}
 	raid_bdev->strip_size_shift = spdk_u32log2(raid_bdev->strip_size);
 	raid_bdev->blocklen_shift = spdk_u32log2(blocklen);
-
+	raid_bdev->bdev.blocklen = blocklen;
-	raid_bdev_gen = &raid_bdev->bdev;
-	raid_bdev_gen->blocklen = blocklen;
 
 	rc = raid_bdev_configure_md(raid_bdev);
 	if (rc != 0) {
@@ -1201,26 +1369,14 @@ raid_bdev_configure(struct raid_bdev *raid_bdev)
 		SPDK_ERRLOG("raid module startup callback failed\n");
 		return rc;
 	}
-	raid_bdev->state = RAID_BDEV_STATE_ONLINE;
+
-	SPDK_DEBUGLOG(bdev_raid, "io device register %p\n", raid_bdev);
+	if (raid_bdev->sb != NULL) {
-	SPDK_DEBUGLOG(bdev_raid, "blockcnt %" PRIu64 ", blocklen %u\n",
+		raid_bdev_init_superblock(raid_bdev);
-		      raid_bdev_gen->blockcnt, raid_bdev_gen->blocklen);
+
-	spdk_io_device_register(raid_bdev, raid_bdev_create_cb, raid_bdev_destroy_cb,
+		return raid_bdev_write_superblock(raid_bdev, raid_bdev_configure_write_sb_cb);
-				sizeof(struct raid_bdev_io_channel),
-				raid_bdev->bdev.name);
-	rc = spdk_bdev_register(raid_bdev_gen);
-	if (rc != 0) {
-		SPDK_ERRLOG("Unable to register raid bdev and stay at configuring state\n");
-		if (raid_bdev->module->stop != NULL) {
-			raid_bdev->module->stop(raid_bdev);
-		}
-		spdk_io_device_unregister(raid_bdev, NULL);
-		raid_bdev->state = RAID_BDEV_STATE_CONFIGURING;
-		return rc;
 	}
-	SPDK_DEBUGLOG(bdev_raid, "raid bdev generic %p\n", raid_bdev_gen);
+
-	SPDK_DEBUGLOG(bdev_raid, "raid bdev is created with name %s, raid_bdev %p\n",
+	raid_bdev_configure_cont(raid_bdev);
-		      raid_bdev_gen->name, raid_bdev);
 
 	return 0;
 }
@@ -1663,6 +1819,12 @@ raid_bdev_configure_base_bdev(struct raid_base_bdev_info *base_info)
 
 	bdev = spdk_bdev_desc_get_bdev(desc);
 
+	if (raid_bdev->sb != NULL && spdk_uuid_is_null(spdk_bdev_get_uuid(bdev))) {
+		SPDK_ERRLOG("Base bdev '%s' does not have a valid UUID\n", base_info->name);
+		spdk_bdev_close(desc);
+		return -EINVAL;
+	}
+
 	rc = spdk_bdev_module_claim_bdev(bdev, NULL, &g_raid_if);
 	if (rc != 0) {
 		SPDK_ERRLOG("Unable to claim this bdev as it is already claimed\n");
@@ -1674,6 +1836,14 @@ raid_bdev_configure_base_bdev(struct raid_base_bdev_info *base_info)
 
 	assert(raid_bdev->state != RAID_BDEV_STATE_ONLINE);
 
+	base_info->app_thread_ch = spdk_bdev_get_io_channel(desc);
+	if (base_info->app_thread_ch == NULL) {
+		SPDK_ERRLOG("Failed to get io channel\n");
+		spdk_bdev_module_release_bdev(bdev);
+		spdk_bdev_close(desc);
+		return -ENOMEM;
+	}
+
 	base_info->bdev = bdev;
 	base_info->desc = desc;
 	base_info->blockcnt = bdev->blockcnt;
@@ -1682,7 +1852,7 @@ raid_bdev_configure_base_bdev(struct raid_base_bdev_info *base_info)
 	raid_bdev->num_base_bdevs_discovered++;
 	assert(raid_bdev->num_base_bdevs_discovered <= raid_bdev->num_base_bdevs);
 
-	if (raid_bdev->superblock_enabled) {
+	if (raid_bdev->sb != NULL) {
 		assert((RAID_BDEV_MIN_DATA_OFFSET_SIZE % bdev->blocklen) == 0);
 		base_info->data_offset = RAID_BDEV_MIN_DATA_OFFSET_SIZE / bdev->blocklen;
 

module/bdev/raid/bdev_raid.h

@@ -9,8 +9,13 @@
 #include "spdk/bdev_module.h"
 #include "spdk/uuid.h"
 
+#include "bdev_raid_sb.h"
+
 #define RAID_BDEV_MIN_DATA_OFFSET_SIZE	(1024*1024) /* 1 MiB */
 
+SPDK_STATIC_ASSERT(RAID_BDEV_SB_MAX_LENGTH < RAID_BDEV_MIN_DATA_OFFSET_SIZE,
+		   "Incorrect min data offset");
+
 enum raid_level {
 	INVALID_RAID_LEVEL	= -1,
 	RAID0			= 0,
@@ -76,6 +81,9 @@ struct raid_base_bdev_info {
 
 	/* Hold the number of blocks to know how large the base bdev is resized. */
 	uint64_t		blockcnt;
+
+	/* io channel for the app thread */
+	struct spdk_io_channel	*app_thread_ch;
 };
 
 /*
@@ -148,9 +156,6 @@ struct raid_bdev {
 	/* Set to true if destroy of this raid bdev is started. */
 	bool				destroy_started;
 
-	/* Set to true if superblock metadata is enabled on this raid bdev */
-	bool				superblock_enabled;
-
 	/* Module for RAID-level specific operations */
 	struct raid_bdev_module		*module;
 
@@ -168,6 +173,12 @@ struct raid_bdev {
 
 	/* Device mutex */
 	pthread_mutex_t			mutex;
+
+	/* Superblock */
+	struct raid_bdev_superblock	*sb;
+
+	/* Superblock write context */
+	void				*sb_write_ctx;
 };
 
 #define RAID_FOR_EACH_BASE_BDEV(r, i) \

module/bdev/raid/bdev_raid_sb.c

@@ -0,0 +1,67 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (C) 2022 Intel Corporation.
+ *   All rights reserved.
+ */
+
+#include "spdk/bdev_module.h"
+#include "spdk/crc32.h"
+#include "spdk/env.h"
+#include "spdk/log.h"
+#include "spdk/string.h"
+#include "spdk/util.h"
+
+#include "bdev_raid_sb.h"
+
+struct raid_bdev_save_sb_ctx {
+	raid_bdev_save_sb_cb cb;
+	void *cb_ctx;
+};
+
+static void
+raid_bdev_write_sb_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
+{
+	struct raid_bdev_save_sb_ctx *ctx = cb_arg;
+
+	spdk_bdev_free_io(bdev_io);
+
+	if (ctx->cb) {
+		ctx->cb(success ? 0 : -EIO, ctx->cb_ctx);
+	}
+
+	free(ctx);
+}
+
+int
+raid_bdev_save_base_bdev_superblock(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
+				    const struct raid_bdev_superblock *sb,
+				    raid_bdev_save_sb_cb cb, void *cb_ctx)
+{
+	struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
+	uint64_t nbytes = SPDK_ALIGN_CEIL(sb->length, spdk_bdev_get_block_size(bdev));
+	struct raid_bdev_save_sb_ctx *ctx;
+	int rc;
+
+	ctx = calloc(1, sizeof(*ctx));
+	if (!ctx) {
+		return -ENOMEM;
+	}
+
+	ctx->cb = cb;
+	ctx->cb_ctx = cb_ctx;
+
+	rc = spdk_bdev_write(desc, ch, (void *)sb, 0, nbytes, raid_bdev_write_sb_cb, ctx);
+	if (rc) {
+		free(ctx);
+	}
+
+	return rc;
+}
+
+void
+raid_bdev_sb_update_crc(struct raid_bdev_superblock *sb)
+{
+	sb->crc = 0;
+	sb->crc = spdk_crc32c_update(sb, sb->length, 0);
+}
+
+SPDK_LOG_REGISTER_COMPONENT(bdev_raid_sb)

module/bdev/raid/bdev_raid_sb.h

@@ -0,0 +1,95 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (C) 2022 Intel Corporation.
+ *   All rights reserved.
+ */
+
+#ifndef SPDK_BDEV_RAID_SB_H_
+#define SPDK_BDEV_RAID_SB_H_
+
+#include "spdk/stdinc.h"
+#include "spdk/util.h"
+#include "spdk/uuid.h"
+
+#define RAID_BDEV_SB_VERSION_MAJOR	1
+#define RAID_BDEV_SB_VERSION_MINOR	0
+
+#define RAID_BDEV_SB_NAME_SIZE		32
+
+#define RAID_BDEV_SB_MAX_LENGTH \
+	SPDK_ALIGN_CEIL((sizeof(struct raid_bdev_superblock) + UINT8_MAX * sizeof(struct raid_bdev_sb_base_bdev)), 0x1000)
+
+enum raid_bdev_sb_base_bdev_state {
+	RAID_SB_BASE_BDEV_MISSING	= 0,
+	RAID_SB_BASE_BDEV_CONFIGURED	= 1,
+	RAID_SB_BASE_BDEV_FAILED	= 2,
+	RAID_SB_BASE_BDEV_SPARE		= 3,
+};
+
+struct raid_bdev_sb_base_bdev {
+	/* uuid of the base bdev */
+	struct spdk_uuid	uuid;
+	/* offset in blocks from base device start to the start of raid data area */
+	uint64_t		data_offset;
+	/* size in blocks of the base device raid data area */
+	uint64_t		data_size;
+	/* state of the base bdev */
+	uint32_t		state;
+	/* feature/status flags */
+	uint32_t		flags;
+	/* slot number of this base bdev in the raid */
+	uint8_t			slot;
+
+	uint8_t			reserved[23];
+};
+SPDK_STATIC_ASSERT(sizeof(struct raid_bdev_sb_base_bdev) == 64, "incorrect size");
+
+struct raid_bdev_superblock {
+#define RAID_BDEV_SB_SIG "SPDKRAID"
+	uint8_t			signature[8];
+	struct {
+		/* incremented when a breaking change in the superblock structure is made */
+		uint16_t	major;
+		/* incremented for changes in the superblock that are backward compatible */
+		uint16_t	minor;
+	} version;
+	/* length in bytes of the entire superblock */
+	uint32_t		length;
+	/* crc32c checksum of the entire superblock */
+	uint32_t		crc;
+	/* feature/status flags */
+	uint32_t		flags;
+	/* unique id of the raid bdev */
+	struct spdk_uuid	uuid;
+	/* name of the raid bdev */
+	uint8_t			name[RAID_BDEV_SB_NAME_SIZE];
+	/* size of the raid bdev in blocks */
+	uint64_t		raid_size;
+	/* the raid bdev block size - must be the same for all base bdevs */
+	uint32_t		block_size;
+	/* the raid level */
+	uint32_t		level;
+	/* strip (chunk) size in blocks */
+	uint32_t		strip_size;
+	/* state of the raid */
+	uint32_t		state;
+	/* sequence number, incremented on every superblock update */
+	uint64_t		seq_number;
+	/* number of raid base devices */
+	uint8_t			num_base_bdevs;
+
+	uint8_t			reserved[86];
+
+	/* size of the base bdevs array */
+	uint8_t			base_bdevs_size;
+	/* array of base bdev descriptors */
+	struct raid_bdev_sb_base_bdev base_bdevs[];
+};
+SPDK_STATIC_ASSERT(sizeof(struct raid_bdev_superblock) == 192, "incorrect size");
+
+typedef void (*raid_bdev_save_sb_cb)(int status, void *ctx);
+
+int raid_bdev_save_base_bdev_superblock(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
+					const struct raid_bdev_superblock *sb, raid_bdev_save_sb_cb cb, void *cb_ctx);
+void raid_bdev_sb_update_crc(struct raid_bdev_superblock *sb);
+
+#endif /* SPDK_BDEV_RAID_SB_H_ */

test/unit/lib/bdev/raid/Makefile

@@ -6,7 +6,7 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-DIRS-y = bdev_raid.c concat.c raid1.c
+DIRS-y = bdev_raid.c bdev_raid_sb.c concat.c raid1.c
 
 DIRS-$(CONFIG_RAID5F) += raid5f.c
 

test/unit/lib/bdev/raid/bdev_raid.c/bdev_raid_ut.c

@@ -125,6 +125,24 @@ DEFINE_STUB(spdk_bdev_get_dif_type, enum spdk_dif_type, (const struct spdk_bdev
 	    SPDK_DIF_DISABLE);
 DEFINE_STUB(spdk_bdev_is_dif_head_of_md, bool, (const struct spdk_bdev *bdev), false);
 DEFINE_STUB(spdk_bdev_notify_blockcnt_change, int, (struct spdk_bdev *bdev, uint64_t size), 0);
+DEFINE_STUB_V(raid_bdev_sb_update_crc, (struct raid_bdev_superblock *sb));
+
+int
+raid_bdev_save_base_bdev_superblock(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
+				    const struct raid_bdev_superblock *sb, raid_bdev_save_sb_cb cb, void *cb_ctx)
+{
+	if (cb) {
+		cb(0, cb_ctx);
+	}
+
+	return 0;
+}
+
+const struct spdk_uuid *
+spdk_bdev_get_uuid(const struct spdk_bdev *bdev)
+{
+	return &bdev->uuid;
+}
 
 struct spdk_io_channel *
 spdk_bdev_get_io_channel(struct spdk_bdev_desc *desc)
@@ -438,12 +456,6 @@ spdk_bdev_desc_get_bdev(struct spdk_bdev_desc *desc)
 	return (void *)desc;
 }
 
-char *
-spdk_sprintf_alloc(const char *format, ...)
-{
-	return strdup(format);
-}
-
 int
 spdk_json_write_named_uint32(struct spdk_json_write_ctx *w, const char *name, uint32_t val)
 {
@@ -919,8 +931,8 @@ verify_raid_bdev(struct rpc_bdev_raid_create *r, bool presence, uint32_t raid_st
 				bdev = spdk_bdev_get_by_name(base_info->bdev->name);
 				CU_ASSERT(bdev != NULL);
 				CU_ASSERT(base_info->remove_scheduled == false);
-				CU_ASSERT((pbdev->superblock_enabled == true && base_info->data_offset != 0) ||
+				CU_ASSERT((pbdev->sb != NULL && base_info->data_offset != 0) ||
-					  (pbdev->superblock_enabled == false && base_info->data_offset == 0));
+					  (pbdev->sb == NULL && base_info->data_offset == 0));
 				CU_ASSERT(base_info->data_offset + base_info->data_size == bdev->blockcnt);
 
 				if (bdev && base_info->data_size < min_blockcnt) {
@@ -989,6 +1001,7 @@ create_base_bdevs(uint32_t bbdev_start_idx)
 		base_bdev = calloc(1, sizeof(struct spdk_bdev));
 		SPDK_CU_ASSERT_FATAL(base_bdev != NULL);
 		base_bdev->name = strdup(name);
+		spdk_uuid_generate(&base_bdev->uuid);
 		SPDK_CU_ASSERT_FATAL(base_bdev->name != NULL);
 		base_bdev->blocklen = g_block_len;
 		base_bdev->blockcnt = BLOCK_CNT;

test/unit/lib/bdev/raid/bdev_raid_sb.c/Makefile

@@ -0,0 +1,10 @@
+#  SPDX-License-Identifier: BSD-3-Clause
+#  Copyright (C) 2022 Intel Corporation.
+#  All rights reserved.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../../../..)
+
+TEST_FILE = bdev_raid_sb_ut.c
+
+include $(SPDK_ROOT_DIR)/mk/spdk.unittest.mk

test/unit/lib/bdev/raid/bdev_raid_sb.c/bdev_raid_sb_ut.c

@@ -0,0 +1,108 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (C) 2022 Intel Corporation.
+ *   All rights reserved.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk_cunit.h"
+#include "spdk/env.h"
+#include "spdk_internal/mock.h"
+
+#include "common/lib/test_env.c"
+#include "bdev/raid/bdev_raid_sb.c"
+
+#define TEST_BUF_ALIGN	64
+#define TEST_BLOCK_SIZE	512
+
+DEFINE_STUB(spdk_bdev_desc_get_bdev, struct spdk_bdev *, (struct spdk_bdev_desc *desc), NULL);
+DEFINE_STUB(spdk_bdev_get_block_size, uint32_t, (const struct spdk_bdev *bdev), TEST_BLOCK_SIZE);
+DEFINE_STUB_V(spdk_bdev_free_io, (struct spdk_bdev_io *g_bdev_io));
+
+void *g_buf;
+
+static int
+test_setup(void)
+{
+	g_buf = spdk_dma_zmalloc(RAID_BDEV_SB_MAX_LENGTH, TEST_BUF_ALIGN, NULL);
+	if (!g_buf) {
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int
+test_cleanup(void)
+{
+	spdk_dma_free(g_buf);
+
+	return 0;
+}
+
+int
+spdk_bdev_write(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
+		void *buf, uint64_t offset, uint64_t nbytes,
+		spdk_bdev_io_completion_cb cb, void *cb_arg)
+{
+	struct raid_bdev_superblock *sb = buf;
+
+	CU_ASSERT(offset == 0);
+	CU_ASSERT(nbytes / TEST_BLOCK_SIZE == spdk_divide_round_up(sb->length, TEST_BLOCK_SIZE));
+
+	cb(NULL, true, cb_arg);
+	return 0;
+}
+
+static void
+prepare_sb(struct raid_bdev_superblock *sb)
+{
+	/* prepare a simplest valid sb */
+	memset(sb, 0, RAID_BDEV_SB_MAX_LENGTH);
+	memcpy(sb->signature, RAID_BDEV_SB_SIG, sizeof(sb->signature));
+	sb->version.major = RAID_BDEV_SB_VERSION_MAJOR;
+	sb->version.minor = RAID_BDEV_SB_VERSION_MINOR;
+	sb->length = sizeof(*sb);
+	sb->crc = spdk_crc32c_update(sb, sb->length, 0);
+}
+
+static void
+save_sb_cb(int status, void *ctx)
+{
+	int *status_out = ctx;
+
+	*status_out = status;
+}
+
+static void
+test_raid_bdev_save_base_bdev_superblock(void)
+{
+	struct raid_bdev_superblock *sb = g_buf;
+	int rc;
+	int status;
+
+	prepare_sb(sb);
+
+	status = INT_MAX;
+	rc = raid_bdev_save_base_bdev_superblock(NULL, NULL, sb, save_sb_cb, &status);
+	CU_ASSERT(rc == 0);
+	CU_ASSERT(status == 0);
+}
+
+int
+main(int argc, char **argv)
+{
+	CU_pSuite suite = NULL;
+	unsigned int num_failures;
+
+	CU_set_error_action(CUEA_ABORT);
+	CU_initialize_registry();
+
+	suite = CU_add_suite("raid_sb", test_setup, test_cleanup);
+	CU_ADD_TEST(suite, test_raid_bdev_save_base_bdev_superblock);
+
+	CU_basic_set_mode(CU_BRM_VERBOSE);
+	CU_basic_run_tests();
+	num_failures = CU_get_number_of_failures();
+	CU_cleanup_registry();
+	return num_failures;
+}

test/unit/lib/bdev/raid/common.c

@@ -86,7 +86,6 @@ raid_test_create_raid_bdev(struct raid_params *params, struct raid_bdev_module *
 		CU_FAIL_FATAL("unsupported raid constraint type");
 	};
 
-	raid_bdev->superblock_enabled = false;
 	raid_bdev->base_bdev_info = calloc(raid_bdev->num_base_bdevs,
 					   sizeof(struct raid_base_bdev_info));
 	SPDK_CU_ASSERT_FATAL(raid_bdev->base_bdev_info != NULL);

test/unit/unittest.sh

@@ -20,6 +20,7 @@ function unittest_bdev() {
 	$valgrind $testdir/lib/bdev/bdev.c/bdev_ut
 	$valgrind $testdir/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut
 	$valgrind $testdir/lib/bdev/raid/bdev_raid.c/bdev_raid_ut
+	$valgrind $testdir/lib/bdev/raid/bdev_raid_sb.c/bdev_raid_sb_ut
 	$valgrind $testdir/lib/bdev/raid/concat.c/concat_ut
 	$valgrind $testdir/lib/bdev/raid/raid1.c/raid1_ut
 	$valgrind $testdir/lib/bdev/bdev_zone.c/bdev_zone_ut