lib/ftl: retrieve caching bdev from configuration

Added means to configure libftl to use (optionally) another bdev as
persistent write buffer cache.

Signed-off-by: Konrad Sztyber <konrad.sztyber@intel.com>
Change-Id: I97028a681be168d9386eac8a226631ff772f803b
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/448629
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

include/spdk/ftl.h

@@ -39,6 +39,7 @@
 #include "spdk/nvme_ocssd.h"
 #include "spdk/uuid.h"
 #include "spdk/thread.h"
+#include "spdk/bdev.h"
 
 struct spdk_ftl_dev;
 
@@ -103,6 +104,8 @@ struct spdk_ftl_dev_init_opts {
 	struct spdk_nvme_ctrlr			*ctrlr;
 	/* Controller's transport ID */
 	struct spdk_nvme_transport_id		trid;
+	/* Write buffer cache */
+	struct spdk_bdev_desc			*cache_bdev_desc;
 
 	/* Thread responsible for core tasks execution */
 	struct spdk_thread			*core_thread;
@@ -130,6 +133,8 @@ struct spdk_ftl_attrs {
 	uint64_t				lbk_cnt;
 	/* Logical block size */
 	size_t					lbk_size;
+	/* Write buffer cache */
+	struct spdk_bdev_desc			*cache_bdev_desc;
 };
 
 struct ftl_module_init_opts {

lib/bdev/nvme/bdev_ftl.c

@@ -56,6 +56,8 @@ struct ftl_bdev {
 
 	struct spdk_ftl_dev		*dev;
 
+	struct spdk_bdev_desc		*cache_bdev_desc;
+
 	ftl_bdev_init_fn		init_cb;
 
 	void				*init_arg;
@@ -184,6 +186,11 @@ bdev_ftl_free_cb(void *ctx, int status)
 
 	bdev_ftl_remove_ctrlr(ftl_bdev->ctrlr);
 
+	if (ftl_bdev->cache_bdev_desc) {
+		spdk_bdev_module_release_bdev(spdk_bdev_desc_get_bdev(ftl_bdev->cache_bdev_desc));
+		spdk_bdev_close(ftl_bdev->cache_bdev_desc);
+	}
+
 	spdk_bdev_destruct_done(&ftl_bdev->bdev, status);
 	free(ftl_bdev->bdev.name);
 	free(ftl_bdev);
@@ -392,7 +399,7 @@ bdev_ftl_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w
 {
 	struct ftl_bdev *ftl_bdev = bdev->ctxt;
 	struct spdk_ftl_attrs attrs;
-	const char *trtype_str;
+	const char *trtype_str, *cache_bdev;
 	char uuid[SPDK_UUID_STRING_LEN];
 
 	spdk_ftl_dev_get_attrs(ftl_bdev->dev, &attrs);
@@ -408,16 +415,20 @@ bdev_ftl_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w
 	if (trtype_str) {
 		spdk_json_write_named_string(w, "trtype", trtype_str);
 	}
-	spdk_json_write_named_string(w, "traddr", ftl_bdev->ctrlr->trid.traddr);
 
+	spdk_json_write_named_string(w, "traddr", ftl_bdev->ctrlr->trid.traddr);
 	spdk_json_write_named_string_fmt(w, "punits", "%d-%d", attrs.range.begin, attrs.range.end);
 
 	spdk_uuid_fmt_lower(uuid, sizeof(uuid), &attrs.uuid);
 	spdk_json_write_named_string(w, "uuid", uuid);
 
-	spdk_json_write_object_end(w);
+	if (ftl_bdev->cache_bdev_desc) {
+		cache_bdev = spdk_bdev_get_name(spdk_bdev_desc_get_bdev(ftl_bdev->cache_bdev_desc));
+		spdk_json_write_named_string(w, "cache", cache_bdev);
+	}
 
 	spdk_json_write_object_end(w);
+	spdk_json_write_object_end(w);
 }
 
 static const struct spdk_bdev_fn_table ftl_fn_table = {
@@ -515,6 +526,7 @@ bdev_ftl_read_bdev_config(struct spdk_conf_section *sp,
 			rc = -1;
 			break;
 		}
+
 		opts->name = val;
 
 		val = spdk_conf_section_get_nmval(sp, "TransportID", i, 2);
@@ -549,6 +561,19 @@ bdev_ftl_read_bdev_config(struct spdk_conf_section *sp,
 		} else {
 			opts->mode = 0;
 		}
+
+		val = spdk_conf_section_get_nmval(sp, "TransportID", i, 4);
+		if (!val) {
+			continue;
+		}
+
+		if (!spdk_bdev_get_by_name(val)) {
+			SPDK_ERRLOG("Invalid cache bdev name: %s for TransportID: %s\n", val, trid);
+			rc = -1;
+			break;
+		}
+
+		opts->cache_bdev = val;
 	}
 
 	if (!rc) {
@@ -608,6 +633,12 @@ bdev_ftl_io_channel_destroy_cb(void *io_device, void *ctx_buf)
 	spdk_put_io_channel(ch->ioch);
 }
 
+static void
+bdev_ftl_cache_removed_cb(void *ctx)
+{
+	assert(0 && "Removed cached bdev\n");
+}
+
 static void
 bdev_ftl_create_cb(struct spdk_ftl_dev *dev, void *ctx, int status)
 {
@@ -669,6 +700,11 @@ error_unregister:
 error_dev:
 	bdev_ftl_remove_ctrlr(ftl_bdev->ctrlr);
 
+	if (ftl_bdev->cache_bdev_desc) {
+		spdk_bdev_module_release_bdev(spdk_bdev_desc_get_bdev(ftl_bdev->cache_bdev_desc));
+		spdk_bdev_close(ftl_bdev->cache_bdev_desc);
+	}
+
 	free(ftl_bdev->bdev.name);
 	free(ftl_bdev);
 
@@ -680,6 +716,7 @@ bdev_ftl_create(struct spdk_nvme_ctrlr *ctrlr, const struct ftl_bdev_init_opts *
 		ftl_bdev_init_fn cb, void *cb_arg)
 {
 	struct ftl_bdev *ftl_bdev = NULL;
+	struct spdk_bdev *cache_bdev = NULL;
 	struct nvme_bdev_ctrlr *ftl_ctrlr;
 	struct spdk_ftl_dev_init_opts opts = {};
 	int rc;
@@ -703,6 +740,29 @@ bdev_ftl_create(struct spdk_nvme_ctrlr *ctrlr, const struct ftl_bdev_init_opts *
 		goto error_ctrlr;
 	}
 
+	if (bdev_opts->cache_bdev) {
+		cache_bdev = spdk_bdev_get_by_name(bdev_opts->cache_bdev);
+		if (!cache_bdev) {
+			SPDK_ERRLOG("Unable to find bdev: %s\n", bdev_opts->cache_bdev);
+			rc = -ENOENT;
+			goto error_name;
+		}
+
+		if (spdk_bdev_open(cache_bdev, true, bdev_ftl_cache_removed_cb,
+				   ftl_bdev, &ftl_bdev->cache_bdev_desc)) {
+			SPDK_ERRLOG("Unable to open cache bdev: %s\n", bdev_opts->cache_bdev);
+			rc = -EPERM;
+			goto error_name;
+		}
+
+		if (spdk_bdev_module_claim_bdev(cache_bdev, ftl_bdev->cache_bdev_desc, &g_ftl_if)) {
+			SPDK_ERRLOG("Unable to claim cache bdev %s\n", bdev_opts->cache_bdev);
+			spdk_bdev_close(ftl_bdev->cache_bdev_desc);
+			rc = -EPERM;
+			goto error_name;
+		}
+	}
+
 	ftl_bdev->ctrlr = ftl_ctrlr;
 	ftl_bdev->init_cb = cb;
 	ftl_bdev->init_arg = cb_arg;
@@ -713,6 +773,7 @@ bdev_ftl_create(struct spdk_nvme_ctrlr *ctrlr, const struct ftl_bdev_init_opts *
 	opts.mode = bdev_opts->mode;
 	opts.uuid = bdev_opts->uuid;
 	opts.name = ftl_bdev->bdev.name;
+	opts.cache_bdev_desc = ftl_bdev->cache_bdev_desc;
 	opts.conf = NULL;
 
 	/* TODO: set threads based on config */
@@ -721,11 +782,16 @@ bdev_ftl_create(struct spdk_nvme_ctrlr *ctrlr, const struct ftl_bdev_init_opts *
 	rc = spdk_ftl_dev_init(&opts, bdev_ftl_create_cb, ftl_bdev);
 	if (rc) {
 		SPDK_ERRLOG("Could not create FTL device\n");
-		goto error_name;
+		goto error_cache;
 	}
 
 	return 0;
 
+error_cache:
+	if (ftl_bdev->cache_bdev_desc) {
+		spdk_bdev_module_release_bdev(cache_bdev);
+		spdk_bdev_close(ftl_bdev->cache_bdev_desc);
+	}
 error_name:
 	free(ftl_bdev->bdev.name);
 error_ctrlr:

lib/bdev/nvme/bdev_ftl.h

@@ -58,6 +58,8 @@ struct ftl_bdev_init_opts {
 	struct spdk_ftl_punit_range		range;
 	/* Bdev's name */
 	const char				*name;
+	/* Write buffer bdev's name */
+	const char				*cache_bdev;
 	/* Bdev's mode */
 	uint32_t				mode;
 	/* UUID if device is restored from SSD */

lib/bdev/nvme/bdev_ftl_rpc.c

@@ -45,6 +45,7 @@ struct rpc_construct_ftl {
 	char *traddr;
 	char *punits;
 	char *uuid;
+	char *cache_bdev;
 };
 
 static void
@@ -55,6 +56,7 @@ free_rpc_construct_ftl(struct rpc_construct_ftl *req)
 	free(req->traddr);
 	free(req->punits);
 	free(req->uuid);
+	free(req->cache_bdev);
 }
 
 static const struct spdk_json_object_decoder rpc_construct_ftl_decoders[] = {
@@ -63,6 +65,7 @@ static const struct spdk_json_object_decoder rpc_construct_ftl_decoders[] = {
 	{"traddr", offsetof(struct rpc_construct_ftl, traddr), spdk_json_decode_string},
 	{"punits", offsetof(struct rpc_construct_ftl, punits), spdk_json_decode_string},
 	{"uuid", offsetof(struct rpc_construct_ftl, uuid), spdk_json_decode_string, true},
+	{"cache", offsetof(struct rpc_construct_ftl, cache_bdev), spdk_json_decode_string, true},
 };
 
 #define FTL_RANGE_MAX_LENGTH 32
@@ -111,8 +114,15 @@ spdk_rpc_construct_ftl_bdev(struct spdk_jsonrpc_request *request,
 		goto invalid;
 	}
 
+	if (req.cache_bdev && !spdk_bdev_get_by_name(req.cache_bdev)) {
+		spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+						     "No such bdev: %s", req.cache_bdev);
+		goto invalid;
+	}
+
 	opts.name = req.name;
 	opts.mode = SPDK_FTL_MODE_CREATE;
+	opts.cache_bdev = req.cache_bdev;
 
 	/* Parse trtype */
 	rc = spdk_nvme_transport_id_parse_trtype(&opts.trid.trtype, req.trtype);

lib/ftl/ftl_core.c

@@ -1310,6 +1310,7 @@ spdk_ftl_dev_get_attrs(const struct spdk_ftl_dev *dev, struct spdk_ftl_attrs *at
 	attrs->lbk_cnt = dev->num_lbas;
 	attrs->lbk_size = FTL_BLOCK_SIZE;
 	attrs->range = dev->range;
+	attrs->cache_bdev_desc = dev->cache_bdev_desc;
 }
 
 static void

lib/ftl/ftl_core.h

@@ -43,6 +43,7 @@
 #include "spdk_internal/log.h"
 #include "spdk/queue.h"
 #include "spdk/ftl.h"
+#include "spdk/bdev.h"
 
 #include "ftl_ppa.h"
 #include "ftl_io.h"
@@ -136,6 +137,8 @@ struct spdk_ftl_dev {
 	struct spdk_nvme_ns			*ns;
 	/* NVMe transport ID */
 	struct spdk_nvme_transport_id		trid;
+	/* Write buffer cache */
+	struct spdk_bdev_desc			*cache_bdev_desc;
 
 	/* LBA map memory pool */
 	struct spdk_mempool			*lba_pool;

lib/ftl/ftl_init.c

@@ -844,6 +844,8 @@ spdk_ftl_dev_init(const struct spdk_ftl_dev_init_opts *opts, spdk_ftl_init_fn cb
 	dev->init_arg = cb_arg;
 	dev->range = opts->range;
 	dev->limit = SPDK_FTL_LIMIT_MAX;
+	dev->cache_bdev_desc = opts->cache_bdev_desc;
+
 	dev->name = strdup(opts->name);
 	if (!dev->name) {
 		SPDK_ERRLOG("Unable to set device name\n");

scripts/rpc.py

@@ -1295,7 +1295,8 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
                                                trtype=args.trtype,
                                                traddr=args.traddr,
                                                punits=args.punits,
-                                               uuid=args.uuid))
+                                               uuid=args.uuid,
+                                               cache=args.cache))
 
     p = subparsers.add_parser('construct_ftl_bdev',
                               help='Add FTL bdev')
@@ -1308,6 +1309,7 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
                    required=True)
     p.add_argument('-u', '--uuid', help='UUID of restored bdev (not applicable when creating new '
                    'instance): e.g. b286d19a-0059-4709-abcd-9f7732b1567d (optional)')
+    p.add_argument('-c', '--cache', help='Name of the bdev to be used as a write buffer cache (optional)')
     p.set_defaults(func=construct_ftl_bdev)
 
     def delete_ftl_bdev(args):

scripts/rpc/bdev.py

@@ -551,7 +551,7 @@ def destruct_split_vbdev(client, base_bdev):
     return client.call('destruct_split_vbdev', params)
 
 
-def construct_ftl_bdev(client, name, trtype, traddr, punits, uuid=None):
+def construct_ftl_bdev(client, name, trtype, traddr, punits, uuid=None, cache=None):
     """Construct FTL bdev
 
     Args:
@@ -560,6 +560,7 @@ def construct_ftl_bdev(client, name, trtype, traddr, punits, uuid=None):
         traddr: transport address
         punit: parallel unit range
         uuid: UUID of the device
+        cache: name of the write buffer bdev
     """
     params = {'name': name,
               'trtype': trtype,
@@ -567,6 +568,9 @@ def construct_ftl_bdev(client, name, trtype, traddr, punits, uuid=None):
               'punits': punits}
     if uuid:
         params['uuid'] = uuid
+    if cache:
+        params['cache'] = cache
+
     return client.call('construct_ftl_bdev', params)