diff --git a/include/spdk/ftl.h b/include/spdk/ftl.h index 326925f63..5b90a305e 100644 --- a/include/spdk/ftl.h +++ b/include/spdk/ftl.h @@ -39,6 +39,7 @@ #include "spdk/nvme_ocssd.h" #include "spdk/uuid.h" #include "spdk/thread.h" +#include "spdk/bdev.h" struct spdk_ftl_dev; @@ -103,6 +104,8 @@ struct spdk_ftl_dev_init_opts { struct spdk_nvme_ctrlr *ctrlr; /* Controller's transport ID */ struct spdk_nvme_transport_id trid; + /* Write buffer cache */ + struct spdk_bdev_desc *cache_bdev_desc; /* Thread responsible for core tasks execution */ struct spdk_thread *core_thread; @@ -130,6 +133,8 @@ struct spdk_ftl_attrs { uint64_t lbk_cnt; /* Logical block size */ size_t lbk_size; + /* Write buffer cache */ + struct spdk_bdev_desc *cache_bdev_desc; }; struct ftl_module_init_opts { diff --git a/lib/bdev/nvme/bdev_ftl.c b/lib/bdev/nvme/bdev_ftl.c index cc202745a..ca8f43f01 100644 --- a/lib/bdev/nvme/bdev_ftl.c +++ b/lib/bdev/nvme/bdev_ftl.c @@ -56,6 +56,8 @@ struct ftl_bdev { struct spdk_ftl_dev *dev; + struct spdk_bdev_desc *cache_bdev_desc; + ftl_bdev_init_fn init_cb; void *init_arg; @@ -184,6 +186,11 @@ bdev_ftl_free_cb(void *ctx, int status) bdev_ftl_remove_ctrlr(ftl_bdev->ctrlr); + if (ftl_bdev->cache_bdev_desc) { + spdk_bdev_module_release_bdev(spdk_bdev_desc_get_bdev(ftl_bdev->cache_bdev_desc)); + spdk_bdev_close(ftl_bdev->cache_bdev_desc); + } + spdk_bdev_destruct_done(&ftl_bdev->bdev, status); free(ftl_bdev->bdev.name); free(ftl_bdev); @@ -392,7 +399,7 @@ bdev_ftl_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w { struct ftl_bdev *ftl_bdev = bdev->ctxt; struct spdk_ftl_attrs attrs; - const char *trtype_str; + const char *trtype_str, *cache_bdev; char uuid[SPDK_UUID_STRING_LEN]; spdk_ftl_dev_get_attrs(ftl_bdev->dev, &attrs); @@ -408,16 +415,20 @@ bdev_ftl_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w if (trtype_str) { spdk_json_write_named_string(w, "trtype", trtype_str); } - spdk_json_write_named_string(w, "traddr", ftl_bdev->ctrlr->trid.traddr); + spdk_json_write_named_string(w, "traddr", ftl_bdev->ctrlr->trid.traddr); spdk_json_write_named_string_fmt(w, "punits", "%d-%d", attrs.range.begin, attrs.range.end); spdk_uuid_fmt_lower(uuid, sizeof(uuid), &attrs.uuid); spdk_json_write_named_string(w, "uuid", uuid); - spdk_json_write_object_end(w); + if (ftl_bdev->cache_bdev_desc) { + cache_bdev = spdk_bdev_get_name(spdk_bdev_desc_get_bdev(ftl_bdev->cache_bdev_desc)); + spdk_json_write_named_string(w, "cache", cache_bdev); + } spdk_json_write_object_end(w); + spdk_json_write_object_end(w); } static const struct spdk_bdev_fn_table ftl_fn_table = { @@ -515,6 +526,7 @@ bdev_ftl_read_bdev_config(struct spdk_conf_section *sp, rc = -1; break; } + opts->name = val; val = spdk_conf_section_get_nmval(sp, "TransportID", i, 2); @@ -549,6 +561,19 @@ bdev_ftl_read_bdev_config(struct spdk_conf_section *sp, } else { opts->mode = 0; } + + val = spdk_conf_section_get_nmval(sp, "TransportID", i, 4); + if (!val) { + continue; + } + + if (!spdk_bdev_get_by_name(val)) { + SPDK_ERRLOG("Invalid cache bdev name: %s for TransportID: %s\n", val, trid); + rc = -1; + break; + } + + opts->cache_bdev = val; } if (!rc) { @@ -608,6 +633,12 @@ bdev_ftl_io_channel_destroy_cb(void *io_device, void *ctx_buf) spdk_put_io_channel(ch->ioch); } +static void +bdev_ftl_cache_removed_cb(void *ctx) +{ + assert(0 && "Removed cached bdev\n"); +} + static void bdev_ftl_create_cb(struct spdk_ftl_dev *dev, void *ctx, int status) { @@ -669,6 +700,11 @@ error_unregister: error_dev: bdev_ftl_remove_ctrlr(ftl_bdev->ctrlr); + if (ftl_bdev->cache_bdev_desc) { + spdk_bdev_module_release_bdev(spdk_bdev_desc_get_bdev(ftl_bdev->cache_bdev_desc)); + spdk_bdev_close(ftl_bdev->cache_bdev_desc); + } + free(ftl_bdev->bdev.name); free(ftl_bdev); @@ -680,6 +716,7 @@ bdev_ftl_create(struct spdk_nvme_ctrlr *ctrlr, const struct ftl_bdev_init_opts * ftl_bdev_init_fn cb, void *cb_arg) { struct ftl_bdev *ftl_bdev = NULL; + struct spdk_bdev *cache_bdev = NULL; struct nvme_bdev_ctrlr *ftl_ctrlr; struct spdk_ftl_dev_init_opts opts = {}; int rc; @@ -703,6 +740,29 @@ bdev_ftl_create(struct spdk_nvme_ctrlr *ctrlr, const struct ftl_bdev_init_opts * goto error_ctrlr; } + if (bdev_opts->cache_bdev) { + cache_bdev = spdk_bdev_get_by_name(bdev_opts->cache_bdev); + if (!cache_bdev) { + SPDK_ERRLOG("Unable to find bdev: %s\n", bdev_opts->cache_bdev); + rc = -ENOENT; + goto error_name; + } + + if (spdk_bdev_open(cache_bdev, true, bdev_ftl_cache_removed_cb, + ftl_bdev, &ftl_bdev->cache_bdev_desc)) { + SPDK_ERRLOG("Unable to open cache bdev: %s\n", bdev_opts->cache_bdev); + rc = -EPERM; + goto error_name; + } + + if (spdk_bdev_module_claim_bdev(cache_bdev, ftl_bdev->cache_bdev_desc, &g_ftl_if)) { + SPDK_ERRLOG("Unable to claim cache bdev %s\n", bdev_opts->cache_bdev); + spdk_bdev_close(ftl_bdev->cache_bdev_desc); + rc = -EPERM; + goto error_name; + } + } + ftl_bdev->ctrlr = ftl_ctrlr; ftl_bdev->init_cb = cb; ftl_bdev->init_arg = cb_arg; @@ -713,6 +773,7 @@ bdev_ftl_create(struct spdk_nvme_ctrlr *ctrlr, const struct ftl_bdev_init_opts * opts.mode = bdev_opts->mode; opts.uuid = bdev_opts->uuid; opts.name = ftl_bdev->bdev.name; + opts.cache_bdev_desc = ftl_bdev->cache_bdev_desc; opts.conf = NULL; /* TODO: set threads based on config */ @@ -721,11 +782,16 @@ bdev_ftl_create(struct spdk_nvme_ctrlr *ctrlr, const struct ftl_bdev_init_opts * rc = spdk_ftl_dev_init(&opts, bdev_ftl_create_cb, ftl_bdev); if (rc) { SPDK_ERRLOG("Could not create FTL device\n"); - goto error_name; + goto error_cache; } return 0; +error_cache: + if (ftl_bdev->cache_bdev_desc) { + spdk_bdev_module_release_bdev(cache_bdev); + spdk_bdev_close(ftl_bdev->cache_bdev_desc); + } error_name: free(ftl_bdev->bdev.name); error_ctrlr: diff --git a/lib/bdev/nvme/bdev_ftl.h b/lib/bdev/nvme/bdev_ftl.h index 2f4a16ae9..ad5d91584 100644 --- a/lib/bdev/nvme/bdev_ftl.h +++ b/lib/bdev/nvme/bdev_ftl.h @@ -58,6 +58,8 @@ struct ftl_bdev_init_opts { struct spdk_ftl_punit_range range; /* Bdev's name */ const char *name; + /* Write buffer bdev's name */ + const char *cache_bdev; /* Bdev's mode */ uint32_t mode; /* UUID if device is restored from SSD */ diff --git a/lib/bdev/nvme/bdev_ftl_rpc.c b/lib/bdev/nvme/bdev_ftl_rpc.c index 87cb75583..f4b732674 100644 --- a/lib/bdev/nvme/bdev_ftl_rpc.c +++ b/lib/bdev/nvme/bdev_ftl_rpc.c @@ -45,6 +45,7 @@ struct rpc_construct_ftl { char *traddr; char *punits; char *uuid; + char *cache_bdev; }; static void @@ -55,6 +56,7 @@ free_rpc_construct_ftl(struct rpc_construct_ftl *req) free(req->traddr); free(req->punits); free(req->uuid); + free(req->cache_bdev); } static const struct spdk_json_object_decoder rpc_construct_ftl_decoders[] = { @@ -63,6 +65,7 @@ static const struct spdk_json_object_decoder rpc_construct_ftl_decoders[] = { {"traddr", offsetof(struct rpc_construct_ftl, traddr), spdk_json_decode_string}, {"punits", offsetof(struct rpc_construct_ftl, punits), spdk_json_decode_string}, {"uuid", offsetof(struct rpc_construct_ftl, uuid), spdk_json_decode_string, true}, + {"cache", offsetof(struct rpc_construct_ftl, cache_bdev), spdk_json_decode_string, true}, }; #define FTL_RANGE_MAX_LENGTH 32 @@ -111,8 +114,15 @@ spdk_rpc_construct_ftl_bdev(struct spdk_jsonrpc_request *request, goto invalid; } + if (req.cache_bdev && !spdk_bdev_get_by_name(req.cache_bdev)) { + spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "No such bdev: %s", req.cache_bdev); + goto invalid; + } + opts.name = req.name; opts.mode = SPDK_FTL_MODE_CREATE; + opts.cache_bdev = req.cache_bdev; /* Parse trtype */ rc = spdk_nvme_transport_id_parse_trtype(&opts.trid.trtype, req.trtype); diff --git a/lib/ftl/ftl_core.c b/lib/ftl/ftl_core.c index 412f72a55..04ac9ebf7 100644 --- a/lib/ftl/ftl_core.c +++ b/lib/ftl/ftl_core.c @@ -1310,6 +1310,7 @@ spdk_ftl_dev_get_attrs(const struct spdk_ftl_dev *dev, struct spdk_ftl_attrs *at attrs->lbk_cnt = dev->num_lbas; attrs->lbk_size = FTL_BLOCK_SIZE; attrs->range = dev->range; + attrs->cache_bdev_desc = dev->cache_bdev_desc; } static void diff --git a/lib/ftl/ftl_core.h b/lib/ftl/ftl_core.h index 0c35006a9..e2c8a0440 100644 --- a/lib/ftl/ftl_core.h +++ b/lib/ftl/ftl_core.h @@ -43,6 +43,7 @@ #include "spdk_internal/log.h" #include "spdk/queue.h" #include "spdk/ftl.h" +#include "spdk/bdev.h" #include "ftl_ppa.h" #include "ftl_io.h" @@ -136,6 +137,8 @@ struct spdk_ftl_dev { struct spdk_nvme_ns *ns; /* NVMe transport ID */ struct spdk_nvme_transport_id trid; + /* Write buffer cache */ + struct spdk_bdev_desc *cache_bdev_desc; /* LBA map memory pool */ struct spdk_mempool *lba_pool; diff --git a/lib/ftl/ftl_init.c b/lib/ftl/ftl_init.c index 891ec9e7c..1bd36eaba 100644 --- a/lib/ftl/ftl_init.c +++ b/lib/ftl/ftl_init.c @@ -844,6 +844,8 @@ spdk_ftl_dev_init(const struct spdk_ftl_dev_init_opts *opts, spdk_ftl_init_fn cb dev->init_arg = cb_arg; dev->range = opts->range; dev->limit = SPDK_FTL_LIMIT_MAX; + dev->cache_bdev_desc = opts->cache_bdev_desc; + dev->name = strdup(opts->name); if (!dev->name) { SPDK_ERRLOG("Unable to set device name\n"); diff --git a/scripts/rpc.py b/scripts/rpc.py index 5e8ec45f2..2f6661ba9 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -1295,7 +1295,8 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse trtype=args.trtype, traddr=args.traddr, punits=args.punits, - uuid=args.uuid)) + uuid=args.uuid, + cache=args.cache)) p = subparsers.add_parser('construct_ftl_bdev', help='Add FTL bdev') @@ -1308,6 +1309,7 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse required=True) p.add_argument('-u', '--uuid', help='UUID of restored bdev (not applicable when creating new ' 'instance): e.g. b286d19a-0059-4709-abcd-9f7732b1567d (optional)') + p.add_argument('-c', '--cache', help='Name of the bdev to be used as a write buffer cache (optional)') p.set_defaults(func=construct_ftl_bdev) def delete_ftl_bdev(args): diff --git a/scripts/rpc/bdev.py b/scripts/rpc/bdev.py index 1334f459e..b702e1895 100644 --- a/scripts/rpc/bdev.py +++ b/scripts/rpc/bdev.py @@ -551,7 +551,7 @@ def destruct_split_vbdev(client, base_bdev): return client.call('destruct_split_vbdev', params) -def construct_ftl_bdev(client, name, trtype, traddr, punits, uuid=None): +def construct_ftl_bdev(client, name, trtype, traddr, punits, uuid=None, cache=None): """Construct FTL bdev Args: @@ -560,6 +560,7 @@ def construct_ftl_bdev(client, name, trtype, traddr, punits, uuid=None): traddr: transport address punit: parallel unit range uuid: UUID of the device + cache: name of the write buffer bdev """ params = {'name': name, 'trtype': trtype, @@ -567,6 +568,9 @@ def construct_ftl_bdev(client, name, trtype, traddr, punits, uuid=None): 'punits': punits} if uuid: params['uuid'] = uuid + if cache: + params['cache'] = cache + return client.call('construct_ftl_bdev', params)