diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index a0dfdd087..a1aca28bc 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -4742,6 +4742,7 @@ cache | Required | string | Name of the cache device uuid | Optional | string | UUID of restored bdev (not applicable when creating new instance) core_mask | Optional | string | CPU core(s) possible for placement of the ftl core thread, application main thread by default overprovisioning | Optional | int | Percentage of base device used for relocation, 20% by default +fast_shutdown | Optional | bool | When set FTL will minimize persisted data on target application shutdown and rely on shared memory during next load #### Result @@ -4796,6 +4797,7 @@ cache | Required | string | Name of the cache device uuid | Required | string | UUID of restored bdev core_mask | Optional | string | CPU core(s) possible for placement of the ftl core thread, application main thread by default overprovisioning | Optional | int | Percentage of base device used for relocation, 20% by default +fast_shutdown | Optional | bool | When set FTL will minimize persisted data on target application shutdown and rely on shared memory during next load #### Result @@ -4845,6 +4847,7 @@ This RPC is subject to change. Name | Optional | Type | Description ----------------------- | -------- | ----------- | ----------- name | Required | string | Bdev name +fast_shutdown | Optional | bool | When set FTL will minimize persisted data during deletion and rely on shared memory during next load #### Example @@ -4882,6 +4885,7 @@ This RPC is subject to change. Name | Optional | Type | Description ----------------------- | -------- | ----------- | ----------- name | Required | string | Bdev name +fast_shutdown | Optional | bool | When set FTL will minimize persisted data during deletion and rely on shared memory during next load #### Example diff --git a/include/spdk/ftl.h b/include/spdk/ftl.h index 79973f5d7..0ba1ce368 100644 --- a/include/spdk/ftl.h +++ b/include/spdk/ftl.h @@ -59,6 +59,8 @@ struct spdk_ftl_conf { /* Name of cache block device (must support extended metadata) */ char *cache_bdev; + /* Enable fast shutdown path */ + bool fast_shutdown; }; enum spdk_ftl_mode { @@ -205,6 +207,14 @@ int spdk_ftl_writev(struct spdk_ftl_dev *dev, struct ftl_io *io, struct spdk_io_ */ size_t spdk_ftl_io_size(void); +/** + * Enable fast shutdown. + * + * During fast shutdown FTL will keep the necessary metadata in shared memory instead + * of serializing it to storage. This allows for shorter downtime during upgrade process. + */ +void spdk_ftl_dev_set_fast_shutdown(struct spdk_ftl_dev *dev, bool fast_shutdown); + #ifdef __cplusplus } #endif diff --git a/lib/ftl/ftl_core.c b/lib/ftl/ftl_core.c index d725b061a..890b0fbb6 100644 --- a/lib/ftl/ftl_core.c +++ b/lib/ftl/ftl_core.c @@ -564,6 +564,13 @@ spdk_ftl_fini(void) spdk_free(g_ftl_read_buf); } +void +spdk_ftl_dev_set_fast_shutdown(struct spdk_ftl_dev *dev, bool fast_shutdown) +{ + assert(dev); + dev->conf.fast_shutdown = fast_shutdown; +} + struct spdk_io_channel * spdk_ftl_get_io_channel(struct spdk_ftl_dev *dev) { diff --git a/lib/ftl/mngt/ftl_mngt_md.c b/lib/ftl/mngt/ftl_mngt_md.c index ed79e99a7..9cf7e2c9d 100644 --- a/lib/ftl/mngt/ftl_mngt_md.c +++ b/lib/ftl/mngt/ftl_mngt_md.c @@ -84,7 +84,7 @@ ftl_mngt_deinit_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) for (i = 0; i < FTL_LAYOUT_REGION_TYPE_MAX; i++, region++) { if (layout->md[i]) { - ftl_md_destroy(layout->md[i]); + ftl_md_destroy(layout->md[i], ftl_md_destroy_region_flags(dev, layout->region[i].type)); layout->md[i] = NULL; } } @@ -238,7 +238,7 @@ ftl_mngt_md_deinit_vss_emu(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mn struct ftl_layout *layout = &dev->layout; if (layout->md[FTL_LAYOUT_REGION_TYPE_VSS]) { - ftl_md_destroy(layout->md[FTL_LAYOUT_REGION_TYPE_VSS]); + ftl_md_destroy(layout->md[FTL_LAYOUT_REGION_TYPE_VSS], 0); layout->md[FTL_LAYOUT_REGION_TYPE_VSS] = NULL; } @@ -314,16 +314,17 @@ ftl_mngt_superblock_deinit(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mn struct ftl_layout *layout = &dev->layout; if (layout->md[FTL_LAYOUT_REGION_TYPE_SB]) { - ftl_md_destroy(layout->md[FTL_LAYOUT_REGION_TYPE_SB]); + ftl_md_destroy(layout->md[FTL_LAYOUT_REGION_TYPE_SB], + ftl_md_destroy_region_flags(dev, FTL_LAYOUT_REGION_TYPE_SB)); layout->md[FTL_LAYOUT_REGION_TYPE_SB] = NULL; } if (layout->md[FTL_LAYOUT_REGION_TYPE_SB_BASE]) { - ftl_md_destroy(layout->md[FTL_LAYOUT_REGION_TYPE_SB_BASE]); + ftl_md_destroy(layout->md[FTL_LAYOUT_REGION_TYPE_SB_BASE], 0); layout->md[FTL_LAYOUT_REGION_TYPE_SB_BASE] = NULL; } - ftl_md_destroy(dev->sb_shm_md); + ftl_md_destroy(dev->sb_shm_md, ftl_md_destroy_shm_flags(dev)); dev->sb_shm_md = NULL; dev->sb_shm = NULL; diff --git a/lib/ftl/mngt/ftl_mngt_misc.c b/lib/ftl/mngt/ftl_mngt_misc.c index 858ecded5..b6c0870f3 100644 --- a/lib/ftl/mngt/ftl_mngt_misc.c +++ b/lib/ftl/mngt/ftl_mngt_misc.c @@ -85,7 +85,7 @@ ftl_mngt_deinit_mem_pools(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mng } if (dev->p2l_pool_md) { - ftl_md_destroy(dev->p2l_pool_md); + ftl_md_destroy(dev->p2l_pool_md, ftl_md_destroy_shm_flags(dev)); dev->p2l_pool_md = NULL; } diff --git a/lib/ftl/spdk_ftl.map b/lib/ftl/spdk_ftl.map index 5bc082e93..b1909de7d 100644 --- a/lib/ftl/spdk_ftl.map +++ b/lib/ftl/spdk_ftl.map @@ -15,6 +15,7 @@ spdk_ftl_io_size; spdk_ftl_readv; spdk_ftl_writev; + spdk_ftl_dev_set_fast_shutdown; local: *; }; diff --git a/lib/ftl/utils/ftl_conf.c b/lib/ftl/utils/ftl_conf.c index 9878f52c1..a1ee71185 100644 --- a/lib/ftl/utils/ftl_conf.c +++ b/lib/ftl/utils/ftl_conf.c @@ -23,6 +23,7 @@ static const struct spdk_ftl_conf g_default_conf = { .nv_cache = { .chunk_compaction_threshold = 80, }, + .fast_shutdown = true, }; void diff --git a/lib/ftl/utils/ftl_md.c b/lib/ftl/utils/ftl_md.c index 56e388932..5872c688f 100644 --- a/lib/ftl/utils/ftl_md.c +++ b/lib/ftl/utils/ftl_md.c @@ -229,7 +229,7 @@ err_shm: } static void -ftl_md_destroy_shm(struct ftl_md *md) +ftl_md_destroy_shm(struct ftl_md *md, int flags) { if (!md->data) { return; @@ -253,6 +253,11 @@ ftl_md_destroy_shm(struct ftl_md *md) md->data = NULL; md->vss_data = NULL; + /* If specified, keep the object in SHM */ + if (flags & FTL_MD_DESTROY_SHM_KEEP) { + return; + } + /* Otherwise destroy/unlink the object */ assert(md->name[0] != 0 && md->shm_unlink != NULL); md->shm_unlink(md->name); @@ -306,7 +311,7 @@ struct ftl_md *ftl_md_create(struct spdk_ftl_dev *dev, uint64_t blocks, return md; err: - ftl_md_destroy(md); + ftl_md_destroy(md, ftl_md_destroy_region_flags(dev, region->type)); return NULL; } @@ -327,13 +332,13 @@ ftl_md_unlink(struct spdk_ftl_dev *dev, const char *name, int flags) } void -ftl_md_destroy(struct ftl_md *md) +ftl_md_destroy(struct ftl_md *md, int flags) { if (!md) { return; } - ftl_md_free_buf(md); + ftl_md_free_buf(md, flags); spdk_free(md->entry_vss_dma_buf); @@ -342,16 +347,17 @@ ftl_md_destroy(struct ftl_md *md) } void -ftl_md_free_buf(struct ftl_md *md) +ftl_md_free_buf(struct ftl_md *md, int flags) { if (!md) { return; } if (md->shm_fd < 0) { + assert(flags == 0); ftl_md_destroy_heap(md); } else { - ftl_md_destroy_shm(md); + ftl_md_destroy_shm(md, flags); } } @@ -1124,6 +1130,24 @@ ftl_md_create_region_flags(struct spdk_ftl_dev *dev, int region_type) return flags; } +int +ftl_md_destroy_region_flags(struct spdk_ftl_dev *dev, int region_type) +{ + switch (region_type) { + case FTL_LAYOUT_REGION_TYPE_SB: + case FTL_LAYOUT_REGION_TYPE_BAND_MD: + case FTL_LAYOUT_REGION_TYPE_NVC_MD: + if (dev->conf.fast_shutdown) { + return FTL_MD_DESTROY_SHM_KEEP; + } + break; + + default: + break; + } + return 0; +} + int ftl_md_create_shm_flags(struct spdk_ftl_dev *dev) { @@ -1131,3 +1155,9 @@ ftl_md_create_shm_flags(struct spdk_ftl_dev *dev) return flags; } + +int +ftl_md_destroy_shm_flags(struct spdk_ftl_dev *dev) +{ + return (dev->conf.fast_shutdown) ? FTL_MD_DESTROY_SHM_KEEP : 0; +} diff --git a/lib/ftl/utils/ftl_md.h b/lib/ftl/utils/ftl_md.h index 2835dc27b..47f151d0e 100644 --- a/lib/ftl/utils/ftl_md.h +++ b/lib/ftl/utils/ftl_md.h @@ -175,19 +175,29 @@ struct ftl_md *ftl_md_create(struct spdk_ftl_dev *dev, uint64_t blocks, */ int ftl_md_unlink(struct spdk_ftl_dev *dev, const char *name, int flags); +/** + * FTL metadata destroy flags + */ +enum ftl_md_destroy_flags { + /** FTL metadata data buf will be kept in SHM */ + FTL_MD_DESTROY_SHM_KEEP = 0x1, +}; + /** * @brief Destroys metadata * * @param md Metadata to be destroyed + * @param flags Bit flags of type ftl_md_destroy_flags */ -void ftl_md_destroy(struct ftl_md *md); +void ftl_md_destroy(struct ftl_md *md, int flags); /** * @brief Free the data buf associated with the metadata * * @param md Metadata object + * @param flags Bit flags of type ftl_md_destroy_flags */ -void ftl_md_free_buf(struct ftl_md *md); +void ftl_md_free_buf(struct ftl_md *md, int flags); /** * @brief Sets the region of a device on which to perform IO when persisting, @@ -329,13 +339,26 @@ uint64_t ftl_md_xfer_blocks(struct spdk_ftl_dev *dev); * 1. superblock upon SPDK_FTL_MODE_CREATE flag set, * 2. other regions if not in a fast startup mode. * - * @param dev The FTL device + * @param dev The FTL device * @param region_type MD region type * * @return MD creation flags */ int ftl_md_create_region_flags(struct spdk_ftl_dev *dev, int region_type); +/** + * @brief Return the md destroy flags for a given md region type + * + * In a fast shutdown mode, returns FTL_MD_DESTROY_SHM_KEEP. + * Otherwise the SHM is unlinked. + * + * @param dev The FTL device + * @param region_type MD region type + * + * #return MD destroy flags + */ +int ftl_md_destroy_region_flags(struct spdk_ftl_dev *dev, int region_type); + /** * @brief Return the SHM-backed md creation flags * @@ -348,4 +371,15 @@ int ftl_md_create_region_flags(struct spdk_ftl_dev *dev, int region_type); */ int ftl_md_create_shm_flags(struct spdk_ftl_dev *dev); +/** + * @brief Return the md destroy flags + * + * In a fast shutdown mode, returns FTL_MD_DESTROY_SHM_KEEP. + * Otherwise the SHM is unlinked. + * + * @param dev The FTL device + * + * @return MD destroy flags + */ +int ftl_md_destroy_shm_flags(struct spdk_ftl_dev *dev); #endif /* FTL_MD_H */ diff --git a/module/bdev/ftl/bdev_ftl.c b/module/bdev/ftl/bdev_ftl.c index 2bac7ec9a..11e243864 100644 --- a/module/bdev/ftl/bdev_ftl.c +++ b/module/bdev/ftl/bdev_ftl.c @@ -213,6 +213,8 @@ bdev_ftl_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w spdk_uuid_fmt_lower(uuid, sizeof(uuid), &conf.uuid); spdk_json_write_named_string(w, "uuid", uuid); + spdk_json_write_named_bool(w, "fast_shutdown", conf.fast_shutdown); + spdk_json_write_named_string(w, "base_bdev", conf.base_bdev); if (conf.cache_bdev) { @@ -331,6 +333,9 @@ bdev_ftl_create_cb(struct spdk_ftl_dev *dev, void *ctx, int status) error: if (ftl_bdev->dev) { + /* Cleanup all FTL */ + spdk_ftl_dev_set_fast_shutdown(ftl_bdev->dev, false); + /* FTL was created, but we have got an error, so we need to delete it */ spdk_ftl_dev_free(dev, bdev_ftl_create_err_cleanup_cb, ftl_bdev); } else { @@ -430,15 +435,47 @@ bdev_ftl_initialize(void) return spdk_ftl_init(); } -void -bdev_ftl_delete_bdev(const char *name, spdk_bdev_unregister_cb cb_fn, void *cb_arg) +static void +bdev_ftl_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx) { +} + +void +bdev_ftl_delete_bdev(const char *name, bool fast_shutdown, spdk_bdev_unregister_cb cb_fn, + void *cb_arg) +{ + struct spdk_bdev_desc *ftl_bdev_desc; + struct spdk_bdev *bdev; + struct ftl_bdev *ftl; int rc; + rc = spdk_bdev_open_ext(name, false, bdev_ftl_event_cb, NULL, &ftl_bdev_desc); + + if (rc) { + goto not_found; + } + + bdev = spdk_bdev_desc_get_bdev(ftl_bdev_desc); + + if (bdev->module != &g_ftl_if) { + goto bdev_opened; + } + + ftl = bdev->ctxt; + assert(ftl); + spdk_ftl_dev_set_fast_shutdown(ftl->dev, fast_shutdown); + spdk_bdev_close(ftl_bdev_desc); + rc = spdk_bdev_unregister_by_name(name, &g_ftl_if, cb_fn, cb_arg); if (rc) { cb_fn(cb_arg, rc); } + + return; +bdev_opened: + spdk_bdev_close(ftl_bdev_desc); +not_found: + cb_fn(cb_arg, -ENODEV); } static void diff --git a/module/bdev/ftl/bdev_ftl.h b/module/bdev/ftl/bdev_ftl.h index db906d863..5f1529525 100644 --- a/module/bdev/ftl/bdev_ftl.h +++ b/module/bdev/ftl/bdev_ftl.h @@ -21,7 +21,8 @@ typedef void (*ftl_bdev_init_fn)(const struct ftl_bdev_info *, void *, int); typedef void (*ftl_bdev_thread_fn)(void *); int bdev_ftl_create_bdev(const struct spdk_ftl_conf *conf, ftl_bdev_init_fn cb, void *cb_arg); -void bdev_ftl_delete_bdev(const char *name, spdk_bdev_unregister_cb cb_fn, void *cb_arg); +void bdev_ftl_delete_bdev(const char *name, bool fast_shutdown, spdk_bdev_unregister_cb cb_fn, + void *cb_arg); int bdev_ftl_defer_init(const struct spdk_ftl_conf *conf); #endif /* SPDK_BDEV_FTL_H */ diff --git a/module/bdev/ftl/bdev_ftl_rpc.c b/module/bdev/ftl/bdev_ftl_rpc.c index 4929630df..125a62800 100644 --- a/module/bdev/ftl/bdev_ftl_rpc.c +++ b/module/bdev/ftl/bdev_ftl_rpc.c @@ -41,6 +41,10 @@ static const struct spdk_json_object_decoder rpc_bdev_ftl_create_decoders[] = { "core_mask", offsetof(struct spdk_ftl_conf, core_mask), spdk_json_decode_string, true }, + { + "fast_shutdown", offsetof(struct spdk_ftl_conf, fast_shutdown), + spdk_json_decode_bool, true + }, }; static void @@ -118,10 +122,15 @@ SPDK_RPC_REGISTER("bdev_ftl_load", rpc_bdev_ftl_load, SPDK_RPC_RUNTIME) struct rpc_delete_ftl { char *name; + bool fast_shutdown; }; static const struct spdk_json_object_decoder rpc_delete_ftl_decoders[] = { {"name", offsetof(struct rpc_delete_ftl, name), spdk_json_decode_string}, + { + "fast_shutdown", offsetof(struct rpc_delete_ftl, fast_shutdown), + spdk_json_decode_bool, true + }, }; static void @@ -146,7 +155,7 @@ rpc_bdev_ftl_delete(struct spdk_jsonrpc_request *request, goto invalid; } - bdev_ftl_delete_bdev(attrs.name, rpc_bdev_ftl_delete_cb, request); + bdev_ftl_delete_bdev(attrs.name, attrs.fast_shutdown, rpc_bdev_ftl_delete_cb, request); invalid: free(attrs.name); } diff --git a/python/spdk/rpc/bdev.py b/python/spdk/rpc/bdev.py index db77a232e..ea0eb7826 100644 --- a/python/spdk/rpc/bdev.py +++ b/python/spdk/rpc/bdev.py @@ -1378,7 +1378,7 @@ def bdev_ftl_load(client, name, base_bdev, **kwargs): return client.call('bdev_ftl_load', params) -def bdev_ftl_unload(client, name): +def bdev_ftl_unload(client, name, fast_shutdown): """Unload FTL bdev Args: @@ -1389,13 +1389,14 @@ def bdev_ftl_unload(client, name): return client.call('bdev_ftl_unload', params) -def bdev_ftl_delete(client, name): +def bdev_ftl_delete(client, name, fast_shutdown): """Delete FTL bdev Args: name: name of the bdev """ - params = {'name': name} + params = {'name': name, + 'fast_shutdown': fast_shutdown} return client.call('bdev_ftl_delete', params) diff --git a/scripts/rpc.py b/scripts/rpc.py index cd3309913..f71bcd103 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -2008,7 +2008,8 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse uuid=args.uuid, cache=args.cache, overprovisioning=args.overprovisioning, - core_mask=args.core_mask)) + core_mask=args.core_mask, + fast_shutdown=args.fast_shutdown)) p = subparsers.add_parser('bdev_ftl_create', help='Add FTL bdev') p.add_argument('-b', '--name', help="Name of the bdev", required=True) @@ -2022,6 +2023,7 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse ' to user (optional); default 20', type=int) p.add_argument('--core-mask', help='CPU core mask - which cores will be used for ftl core thread, ' 'by default core thread will be set to the main application core (optional)') + p.add_argument('-f', '--fast-shutdown', help="Enable fast shutdown", action='store_true') p.set_defaults(func=bdev_ftl_create) def bdev_ftl_load(args): @@ -2031,7 +2033,8 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse uuid=args.uuid, cache=args.cache, overprovisioning=args.overprovisioning, - core_mask=args.core_mask)) + core_mask=args.core_mask, + fast_shutdown=args.fast_shutdown)) p = subparsers.add_parser('bdev_ftl_load', help='Load FTL bdev') p.add_argument('-b', '--name', help="Name of the bdev", required=True) @@ -2044,6 +2047,7 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse ' to user (optional); default 20', type=int) p.add_argument('--core-mask', help='CPU core mask - which cores will be used for ftl core thread, ' 'by default core thread will be set to the main application core (optional)') + p.add_argument('-f', '--fast-shutdown', help="Enable fast shutdown", action='store_true') p.set_defaults(func=bdev_ftl_load) def bdev_ftl_unload(args): @@ -2051,13 +2055,15 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse p = subparsers.add_parser('bdev_ftl_unload', help='Unload FTL bdev') p.add_argument('-b', '--name', help="Name of the bdev", required=True) + p.add_argument('-f', '--fast-shutdown', help="Fast shutdown", action='store_true') p.set_defaults(func=bdev_ftl_unload) def bdev_ftl_delete(args): - print_dict(rpc.bdev.bdev_ftl_delete(args.client, name=args.name)) + print_dict(rpc.bdev.bdev_ftl_delete(args.client, name=args.name, fast_shutdown=args.fast_shutdown)) p = subparsers.add_parser('bdev_ftl_delete', help='Delete FTL bdev') p.add_argument('-b', '--name', help="Name of the bdev", required=True) + p.add_argument('-f', '--fast-shutdown', help="Fast shutdown", action='store_true') p.set_defaults(func=bdev_ftl_delete) # vmd