diff --git a/CHANGELOG.md b/CHANGELOG.md index bb55cba7d..49fd90f29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ ## v23.05: (Upcoming Release) +### bdev + +A new API `spdk_bdev_module_claim_bdev_desc` was added. Unlike `spdk_bdev_module_claim_bdev`, this +function requires a bdev descriptor to be passed and the claim is automatically released when the +descriptor is closed. It allows bdev modules to claim bdevs as a single writer, multiple writers, or +multiple readers. + ### env New function `spdk_env_get_main_core` was added. diff --git a/doc/bdev_module.md b/doc/bdev_module.md index 3234e8cb3..666fc1b75 100644 --- a/doc/bdev_module.md +++ b/doc/bdev_module.md @@ -155,21 +155,42 @@ bdevs, but take the one additional step of claiming the bdev. The module can open the underlying bdevs it wishes to route I/O to using spdk_bdev_open_ext(), where the string name is provided by the user via an RPC. -This will return a bdev descriptor that may be used with future calls to -spdk_bdev_module_claim_bdev() and spdk_bdev_get_io_channel(). The bdev structure -required by spdk_bdev_module_claim_bdev() may be obtained with -spdk_bdev_module_open_ext() followed by spdk_bdev_desc_get_bdev(). +To ensure that other consumers do not modify the underlying bdev in an unexpected +way, the virtual bdev should take a claim on the underlying bdev before +reading from or writing to the underlying bdev. + +There are two slightly different APIs for taking and releasing claims. The +preferred interface uses `spdk_bdev_module_claim_bdev_desc()`. This method allows +claims that ensure there is a single writer with +`SPDK_BDEV_CLAIM_READ_WRITE_ONCE`, cooperating shared writers with +`SPDK_BDEV_CLAIM_READ_WRITE_MANY`, and shared readers that prevent any +writers with `SPDK_BDEV_CLAIM_READ_ONLY_MANY`. In all cases, +`spdk_bdev_open_ext()` may be used to open the underlying bdev read-only. If a +read-only bdev descriptor successfully claims a bdev with +`SPDK_BDEV_CLAIM_READ_WRITE_ONCE` or `SPDK_BDEV_CLAIM_READ_WRITE_MANY` +the bdev descriptor is promoted to read-write. +Any claim that is obtained with `spdk_bdev_module_claim_bdev_desc()` is +automatically released upon closing the bdev descriptor used to obtain the +claim. Shared claims continue to block new incompatible claims and new writers +until the last claim is released. + +The non-preferred interface for obtaining a claim allows the caller to obtain +an exclusive writer claim with `spdk_bdev_module_claim_bdev()`. It may be +be released with `spdk_bdev_module_release_bdev()`. If a read-only bdev +descriptor is passed, it is promoted to read-write. NULL may be passed instead +of a bdev descriptor to avoid promotion and to block new writers. New code +should use `spdk_bdev_module_claim_bdev_desc()` with the claim type that is +tailored to the virtual bdev's needs. The descriptor obtained from the successful spdk_bdev_open_ext() may be used with spdk_bdev_get_io_channel() to obtain I/O channels for the bdev. This is likely done in response to the virtual bdev's `get_io_channel` callback. -Channels may be obtained before and/or after claiming the underlying -bdev. Claims are described below. +Channels may be obtained before and/or after claiming the underlying bdev, but +beware there may be other unknown writers until the underlying bdev has been +claimed. -The final step is to have the module use its open descriptor to call -spdk_bdev_module_claim_bdev(), indicating that it is consuming the underlying -bdev. This prevents other users from opening descriptors with write permissions. -This effectively 'promotes' the descriptor to write-exclusive and is an -operation only available to bdev modules. If a virtual bdev module wishes to -prevent a descriptor from being upgraded to read-write, -spdk_bdev_module_claim_bdev() may be called with a NULL descriptor. +When a virtual bdev module claims an underlying bdev from its `examine_config` +callback, it causes the `examine_disk` callback to only be called for this +module and any others that establish a shared claim. If no claims are taken by +`examine_config` callbacks, all virtual bdevs' `examine_disk` callbacks are +called. diff --git a/include/spdk/bdev_module.h b/include/spdk/bdev_module.h index 8101ebe60..d350d29fd 100644 --- a/include/spdk/bdev_module.h +++ b/include/spdk/bdev_module.h @@ -28,6 +28,8 @@ extern "C" { #endif +#define SPDK_BDEV_CLAIM_NAME_LEN 32 + /* This parameter is best defined for bdevs that share an underlying bdev, * such as multiple lvol bdevs sharing an nvme device, to avoid unnecessarily * resetting the underlying bdev and affecting other bdevs that are sharing it. */ @@ -167,11 +169,94 @@ enum spdk_bdev_claim_type { SPDK_BDEV_CLAIM_NONE = 0, /** - * Exclusive writer. + * Exclusive writer, with allowances for legacy behavior. This matches the behavior of + * `spdk_bdev_module_claim_bdev()` as of SPDK 22.09. New consumer should use + * SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE instead. */ - SPDK_BDEV_CLAIM_EXCL_WRITE + SPDK_BDEV_CLAIM_EXCL_WRITE, + + /** + * The descriptor passed with this claim request is the only writer. Other claimless readers + * are allowed. + */ + SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, + + /** + * Any number of readers, no writers. Readers without a claim are allowed. + */ + SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE, + + /** + * Any number of writers with matching shared_claim_key. After the first writer establishes + * a claim, future aspiring writers should open read-only and pass the read-only descriptor. + * If the shared claim is granted to the aspiring writer, the descriptor will be upgraded to + * read-write. + */ + SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED }; +/** Options used when requesting a claim. */ +struct spdk_bdev_claim_opts { + /* Size of this structure in bytes */ + size_t opts_size; + /** + * An arbitrary name for the claim. If set, it should be a string suitable for printing in + * error messages. Must be '\0' terminated. + */ + char name[SPDK_BDEV_CLAIM_NAME_LEN]; + /** + * Used with SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED claims. Any non-zero value is considered + * a key. + */ + uint64_t shared_claim_key; +}; +SPDK_STATIC_ASSERT(sizeof(struct spdk_bdev_claim_opts) == 48, "Incorrect size"); + +/** + * Retrieve the name of the bdev module claim type. The mapping between claim types and their names + * is: + * + * SPDK_BDEV_CLAIM_NONE "not_claimed" + * SPDK_BDEV_CLAIM_EXCL_WRITE "exclusive_write" + * SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE "read_many_write_one" + * SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE "read_many_write_none" + * SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED "read_many_write_shared" + * + * Any other value will return "invalid_claim". + * + * \param claim_type The claim type. + * \return A string that describes the claim type. + */ +const char *spdk_bdev_claim_get_name(enum spdk_bdev_claim_type claim_type); + +/** + * Initialize bdev module claim options structure. + * + * \param opts The structure to initialize. + * \param size The size of *opts. + */ +void spdk_bdev_claim_opts_init(struct spdk_bdev_claim_opts *opts, size_t size); + +/** + * Claim the bdev referenced by the open descriptor. The claim is released as the descriptor is + * closed. + * + * \param desc An open bdev descriptor. Some claim types may upgrade this from read-only to + * read-write. + * \param type The type of claim to establish. + * \param opts NULL or options required by the particular claim type. + * \param module The bdev module making this claim. + * \return 0 on success + * \return -ENOMEM if insufficient memory to track the claim + * \return -EBUSY if the claim cannot be granted due to a conflict + * \return -EINVAL if the claim type required options that were not passed or required parameters + * were NULL. + */ +int spdk_bdev_module_claim_bdev_desc(struct spdk_bdev_desc *desc, + enum spdk_bdev_claim_type type, + struct spdk_bdev_claim_opts *opts, + struct spdk_bdev_module *module); + /** * Called by a bdev module to lay exclusive claim to a bdev. * @@ -311,6 +396,13 @@ struct spdk_bdev_alias { TAILQ_ENTRY(spdk_bdev_alias) tailq; }; +struct spdk_bdev_module_claim { + struct spdk_bdev_module *module; + struct spdk_bdev_desc *desc; + char name[SPDK_BDEV_CLAIM_NAME_LEN]; + TAILQ_ENTRY(spdk_bdev_module_claim) link; +}; + typedef TAILQ_HEAD(, spdk_bdev_io) bdev_io_tailq_t; typedef STAILQ_HEAD(, spdk_bdev_io) bdev_io_stailq_t; typedef TAILQ_HEAD(, lba_range) lba_range_tailq_t; @@ -541,14 +633,23 @@ struct spdk_bdev { /** Which module has claimed this bdev. Must hold spinlock on all updates. */ union __bdev_internal_claim { + /** Claims acquired with spdk_bdev_module_claim_bdev() */ struct __bdev_internal_claim_v1 { /** * Pointer to the module that has claimed this bdev for purposes of - * creating virtual bdevs on top of it. Set to NULL if the bdev has - * not been claimed. + * creating virtual bdevs on top of it. Set to NULL and set + * claim_type to SPDK_BDEV_CLAIM_NONE if the bdev has not been + * claimed. */ struct spdk_bdev_module *module; } v1; + /** Claims acquired with spdk_bdev_module_claim_bdev_desc() */ + struct __bdev_internal_claim_v2 { + /** The claims on this bdev */ + TAILQ_HEAD(v2_claims, spdk_bdev_module_claim) claims; + /** See spdk_bdev_claim_opts.shared_claim_key */ + uint64_t key; + } v2; } claim; /** Callback function that will be called after bdev destruct is completed. */ diff --git a/lib/bdev/bdev.c b/lib/bdev/bdev.c index 23ad70388..d7cf8ae23 100644 --- a/lib/bdev/bdev.c +++ b/lib/bdev/bdev.c @@ -331,6 +331,7 @@ struct spdk_bdev_desc { spdk_bdev_io_timeout_cb cb_fn; void *cb_arg; struct spdk_poller *io_timeout_poller; + struct spdk_bdev_module_claim *claim; }; struct spdk_bdev_iostat_ctx { @@ -393,6 +394,9 @@ static inline void bdev_io_complete(void *ctx); static bool bdev_abort_queued_io(bdev_io_tailq_t *queue, struct spdk_bdev_io *bio_to_abort); static bool bdev_abort_buf_io(struct spdk_bdev_mgmt_channel *ch, struct spdk_bdev_io *bio_to_abort); +static bool claim_type_is_v2(enum spdk_bdev_claim_type type); +static void bdev_desc_release_claims(struct spdk_bdev_desc *desc); + void spdk_bdev_get_opts(struct spdk_bdev_opts *opts, size_t opts_size) { @@ -1840,8 +1844,7 @@ bdev_finish_unregister_bdevs_iter(void *cb_arg, int bdeverrno) bdev; bdev = TAILQ_PREV(bdev, spdk_bdev_list, internal.link)) { spdk_spin_lock(&bdev->internal.spinlock); if (bdev->internal.claim_type != SPDK_BDEV_CLAIM_NONE) { - SPDK_DEBUGLOG(bdev, "Skipping claimed bdev '%s'(<-'%s').\n", - bdev->name, bdev->internal.claim.v1.module->name); + LOG_ALREADY_CLAIMED_DEBUG("claimed, skipping", bdev); spdk_spin_unlock(&bdev->internal.spinlock); continue; } @@ -7060,12 +7063,14 @@ log_already_claimed(enum spdk_log_level level, const int line, const char *func, const char *typename, *modname; extern struct spdk_log_flag SPDK_LOG_bdev; + assert(spdk_spin_held(&bdev->internal.spinlock)); + if (level >= SPDK_LOG_INFO && !SPDK_LOG_bdev.enabled) { return; } type = bdev->internal.claim_type; - typename = "exclusive_write"; + typename = spdk_bdev_claim_get_name(type); if (type == SPDK_BDEV_CLAIM_EXCL_WRITE) { modname = bdev->internal.claim.v1.module->name; @@ -7074,6 +7079,17 @@ log_already_claimed(enum spdk_log_level level, const int line, const char *func, return; } + if (claim_type_is_v2(type)) { + struct spdk_bdev_module_claim *claim; + + TAILQ_FOREACH(claim, &bdev->internal.claim.v2.claims, link) { + modname = claim->module->name; + spdk_log(level, __FILE__, line, func, "bdev %s %s: type %s by module %s\n", + bdev->name, detail, typename, modname); + } + return; + } + assert(false); } @@ -7218,6 +7234,10 @@ bdev_close(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc) desc->closed = true; + if (desc->claim != NULL) { + bdev_desc_release_claims(desc); + } + if (0 == desc->refs) { spdk_spin_unlock(&desc->spinlock); bdev_desc_free(desc); @@ -7361,6 +7381,373 @@ spdk_bdev_module_release_bdev(struct spdk_bdev *bdev) spdk_spin_unlock(&bdev->internal.spinlock); } +/* + * Start claims v2 + */ + +const char * +spdk_bdev_claim_get_name(enum spdk_bdev_claim_type type) +{ + switch (type) { + case SPDK_BDEV_CLAIM_NONE: + return "not_claimed"; + case SPDK_BDEV_CLAIM_EXCL_WRITE: + return "exclusive_write"; + case SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE: + return "read_many_write_one"; + case SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE: + return "read_many_write_none"; + case SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED: + return "read_many_write_many"; + default: + break; + } + return "invalid_claim"; +} + +static bool +claim_type_is_v2(enum spdk_bdev_claim_type type) +{ + switch (type) { + case SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE: + case SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE: + case SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED: + return true; + default: + break; + } + return false; +} + +/* Returns true if taking a claim with desc->write == false should make the descriptor writable. */ +static bool +claim_type_promotes_to_write(enum spdk_bdev_claim_type type) +{ + switch (type) { + case SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE: + case SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED: + return true; + default: + break; + } + return false; +} + +void +spdk_bdev_claim_opts_init(struct spdk_bdev_claim_opts *opts, size_t size) +{ + if (opts == NULL) { + SPDK_ERRLOG("opts should not be NULL\n"); + assert(opts != NULL); + return; + } + if (size == 0) { + SPDK_ERRLOG("size should not be zero\n"); + assert(size != 0); + return; + } + + memset(opts, 0, size); + opts->opts_size = size; + +#define FIELD_OK(field) \ + offsetof(struct spdk_bdev_claim_opts, field) + sizeof(opts->field) <= size + +#define SET_FIELD(field, value) \ + if (FIELD_OK(field)) { \ + opts->field = value; \ + } \ + + SET_FIELD(shared_claim_key, 0); + +#undef FIELD_OK +#undef SET_FIELD +} + +static int +claim_opts_copy(struct spdk_bdev_claim_opts *src, struct spdk_bdev_claim_opts *dst) +{ + if (src->opts_size == 0) { + SPDK_ERRLOG("size should not be zero\n"); + return -1; + } + + memset(dst, 0, sizeof(*dst)); + dst->opts_size = src->opts_size; + +#define FIELD_OK(field) \ + offsetof(struct spdk_bdev_claim_opts, field) + sizeof(src->field) <= src->opts_size + +#define SET_FIELD(field) \ + if (FIELD_OK(field)) { \ + dst->field = src->field; \ + } \ + + if (FIELD_OK(name)) { + snprintf(dst->name, sizeof(dst->name), "%s", src->name); + } + + SET_FIELD(shared_claim_key); + + /* You should not remove this statement, but need to update the assert statement + * if you add a new field, and also add a corresponding SET_FIELD statement */ + SPDK_STATIC_ASSERT(sizeof(struct spdk_bdev_claim_opts) == 48, "Incorrect size"); + +#undef FIELD_OK +#undef SET_FIELD + return 0; +} + +/* Returns 0 if a read-write-once claim can be taken. */ +static int +claim_verify_rwo(struct spdk_bdev_desc *desc, enum spdk_bdev_claim_type type, + struct spdk_bdev_claim_opts *opts, struct spdk_bdev_module *module) +{ + struct spdk_bdev *bdev = desc->bdev; + struct spdk_bdev_desc *open_desc; + + assert(spdk_spin_held(&bdev->internal.spinlock)); + assert(type == SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE); + + if (opts->shared_claim_key != 0) { + SPDK_ERRLOG("%s: key option not supported with read-write-once claims\n", + bdev->name); + return -EINVAL; + } + if (bdev->internal.claim_type != SPDK_BDEV_CLAIM_NONE) { + LOG_ALREADY_CLAIMED_ERROR("already claimed", bdev); + return -EPERM; + } + if (desc->claim != NULL) { + SPDK_NOTICELOG("%s: descriptor already claimed bdev with module %s\n", + bdev->name, desc->claim->module->name); + return -EPERM; + } + TAILQ_FOREACH(open_desc, &bdev->internal.open_descs, link) { + if (desc != open_desc && open_desc->write) { + SPDK_NOTICELOG("%s: Cannot obtain read-write-once claim while " + "another descriptor is open for writing\n", + bdev->name); + return -EPERM; + } + } + + return 0; +} + +/* Returns 0 if a read-only-many claim can be taken. */ +static int +claim_verify_rom(struct spdk_bdev_desc *desc, enum spdk_bdev_claim_type type, + struct spdk_bdev_claim_opts *opts, struct spdk_bdev_module *module) +{ + struct spdk_bdev *bdev = desc->bdev; + struct spdk_bdev_desc *open_desc; + + assert(spdk_spin_held(&bdev->internal.spinlock)); + assert(type == SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE); + assert(desc->claim == NULL); + + if (desc->write) { + SPDK_ERRLOG("%s: Cannot obtain read-only-many claim with writable descriptor\n", + bdev->name); + return -EINVAL; + } + if (opts->shared_claim_key != 0) { + SPDK_ERRLOG("%s: key option not supported with read-only-may claims\n", bdev->name); + return -EINVAL; + } + if (bdev->internal.claim_type == SPDK_BDEV_CLAIM_NONE) { + TAILQ_FOREACH(open_desc, &bdev->internal.open_descs, link) { + if (open_desc->write) { + SPDK_NOTICELOG("%s: Cannot obtain read-only-many claim while " + "another descriptor is open for writing\n", + bdev->name); + return -EPERM; + } + } + } + + return 0; +} + +/* Returns 0 if a read-write-many claim can be taken. */ +static int +claim_verify_rwm(struct spdk_bdev_desc *desc, enum spdk_bdev_claim_type type, + struct spdk_bdev_claim_opts *opts, struct spdk_bdev_module *module) +{ + struct spdk_bdev *bdev = desc->bdev; + struct spdk_bdev_desc *open_desc; + + assert(spdk_spin_held(&bdev->internal.spinlock)); + assert(type == SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED); + assert(desc->claim == NULL); + + if (opts->shared_claim_key == 0) { + SPDK_ERRLOG("%s: shared_claim_key option required with read-write-may claims\n", + bdev->name); + return -EINVAL; + } + switch (bdev->internal.claim_type) { + case SPDK_BDEV_CLAIM_NONE: + TAILQ_FOREACH(open_desc, &bdev->internal.open_descs, link) { + if (open_desc == desc) { + continue; + } + if (open_desc->write) { + SPDK_NOTICELOG("%s: Cannot obtain read-write-many claim while " + "another descriptor is open for writing without a " + "claim\n", bdev->name); + return -EPERM; + } + } + break; + case SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED: + if (opts->shared_claim_key != bdev->internal.claim.v2.key) { + LOG_ALREADY_CLAIMED_ERROR("already claimed with another key", bdev); + return -EPERM; + } + break; + default: + LOG_ALREADY_CLAIMED_ERROR("already claimed", bdev); + return -EBUSY; + } + + return 0; +} + +/* Updates desc and its bdev with a v2 claim. */ +static int +claim_bdev(struct spdk_bdev_desc *desc, enum spdk_bdev_claim_type type, + struct spdk_bdev_claim_opts *opts, struct spdk_bdev_module *module) +{ + struct spdk_bdev *bdev = desc->bdev; + struct spdk_bdev_module_claim *claim; + + assert(spdk_spin_held(&bdev->internal.spinlock)); + assert(claim_type_is_v2(type)); + assert(desc->claim == NULL); + + claim = calloc(1, sizeof(*desc->claim)); + if (claim == NULL) { + SPDK_ERRLOG("%s: out of memory while allocating claim\n", bdev->name); + return -ENOMEM; + } + claim->module = module; + claim->desc = desc; + SPDK_STATIC_ASSERT(sizeof(claim->name) == sizeof(opts->name), "sizes must match"); + memcpy(claim->name, opts->name, sizeof(claim->name)); + desc->claim = claim; + + if (bdev->internal.claim_type == SPDK_BDEV_CLAIM_NONE) { + bdev->internal.claim_type = type; + TAILQ_INIT(&bdev->internal.claim.v2.claims); + bdev->internal.claim.v2.key = opts->shared_claim_key; + } + assert(type == bdev->internal.claim_type); + + TAILQ_INSERT_TAIL(&bdev->internal.claim.v2.claims, claim, link); + + if (!desc->write && claim_type_promotes_to_write(type)) { + desc->write = true; + } + + return 0; +} + +int +spdk_bdev_module_claim_bdev_desc(struct spdk_bdev_desc *desc, enum spdk_bdev_claim_type type, + struct spdk_bdev_claim_opts *_opts, + struct spdk_bdev_module *module) +{ + struct spdk_bdev *bdev = desc->bdev; + struct spdk_bdev_claim_opts opts; + int rc = 0; + + if (desc == NULL) { + SPDK_ERRLOG("descriptor must not be NULL\n"); + return -EINVAL; + } + + if (_opts == NULL) { + spdk_bdev_claim_opts_init(&opts, sizeof(opts)); + } else if (claim_opts_copy(_opts, &opts) != 0) { + return -EINVAL; + } + + spdk_spin_lock(&bdev->internal.spinlock); + + if (bdev->internal.claim_type != SPDK_BDEV_CLAIM_NONE && + bdev->internal.claim_type != type) { + LOG_ALREADY_CLAIMED_ERROR("already claimed", bdev); + spdk_spin_unlock(&bdev->internal.spinlock); + return -EPERM; + } + + if (claim_type_is_v2(type) && desc->claim != NULL) { + SPDK_ERRLOG("%s: descriptor already has %s claim with name '%s'\n", + bdev->name, spdk_bdev_claim_get_name(type), desc->claim->name); + spdk_spin_unlock(&bdev->internal.spinlock); + return -EPERM; + } + + switch (type) { + case SPDK_BDEV_CLAIM_EXCL_WRITE: + spdk_spin_unlock(&bdev->internal.spinlock); + return spdk_bdev_module_claim_bdev(bdev, desc, module); + case SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE: + rc = claim_verify_rwo(desc, type, &opts, module); + break; + case SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE: + rc = claim_verify_rom(desc, type, &opts, module); + break; + case SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED: + rc = claim_verify_rwm(desc, type, &opts, module); + break; + default: + SPDK_ERRLOG("%s: claim type %d not supported\n", bdev->name, type); + rc = -ENOTSUP; + } + + if (rc == 0) { + rc = claim_bdev(desc, type, &opts, module); + } + + spdk_spin_unlock(&bdev->internal.spinlock); + return rc; +} + +static void +claim_reset(struct spdk_bdev *bdev) +{ + assert(spdk_spin_held(&bdev->internal.spinlock)); + assert(claim_type_is_v2(bdev->internal.claim_type)); + assert(TAILQ_EMPTY(&bdev->internal.claim.v2.claims)); + + memset(&bdev->internal.claim, 0, sizeof(bdev->internal.claim)); + bdev->internal.claim_type = SPDK_BDEV_CLAIM_NONE; +} + +static void +bdev_desc_release_claims(struct spdk_bdev_desc *desc) +{ + struct spdk_bdev *bdev = desc->bdev; + + assert(spdk_spin_held(&bdev->internal.spinlock)); + assert(claim_type_is_v2(bdev->internal.claim_type)); + + TAILQ_REMOVE(&bdev->internal.claim.v2.claims, desc->claim, link); + free(desc->claim); + desc->claim = NULL; + + if (TAILQ_EMPTY(&bdev->internal.claim.v2.claims)) { + claim_reset(bdev); + } +} + +/* + * End claims v2 + */ + struct spdk_bdev * spdk_bdev_desc_get_bdev(struct spdk_bdev_desc *desc) { diff --git a/lib/bdev/bdev_rpc.c b/lib/bdev/bdev_rpc.c index a718c856f..dfac33d4f 100644 --- a/lib/bdev/bdev_rpc.c +++ b/lib/bdev/bdev_rpc.c @@ -716,6 +716,10 @@ rpc_dump_bdev_info(void *ctx, struct spdk_bdev *bdev) spdk_json_write_named_bool(w, "claimed", (bdev->internal.claim_type != SPDK_BDEV_CLAIM_NONE)); + if (bdev->internal.claim_type != SPDK_BDEV_CLAIM_NONE) { + spdk_json_write_named_string(w, "claim_type", + spdk_bdev_claim_get_name(bdev->internal.claim_type)); + } spdk_json_write_named_bool(w, "zoned", bdev->zoned); if (bdev->zoned) { diff --git a/lib/bdev/spdk_bdev.map b/lib/bdev/spdk_bdev.map index f40a35f66..c836246a3 100644 --- a/lib/bdev/spdk_bdev.map +++ b/lib/bdev/spdk_bdev.map @@ -120,6 +120,9 @@ spdk_bdev_module_fini_start_done; spdk_bdev_module_claim_bdev; spdk_bdev_module_release_bdev; + spdk_bdev_claim_get_name; + spdk_bdev_claim_opts_init; + spdk_bdev_module_claim_bdev_desc; spdk_bdev_alias_add; spdk_bdev_alias_del; spdk_bdev_alias_del_all; diff --git a/test/unit/lib/bdev/bdev.c/bdev_ut.c b/test/unit/lib/bdev/bdev.c/bdev_ut.c index 25bd02f2a..775263efd 100644 --- a/test/unit/lib/bdev/bdev.c/bdev_ut.c +++ b/test/unit/lib/bdev/bdev.c/bdev_ut.c @@ -6216,7 +6216,7 @@ bdev_copy_split_test(void) } static void -examine_claim(struct spdk_bdev *bdev) +examine_claim_v1(struct spdk_bdev *bdev) { int rc; @@ -6249,7 +6249,7 @@ examine_locks(void) /* Exercise the other path that is taken when examine_config() takes a claim. */ memset(&ctx, 0, sizeof(ctx)); - ctx.examine_config = examine_claim; + ctx.examine_config = examine_claim_v1; ctx.examine_disk = examine_no_lock_held; bdev = allocate_bdev_ctx("bdev0", &ctx); CU_ASSERT(ctx.examine_config_count == 1); @@ -6262,6 +6262,553 @@ examine_locks(void) free_bdev(bdev); } +#define UT_ASSERT_CLAIM_V2_COUNT(bdev, expect) \ + do { \ + uint32_t len = 0; \ + struct spdk_bdev_module_claim *claim; \ + TAILQ_FOREACH(claim, &bdev->internal.claim.v2.claims, link) { \ + len++; \ + } \ + CU_ASSERT(len == expect); \ + } while (0) + +static void +claim_v2_rwo(void) +{ + struct spdk_bdev *bdev; + struct spdk_bdev_desc *desc; + struct spdk_bdev_desc *desc2; + struct spdk_bdev_claim_opts opts; + int rc; + + bdev = allocate_bdev("bdev0"); + + /* Claim without options */ + desc = NULL; + rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); + CU_ASSERT(rc == 0); + SPDK_CU_ASSERT_FATAL(desc != NULL); + rc = spdk_bdev_module_claim_bdev_desc(desc, SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, NULL, + &bdev_ut_if); + CU_ASSERT(rc == 0); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE); + CU_ASSERT(desc->claim != NULL); + CU_ASSERT(desc->claim->module == &bdev_ut_if); + CU_ASSERT(strcmp(desc->claim->name, "") == 0); + CU_ASSERT(TAILQ_FIRST(&bdev->internal.claim.v2.claims) == desc->claim); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 1); + + /* Release the claim by closing the descriptor */ + spdk_bdev_close(desc); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_NONE); + CU_ASSERT(TAILQ_EMPTY(&bdev->internal.open_descs)); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 0); + + /* Claim with options */ + spdk_bdev_claim_opts_init(&opts, sizeof(opts)); + snprintf(opts.name, sizeof(opts.name), "%s", "claim with options"); + desc = NULL; + rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); + CU_ASSERT(rc == 0); + SPDK_CU_ASSERT_FATAL(desc != NULL); + rc = spdk_bdev_module_claim_bdev_desc(desc, SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, &opts, + &bdev_ut_if); + CU_ASSERT(rc == 0); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE); + CU_ASSERT(desc->claim != NULL); + CU_ASSERT(desc->claim->module == &bdev_ut_if); + CU_ASSERT(strcmp(desc->claim->name, "claim with options") == 0); + memset(&opts, 0, sizeof(opts)); + CU_ASSERT(strcmp(desc->claim->name, "claim with options") == 0); + CU_ASSERT(TAILQ_FIRST(&bdev->internal.claim.v2.claims) == desc->claim); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 1); + + /* The claim blocks new writers. */ + desc2 = NULL; + rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc2); + CU_ASSERT(rc == -EPERM); + CU_ASSERT(desc2 == NULL); + + /* New readers are allowed */ + desc2 = NULL; + rc = spdk_bdev_open_ext("bdev0", false, bdev_ut_event_cb, NULL, &desc2); + CU_ASSERT(rc == 0); + CU_ASSERT(desc2 != NULL); + CU_ASSERT(!desc2->write); + + /* No new v2 RWO claims are allowed */ + rc = spdk_bdev_module_claim_bdev_desc(desc2, SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, NULL, + &bdev_ut_if); + CU_ASSERT(rc == -EPERM); + + /* No new v2 ROM claims are allowed */ + CU_ASSERT(!desc2->write); + rc = spdk_bdev_module_claim_bdev_desc(desc2, SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE, NULL, + &bdev_ut_if); + CU_ASSERT(rc == -EPERM); + CU_ASSERT(!desc2->write); + + /* No new v2 RWM claims are allowed */ + spdk_bdev_claim_opts_init(&opts, sizeof(opts)); + opts.shared_claim_key = (uint64_t)&opts; + rc = spdk_bdev_module_claim_bdev_desc(desc2, SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED, &opts, + &bdev_ut_if); + CU_ASSERT(rc == -EPERM); + CU_ASSERT(!desc2->write); + + /* No new v1 claims are allowed */ + rc = spdk_bdev_module_claim_bdev(bdev, NULL, &bdev_ut_if); + CU_ASSERT(rc == -EPERM); + + /* None of the above changed the existing claim */ + CU_ASSERT(TAILQ_FIRST(&bdev->internal.claim.v2.claims) == desc->claim); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 1); + + /* Closing the first descriptor now allows a new claim and it is promoted to rw. */ + spdk_bdev_close(desc); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_NONE); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 0); + CU_ASSERT(!desc2->write); + rc = spdk_bdev_module_claim_bdev_desc(desc2, SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, NULL, + &bdev_ut_if); + CU_ASSERT(rc == 0); + CU_ASSERT(desc2->claim != NULL); + CU_ASSERT(desc2->write); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE); + CU_ASSERT(TAILQ_FIRST(&bdev->internal.claim.v2.claims) == desc2->claim); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 1); + spdk_bdev_close(desc2); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_NONE); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 0); + + /* Cannot claim with a key */ + desc = NULL; + rc = spdk_bdev_open_ext("bdev0", false, bdev_ut_event_cb, NULL, &desc); + CU_ASSERT(rc == 0); + SPDK_CU_ASSERT_FATAL(desc != NULL); + spdk_bdev_claim_opts_init(&opts, sizeof(opts)); + opts.shared_claim_key = (uint64_t)&opts; + rc = spdk_bdev_module_claim_bdev_desc(desc, SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, &opts, + &bdev_ut_if); + CU_ASSERT(rc == -EINVAL); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_NONE); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 0); + spdk_bdev_close(desc); + + /* Clean up */ + free_bdev(bdev); +} + +static void +claim_v2_rom(void) +{ + struct spdk_bdev *bdev; + struct spdk_bdev_desc *desc; + struct spdk_bdev_desc *desc2; + struct spdk_bdev_claim_opts opts; + int rc; + + bdev = allocate_bdev("bdev0"); + + /* Claim without options */ + desc = NULL; + rc = spdk_bdev_open_ext("bdev0", false, bdev_ut_event_cb, NULL, &desc); + CU_ASSERT(rc == 0); + SPDK_CU_ASSERT_FATAL(desc != NULL); + rc = spdk_bdev_module_claim_bdev_desc(desc, SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE, NULL, + &bdev_ut_if); + CU_ASSERT(rc == 0); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE); + CU_ASSERT(desc->claim != NULL); + CU_ASSERT(desc->claim->module == &bdev_ut_if); + CU_ASSERT(strcmp(desc->claim->name, "") == 0); + CU_ASSERT(TAILQ_FIRST(&bdev->internal.claim.v2.claims) == desc->claim); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 1); + + /* Release the claim by closing the descriptor */ + spdk_bdev_close(desc); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_NONE); + CU_ASSERT(TAILQ_EMPTY(&bdev->internal.open_descs)); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_NONE); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 0); + + /* Claim with options */ + spdk_bdev_claim_opts_init(&opts, sizeof(opts)); + snprintf(opts.name, sizeof(opts.name), "%s", "claim with options"); + desc = NULL; + rc = spdk_bdev_open_ext("bdev0", false, bdev_ut_event_cb, NULL, &desc); + CU_ASSERT(rc == 0); + SPDK_CU_ASSERT_FATAL(desc != NULL); + rc = spdk_bdev_module_claim_bdev_desc(desc, SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE, &opts, + &bdev_ut_if); + CU_ASSERT(rc == 0); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE); + SPDK_CU_ASSERT_FATAL(desc->claim != NULL); + CU_ASSERT(desc->claim->module == &bdev_ut_if); + CU_ASSERT(strcmp(desc->claim->name, "claim with options") == 0); + memset(&opts, 0, sizeof(opts)); + CU_ASSERT(strcmp(desc->claim->name, "claim with options") == 0); + CU_ASSERT(TAILQ_FIRST(&bdev->internal.claim.v2.claims) == desc->claim); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 1); + + /* The claim blocks new writers. */ + desc2 = NULL; + rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc2); + CU_ASSERT(rc == -EPERM); + CU_ASSERT(desc2 == NULL); + + /* New readers are allowed */ + desc2 = NULL; + rc = spdk_bdev_open_ext("bdev0", false, bdev_ut_event_cb, NULL, &desc2); + CU_ASSERT(rc == 0); + CU_ASSERT(desc2 != NULL); + CU_ASSERT(!desc2->write); + + /* No new v2 RWO claims are allowed */ + rc = spdk_bdev_module_claim_bdev_desc(desc2, SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, NULL, + &bdev_ut_if); + CU_ASSERT(rc == -EPERM); + + /* No new v2 RWM claims are allowed */ + spdk_bdev_claim_opts_init(&opts, sizeof(opts)); + opts.shared_claim_key = (uint64_t)&opts; + rc = spdk_bdev_module_claim_bdev_desc(desc2, SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED, &opts, + &bdev_ut_if); + CU_ASSERT(rc == -EPERM); + CU_ASSERT(!desc2->write); + + /* No new v1 claims are allowed */ + rc = spdk_bdev_module_claim_bdev(bdev, NULL, &bdev_ut_if); + CU_ASSERT(rc == -EPERM); + + /* None of the above messed up the existing claim */ + CU_ASSERT(TAILQ_FIRST(&bdev->internal.claim.v2.claims) == desc->claim); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 1); + + /* New v2 ROM claims are allowed and the descriptor stays read-only. */ + CU_ASSERT(!desc2->write); + rc = spdk_bdev_module_claim_bdev_desc(desc2, SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE, NULL, + &bdev_ut_if); + CU_ASSERT(rc == 0); + CU_ASSERT(!desc2->write); + CU_ASSERT(TAILQ_FIRST(&bdev->internal.claim.v2.claims) == desc->claim); + CU_ASSERT(TAILQ_NEXT(desc->claim, link) == desc2->claim); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 2); + + /* Claim remains when closing the first descriptor */ + spdk_bdev_close(desc); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE); + CU_ASSERT(!TAILQ_EMPTY(&bdev->internal.open_descs)); + CU_ASSERT(TAILQ_FIRST(&bdev->internal.claim.v2.claims) == desc2->claim); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 1); + + /* Claim removed when closing the other descriptor */ + spdk_bdev_close(desc2); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_NONE); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 0); + CU_ASSERT(TAILQ_EMPTY(&bdev->internal.open_descs)); + + /* Cannot claim with a key */ + desc = NULL; + rc = spdk_bdev_open_ext("bdev0", false, bdev_ut_event_cb, NULL, &desc); + CU_ASSERT(rc == 0); + SPDK_CU_ASSERT_FATAL(desc != NULL); + spdk_bdev_claim_opts_init(&opts, sizeof(opts)); + opts.shared_claim_key = (uint64_t)&opts; + rc = spdk_bdev_module_claim_bdev_desc(desc, SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE, &opts, + &bdev_ut_if); + CU_ASSERT(rc == -EINVAL); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_NONE); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 0); + spdk_bdev_close(desc); + + /* Cannot claim with a read-write descriptor */ + desc = NULL; + rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); + CU_ASSERT(rc == 0); + SPDK_CU_ASSERT_FATAL(desc != NULL); + rc = spdk_bdev_module_claim_bdev_desc(desc, SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE, NULL, + &bdev_ut_if); + CU_ASSERT(rc == -EINVAL); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_NONE); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 0); + spdk_bdev_close(desc); + CU_ASSERT(TAILQ_EMPTY(&bdev->internal.open_descs)); + + /* Clean up */ + free_bdev(bdev); +} + +static void +claim_v2_rwm(void) +{ + struct spdk_bdev *bdev; + struct spdk_bdev_desc *desc; + struct spdk_bdev_desc *desc2; + struct spdk_bdev_claim_opts opts; + char good_key, bad_key; + int rc; + + bdev = allocate_bdev("bdev0"); + + /* Claim without options should fail */ + desc = NULL; + rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); + CU_ASSERT(rc == 0); + SPDK_CU_ASSERT_FATAL(desc != NULL); + rc = spdk_bdev_module_claim_bdev_desc(desc, SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED, NULL, + &bdev_ut_if); + CU_ASSERT(rc == -EINVAL); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_NONE); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 0); + CU_ASSERT(desc->claim == NULL); + + /* Claim with options */ + spdk_bdev_claim_opts_init(&opts, sizeof(opts)); + snprintf(opts.name, sizeof(opts.name), "%s", "claim with options"); + opts.shared_claim_key = (uint64_t)&good_key; + rc = spdk_bdev_module_claim_bdev_desc(desc, SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED, &opts, + &bdev_ut_if); + CU_ASSERT(rc == 0); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED); + SPDK_CU_ASSERT_FATAL(desc->claim != NULL); + CU_ASSERT(desc->claim->module == &bdev_ut_if); + CU_ASSERT(strcmp(desc->claim->name, "claim with options") == 0); + memset(&opts, 0, sizeof(opts)); + CU_ASSERT(strcmp(desc->claim->name, "claim with options") == 0); + CU_ASSERT(TAILQ_FIRST(&bdev->internal.claim.v2.claims) == desc->claim); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 1); + + /* The claim blocks new writers. */ + desc2 = NULL; + rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc2); + CU_ASSERT(rc == -EPERM); + CU_ASSERT(desc2 == NULL); + + /* New readers are allowed */ + desc2 = NULL; + rc = spdk_bdev_open_ext("bdev0", false, bdev_ut_event_cb, NULL, &desc2); + CU_ASSERT(rc == 0); + CU_ASSERT(desc2 != NULL); + CU_ASSERT(!desc2->write); + + /* No new v2 RWO claims are allowed */ + rc = spdk_bdev_module_claim_bdev_desc(desc2, SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, NULL, + &bdev_ut_if); + CU_ASSERT(rc == -EPERM); + + /* No new v2 ROM claims are allowed and the descriptor stays read-only. */ + CU_ASSERT(!desc2->write); + rc = spdk_bdev_module_claim_bdev_desc(desc2, SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE, NULL, + &bdev_ut_if); + CU_ASSERT(rc == -EPERM); + CU_ASSERT(!desc2->write); + + /* No new v1 claims are allowed */ + rc = spdk_bdev_module_claim_bdev(bdev, NULL, &bdev_ut_if); + CU_ASSERT(rc == -EPERM); + + /* No new v2 RWM claims are allowed if the key does not match */ + spdk_bdev_claim_opts_init(&opts, sizeof(opts)); + opts.shared_claim_key = (uint64_t)&bad_key; + rc = spdk_bdev_module_claim_bdev_desc(desc2, SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED, &opts, + &bdev_ut_if); + CU_ASSERT(rc == -EPERM); + CU_ASSERT(!desc2->write); + + /* None of the above messed up the existing claim */ + CU_ASSERT(TAILQ_FIRST(&bdev->internal.claim.v2.claims) == desc->claim); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 1); + + /* New v2 RWM claims are allowed and the descriptor is promoted if the key matches. */ + spdk_bdev_claim_opts_init(&opts, sizeof(opts)); + opts.shared_claim_key = (uint64_t)&good_key; + CU_ASSERT(!desc2->write); + rc = spdk_bdev_module_claim_bdev_desc(desc2, SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED, &opts, + &bdev_ut_if); + CU_ASSERT(rc == 0); + CU_ASSERT(desc2->write); + CU_ASSERT(TAILQ_NEXT(desc->claim, link) == desc2->claim); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 2); + + /* Claim remains when closing the first descriptor */ + spdk_bdev_close(desc); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED); + CU_ASSERT(!TAILQ_EMPTY(&bdev->internal.open_descs)); + CU_ASSERT(TAILQ_FIRST(&bdev->internal.claim.v2.claims) == desc2->claim); + UT_ASSERT_CLAIM_V2_COUNT(bdev, 1); + + /* Claim removed when closing the other descriptor */ + spdk_bdev_close(desc2); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_NONE); + CU_ASSERT(TAILQ_EMPTY(&bdev->internal.open_descs)); + + /* Cannot claim without a key */ + desc = NULL; + rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); + CU_ASSERT(rc == 0); + SPDK_CU_ASSERT_FATAL(desc != NULL); + spdk_bdev_claim_opts_init(&opts, sizeof(opts)); + rc = spdk_bdev_module_claim_bdev_desc(desc, SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED, &opts, + &bdev_ut_if); + CU_ASSERT(rc == -EINVAL); + spdk_bdev_close(desc); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_NONE); + CU_ASSERT(TAILQ_EMPTY(&bdev->internal.open_descs)); + + /* Clean up */ + free_bdev(bdev); +} + +static void +claim_v2_existing_writer(void) +{ + struct spdk_bdev *bdev; + struct spdk_bdev_desc *desc; + struct spdk_bdev_desc *desc2; + struct spdk_bdev_claim_opts opts; + enum spdk_bdev_claim_type type; + enum spdk_bdev_claim_type types[] = { + SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, + SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED, + SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE + }; + size_t i; + int rc; + + bdev = allocate_bdev("bdev0"); + + desc = NULL; + rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc); + CU_ASSERT(rc == 0); + SPDK_CU_ASSERT_FATAL(desc != NULL); + desc2 = NULL; + rc = spdk_bdev_open_ext("bdev0", true, bdev_ut_event_cb, NULL, &desc2); + CU_ASSERT(rc == 0); + SPDK_CU_ASSERT_FATAL(desc2 != NULL); + + for (i = 0; i < SPDK_COUNTOF(types); i++) { + type = types[i]; + spdk_bdev_claim_opts_init(&opts, sizeof(opts)); + if (type == SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED) { + opts.shared_claim_key = (uint64_t)&opts; + } + rc = spdk_bdev_module_claim_bdev_desc(desc, type, &opts, &bdev_ut_if); + if (type == SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE) { + CU_ASSERT(rc == -EINVAL); + } else { + CU_ASSERT(rc == -EPERM); + } + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_NONE); + rc = spdk_bdev_module_claim_bdev_desc(desc2, type, &opts, &bdev_ut_if); + if (type == SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE) { + CU_ASSERT(rc == -EINVAL); + } else { + CU_ASSERT(rc == -EPERM); + } + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_NONE); + } + + spdk_bdev_close(desc); + spdk_bdev_close(desc2); + + /* Clean up */ + free_bdev(bdev); +} + +static void +claim_v2_existing_v1(void) +{ + struct spdk_bdev *bdev; + struct spdk_bdev_desc *desc; + struct spdk_bdev_claim_opts opts; + enum spdk_bdev_claim_type type; + enum spdk_bdev_claim_type types[] = { + SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, + SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED, + SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE + }; + size_t i; + int rc; + + bdev = allocate_bdev("bdev0"); + + rc = spdk_bdev_module_claim_bdev(bdev, NULL, &bdev_ut_if); + CU_ASSERT(rc == 0); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_EXCL_WRITE); + + desc = NULL; + rc = spdk_bdev_open_ext("bdev0", false, bdev_ut_event_cb, NULL, &desc); + CU_ASSERT(rc == 0); + SPDK_CU_ASSERT_FATAL(desc != NULL); + + for (i = 0; i < SPDK_COUNTOF(types); i++) { + type = types[i]; + spdk_bdev_claim_opts_init(&opts, sizeof(opts)); + if (type == SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED) { + opts.shared_claim_key = (uint64_t)&opts; + } + rc = spdk_bdev_module_claim_bdev_desc(desc, type, &opts, &bdev_ut_if); + CU_ASSERT(rc == -EPERM); + CU_ASSERT(bdev->internal.claim_type == SPDK_BDEV_CLAIM_EXCL_WRITE); + } + + spdk_bdev_module_release_bdev(bdev); + spdk_bdev_close(desc); + + /* Clean up */ + free_bdev(bdev); +} + +static void +claim_v1_existing_v2(void) +{ + struct spdk_bdev *bdev; + struct spdk_bdev_desc *desc; + struct spdk_bdev_claim_opts opts; + enum spdk_bdev_claim_type type; + enum spdk_bdev_claim_type types[] = { + SPDK_BDEV_CLAIM_READ_MANY_WRITE_ONE, + SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED, + SPDK_BDEV_CLAIM_READ_MANY_WRITE_NONE + }; + size_t i; + int rc; + + bdev = allocate_bdev("bdev0"); + + for (i = 0; i < SPDK_COUNTOF(types); i++) { + type = types[i]; + + desc = NULL; + rc = spdk_bdev_open_ext("bdev0", false, bdev_ut_event_cb, NULL, &desc); + CU_ASSERT(rc == 0); + SPDK_CU_ASSERT_FATAL(desc != NULL); + + /* Get a v2 claim */ + spdk_bdev_claim_opts_init(&opts, sizeof(opts)); + if (type == SPDK_BDEV_CLAIM_READ_MANY_WRITE_SHARED) { + opts.shared_claim_key = (uint64_t)&opts; + } + rc = spdk_bdev_module_claim_bdev_desc(desc, type, &opts, &bdev_ut_if); + CU_ASSERT(rc == 0); + + /* Fail to get a v1 claim */ + rc = spdk_bdev_module_claim_bdev(bdev, NULL, &bdev_ut_if); + CU_ASSERT(rc == -EPERM); + + spdk_bdev_close(desc); + + /* Now v1 succeeds */ + rc = spdk_bdev_module_claim_bdev(bdev, NULL, &bdev_ut_if); + CU_ASSERT(rc == 0) + spdk_bdev_module_release_bdev(bdev); + } + + /* Clean up */ + free_bdev(bdev); +} + int main(int argc, char **argv) { @@ -6325,6 +6872,12 @@ main(int argc, char **argv) CU_ADD_TEST(suite, bdev_copy); CU_ADD_TEST(suite, bdev_copy_split_test); CU_ADD_TEST(suite, examine_locks); + CU_ADD_TEST(suite, claim_v2_rwo); + CU_ADD_TEST(suite, claim_v2_rom); + CU_ADD_TEST(suite, claim_v2_rwm); + CU_ADD_TEST(suite, claim_v2_existing_writer); + CU_ADD_TEST(suite, claim_v2_existing_v1); + CU_ADD_TEST(suite, claim_v1_existing_v2); allocate_cores(1); allocate_threads(1);