From 249a68e92b57da9969b12921c183255dbda8528e Mon Sep 17 00:00:00 2001 From: Daniel Verkamp Date: Tue, 10 Jan 2017 09:54:23 -0700 Subject: [PATCH] bdev: add API to claim block devices Claim the block devices used by iSCSI LUNs and NVMe-oF subsystems so they can't accidentally be reused. This will also be used by virtual block devices to allow layering of bdevs. Change-Id: I5384923fbf24f13f4ce720a797c5a628053d49f4 Signed-off-by: Daniel Verkamp --- include/spdk/bdev.h | 26 ++++++++++++++++++ lib/bdev/bdev.c | 37 ++++++++++++++++++++++++++ lib/nvmf/subsystem.c | 7 +++++ lib/nvmf/virtual.c | 6 ++++- lib/scsi/lun.c | 7 +++++ test/lib/bdev/bdevio/bdevio.c | 3 ++- test/lib/bdev/bdevperf/bdevperf.c | 3 ++- test/lib/nvmf/subsystem/subsystem_ut.c | 6 +++++ test/lib/scsi/lun/lun_ut.c | 11 ++++++++ 9 files changed, 103 insertions(+), 3 deletions(-) diff --git a/include/spdk/bdev.h b/include/spdk/bdev.h index 7efc70718..dd4bdfc52 100644 --- a/include/spdk/bdev.h +++ b/include/spdk/bdev.h @@ -43,6 +43,7 @@ #include /* for offsetof */ #include /* for struct iovec */ #include +#include #include "spdk/event.h" #include "spdk/queue.h" @@ -100,6 +101,9 @@ struct spdk_bdev { /** generation value used by block device reset */ uint32_t gencnt; + /** Mutex protecting claimed */ + pthread_mutex_t mutex; + /** True if another blockdev or a LUN is using this device */ bool claimed; @@ -285,6 +289,28 @@ void spdk_bdev_unregister(struct spdk_bdev *bdev); struct spdk_bdev *spdk_bdev_first(void); struct spdk_bdev *spdk_bdev_next(struct spdk_bdev *prev); +/** + * Claim ownership of a block device. + * + * User applications and virtual blockdevs may use this to mediate access to bdevs. + * + * When the ownership of the bdev is no longer needed, the user should call spdk_bdev_unclaim(). + * + * \param bdev Block device to claim. + * \return true if the caller claimed the bdev, or false if it was already claimed by another user. + */ +bool spdk_bdev_claim(struct spdk_bdev *bdev); + +/** + * Release claim of ownership of a block device. + * + * When a bdev reference acquired with spdk_bdev_claim() is no longer needed, the user should + * release the claim using spdk_bdev_unclaim(). + * + * \param bdev Block device to release. + */ +void spdk_bdev_unclaim(struct spdk_bdev *bdev); + bool spdk_bdev_io_type_supported(struct spdk_bdev *bdev, enum spdk_bdev_io_type io_type); int spdk_bdev_dump_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w); diff --git a/lib/bdev/bdev.c b/lib/bdev/bdev.c index 1f71efa2e..bae326b0c 100644 --- a/lib/bdev/bdev.c +++ b/lib/bdev/bdev.c @@ -854,6 +854,9 @@ spdk_bdev_register(struct spdk_bdev *bdev) /* initialize the reset generation value to zero */ bdev->gencnt = 0; + pthread_mutex_init(&bdev->mutex, NULL); + bdev->claimed = false; + SPDK_TRACELOG(SPDK_TRACE_DEBUG, "Inserting bdev %s into list\n", bdev->name); TAILQ_INSERT_TAIL(&spdk_bdev_list, bdev, link); } @@ -866,12 +869,46 @@ spdk_bdev_unregister(struct spdk_bdev *bdev) SPDK_TRACELOG(SPDK_TRACE_DEBUG, "Removing bdev %s from list\n", bdev->name); TAILQ_REMOVE(&spdk_bdev_list, bdev, link); + pthread_mutex_destroy(&bdev->mutex); + rc = bdev->fn_table->destruct(bdev->ctxt); if (rc < 0) { SPDK_ERRLOG("destruct failed\n"); } } +bool +spdk_bdev_claim(struct spdk_bdev *bdev) +{ + bool success; + + pthread_mutex_lock(&bdev->mutex); + + if (!bdev->claimed) { + /* Take ownership of bdev. */ + bdev->claimed = true; + success = true; + } else { + /* bdev is already claimed. */ + success = false; + } + + pthread_mutex_unlock(&bdev->mutex); + + return success; +} + +void +spdk_bdev_unclaim(struct spdk_bdev *bdev) +{ + pthread_mutex_lock(&bdev->mutex); + + assert(bdev->claimed); + bdev->claimed = false; + + pthread_mutex_unlock(&bdev->mutex); +} + void spdk_bdev_io_get_rbuf(struct spdk_bdev_io *bdev_io, spdk_bdev_io_get_rbuf_cb cb) { diff --git a/lib/nvmf/subsystem.c b/lib/nvmf/subsystem.c index b63131fc0..a9625ee19 100644 --- a/lib/nvmf/subsystem.c +++ b/lib/nvmf/subsystem.c @@ -43,6 +43,7 @@ #include "spdk/trace.h" #include "spdk/nvmf_spec.h" +#include "spdk_internal/bdev.h" #include "spdk_internal/log.h" static TAILQ_HEAD(, spdk_nvmf_subsystem) g_subsystems = TAILQ_HEAD_INITIALIZER(g_subsystems); @@ -420,6 +421,12 @@ spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bd { int i = 0; + if (!spdk_bdev_claim(bdev)) { + SPDK_ERRLOG("Subsystem %s: bdev %s is already claimed\n", + subsystem->subnqn, bdev->name); + return -1; + } + assert(subsystem->mode == NVMF_SUBSYSTEM_MODE_VIRTUAL); while (i < MAX_VIRTUAL_NAMESPACE && subsystem->dev.virt.ns_list[i]) { i++; diff --git a/lib/nvmf/virtual.c b/lib/nvmf/virtual.c index 7672760cb..4a48badb0 100644 --- a/lib/nvmf/virtual.c +++ b/lib/nvmf/virtual.c @@ -534,7 +534,11 @@ nvmf_virtual_ctrlr_process_io_cmd(struct spdk_nvmf_request *req) static void nvmf_virtual_ctrlr_detach(struct spdk_nvmf_subsystem *subsystem) { - return; + uint32_t i; + + for (i = 0; i < subsystem->dev.virt.ns_count; i++) { + spdk_bdev_unclaim(subsystem->dev.virt.ns_list[i]); + } } const struct spdk_nvmf_ctrlr_ops spdk_nvmf_virtual_ctrlr_ops = { diff --git a/lib/scsi/lun.c b/lib/scsi/lun.c index e698c62b5..512a7b0e9 100644 --- a/lib/scsi/lun.c +++ b/lib/scsi/lun.c @@ -289,6 +289,12 @@ spdk_scsi_lun_construct(const char *name, struct spdk_bdev *bdev) return NULL; } + if (!spdk_bdev_claim(bdev)) { + SPDK_ERRLOG("LUN %s: bdev %s is already claimed\n", name, bdev->name); + free(lun); + return NULL; + } + TAILQ_INIT(&lun->tasks); TAILQ_INIT(&lun->pending_tasks); @@ -308,6 +314,7 @@ spdk_scsi_lun_construct(const char *name, struct spdk_bdev *bdev) static int spdk_scsi_lun_destruct(struct spdk_scsi_lun *lun) { + spdk_bdev_unclaim(lun->bdev); spdk_scsi_lun_db_delete(lun); free(lun); diff --git a/test/lib/bdev/bdevio/bdevio.c b/test/lib/bdev/bdevio/bdevio.c index f9372f049..6c48de4da 100644 --- a/test/lib/bdev/bdevio/bdevio.c +++ b/test/lib/bdev/bdevio/bdevio.c @@ -110,7 +110,7 @@ bdevio_construct_targets(void) bdev = spdk_bdev_first(); while (bdev != NULL) { - if (bdev->claimed) { + if (!spdk_bdev_claim(bdev)) { bdev = spdk_bdev_next(bdev); continue; } @@ -147,6 +147,7 @@ bdevio_cleanup_targets(void) target = g_io_targets; while (target != NULL) { execute_spdk_function(__put_io_channel, target, NULL); + spdk_bdev_unclaim(target->bdev); g_io_targets = target->next; free(target); target = g_io_targets; diff --git a/test/lib/bdev/bdevperf/bdevperf.c b/test/lib/bdev/bdevperf/bdevperf.c index 97fd64af6..a356e8ae4 100644 --- a/test/lib/bdev/bdevperf/bdevperf.c +++ b/test/lib/bdev/bdevperf/bdevperf.c @@ -120,7 +120,7 @@ bdevperf_construct_targets(void) bdev = spdk_bdev_first(); while (bdev != NULL) { - if (bdev->claimed) { + if (!spdk_bdev_claim(bdev)) { bdev = spdk_bdev_next(bdev); continue; } @@ -168,6 +168,7 @@ end_run(void *arg1, void *arg2) struct io_target *target = arg1; spdk_put_io_channel(target->ch); + spdk_bdev_unclaim(target->bdev); if (--g_target_count == 0) { if (g_show_performance_real_time) { spdk_poller_unregister(&g_perf_timer, NULL); diff --git a/test/lib/nvmf/subsystem/subsystem_ut.c b/test/lib/nvmf/subsystem/subsystem_ut.c index 26a667f25..e58831796 100644 --- a/test/lib/nvmf/subsystem/subsystem_ut.c +++ b/test/lib/nvmf/subsystem/subsystem_ut.c @@ -111,6 +111,12 @@ spdk_nvmf_session_poll(struct spdk_nvmf_session *session) return -1; } +bool +spdk_bdev_claim(struct spdk_bdev *bdev) +{ + return true; +} + static void nvmf_test_create_subsystem(void) { diff --git a/test/lib/scsi/lun/lun_ut.c b/test/lib/scsi/lun/lun_ut.c index c28d5ac69..9ef6e1d88 100644 --- a/test/lib/scsi/lun/lun_ut.c +++ b/test/lib/scsi/lun/lun_ut.c @@ -108,6 +108,17 @@ spdk_bdev_free_io(struct spdk_bdev_io *bdev_io) return -1; } +bool +spdk_bdev_claim(struct spdk_bdev *bdev) +{ + return true; +} + +void +spdk_bdev_unclaim(struct spdk_bdev *bdev) +{ +} + void spdk_scsi_dev_queue_mgmt_task(struct spdk_scsi_dev *dev, struct spdk_scsi_task *task) {