From 67e96cca4b7d9bf8e48473896e9aea8206bb1af4 Mon Sep 17 00:00:00 2001 From: Keith Lucas Date: Tue, 8 Feb 2022 21:53:31 -0500 Subject: [PATCH] Fix remote replicas. Signed-off-by: Keith Lucas --- module/bdev/longhorn/bdev_longhorn.c | 59 +++++++++++++++++++---- module/bdev/longhorn/bdev_longhorn.h | 7 ++- module/bdev/longhorn/bdev_longhorn_impl.c | 3 +- module/bdev/longhorn/bdev_longhorn_nvmf.c | 12 ++++- module/bdev/longhorn/bdev_longhorn_nvmf.h | 3 ++ 5 files changed, 70 insertions(+), 14 deletions(-) diff --git a/module/bdev/longhorn/bdev_longhorn.c b/module/bdev/longhorn/bdev_longhorn.c index 430d51493..4f3c4a6ca 100644 --- a/module/bdev/longhorn/bdev_longhorn.c +++ b/module/bdev/longhorn/bdev_longhorn.c @@ -254,7 +254,6 @@ longhorn_bdev_destroy_cb(void *io_device, void *ctx_buf) struct longhorn_bdev_io_channel *longhorn_ch = ctx_buf; struct longhorn_base_io_channel *base_channel; struct longhorn_base_io_channel *next; - uint8_t i; SPDK_DEBUGLOG(bdev_longhorn, "longhorn_bdev_destroy_cb\n"); SPDK_ERRLOG("longhorn_bdev_destroy_cb, %p\n", longhorn_ch); @@ -976,8 +975,6 @@ int longhorn_bdev_config_add_base_bdev(struct longhorn_bdev_config *longhorn_cfg, const char *base_bdev_name, uint8_t slot) { - uint8_t i; - struct longhorn_bdev_config *tmp; char *bdev_name; if (slot >= longhorn_cfg->num_base_bdevs) { @@ -1622,13 +1619,41 @@ longhorn_bdev_add_base_device(struct longhorn_bdev *longhorn_bdev, } struct replica_add_ctx { + struct longhorn_bdev *longhorn_bdev; struct longhorn_base_bdev_info *base_info; + + char *prefix; }; +static void +longhorn_replica_attach_cb(const char **bdev_names, size_t bdev_cnt, + int status, void *arg) +{ + struct replica_add_ctx *ctx = arg; + + if (bdev_cnt > 0) { + ctx->base_info->bdev_name = strdup(bdev_names[0]); + + longhorn_bdev_add_base_device(ctx->longhorn_bdev, ctx->base_info); + } + + free(ctx->prefix); + free(ctx); + +} + +static char * +generate_prefix(const char *name) { + static int n = 0; + + return spdk_sprintf_alloc("%s_nvmf%d_", name, n++); +} + int longhorn_bdev_add_replica(const char *name, char *lvs, char *addr, uint16_t nvmf_port, uint16_t comm_port) { struct longhorn_bdev *longhorn_bdev; struct longhorn_base_bdev_info *base_info; + struct replica_add_ctx *ctx; longhorn_bdev = longhorn_bdev_find_by_name(name); if (!longhorn_bdev) { @@ -1649,7 +1674,22 @@ longhorn_bdev_add_replica(const char *name, char *lvs, char *addr, uint16_t nvmf base_info->remote_addr = strdup(addr); base_info->nvmf_port = nvmf_port; base_info->comm_port = comm_port; + + base_info->remote_nqn = longhorn_generate_replica_nqn(lvs, name); + + ctx = calloc(1, sizeof(*ctx)); + + ctx->longhorn_bdev = longhorn_bdev; + ctx->base_info = base_info; + ctx->prefix = generate_prefix(name); + + SPDK_ERRLOG("attempting to attach %s\n", base_info->remote_nqn); + + longhorn_attach_nvmf(ctx->prefix, base_info->remote_nqn, addr, + nvmf_port, longhorn_replica_attach_cb, ctx); } + + return 0; } @@ -1707,8 +1747,6 @@ longhorn_bdev_add_base_devices(struct longhorn_bdev_config *longhorn_cfg) static void longhorn_bdev_examine(struct spdk_bdev *bdev) { - struct longhorn_bdev_config *longhorn_cfg; - uint8_t base_bdev_slot; #if 0 if (longhorn_bdev_can_claim_bdev(bdev->name, &longhorn_cfg, &base_bdev_slot)) { @@ -1858,9 +1896,9 @@ int longhorn_bdev_remove_replica(char *name, char *lvs, char *addr, uint16_t nvm int longhorn_volume_add_replica(char *name, char *lvs, char *addr, uint16_t nvmf_port, uint16_t comm_port) { struct longhorn_bdev *longhorn_bdev; - struct longhorn_base_bdev_info *base_info; - struct longhorn_bdev_io_channel *io_channel; - struct io_channel_remove_ctx *ctx; + //struct longhorn_base_bdev_info *base_info; + //struct longhorn_bdev_io_channel *io_channel; + //struct io_channel_remove_ctx *ctx; int rc; longhorn_bdev = longhorn_bdev_find_by_name(name); @@ -1877,11 +1915,14 @@ int longhorn_volume_add_replica(char *name, char *lvs, char *addr, uint16_t nvmf } + return -errno; } + + return 0; } -void longhorn_unpause(struct longhorn_bdev *longhorn_bdev) +int longhorn_unpause(struct longhorn_bdev *longhorn_bdev) { int rc; struct longhorn_bdev_io_channel *io_channel; diff --git a/module/bdev/longhorn/bdev_longhorn.h b/module/bdev/longhorn/bdev_longhorn.h index 2c056cab6..5105c2272 100644 --- a/module/bdev/longhorn/bdev_longhorn.h +++ b/module/bdev/longhorn/bdev_longhorn.h @@ -100,6 +100,7 @@ struct longhorn_base_bdev_info { uint16_t nvmf_port; uint16_t comm_port; char *bdev_name; + char *remote_nqn; TAILQ_ENTRY(longhorn_base_bdev_info) infos; }; @@ -223,6 +224,9 @@ struct longhorn_bdev { /* Set to true if destroy of this longhorn bdev is started. */ bool destroy_started; + bool op_in_progress; + + atomic_int io_ops; atomic_int channels_to_pause; @@ -317,9 +321,10 @@ void bdev_longhorn_pause_io(void *cb_arg); void bdev_longhorn_unpause_io(void *cb_arg); struct longhorn_bdev *longhorn_bdev_find_by_name(const char *longhorn_name); -void longhorn_unpause(struct longhorn_bdev *longhorn_bdev); +int longhorn_unpause(struct longhorn_bdev *longhorn_bdev); void longhorn_volume_add_pause_cb(struct longhorn_bdev *longhorn_dev, longhorn_pause_cb cb_fn, void *cb_arg); +int longhorn_volume_add_replica(char *name, char *lvs, char *addr, uint16_t nvmf_port, uint16_t comm_port); #endif /* SPDK_BDEV_RAID_INTERNAL_H */ diff --git a/module/bdev/longhorn/bdev_longhorn_impl.c b/module/bdev/longhorn/bdev_longhorn_impl.c index 69d0e10c4..62c505738 100644 --- a/module/bdev/longhorn/bdev_longhorn_impl.c +++ b/module/bdev/longhorn/bdev_longhorn_impl.c @@ -159,10 +159,9 @@ longhorn_submit_write_request(struct longhorn_bdev_io *longhorn_io) struct longhorn_base_io_channel *base_channel; assert(longhorn_ch != NULL); - assert(longhorn_ch->base_channel); if (longhorn_io->base_bdev_io_remaining == 0) { - longhorn_io->base_bdev_io_remaining = longhorn_bdev->num_base_bdevs; + longhorn_io->base_bdev_io_remaining = longhorn_ch->num_channels; } TAILQ_FOREACH(base_channel, &longhorn_ch->base_channels, channels) { diff --git a/module/bdev/longhorn/bdev_longhorn_nvmf.c b/module/bdev/longhorn/bdev_longhorn_nvmf.c index 79a24a817..07d26cbd2 100644 --- a/module/bdev/longhorn/bdev_longhorn_nvmf.c +++ b/module/bdev/longhorn/bdev_longhorn_nvmf.c @@ -1,4 +1,5 @@ #include "spdk/nvmf.h" +#include "spdk/string.h" #include "spdk/util.h" #include "bdev_longhorn_nvmf.h" @@ -119,7 +120,7 @@ static void add_ns_pause_cb(struct spdk_nvmf_subsystem *subsystem, void *cb_arg, } void longhorn_nvmf_subsystem_add_ns(const char *nqn, const char *bdev_name) { - struct spdk_nvmf_tgt *tgt; + struct spdk_nvmf_tgt *tgt = NULL; struct spdk_nvmf_subsystem *subsystem; subsystem = spdk_nvmf_tgt_find_subsystem(tgt, nqn); @@ -218,7 +219,9 @@ void longhorn_attach_nvmf(const char *bdev_name_prefix, const char *nqn, ctx->cb_fn = cb_fn; ctx->cb_arg = cb_arg; - bdev_nvme_create(&trid, &hostid, bdev_name_prefix, ctx->names, ctx->count, + spdk_nvme_ctrlr_get_default_ctrlr_opts(&ctx->opts, sizeof(ctx->opts)); + + bdev_nvme_create(trid, &hostid, bdev_name_prefix, ctx->names, ctx->count, prchk_flags, longhorn_nvme_create_cb, ctx, &ctx->opts); } @@ -267,6 +270,11 @@ void longhorn_set_external_addr(const char *addr, } +char * +longhorn_generate_replica_nqn(const char *lvs, const char *name) { + char *nqn = spdk_sprintf_alloc(REPLICA_FORMAT, lvs, name); + return nqn; +} diff --git a/module/bdev/longhorn/bdev_longhorn_nvmf.h b/module/bdev/longhorn/bdev_longhorn_nvmf.h index cf4a82cdd..f4560e5cb 100644 --- a/module/bdev/longhorn/bdev_longhorn_nvmf.h +++ b/module/bdev/longhorn/bdev_longhorn_nvmf.h @@ -24,5 +24,8 @@ void longhorn_set_external_addr(const char *addr, typedef void (*longhorn_attach_nvmf_cb)(const char **bdev_names, size_t bdev_cnt, int status, void *arg); void longhorn_attach_nvmf(const char *bdev_name_prefix, const char *nqn, const char *addr, uint16_t port, longhorn_attach_nvmf_cb cb_fn, void *cb_arg); +char * +longhorn_generate_replica_nqn(const char *lvs, const char *name); + #endif /* _BDEV_LONGHORN_NVMF_H_ */