Fix remote replicas.

Signed-off-by: Keith Lucas <keith.lucas@suse.com>

module/bdev/longhorn/bdev_longhorn.c

@@ -254,7 +254,6 @@ longhorn_bdev_destroy_cb(void *io_device, void *ctx_buf)
 	struct longhorn_bdev_io_channel *longhorn_ch = ctx_buf;
 	struct longhorn_base_io_channel *base_channel;
 	struct longhorn_base_io_channel *next;
-	uint8_t i;
 
 	SPDK_DEBUGLOG(bdev_longhorn, "longhorn_bdev_destroy_cb\n");
 	SPDK_ERRLOG("longhorn_bdev_destroy_cb, %p\n", longhorn_ch);
@@ -976,8 +975,6 @@ int
 longhorn_bdev_config_add_base_bdev(struct longhorn_bdev_config *longhorn_cfg, const char *base_bdev_name,
 			       uint8_t slot)
 {
-	uint8_t i;
-	struct longhorn_bdev_config *tmp;
 	char *bdev_name;
 
 	if (slot >= longhorn_cfg->num_base_bdevs) {
@@ -1622,13 +1619,41 @@ longhorn_bdev_add_base_device(struct longhorn_bdev *longhorn_bdev,
 }
 
 struct replica_add_ctx {
+	struct longhorn_bdev	*longhorn_bdev;
 	struct longhorn_base_bdev_info *base_info;
+
+	char *prefix;
 };
 
+static void
+longhorn_replica_attach_cb(const char **bdev_names, size_t bdev_cnt,
+			   int status, void *arg) 
+{
+	struct replica_add_ctx *ctx = arg;
+
+	if (bdev_cnt > 0) {
+		ctx->base_info->bdev_name = strdup(bdev_names[0]);
+
+		longhorn_bdev_add_base_device(ctx->longhorn_bdev, ctx->base_info);
+	}
+
+	free(ctx->prefix);
+	free(ctx);
+
+}
+
+static char *
+generate_prefix(const char *name) {
+	static int n = 0;
+
+	return spdk_sprintf_alloc("%s_nvmf%d_", name, n++);
+}
+
 int 
 longhorn_bdev_add_replica(const char *name, char *lvs, char *addr, uint16_t nvmf_port, uint16_t comm_port) {
 	struct longhorn_bdev	*longhorn_bdev;
 	struct longhorn_base_bdev_info *base_info;
+	struct replica_add_ctx *ctx;
 
 	longhorn_bdev = longhorn_bdev_find_by_name(name);
 	if (!longhorn_bdev) {
@@ -1649,7 +1674,22 @@ longhorn_bdev_add_replica(const char *name, char *lvs, char *addr, uint16_t nvmf
 		base_info->remote_addr = strdup(addr);
 		base_info->nvmf_port = nvmf_port;
 		base_info->comm_port = comm_port;
+
+		base_info->remote_nqn = longhorn_generate_replica_nqn(lvs, name);
+
+		ctx = calloc(1, sizeof(*ctx));
+		
+		ctx->longhorn_bdev = longhorn_bdev;
+		ctx->base_info = base_info;
+		ctx->prefix = generate_prefix(name);
+
+		SPDK_ERRLOG("attempting to attach %s\n", base_info->remote_nqn);
+
+		longhorn_attach_nvmf(ctx->prefix, base_info->remote_nqn, addr, 
+				     nvmf_port, longhorn_replica_attach_cb, ctx);
 	}
+
+	return 0;
 }
 	
 
@@ -1707,8 +1747,6 @@ longhorn_bdev_add_base_devices(struct longhorn_bdev_config *longhorn_cfg)
 static void
 longhorn_bdev_examine(struct spdk_bdev *bdev)
 {
-	struct longhorn_bdev_config	*longhorn_cfg;
-	uint8_t			base_bdev_slot;
 
 #if 0
 	if (longhorn_bdev_can_claim_bdev(bdev->name, &longhorn_cfg, &base_bdev_slot)) {
@@ -1858,9 +1896,9 @@ int longhorn_bdev_remove_replica(char *name, char *lvs, char *addr, uint16_t nvm
 
 int longhorn_volume_add_replica(char *name, char *lvs, char *addr, uint16_t nvmf_port, uint16_t comm_port) {
 	struct longhorn_bdev	*longhorn_bdev;
-	struct longhorn_base_bdev_info	*base_info;
-	struct longhorn_bdev_io_channel *io_channel;
-	struct io_channel_remove_ctx *ctx;
+	//struct longhorn_base_bdev_info	*base_info;
+	//struct longhorn_bdev_io_channel *io_channel;
+	//struct io_channel_remove_ctx *ctx;
 	int			rc;
 
 	longhorn_bdev = longhorn_bdev_find_by_name(name);
@@ -1877,11 +1915,14 @@ int longhorn_volume_add_replica(char *name, char *lvs, char *addr, uint16_t nvmf
 		}
 
 
+
 		return -errno;
 	}
+
+	return 0;
 }
 
-void longhorn_unpause(struct longhorn_bdev *longhorn_bdev)
+int longhorn_unpause(struct longhorn_bdev *longhorn_bdev)
 {
 	int			rc;
 	struct longhorn_bdev_io_channel *io_channel;

module/bdev/longhorn/bdev_longhorn.h

@@ -100,6 +100,7 @@ struct longhorn_base_bdev_info {
 	uint16_t nvmf_port;
 	uint16_t comm_port;
 	char *bdev_name;
+	char *remote_nqn;
 	
 	TAILQ_ENTRY(longhorn_base_bdev_info) infos;
 };
@@ -223,6 +224,9 @@ struct longhorn_bdev {
 	/* Set to true if destroy of this longhorn bdev is started. */
 	bool				destroy_started;
 
+	bool op_in_progress;
+
+
 	atomic_int                      io_ops;
 	atomic_int                      channels_to_pause;
 
@@ -317,9 +321,10 @@ void bdev_longhorn_pause_io(void *cb_arg);
 void bdev_longhorn_unpause_io(void *cb_arg);
 struct longhorn_bdev *longhorn_bdev_find_by_name(const char *longhorn_name);
 
-void longhorn_unpause(struct longhorn_bdev *longhorn_bdev);
+int longhorn_unpause(struct longhorn_bdev *longhorn_bdev);
 void longhorn_volume_add_pause_cb(struct longhorn_bdev *longhorn_dev,
                                   longhorn_pause_cb cb_fn,
                                   void *cb_arg);
+int longhorn_volume_add_replica(char *name, char *lvs, char *addr, uint16_t nvmf_port, uint16_t comm_port);
 
 #endif /* SPDK_BDEV_RAID_INTERNAL_H */

module/bdev/longhorn/bdev_longhorn_impl.c

@@ -159,10 +159,9 @@ longhorn_submit_write_request(struct longhorn_bdev_io *longhorn_io)
 	struct longhorn_base_io_channel *base_channel;
 
 	assert(longhorn_ch != NULL);
-	assert(longhorn_ch->base_channel);
 
 	if (longhorn_io->base_bdev_io_remaining == 0) {
-		longhorn_io->base_bdev_io_remaining = longhorn_bdev->num_base_bdevs;
+		longhorn_io->base_bdev_io_remaining = longhorn_ch->num_channels;
 	}
 
 	TAILQ_FOREACH(base_channel, &longhorn_ch->base_channels, channels) {

module/bdev/longhorn/bdev_longhorn_nvmf.c

@@ -1,4 +1,5 @@
 #include "spdk/nvmf.h"
+#include "spdk/string.h"
 #include "spdk/util.h"
 
 #include "bdev_longhorn_nvmf.h"
@@ -119,7 +120,7 @@ static void add_ns_pause_cb(struct spdk_nvmf_subsystem *subsystem, void *cb_arg,
 }
 
 void longhorn_nvmf_subsystem_add_ns(const char *nqn, const char *bdev_name) { 
-	struct spdk_nvmf_tgt *tgt;
+	struct spdk_nvmf_tgt *tgt = NULL;
 	struct spdk_nvmf_subsystem      *subsystem;
 	
 	subsystem = spdk_nvmf_tgt_find_subsystem(tgt, nqn);
@@ -218,7 +219,9 @@ void longhorn_attach_nvmf(const char *bdev_name_prefix, const char *nqn,
 	ctx->cb_fn = cb_fn;
 	ctx->cb_arg = cb_arg;
 
-	bdev_nvme_create(&trid, &hostid, bdev_name_prefix, ctx->names, ctx->count, 
+	spdk_nvme_ctrlr_get_default_ctrlr_opts(&ctx->opts, sizeof(ctx->opts));
+
+	bdev_nvme_create(trid, &hostid, bdev_name_prefix, ctx->names, ctx->count, 
 			 prchk_flags, longhorn_nvme_create_cb, ctx, &ctx->opts);
 
 }
@@ -267,6 +270,11 @@ void longhorn_set_external_addr(const char *addr,
 
 }
 
+char *
+longhorn_generate_replica_nqn(const char *lvs, const char *name) {
+	char *nqn = spdk_sprintf_alloc(REPLICA_FORMAT, lvs, name);
+	return nqn;
+}
 
 
 

module/bdev/longhorn/bdev_longhorn_nvmf.h

@@ -24,5 +24,8 @@ void longhorn_set_external_addr(const char *addr,
 typedef void (*longhorn_attach_nvmf_cb)(const char **bdev_names, size_t bdev_cnt, int status, void *arg);
 void longhorn_attach_nvmf(const char *bdev_name_prefix, const char *nqn, const char *addr, uint16_t port, longhorn_attach_nvmf_cb cb_fn, void *cb_arg);
 
+char *
+longhorn_generate_replica_nqn(const char *lvs, const char *name);
+
 #endif /* _BDEV_LONGHORN_NVMF_H_ */