ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cleanup log messages.

Browse Source 
Signed-off-by: Keith Lucas <keith.lucas@suse.com>
This commit is contained in:
Keith Lucas 2022-04-05 17:41:49 -04:00
parent 67e96cca4b
commit 7cda68ff38
12 changed files with 1009 additions and 400 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
module/bdev/longhorn/Makefile
View File

@ -37,7 +37,7 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
SO_VER := 3
SO_MINOR := 0
C_SRCS = bdev_longhorn.c bdev_longhorn_rpc.c bdev_longhorn_rebuild.c bdev_longhorn_rebuild_rpc.c bdev_longhorn_remote.c bdev_longhorn_remote_sync.c bdev_longhorn_sync_client.c bdev_longhorn_lvol.c bdev_longhorn_impl.c bdev_longhorn_nvmf.c bdev_longhorn_sync.c bdev_longhorn_replica_rpc.c bdev_longhorn_replica.c bdev_longhorn_snapshot.c bdev_longhorn_snapshot_rpc.c
C_SRCS = bdev_longhorn.c bdev_longhorn_rpc.c bdev_longhorn_rebuild.c bdev_longhorn_rebuild_rpc.c bdev_longhorn_remote.c bdev_longhorn_remote_sync.c bdev_longhorn_sync_client.c bdev_longhorn_lvol.c bdev_longhorn_io.c bdev_longhorn_nvmf.c bdev_longhorn_sync.c bdev_longhorn_replica_rpc.c bdev_longhorn_replica.c bdev_longhorn_snapshot.c bdev_longhorn_snapshot_rpc.c
LIBNAME = bdev_longhorn
SPDK_MAP_FILE = $(SPDK_ROOT_DIR)/mk/spdk_blank.map

501
module/bdev/longhorn/bdev_longhorn.c
View File

@ -32,7 +32,7 @@
*/
#include "bdev_longhorn.h"
#include "bdev_longhorn_impl.h"
#include "bdev_longhorn_io.h"
#include "bdev_longhorn_nvmf.h"
#include "spdk/env.h"
#include "spdk/thread.h"
@ -41,14 +41,10 @@
#include "spdk/util.h"
#include "spdk/json.h"
#include "spdk/string.h"
#include "lib/thread/thread_internal.h"
static bool g_shutdown_started = false;
/* longhorn bdev config as read from config file */
struct longhorn_config g_longhorn_config = {
.longhorn_bdev_config_head = TAILQ_HEAD_INITIALIZER(g_longhorn_config.longhorn_bdev_config_head),
};
/*
* List of longhorn bdev in configured list, these longhorn bdevs are registered with
* bdev layer
@ -94,33 +90,24 @@ longhorn_bdev_create_cb(void *io_device, void *ctx_buf)
struct longhorn_base_bdev_info *base_info;
struct spdk_thread *thread;
struct longhorn_base_io_channel *base_channel;
uint8_t i = 0;
TAILQ_INIT(&longhorn_ch->base_channels);
TAILQ_INIT(&longhorn_ch->io_wait_queue);
thread = spdk_get_thread();
longhorn_ch->thread = thread;
SPDK_DEBUGLOG(bdev_longhorn, "onghorn_bdev_create_cb, %p\n", longhorn_ch);
SPDK_ERRLOG("longhorn_bdev_create_cb, %p, %p (%s)\n", longhorn_ch, thread, spdk_thread_get_name(thread));
SPDK_DEBUGLOG(bdev_longhorn, "Calling longhorn_bdev_create_cb for thread %s (%p)\n", spdk_thread_get_name(thread), thread);
assert(longhorn_bdev != NULL);
assert(longhorn_bdev->state == RAID_BDEV_STATE_ONLINE);
longhorn_ch->num_channels = longhorn_bdev->num_base_bdevs;
longhorn_ch->num_channels = 0;
longhorn_ch->longhorn_bdev = longhorn_bdev;
#if 0
// TODO linked list
longhorn_ch->base_channel = calloc(longhorn_ch->num_channels,
sizeof(struct spdk_io_channel *));
if (!longhorn_ch->base_channel) {
SPDK_ERRLOG("Unable to allocate base bdevs io channel\n");
return -ENOMEM;
}
#endif
TAILQ_FOREACH(base_info, &longhorn_bdev->base_bdevs_head, infos) {
base_channel = calloc(1, sizeof(*base_channel));
/*
* Get the spdk_io_channel for all the base bdevs. This is used during
@ -128,35 +115,18 @@ longhorn_bdev_create_cb(void *io_device, void *ctx_buf)
* bdev io channel.
*/
base_channel->base_channel = spdk_bdev_get_io_channel(base_info->desc);
//longhorn_ch->base_channel[i] = base_info->base_channel;
//base_channel->base_channel = base_info->base_channel;
SPDK_ERRLOG("base_info when creating io_channel %p\n", base_info);
base_channel->base_info = base_info;
if (!base_channel->base_channel) {
SPDK_ERRLOG("Unable to create io channel for base bdev\n");
}
#if 0
if (!base_channel->base_channel[i]) {
uint8_t j;
for (j = 0; j < i; j++) {
spdk_put_io_channel(longhorn_ch->base_channel[j]);
}
free(base_channel->base_channel);
longhorn_ch->base_channel = NULL;
SPDK_ERRLOG("Unable to create io channel for base bdev\n");
return -ENOMEM;
}
#endif
longhorn_ch->num_channels++;
TAILQ_INSERT_TAIL(&longhorn_ch->base_channels,
base_channel, channels);
++i;
}
TAILQ_INSERT_TAIL(&longhorn_bdev->io_channel_head, longhorn_ch, channels);
@ -184,11 +154,10 @@ static void longhorn_check_pause_complete(struct longhorn_bdev *longhorn_bdev)
entry = TAILQ_FIRST(&longhorn_bdev->pause_cbs);
while (entry != NULL) {
if (entry->cb_fn != NULL) {
entry->cb_fn(longhorn_bdev, entry->cb_arg);
} else {
SPDK_ERRLOG("PAUSE CB NULL \n");
SPDK_ERRLOG("PAUSE CB NULL \n");
}
next = TAILQ_NEXT(entry, link);
@ -255,8 +224,7 @@ longhorn_bdev_destroy_cb(void *io_device, void *ctx_buf)
struct longhorn_base_io_channel *base_channel;
struct longhorn_base_io_channel *next;
SPDK_DEBUGLOG(bdev_longhorn, "longhorn_bdev_destroy_cb\n");
SPDK_ERRLOG("longhorn_bdev_destroy_cb, %p\n", longhorn_ch);
SPDK_DEBUGLOG(bdev_longhorn, "longhorn_bdev_destroy_cb, %p\n", longhorn_ch);
assert(longhorn_ch != NULL);
@ -267,7 +235,7 @@ longhorn_bdev_destroy_cb(void *io_device, void *ctx_buf)
next = TAILQ_NEXT(base_channel, channels);
SPDK_ERRLOG("longhorn_bdev_destroy_cb, removing bdev %s\n", base_channel->base_info->bdev->name);
SPDK_DEBUGLOG(bdev_longhorn, "longhorn_bdev_destroy_cb, removing bdev %s\n", base_channel->base_info->bdev->name);
spdk_put_io_channel(base_channel->base_channel);
free(base_channel);
@ -279,19 +247,10 @@ longhorn_bdev_destroy_cb(void *io_device, void *ctx_buf)
longhorn_ch->deleted = true;
longhorn_bdev->num_io_channels--;
SPDK_ERRLOG("removing num io channels %u\n", longhorn_bdev->num_io_channels);
SPDK_DEBUGLOG(bdev_longhorn, "removing num io channels %u\n", longhorn_bdev->num_io_channels);
TAILQ_REMOVE(&longhorn_bdev->io_channel_head, longhorn_ch, channels);
#if 0
for (i = 0; i < longhorn_ch->num_channels; i++) {
/* Free base bdev channels */
assert(longhorn_ch->base_channel[i] != NULL);
spdk_put_io_channel(longhorn_ch->base_channel[i]);
}
free(longhorn_ch->base_channel);
#endif
//onghorn_ch->base_channel = NULL;
}
/*
@ -319,9 +278,12 @@ longhorn_bdev_cleanup(struct longhorn_bdev *longhorn_bdev)
TAILQ_REMOVE(&g_longhorn_bdev_list, longhorn_bdev, global_link);
free(longhorn_bdev->bdev.name);
free(longhorn_bdev->base_bdev_info);
#if 0
if (longhorn_bdev->config) {
longhorn_bdev->config->longhorn_bdev = NULL;
}
#endif
free(longhorn_bdev);
}
@ -500,147 +462,6 @@ longhorn_bdev_queue_io_wait(struct longhorn_bdev_io *longhorn_io, struct spdk_bd
spdk_bdev_queue_io_wait(bdev, ch, &longhorn_io->waitq_entry);
}
static void
longhorn_base_bdev_reset_complete(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
{
struct longhorn_bdev_io *longhorn_io = cb_arg;
spdk_bdev_free_io(bdev_io);
longhorn_bdev_io_complete_part(longhorn_io, 1, success ?
SPDK_BDEV_IO_STATUS_SUCCESS :
SPDK_BDEV_IO_STATUS_FAILED);
}
static void
longhorn_bdev_submit_reset_request(struct longhorn_bdev_io *longhorn_io);
static void
_longhorn_bdev_submit_reset_request(void *_longhorn_io)
{
struct longhorn_bdev_io *longhorn_io = _longhorn_io;
longhorn_bdev_submit_reset_request(longhorn_io);
}
/*
* brief:
* longhorn_bdev_submit_reset_request function submits reset requests
* to member disks; it will submit as many as possible unless a reset fails with -ENOMEM, in
* which case it will queue it for later submission
* params:
* longhorn_io
* returns:
* none
*/
static void
longhorn_bdev_submit_reset_request(struct longhorn_bdev_io *longhorn_io)
{
struct longhorn_bdev_io_channel *longhorn_ch = longhorn_io->longhorn_ch;
struct longhorn_bdev *longhorn_bdev = longhorn_io->longhorn_bdev;
int ret;
struct longhorn_base_bdev_info *base_info;
struct spdk_io_channel *base_ch;
struct longhorn_base_io_channel *base_channel;
if (longhorn_io->base_bdev_io_remaining == 0) {
longhorn_io->base_bdev_io_remaining = longhorn_bdev->num_base_bdevs;
}
TAILQ_FOREACH(base_channel, &longhorn_ch->base_channels, channels) {
//while (longhorn_io->base_bdev_io_submitted < longhorn_bdev->num_base_bdevs) {
base_ch = base_channel->base_channel;
base_info = base_channel->base_info;
ret = spdk_bdev_reset(base_info->desc, base_ch,
longhorn_base_bdev_reset_complete, longhorn_io);
if (ret == 0) {
longhorn_io->base_bdev_io_submitted++;
} else if (ret == -ENOMEM) {
longhorn_bdev_queue_io_wait(longhorn_io, base_info->bdev, base_ch,
_longhorn_bdev_submit_reset_request);
return;
} else {
SPDK_ERRLOG("bdev io submit error not due to ENOMEM, it should not happen\n");
assert(false);
longhorn_bdev_io_complete(longhorn_io, SPDK_BDEV_IO_STATUS_FAILED);
return;
}
}
}
/*
* brief:
* Callback function to spdk_bdev_io_get_buf.
* params:
* ch - pointer to longhorn bdev io channel
* bdev_io - pointer to parent bdev_io on longhorn bdev device
* success - True if buffer is allocated or false otherwise.
* returns:
* none
*/
static void
longhorn_bdev_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io,
bool success)
{
struct longhorn_bdev_io *longhorn_io = (struct longhorn_bdev_io *)bdev_io->driver_ctx;
if (!success) {
longhorn_bdev_io_complete(longhorn_io, SPDK_BDEV_IO_STATUS_FAILED);
return;
}
longhorn_submit_rw_request(longhorn_io);
}
/*
* brief:
* longhorn_bdev_submit_request function is the submit_request function pointer of
* longhorn bdev function table. This is used to submit the io on longhorn_bdev to below
* layers.
* params:
* ch - pointer to longhorn bdev io channel
* bdev_io - pointer to parent bdev_io on longhorn bdev device
* returns:
* none
*/
static void
longhorn_bdev_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
{
struct longhorn_bdev_io *longhorn_io = (struct longhorn_bdev_io *)bdev_io->driver_ctx;
longhorn_io->longhorn_bdev = bdev_io->bdev->ctxt;
longhorn_io->longhorn_ch = spdk_io_channel_get_ctx(ch);
longhorn_io->base_bdev_io_remaining = 0;
longhorn_io->base_bdev_io_submitted = 0;
longhorn_io->base_bdev_io_status = SPDK_BDEV_IO_STATUS_SUCCESS;
switch (bdev_io->type) {
case SPDK_BDEV_IO_TYPE_READ:
spdk_bdev_io_get_buf(bdev_io, longhorn_bdev_get_buf_cb,
bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen);
break;
case SPDK_BDEV_IO_TYPE_WRITE:
longhorn_submit_rw_request(longhorn_io);
break;
case SPDK_BDEV_IO_TYPE_RESET:
longhorn_bdev_submit_reset_request(longhorn_io);
break;
case SPDK_BDEV_IO_TYPE_FLUSH:
case SPDK_BDEV_IO_TYPE_UNMAP:
longhorn_submit_null_payload_request(longhorn_io);
break;
default:
SPDK_ERRLOG("submit request, invalid io type %u\n", bdev_io->type);
longhorn_bdev_io_complete(longhorn_io, SPDK_BDEV_IO_STATUS_FAILED);
break;
}
}
/*
* brief:
* _longhorn_bdev_io_type_supported checks whether io_type is supported in
@ -756,7 +577,7 @@ longhorn_bdev_dump_info_json(void *ctx, struct spdk_json_write_ctx *w)
spdk_json_write_named_uint32(w, "num_base_bdevs_discovered", longhorn_bdev->num_base_bdevs_discovered);
spdk_json_write_name(w, "base_bdevs_list");
spdk_json_write_array_begin(w);
//LONGHORN_FOR_EACH_BASE_BDEV(longhorn_bdev, base_info) {
TAILQ_FOREACH(base_info, &longhorn_bdev->base_bdevs_head, infos) {
if (base_info->bdev) {
spdk_json_write_string(w, base_info->bdev->name);
@ -815,6 +636,8 @@ static const struct spdk_bdev_fn_table g_longhorn_bdev_fn_table = {
.write_config_json = longhorn_bdev_write_config_json,
};
#if 0
/*
* brief:
* longhorn_bdev_config_cleanup function is used to free memory for one longhorn_bdev in configuration
@ -827,8 +650,7 @@ void
longhorn_bdev_config_cleanup(struct longhorn_bdev_config *longhorn_cfg)
{
uint8_t i;
TAILQ_REMOVE(&g_longhorn_config.longhorn_bdev_config_head, longhorn_cfg, link);
TAILQ_REMOVE(&g_longhorn_config.longhorn_bdev_config_head, longhorn_cfg, link);
g_longhorn_config.total_longhorn_bdev--;
if (longhorn_cfg->base_bdev) {
@ -840,6 +662,7 @@ longhorn_bdev_config_cleanup(struct longhorn_bdev_config *longhorn_cfg)
free(longhorn_cfg->name);
free(longhorn_cfg);
}
#endif
/*
* brief:
@ -853,14 +676,17 @@ longhorn_bdev_config_cleanup(struct longhorn_bdev_config *longhorn_cfg)
static void
longhorn_bdev_free(void)
{
#if 0
struct longhorn_bdev_config *longhorn_cfg, *tmp;
SPDK_DEBUGLOG(bdev_longhorn, "longhorn_bdev_free\n");
TAILQ_FOREACH_SAFE(longhorn_cfg, &g_longhorn_config.longhorn_bdev_config_head, link, tmp) {
longhorn_bdev_config_cleanup(longhorn_cfg);
}
#endif
}
#if 0
/* brief
* longhorn_bdev_config_find_by_name is a helper function to find longhorn bdev config
* by name as key.
@ -881,6 +707,7 @@ longhorn_bdev_config_find_by_name(const char *longhorn_name)
return longhorn_cfg;
}
#endif
/* brief
* longhorn_bdev_find_by_name is a helper function to find longhorn bdev
@ -904,6 +731,7 @@ longhorn_bdev_find_by_name(const char *longhorn_name)
}
#if 0
/*
* brief
* longhorn_bdev_config_add function adds config for newly created longhorn bdev.
@ -961,51 +789,8 @@ longhorn_bdev_config_add(const char *longhorn_name, uint8_t num_base_bdevs,
*_longhorn_cfg = longhorn_cfg;
return 0;
}
/*
* brief:
* longhorn_bdev_config_add_base_bdev function add base bdev to longhorn bdev config.
*
* params:
* longhorn_cfg - pointer to longhorn bdev configuration
* base_bdev_name - name of base bdev
* slot - Position to add base bdev
*/
int
longhorn_bdev_config_add_base_bdev(struct longhorn_bdev_config *longhorn_cfg, const char *base_bdev_name,
uint8_t slot)
{
char *bdev_name;
if (slot >= longhorn_cfg->num_base_bdevs) {
return -EINVAL;
}
bdev_name = spdk_sprintf_alloc("%s/%s", base_bdev_name, longhorn_cfg->name);
#if 0
TAILQ_FOREACH(tmp, &g_longhorn_config.longhorn_bdev_config_head, link) {
for (i = 0; i < tmp->num_base_bdevs; i++) {
if (tmp->base_bdev[i].name != NULL) {
if (!strcmp(tmp->base_bdev[i].name, bdev_name)) {
SPDK_ERRLOG("duplicate base bdev name %s mentioned\n",
base_bdev_name);
return -EEXIST;
}
}
}
}
#endif
longhorn_cfg->base_bdev[slot].name = bdev_name;
if (longhorn_cfg->base_bdev[slot].name == NULL) {
SPDK_ERRLOG("unable to allocate memory\n");
return -ENOMEM;
}
return 0;
}
/*
* brief:
* longhorn_bdev_fini_start is called when bdev layer is starting the
@ -1053,6 +838,7 @@ longhorn_bdev_get_ctx_size(void)
return sizeof(struct longhorn_bdev_io);
}
#if 0
/*
* brief:
* longhorn_bdev_can_claim_bdev is the function to check if this base_bdev can be
@ -1091,6 +877,7 @@ longhorn_bdev_can_claim_bdev(const char *bdev_name, struct longhorn_bdev_config
return false;
}
#endif
static struct spdk_bdev_module g_longhorn_if = {
.name = "longhorn",
@ -1130,7 +917,7 @@ longhorn_bdev_init(void)
*/
int
//longhorn_bdev_create(struct longhorn_bdev_config *longhorn_cfg)
longhorn_bdev_create(const char *name, uint8_t num_base_bdevs)
longhorn_bdev_create(const char *name, const char *address, uint8_t num_base_bdevs)
{
struct longhorn_bdev *longhorn_bdev;
struct spdk_bdev *longhorn_bdev_gen;
@ -1142,20 +929,22 @@ longhorn_bdev_create(const char *name, uint8_t num_base_bdevs)
return -ENOMEM;
}
longhorn_bdev->name = strdup(name);
if (address != NULL && address[0] != '\0') {
longhorn_bdev->address = strdup(address);
longhorn_bdev->nqn = spdk_sprintf_alloc(VOLUME_FORMAT, address);
} else {
longhorn_bdev->nqn = spdk_sprintf_alloc(VOLUME_FORMAT, "127.0.0.1");
}
longhorn_bdev->io_ops = 0;
longhorn_bdev->num_base_bdevs = num_base_bdevs;
//longhorn_bdev->base_bdev_info = calloc(longhorn_bdev->num_base_bdevs,
// sizeof(struct longhorn_base_bdev_info));
//if (!longhorn_bdev->base_bdev_info) {
// SPDK_ERRLOG("Unable able to allocate base bdev info\n");
// free(longhorn_bdev);
// return -ENOMEM;
// }
//
pthread_mutex_init(&longhorn_bdev->base_bdevs_mutex, NULL);
TAILQ_INIT(&longhorn_bdev->pause_cbs);
TAILQ_INIT(&longhorn_bdev->base_bdevs_head);
TAILQ_INIT(&longhorn_bdev->io_channel_head);
@ -1225,8 +1014,6 @@ longhorn_bdev_alloc_base_bdev_resource(struct longhorn_bdev *longhorn_bdev, cons
SPDK_DEBUGLOG(bdev_longhorn, "bdev %s is claimed\n", bdev_name);
assert(longhorn_bdev->state != RAID_BDEV_STATE_ONLINE);
//assert(base_bdev_slot < longhorn_bdev->num_base_bdevs);
base_info = calloc(sizeof (struct longhorn_base_bdev_info), 1);
@ -1270,13 +1057,8 @@ longhorn_bdev_configure_base_info(struct longhorn_bdev *longhorn_bdev,
SPDK_DEBUGLOG(bdev_longhorn, "bdev %s is claimed\n", base_info->bdev_name);
assert(longhorn_bdev->state != RAID_BDEV_STATE_ONLINE);
//assert(base_bdev_slot < longhorn_bdev->num_base_bdevs);
//assert(longhorn_bdev->state != RAID_BDEV_STATE_ONLINE);
//base_info = calloc(sizeof (struct longhorn_base_bdev_info), 1);
base_info->thread = spdk_get_thread();
base_info->bdev = bdev;
base_info->desc = desc;
@ -1366,8 +1148,15 @@ longhorn_bdev_configure(struct longhorn_bdev *longhorn_bdev)
longhorn_bdev_gen->name, longhorn_bdev);
nqn = spdk_sprintf_alloc(VOLUME_FORMAT, longhorn_bdev_gen->name);
longhorn_publish_nvmf(longhorn_bdev_gen->name, nqn, "127.0.0.1",
4420, longhorn_bdev_nvmf_cb, NULL);
if (longhorn_bdev->address) {
longhorn_publish_nvmf(longhorn_bdev_gen->name, nqn,
longhorn_bdev->address,
4420, longhorn_bdev_nvmf_cb, NULL);
} else {
longhorn_publish_nvmf(longhorn_bdev_gen->name, nqn, "127.0.0.1",
4420, longhorn_bdev_nvmf_cb, NULL);
}
return 0;
@ -1507,6 +1296,8 @@ longhorn_bdev_event_base_bdev(enum spdk_bdev_event_type type, struct spdk_bdev *
}
}
#if 0
/*
* brief:
* Remove base bdevs from the longhorn bdev one by one. Skip any base bdev which
@ -1574,6 +1365,7 @@ longhorn_bdev_remove_base_devices(struct longhorn_bdev_config *longhorn_cfg,
longhorn_bdev_deconfigure(longhorn_bdev, cb_fn, cb_arg);
}
#endif
/*
* brief:
@ -1608,10 +1400,17 @@ longhorn_bdev_add_base_device(struct longhorn_bdev *longhorn_bdev,
assert(longhorn_bdev->num_base_bdevs_discovered <= longhorn_bdev->num_base_bdevs);
if (longhorn_bdev->num_base_bdevs_discovered == longhorn_bdev->num_base_bdevs) {
rc = longhorn_bdev_configure(longhorn_bdev);
if (rc != 0) {
SPDK_ERRLOG("Failed to configure longhorn bdev\n");
return rc;
if (longhorn_bdev->configured) {
SPDK_DEBUGLOG(bdev_longhorn, "Device already configured\n");
} else {
rc = longhorn_bdev_configure(longhorn_bdev);
if (rc != 0) {
SPDK_ERRLOG("Failed to configure longhorn bdev\n");
return rc;
}
longhorn_bdev->configured = true;
}
}
@ -1650,7 +1449,7 @@ generate_prefix(const char *name) {
}
int
longhorn_bdev_add_replica(const char *name, char *lvs, char *addr, uint16_t nvmf_port, uint16_t comm_port) {
longhorn_bdev_add_replica(const char *name, char *lvs, char *addr, uint16_t nvmf_port, uint16_t comm_port, enum longhorn_base_bdev_state state) {
struct longhorn_bdev *longhorn_bdev;
struct longhorn_base_bdev_info *base_info;
struct replica_add_ctx *ctx;
@ -1664,6 +1463,7 @@ longhorn_bdev_add_replica(const char *name, char *lvs, char *addr, uint16_t nvmf
base_info = calloc(1, sizeof(*base_info));
base_info->lvs = strdup(lvs);
base_info->state = state;
if ((!addr || addr[0] == '\0')) {
base_info->is_local = true;
@ -1683,7 +1483,7 @@ longhorn_bdev_add_replica(const char *name, char *lvs, char *addr, uint16_t nvmf
ctx->base_info = base_info;
ctx->prefix = generate_prefix(name);
SPDK_ERRLOG("attempting to attach %s\n", base_info->remote_nqn);
SPDK_DEBUGLOG(bdev_longhorn, "attempting to attach %s\n", base_info->remote_nqn);
longhorn_attach_nvmf(ctx->prefix, base_info->remote_nqn, addr,
nvmf_port, longhorn_replica_attach_cb, ctx);
@ -1783,6 +1583,8 @@ longhorn_bdev_find_base_bdev(struct longhorn_bdev *longhorn_bdev, char *lvs, cha
struct io_channel_remove_ctx {
struct longhorn_base_bdev_info *base_info;
struct longhorn_bdev_io_channel *io_channel;
atomic_uint *num_io_channels_to_remove;
};
static struct longhorn_base_io_channel *
@ -1808,9 +1610,15 @@ static void longhorn_io_channel_remove_bdev(void *arg) {
if (base_channel != NULL) {
SPDK_ERRLOG("removing %p\n", base_channel);
SPDK_DEBUGLOG(bdev_longhorn, "removing %p\n", base_channel);
TAILQ_REMOVE(&ctx->io_channel->base_channels, base_channel, channels);
//1TAILQ_INSERT_TAIL(&longhorn_ch->base_channels, base_channel, channels);
spdk_put_io_channel(base_channel->base_channel);
ctx->io_channel->num_channels--;
free(base_channel);
}
@ -1818,18 +1626,84 @@ static void longhorn_io_channel_remove_bdev(void *arg) {
ctx->io_channel->last_read_io_ch = NULL;
TAILQ_FOREACH(base_channel, &ctx->io_channel->base_channels, channels) {
SPDK_ERRLOG("Longhorn base bdev '%s' remaining %p\n", base_channel->base_info->lvs, base_channel);
SPDK_DEBUGLOG(bdev_longhorn, "Longhorn base bdev '%s' remaining %p\n", base_channel->base_info->lvs, base_channel);
}
/* TODO If this is the last io_channel to remove,
* * unclaim the bdev
* * free the base_info */
SPDK_DEBUGLOG(bdev_longhorn, "io_channels_to_remove = %u\n", *ctx->num_io_channels_to_remove - 1);
atomic_fetch_sub(ctx->num_io_channels_to_remove, 1);
if (atomic_load(ctx->num_io_channels_to_remove) == 0) {
SPDK_DEBUGLOG(bdev_longhorn, "All io_channels removed\n");
free(ctx->num_io_channels_to_remove);
spdk_bdev_module_release_bdev(ctx->base_info->bdev);
spdk_thread_send_msg(ctx->base_info->thread, _longhorn_bdev_free_base_bdev_resource, ctx->base_info->desc);
free(ctx->base_info);
}
free(ctx);
}
struct io_channel_add_ctx {
struct longhorn_base_bdev_info *base_info;
struct longhorn_bdev_io_channel *io_channel;
atomic_uint *num_io_channels_to_add;
};
static void longhorn_io_channel_add_bdev(void *arg) {
struct io_channel_add_ctx *ctx = arg;
struct longhorn_base_io_channel *base_channel;
base_channel = calloc(1, sizeof(*base_channel));
/*
* Get the spdk_io_channel for all the base bdevs. This is used during
* split logic to send the respective child bdev ios to respective base
* bdev io channel.
*/
base_channel->base_channel = spdk_bdev_get_io_channel(ctx->base_info->desc);
base_channel->base_info = ctx->base_info;
if (!base_channel->base_channel) {
SPDK_ERRLOG("Unable to create io channel for base bdev\n");
}
ctx->io_channel->num_channels++;
TAILQ_INSERT_TAIL(&ctx->io_channel->base_channels,
base_channel, channels);
SPDK_DEBUGLOG(bdev_longhorn, "io_channels_to_add = %u\n", *ctx->num_io_channels_to_add - 1);
atomic_fetch_sub(ctx->num_io_channels_to_add, 1);
if (atomic_load(ctx->num_io_channels_to_add) == 0) {
SPDK_DEBUGLOG(bdev_longhorn, "bdev added to all io_channels\n");
// TODO
}
free(ctx);
}
int longhorn_bdev_remove_replica(char *name, char *lvs, char *addr, uint16_t nvmf_port, uint16_t comm_port) {
struct longhorn_bdev *longhorn_bdev;
@ -1837,6 +1711,7 @@ int longhorn_bdev_remove_replica(char *name, char *lvs, char *addr, uint16_t nvm
struct longhorn_bdev_io_channel *io_channel;
struct io_channel_remove_ctx *ctx;
int rc;
atomic_uint *num_io_channels_to_remove;
longhorn_bdev = longhorn_bdev_find_by_name(name);
if (!longhorn_bdev) {
@ -1868,20 +1743,77 @@ int longhorn_bdev_remove_replica(char *name, char *lvs, char *addr, uint16_t nvm
longhorn_bdev->num_base_bdevs_discovered--;
longhorn_bdev->num_base_bdevs--;
// claim
TAILQ_REMOVE(&longhorn_bdev->base_bdevs_head, base_info, infos);
/* signal each longhorn_io to stop using the bdev */
SPDK_ERRLOG("num io channels %u\n", longhorn_bdev->num_io_channels);
SPDK_DEBUGLOG(bdev_longhorn,"num io channels %u\n", longhorn_bdev->num_io_channels);
num_io_channels_to_remove = calloc(1, sizeof(atomic_int));
atomic_init(num_io_channels_to_remove, longhorn_bdev->num_io_channels);
TAILQ_FOREACH(io_channel, &longhorn_bdev->io_channel_head, channels) {
ctx = calloc(1, sizeof (*ctx));
ctx->base_info = base_info;
ctx->io_channel = io_channel;
ctx->num_io_channels_to_remove = num_io_channels_to_remove;
if (!io_channel->deleted) {
spdk_thread_send_msg(io_channel->thread, longhorn_io_channel_remove_bdev, ctx);
}
}
pthread_mutex_unlock(&longhorn_bdev->base_bdevs_mutex);
return 0;
}
int longhorn_bdev_add_base_replica(struct longhorn_base_bdev_info *base_info)
{
struct longhorn_bdev *longhorn_bdev;
struct longhorn_bdev_io_channel *io_channel;
struct io_channel_add_ctx *ctx;
int rc;
atomic_uint *num_io_channels_to_add;
rc = pthread_mutex_trylock(&longhorn_bdev->base_bdevs_mutex);
if (rc != 0) {
if (errno == EBUSY) {
SPDK_ERRLOG("Longhorn bdev '%s' is busy\n",
longhorn_bdev->bdev.name);
}
return -errno;
}
// claim
TAILQ_REMOVE(&longhorn_bdev->base_bdevs_head, base_info, infos);
/* signal each longhorn_io to stop using the bdev */
SPDK_DEBUGLOG(bdev_longhorn,"num io channels %u\n", longhorn_bdev->num_io_channels);
num_io_channels_to_add = calloc(1, sizeof(atomic_int));
atomic_init(num_io_channels_to_add, longhorn_bdev->num_io_channels);
TAILQ_FOREACH(io_channel, &longhorn_bdev->io_channel_head, channels) {
ctx = calloc(1, sizeof (*ctx));
ctx->base_info = base_info;
ctx->io_channel = io_channel;
ctx->num_io_channels_to_add = num_io_channels_to_add;
if (!io_channel->deleted) {
spdk_thread_send_msg(io_channel->thread, longhorn_io_channel_add_bdev, ctx);
}
}
@ -1893,14 +1825,14 @@ int longhorn_bdev_remove_replica(char *name, char *lvs, char *addr, uint16_t nvm
return 0;
}
int longhorn_volume_add_replica(char *name, char *lvs, char *addr, uint16_t nvmf_port, uint16_t comm_port) {
struct longhorn_bdev *longhorn_bdev;
//struct longhorn_base_bdev_info *base_info;
//struct longhorn_bdev_io_channel *io_channel;
struct longhorn_bdev_io_channel *io_channel;
//struct io_channel_remove_ctx *ctx;
int rc;
/* Create base_info and add to base_info list */
longhorn_bdev = longhorn_bdev_find_by_name(name);
if (!longhorn_bdev) {
SPDK_ERRLOG("Longhorn bdev '%s' is not created yet\n", name);
@ -1915,10 +1847,28 @@ int longhorn_volume_add_replica(char *name, char *lvs, char *addr, uint16_t nvmf
}
return -errno;
}
longhorn_bdev->num_base_bdevs++;
longhorn_bdev_add_replica(name, lvs, addr, nvmf_port, comm_port, LONGHORN_BASE_BDEV_WO);
TAILQ_FOREACH(io_channel, &longhorn_bdev->io_channel_head, channels) {
spdk_thread_send_msg(io_channel->thread, bdev_longhorn_pause_io, io_channel);
}
longhorn_bdev->op_in_progress = true;
longhorn_bdev->op_in_progress = true;
pthread_mutex_unlock(&longhorn_bdev->base_bdevs_mutex);
/* pause writing */
/* snapshot the existing replicas and this new replica. */
/* add new replica to any open io_channels as write only */
/* rebuild all snapshots of the new replica */
/* reparent the new replica lvol */
/* mark the replica as read/write */
return 0;
}
@ -1928,19 +1878,20 @@ int longhorn_unpause(struct longhorn_bdev *longhorn_bdev)
struct longhorn_bdev_io_channel *io_channel;
rc = pthread_mutex_trylock(&longhorn_bdev->base_bdevs_mutex);
rc =pthread_mutex_trylock(&longhorn_bdev->base_bdevs_mutex);
if (rc != 0) {
return -errno;
}
TAILQ_FOREACH(io_channel, &longhorn_bdev->io_channel_head, channels) {
longhorn_pause_queue_playback(io_channel);
spdk_thread_send_msg(io_channel->thread, bdev_longhorn_unpause_io, io_channel);
}
pthread_mutex_unlock(&longhorn_bdev->base_bdevs_mutex);
SPDK_ERRLOG("UNPAUSE COMPLETE \n");
SPDK_DEBUGLOG(bdev_longhorn, "UNPAUSE COMPLETE \n");
return 0;
}

65
module/bdev/longhorn/bdev_longhorn.h
View File

@ -67,6 +67,12 @@ enum longhorn_base_bdev_state {
LONGHORN_BASE_BDEV_ERR
};
enum longhorn_pause_operation {
LONGHORN_PAUSE_OP_NONE,
LONGHORN_PAUSE_OP_SNAPSHOT,
LONGHORN_PAUSE_OP_ADD,
};
/*
* longhorn_base_bdev_info contains information for the base bdevs which are part of some
* longhorn. This structure contains the per base bdev information. Whatever is
@ -162,6 +168,10 @@ struct longhorn_bdev_io_channel {
TAILQ_ENTRY(longhorn_bdev_io_channel) channels;
struct longhorn_base_io_channel *last_read_io_ch;
TAILQ_HEAD(, spdk_bdev_io_wait_entry) io_wait_queue;
uint32_t queue_len;
};
TAILQ_HEAD(io_channels, longhorn_bdev_io_channel);
@ -182,6 +192,9 @@ struct longhorn_pause_cb_entry {
struct longhorn_bdev {
/* longhorn bdev device, this will get registered in bdev layer */
struct spdk_bdev bdev;
char *name;
char *address;
char *nqn;
/* link of longhorn bdev to link it to configured, configuring or offline list */
TAILQ_ENTRY(longhorn_bdev) state_link;
@ -225,56 +238,16 @@ struct longhorn_bdev {
bool destroy_started;
bool op_in_progress;
enum longhorn_pause_operation pause_op;
bool configured;
atomic_int io_ops;
atomic_int channels_to_pause;
};
#define LONGHORN_FOR_EACH_BASE_BDEV(r, i) \
for (i = r->base_bdev_info; i < r->base_bdev_info + r->num_base_bdevs; i++)
/*
* longhorn_base_bdev_config is the per base bdev data structure which contains
* information w.r.t to per base bdev during parsing config
*/
struct longhorn_base_bdev_config {
/* base bdev name from config file */
char *name;
};
/*
* longhorn_bdev_config contains the longhorn bdev config related information after
* parsing the config file
*/
struct longhorn_bdev_config {
/* base bdev config per underlying bdev */
struct longhorn_base_bdev_config *base_bdev;
/* Points to already created longhorn bdev */
struct longhorn_bdev *longhorn_bdev;
char *name;
/* number of base bdevs */
uint8_t num_base_bdevs;
TAILQ_ENTRY(longhorn_bdev_config) link;
};
/*
* longhorn_config is the top level structure representing the longhorn bdev config as read
* from config file for all longhorns
*/
struct longhorn_config {
/* longhorn bdev context from config file */
TAILQ_HEAD(, longhorn_bdev_config) longhorn_bdev_config_head;
/* total longhorn bdev from config file */
uint8_t total_longhorn_bdev;
};
/* TAIL heads for various longhorn bdev lists */
TAILQ_HEAD(longhorn_configured_tailq, longhorn_bdev);
TAILQ_HEAD(longhorn_configuring_tailq, longhorn_bdev);
@ -289,8 +262,7 @@ extern struct longhorn_config g_longhorn_config;
typedef void (*longhorn_bdev_destruct_cb)(void *cb_ctx, int rc);
//int longhorn_bdev_create(struct longhorn_bdev_config *longhorn_cfg);
int longhorn_bdev_create(const char *name, uint8_t num_base_bdevs);
int longhorn_bdev_create(const char *name, const char *address, uint8_t num_base_bdevs);
int longhorn_bdev_add_base_devices(struct longhorn_bdev_config *longhorn_cfg);
void longhorn_bdev_remove_base_devices(struct longhorn_bdev_config *longhorn_cfg,
longhorn_bdev_destruct_cb cb_fn, void *cb_ctx);
@ -311,11 +283,10 @@ longhorn_bdev_queue_io_wait(struct longhorn_bdev_io *longhorn_io, struct spdk_bd
void
longhorn_bdev_io_complete(struct longhorn_bdev_io *longhorn_io, enum spdk_bdev_io_status status);
//int longhorn_bdev_add_base_device(const char *name, const char *bdev_name);
int longhorn_bdev_add_base_device(struct longhorn_bdev *longhorn_bdev, struct longhorn_base_bdev_info *base_info);
int longhorn_bdev_remove_replica(char *name, char *lvs, char *addr, uint16_t nvmf_port, uint16_t comm_port);
int
longhorn_bdev_add_replica(const char *name, char *lvs, char *addr, uint16_t nvmf_port, uint16_t comm_port);
longhorn_bdev_add_replica(const char *name, char *lvs, char *addr, uint16_t nvmf_port, uint16_t comm_port, enum longhorn_base_bdev_state state);
void bdev_longhorn_pause_io(void *cb_arg);
void bdev_longhorn_unpause_io(void *cb_arg);

5
module/bdev/longhorn/bdev_longhorn_impl.c
View File

@ -99,11 +99,7 @@ longhorn_submit_read_request(struct longhorn_bdev_io *longhorn_io)
struct spdk_io_channel *base_ch;
struct longhorn_base_io_channel *base_channel;
SPDK_ERRLOG("longhorn_submit_read_request\n");
assert(longhorn_ch != NULL);
SPDK_ERRLOG("longhorn_submit_read_request\n");
assert(longhorn_ch->base_channel);
SPDK_ERRLOG("longhorn_submit_read_request\n");
if (longhorn_ch->last_read_io_ch) {
@ -258,7 +254,6 @@ longhorn_submit_null_payload_request(struct longhorn_bdev_io *longhorn_io)
struct longhorn_base_io_channel *base_channel;
assert(longhorn_ch != NULL);
assert(longhorn_ch->base_channel);
if (longhorn_io->base_bdev_io_remaining == 0) {
longhorn_io->base_bdev_io_remaining = longhorn_bdev->num_base_bdevs;

542
module/bdev/longhorn/bdev_longhorn_io.c Normal file
View File

@ -0,0 +1,542 @@
/*-
* BSD LICENSE
*
* Copyright (c) SUSE
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of SUSE nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "bdev_longhorn.h"
#include "bdev_longhorn_impl.h"
#include "spdk/env.h"
#include "spdk/thread.h"
#include "spdk/string.h"
#include "spdk/util.h"
#include "spdk/log.h"
/*
* brief:
* longhorn_bdev_io_completion function is called by lower layers to notify longhorn
* module that particular bdev_io is completed.
* params:
* bdev_io - pointer to bdev io submitted to lower layers, like child io
* success - bdev_io status
* cb_arg - function callback context (parent longhorn_bdev_io)
* returns:
* none
*/
static void
longhorn_bdev_io_completion(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
{
struct longhorn_bdev_io *longhorn_io = cb_arg;
spdk_bdev_free_io(bdev_io);
if (success) {
//SPDK_ERRLOG("io op success\n");
longhorn_bdev_io_complete(longhorn_io, SPDK_BDEV_IO_STATUS_SUCCESS);
} else {
//SPDK_ERRLOG("io op failure\n");
longhorn_bdev_io_complete(longhorn_io, SPDK_BDEV_IO_STATUS_FAILED);
}
}
static void
longhorn_base_io_complete(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
{
struct longhorn_bdev_io *longhorn_io = cb_arg;
longhorn_bdev_io_complete_part(longhorn_io, 1, success ?
SPDK_BDEV_IO_STATUS_SUCCESS :
SPDK_BDEV_IO_STATUS_FAILED);
spdk_bdev_free_io(bdev_io);
}
static void
_longhorn_submit_rw_request(void *_longhorn_io)
{
struct longhorn_bdev_io *longhorn_io = _longhorn_io;
longhorn_submit_rw_request(longhorn_io);
}
static struct longhorn_base_io_channel *
_longhorn_get_next_readable_base_channel(struct longhorn_bdev_io_channel *longhorn_ch) {
/* Get next channel */
if (longhorn_ch->last_read_io_ch) {
longhorn_ch->last_read_io_ch = TAILQ_NEXT(longhorn_ch->last_read_io_ch, channels);
}
/* Search for a RW replica */
while (longhorn_ch->last_read_io_ch &&
longhorn_ch->last_read_io_ch->base_info->state != LONGHORN_BASE_BDEV_RW) {
longhorn_ch->last_read_io_ch = TAILQ_NEXT(longhorn_ch->last_read_io_ch, channels);
}
/* We found a RW replica before the end of the list */
if (longhorn_ch->last_read_io_ch) {
return longhorn_ch->last_read_io_ch;
}
/* Search for a RW replica from the beginning of the list */
longhorn_ch->last_read_io_ch = TAILQ_FIRST(&longhorn_ch->base_channels);
while (longhorn_ch->last_read_io_ch &&
longhorn_ch->last_read_io_ch->base_info->state != LONGHORN_BASE_BDEV_RW) {
longhorn_ch->last_read_io_ch = TAILQ_NEXT(longhorn_ch->last_read_io_ch, channels);
}
return longhorn_ch->last_read_io_ch;
}
static void
longhorn_submit_read_request(struct longhorn_bdev_io *longhorn_io)
{
struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(longhorn_io);
struct longhorn_bdev_io_channel *longhorn_ch = longhorn_io->longhorn_ch;
struct longhorn_bdev *longhorn_bdev = longhorn_io->longhorn_bdev;
int ret = 0;
struct longhorn_base_bdev_info *base_info;
struct spdk_io_channel *base_ch;
struct longhorn_base_io_channel *base_channel;
assert(longhorn_ch != NULL);
base_channel = _longhorn_get_next_readable_base_channel(longhorn_ch);
if (!base_channel) {
SPDK_ERRLOG("bdev io submit with no base devices, it should not happen\n");
return;
}
base_ch = base_channel->base_channel;
base_info = base_channel->base_info;
//SPDK_ERRLOG("longhorn_submit_read_request base_info %p\n", base_info);
ret = spdk_bdev_readv_blocks(base_info->desc, base_ch,
bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
bdev_io->u.bdev.offset_blocks, bdev_io->u.bdev.num_blocks, longhorn_bdev_io_completion,
longhorn_io);
if (ret == -ENOMEM) {
longhorn_bdev_queue_io_wait(longhorn_io, base_info->bdev, base_ch,
_longhorn_submit_rw_request);
} else if (ret != 0) {
SPDK_ERRLOG("bdev io submit error not due to ENOMEM, it should not happen\n");
assert(false);
longhorn_bdev_io_complete(longhorn_io, SPDK_BDEV_IO_STATUS_FAILED);
}
}
static void
longhorn_submit_write_request(struct longhorn_bdev_io *longhorn_io)
{
struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(longhorn_io);
struct longhorn_bdev_io_channel *longhorn_ch = longhorn_io->longhorn_ch;
struct longhorn_bdev *longhorn_bdev = longhorn_io->longhorn_bdev;
//uint8_t pd_idx;
int ret = 0;
struct longhorn_base_bdev_info *base_info;
struct spdk_io_channel *base_ch;
struct longhorn_base_io_channel *base_channel;
assert(longhorn_ch != NULL);
if (longhorn_io->base_bdev_io_remaining == 0) {
longhorn_io->base_bdev_io_remaining = longhorn_ch->num_channels;
}
TAILQ_FOREACH(base_channel, &longhorn_ch->base_channels, channels) {
//for (pd_idx = 0; pd_idx < longhorn_bdev->num_base_bdevs; pd_idx++) {
//base_ch = longhorn_ch->base_channel[pd_idx];
//base_info = &longhorn_bdev->base_bdev_info[pd_idx];
base_ch = base_channel->base_channel;
base_info = base_channel->base_info;
ret = spdk_bdev_writev_blocks(base_info->desc, base_ch,
bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
bdev_io->u.bdev.offset_blocks, bdev_io->u.bdev.num_blocks, longhorn_base_io_complete,
longhorn_io);
if (ret == -ENOMEM) {
SPDK_ERRLOG("enqueuing bdev io submit due to ENOMEM\n");
longhorn_bdev_queue_io_wait(longhorn_io, base_info->bdev, base_ch,
_longhorn_submit_rw_request);
} else if (ret != 0) {
SPDK_ERRLOG("bdev io submit error not due to ENOMEM, it should not happen\n");
assert(false);
longhorn_bdev_io_complete(longhorn_io, SPDK_BDEV_IO_STATUS_FAILED);
}
atomic_fetch_add(&longhorn_bdev->io_ops, 1);
atomic_fetch_add(&longhorn_ch->io_ops, 1);
longhorn_io->submitted = true;
}
}
/*
* brief:
* longhorn_submit_rw_request function is used to submit I/O to the correct
* member disk for longhorn bdevs.
* params:
* longhorn_io
* returns:
* none
*/
void
longhorn_submit_rw_request(struct longhorn_bdev_io *longhorn_io)
{
struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(longhorn_io);
if (bdev_io->type == SPDK_BDEV_IO_TYPE_READ) {
longhorn_submit_read_request(longhorn_io);
} else if (bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE) {
longhorn_submit_write_request(longhorn_io);
} else {
SPDK_ERRLOG("Recvd not supported io type %u\n", bdev_io->type);
assert(0);
}
}
static void
_longhorn_submit_null_payload_request(void *_longhorn_io)
{
struct longhorn_bdev_io *longhorn_io = _longhorn_io;
longhorn_submit_null_payload_request(longhorn_io);
}
/*
* brief:
* longhorn_submit_null_payload_request function submits the next batch of
* io requests with range but without payload, like FLUSH and UNMAP, to member disks;
* it will submit as many as possible unless one base io request fails with -ENOMEM,
* in which case it will queue itself for later submission.
* params:
* bdev_io - pointer to parent bdev_io on longhorn bdev device
* returns:
* none
*/
void
longhorn_submit_null_payload_request(struct longhorn_bdev_io *longhorn_io)
{
struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(longhorn_io);
struct longhorn_bdev_io_channel *longhorn_ch = longhorn_io->longhorn_ch;
struct longhorn_bdev *longhorn_bdev = longhorn_io->longhorn_bdev;
uint8_t pd_idx;
int ret = 0;
struct longhorn_base_bdev_info *base_info;
struct spdk_io_channel *base_ch;
struct longhorn_base_io_channel *base_channel;
assert(longhorn_ch != NULL);
if (longhorn_io->base_bdev_io_remaining == 0) {
longhorn_io->base_bdev_io_remaining = longhorn_bdev->num_base_bdevs;
}
TAILQ_FOREACH(base_channel, &longhorn_ch->base_channels, channels) {
//for (pd_idx = 0; pd_idx < longhorn_bdev->num_base_bdevs; pd_idx++) {
//base_ch = longhorn_ch->base_channel[pd_idx];
//base_info = &longhorn_bdev->base_bdev_info[pd_idx];
base_ch = base_channel->base_channel;
base_info = base_channel->base_info;
switch (bdev_io->type) {
case SPDK_BDEV_IO_TYPE_UNMAP:
ret = spdk_bdev_unmap_blocks(base_info->desc, base_ch,
bdev_io->u.bdev.offset_blocks,
bdev_io->u.bdev.num_blocks,
longhorn_base_io_complete, longhorn_io);
break;
case SPDK_BDEV_IO_TYPE_FLUSH:
ret = spdk_bdev_flush_blocks(base_info->desc, base_ch,
bdev_io->u.bdev.offset_blocks,
bdev_io->u.bdev.num_blocks,
longhorn_base_io_complete, longhorn_io);
break;
default:
SPDK_ERRLOG("submit request, invalid io type with null payload %u\n", bdev_io->type);
assert(false);
ret = -EIO;
}
if (ret == -ENOMEM) {
longhorn_bdev_queue_io_wait(longhorn_io, base_info->bdev, base_ch,
_longhorn_submit_null_payload_request);
} else if (ret != 0) {
SPDK_ERRLOG("bdev io submit error not due to ENOMEM, it should not happen\n");
assert(false);
longhorn_bdev_io_complete(longhorn_io, SPDK_BDEV_IO_STATUS_FAILED);
} else {
SPDK_ERRLOG("success\n");
}
}
}
int longhorn_start(struct longhorn_bdev *longhorn_bdev)
{
uint64_t min_blockcnt = UINT64_MAX;
uint64_t min_blocklen = UINT64_MAX;
struct longhorn_base_bdev_info *base_info;
TAILQ_FOREACH(base_info, &longhorn_bdev->base_bdevs_head, infos) {
/* Calculate minimum block count and length from all base bdevs */
min_blockcnt = spdk_min(min_blockcnt, base_info->bdev->blockcnt);
min_blocklen = spdk_min(min_blocklen, base_info->bdev->blocklen);
}
TAILQ_FOREACH(base_info, &longhorn_bdev->base_bdevs_head, infos) {
if (base_info->bdev->blockcnt != min_blockcnt) {
SPDK_ERRLOG("Not all disks on RAID 1 has same block count");
return -EINVAL;
}
if (base_info->bdev->blocklen != min_blocklen) {
SPDK_ERRLOG("Not all disks on RAID 1 has same block length");
return -EINVAL;
}
}
longhorn_bdev->bdev.blockcnt = min_blockcnt;
if (longhorn_bdev->num_base_bdevs > 1) {
longhorn_bdev->bdev.split_on_optimal_io_boundary = true;
} else {
/* Do not need to split reads/writes on single bdev RAID modules. */
longhorn_bdev->bdev.split_on_optimal_io_boundary = false;
}
return 0;
}
static void
longhorn_base_bdev_reset_complete(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
{
struct longhorn_bdev_io *longhorn_io = cb_arg;
spdk_bdev_free_io(bdev_io);
longhorn_bdev_io_complete_part(longhorn_io, 1, success ?
SPDK_BDEV_IO_STATUS_SUCCESS :
SPDK_BDEV_IO_STATUS_FAILED);
}
static void
longhorn_bdev_submit_reset_request(struct longhorn_bdev_io *longhorn_io);
static void
_longhorn_bdev_submit_reset_request(void *_longhorn_io)
{
struct longhorn_bdev_io *longhorn_io = _longhorn_io;
longhorn_bdev_submit_reset_request(longhorn_io);
}
/*
* brief:
* longhorn_bdev_submit_reset_request function submits reset requests
* to member disks; it will submit as many as possible unless a reset fails with -ENOMEM, in
* which case it will queue it for later submission
* params:
* longhorn_io
* returns:
* none
*/
static void
longhorn_bdev_submit_reset_request(struct longhorn_bdev_io *longhorn_io)
{
struct longhorn_bdev_io_channel *longhorn_ch = longhorn_io->longhorn_ch;
struct longhorn_bdev *longhorn_bdev = longhorn_io->longhorn_bdev;
int ret;
struct longhorn_base_bdev_info *base_info;
struct spdk_io_channel *base_ch;
struct longhorn_base_io_channel *base_channel;
if (longhorn_io->base_bdev_io_remaining == 0) {
longhorn_io->base_bdev_io_remaining = longhorn_bdev->num_base_bdevs;
}
TAILQ_FOREACH(base_channel, &longhorn_ch->base_channels, channels) {
base_ch = base_channel->base_channel;
base_info = base_channel->base_info;
ret = spdk_bdev_reset(base_info->desc, base_ch,
longhorn_base_bdev_reset_complete, longhorn_io);
if (ret == 0) {
longhorn_io->base_bdev_io_submitted++;
} else if (ret == -ENOMEM) {
longhorn_bdev_queue_io_wait(longhorn_io, base_info->bdev, base_ch,
_longhorn_bdev_submit_reset_request);
return;
} else {
SPDK_ERRLOG("bdev io submit error not due to ENOMEM, it should not happen\n");
assert(false);
longhorn_bdev_io_complete(longhorn_io, SPDK_BDEV_IO_STATUS_FAILED);
return;
}
}
}
/*
* brief:
* Callback function to spdk_bdev_io_get_buf.
* params:
* ch - pointer to longhorn bdev io channel
* bdev_io - pointer to parent bdev_io on longhorn bdev device
* success - True if buffer is allocated or false otherwise.
* returns:
* none
*/
static void
longhorn_bdev_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io,
bool success)
{
struct longhorn_bdev_io *longhorn_io = (struct longhorn_bdev_io *)bdev_io->driver_ctx;
if (!success) {
longhorn_bdev_io_complete(longhorn_io, SPDK_BDEV_IO_STATUS_FAILED);
return;
}
longhorn_submit_rw_request(longhorn_io);
}
/*
* brief:
* longhorn_bdev_submit_request function is the submit_request function pointer of
* longhorn bdev function table. This is used to submit the io on longhorn_bdev to below
* layers.
* params:
* ch - pointer to longhorn bdev io channel
* bdev_io - pointer to parent bdev_io on longhorn bdev device
* returns:
* none
*/
static void
_longhorn_bdev_submit_request(struct longhorn_bdev_io *longhorn_io)
{
struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(longhorn_io);
switch (bdev_io->type) {
case SPDK_BDEV_IO_TYPE_READ:
spdk_bdev_io_get_buf(bdev_io, longhorn_bdev_get_buf_cb,
bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen);
break;
case SPDK_BDEV_IO_TYPE_WRITE:
longhorn_submit_rw_request(longhorn_io);
break;
case SPDK_BDEV_IO_TYPE_RESET:
longhorn_bdev_submit_reset_request(longhorn_io);
break;
case SPDK_BDEV_IO_TYPE_FLUSH:
case SPDK_BDEV_IO_TYPE_UNMAP:
longhorn_submit_null_payload_request(longhorn_io);
break;
default:
SPDK_ERRLOG("submit request, invalid io type %u\n", bdev_io->type);
longhorn_bdev_io_complete(longhorn_io, SPDK_BDEV_IO_STATUS_FAILED);
break;
}
}
void
longhorn_pause_queue_io_wait(struct longhorn_bdev_io *longhorn_io)
{
longhorn_io->waitq_entry.cb_arg = longhorn_io;
longhorn_io->longhorn_ch->queue_len++;
SPDK_ERRLOG("adding to queue: %d\n", longhorn_io->longhorn_ch->queue_len);
TAILQ_INSERT_TAIL(&longhorn_io->longhorn_ch->io_wait_queue, &longhorn_io->waitq_entry, link);
}
void
longhorn_pause_queue_playback(struct longhorn_bdev_io_channel *longhorn_ch)
{
while (!TAILQ_EMPTY(&longhorn_ch->io_wait_queue)) {
struct spdk_bdev_io_wait_entry *entry;
entry = TAILQ_FIRST(&longhorn_ch->io_wait_queue);
TAILQ_REMOVE(&longhorn_ch->io_wait_queue, entry, link);
_longhorn_bdev_submit_request(entry->cb_arg);
longhorn_ch->queue_len--;
SPDK_ERRLOG("removing to queue: %d\n", longhorn_ch->queue_len);
}
}
void
longhorn_bdev_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
{
struct longhorn_bdev_io *longhorn_io = (struct longhorn_bdev_io *)bdev_io->driver_ctx;
longhorn_io->longhorn_bdev = bdev_io->bdev->ctxt;
longhorn_io->longhorn_ch = spdk_io_channel_get_ctx(ch);
longhorn_io->base_bdev_io_remaining = 0;
longhorn_io->base_bdev_io_submitted = 0;
longhorn_io->base_bdev_io_status = SPDK_BDEV_IO_STATUS_SUCCESS;
if (longhorn_io->longhorn_ch->paused) {
longhorn_pause_queue_io_wait(longhorn_io);
return;
}
_longhorn_bdev_submit_request(longhorn_io);
}

15
module/bdev/longhorn/bdev_longhorn_io.h Normal file
View File

@ -0,0 +1,15 @@
#ifndef _BDEV_LONGHORN_IO__H
#define _BDEV_LONGHORN_IO__H
void
longhorn_pause_queue_playback(struct longhorn_bdev_io_channel *longhorn_ch);
void
longhorn_bdev_submit_request(struct spdk_io_channel *ch,
struct spdk_bdev_io *bdev_io);
int
longhorn_start(struct longhorn_bdev *bdev);
#endif /* _BDEV_LONGHORN_IO__H */

48
module/bdev/longhorn/bdev_longhorn_nvmf.c
View File

@ -133,7 +133,7 @@ struct longhorn_publish_nvmf_ctx {
void *cb_arg;
};
void longhorn_publish_nvmf(const char *bdev_name, const char *nqn, const char *addr, uint16_t port, longhorn_publish_nvmf_cb cb_fn, void *cb_arg) {
static void _longhorn_publish_nvmf(const char *bdev_name, const char *nqn, const char *addr, uint16_t port, longhorn_publish_nvmf_cb cb_fn, void *cb_arg) {
struct spdk_nvmf_tgt *tgt;
struct spdk_nvmf_subsystem *subsystem;
struct spdk_nvmf_ns_opts ns_opts;
@ -163,6 +163,52 @@ void longhorn_publish_nvmf(const char *bdev_name, const char *nqn, const char *a
}
struct longhorn_publish_nvmf_no_transport_ctx {
const char *bdev_name;
const char *nqn;
const char *addr;
uint16_t port;
longhorn_publish_nvmf_cb cb_fn;
void *cb_arg;
};
static void longhorn_publish_no_transport_cb(void *cb_arg) {
struct longhorn_publish_nvmf_no_transport_ctx *ctx = cb_arg;
_longhorn_publish_nvmf(ctx->bdev_name, ctx->nqn, ctx->addr, ctx->port,
ctx->cb_fn, ctx->cb_arg);
free(ctx->bdev_name);
free(ctx->nqn);
free(ctx->addr);
free(ctx);
}
void longhorn_publish_nvmf(const char *bdev_name, const char *nqn, const char *addr, uint16_t port, longhorn_publish_nvmf_cb cb_fn, void *cb_arg) {
if (tcp_transport_created) {
_longhorn_publish_nvmf(bdev_name, nqn, addr, port, cb_fn, cb_arg);
} else {
struct longhorn_publish_nvmf_no_transport_ctx *ctx;
ctx = calloc(1, sizeof(*ctx));
ctx->bdev_name = strdup(bdev_name);
ctx->nqn = strdup(nqn);
ctx->addr = strdup(addr);
ctx->port = port;
ctx->cb_fn = cb_fn;
ctx->cb_arg = cb_arg;
longhorn_nvmf_create_transport(longhorn_publish_no_transport_cb,
ctx);
}
}
#define NVME_MAX_BDEVS_PER_RPC 128
struct longhorn_attach_nvmf_ctx {

4
module/bdev/longhorn/bdev_longhorn_rebuild_rpc.c
View File

@ -72,9 +72,6 @@ rpc_bdev_lvol_list_children(struct spdk_jsonrpc_request *request,
struct spdk_bdev *bdev = NULL;
struct spdk_lvol *lvol = NULL;
struct longhorn_child_blob_context *ctx = NULL;
//spdk_blob_id parent_id;
if (spdk_json_decode_object(params, rpc_lvol_list_children_decoders,
SPDK_COUNTOF(rpc_lvol_list_children_decoders),
@ -82,7 +79,6 @@ rpc_bdev_lvol_list_children(struct spdk_jsonrpc_request *request,
SPDK_DEBUGLOG(bdev_malloc, "spdk_json_decode_object failed\n");
spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
"spdk_json_decode_object failed");
//free_rpc_construct_malloc(&req);
return;
}

80
module/bdev/longhorn/bdev_longhorn_rpc.c
View File

@ -166,8 +166,9 @@ struct rpc_bdev_longhorn_create_base_bdevs {
* Input structure for RPC rpc_bdev_longhorn_create
*/
struct rpc_bdev_longhorn_create {
/* Raid bdev name */
/* longhorn bdev name */
char *name;
char *address;
/* Base bdevs information */
struct rpc_bdev_longhorn_create_base_bdevs base_bdevs;
@ -244,6 +245,7 @@ decode_base_bdevs(const struct spdk_json_val *val, void *out)
*/
static const struct spdk_json_object_decoder rpc_bdev_longhorn_create_decoders[] = {
{"name", offsetof(struct rpc_bdev_longhorn_create, name), spdk_json_decode_string},
{"address", offsetof(struct rpc_bdev_longhorn_create, address), spdk_json_decode_string, true},
{"replicas", offsetof(struct rpc_bdev_longhorn_create, base_bdevs), decode_base_bdevs},
};
@ -262,12 +264,11 @@ rpc_bdev_longhorn_create(struct spdk_jsonrpc_request *request,
const struct spdk_json_val *params)
{
struct rpc_bdev_longhorn_create req = {};
struct longhorn_bdev_config *longhorn_cfg;
//struct longhorn_bdev_config *longhorn_cfg;
int rc;
size_t i;
char *bdev_name;
//struct spdk_json_write_ctx *w;
printf("type %d\n", params->type);
if (spdk_json_decode_object(params, rpc_bdev_longhorn_create_decoders,
SPDK_COUNTOF(rpc_bdev_longhorn_create_decoders),
&req)) {
@ -276,30 +277,7 @@ rpc_bdev_longhorn_create(struct spdk_jsonrpc_request *request,
goto cleanup;
}
#if 0
rc = longhorn_bdev_config_add(req.name, req.base_bdevs.num_base_bdevs,
&longhorn_cfg);
if (rc != 0) {
spdk_jsonrpc_send_error_response_fmt(request, rc,
"Failed to add RAID bdev config %s: %s",
req.name, spdk_strerror(-rc));
goto cleanup;
}
for (i = 0; i < req.base_bdevs.num_base_bdevs; i++) {
rc = longhorn_bdev_config_add_base_bdev(longhorn_cfg, req.base_bdevs.replicas[i].lvs, i);
if (rc != 0) {
longhorn_bdev_config_cleanup(longhorn_cfg);
spdk_jsonrpc_send_error_response_fmt(request, rc,
"Failed to add base bdev %s to RAID bdev config %s: %s",
req.base_bdevs.replicas[i].addr, req.name,
spdk_strerror(-rc));
goto cleanup;
}
}
#endif
rc = longhorn_bdev_create(req.name, req.base_bdevs.num_base_bdevs);
rc = longhorn_bdev_create(req.name, req.address, req.base_bdevs.num_base_bdevs);
if (rc != 0) {
//longhorn_bdev_config_cleanup(longhorn_cfg);
spdk_jsonrpc_send_error_response_fmt(request, rc,
@ -312,27 +290,17 @@ rpc_bdev_longhorn_create(struct spdk_jsonrpc_request *request,
//longhorn_bdev_add_base_device(req.name, bdev_name);
longhorn_bdev_add_replica(req.name, req.base_bdevs.replicas[i].lvs, req.base_bdevs.replicas[i].addr, req.base_bdevs.replicas[i].nvmf_port, req.base_bdevs.replicas[i].comm_port);
longhorn_bdev_add_replica(req.name, req.base_bdevs.replicas[i].lvs, req.base_bdevs.replicas[i].addr, req.base_bdevs.replicas[i].nvmf_port, req.base_bdevs.replicas[i].comm_port, LONGHORN_BASE_BDEV_RW);
free(bdev_name);
}
#if 0
rc = longhorn_bdev_add_base_devices(longhorn_cfg);
if (rc != 0) {
spdk_jsonrpc_send_error_response_fmt(request, rc,
"Failed to add any base bdev to RAID bdev %s: %s",
req.name, spdk_strerror(-rc));
goto cleanup;
}
#endif
spdk_jsonrpc_send_bool_response(request, true);
cleanup:
free_rpc_bdev_longhorn_create(&req);
}
@ -375,7 +343,7 @@ rpc_bdev_longhorn_add_replica_cmd(struct spdk_jsonrpc_request *request,
const struct spdk_json_val *params)
{
struct rpc_bdev_longhorn_add_replica req = {};
struct longhorn_bdev_config *longhorn_cfg;
//struct longhorn_bdev_config *longhorn_cfg;
int rc;
size_t i;
char *bdev_name;
@ -387,6 +355,16 @@ rpc_bdev_longhorn_add_replica_cmd(struct spdk_jsonrpc_request *request,
spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
"longhorn spdk_json_decode_object failed");
}
rc = longhorn_volume_add_replica(req.name, req.replica.lvs, req.replica.addr, req.replica.nvmf_port, req.replica.comm_port);
if (rc != 0) {
spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, "failed to add replica");
} else {
spdk_jsonrpc_send_bool_response(request, true);
}
}
SPDK_RPC_REGISTER("longhorn_volume_add_replica", rpc_bdev_longhorn_add_replica_cmd, SPDK_RPC_RUNTIME)
@ -396,7 +374,7 @@ rpc_bdev_longhorn_remove_replica_cmd(struct spdk_jsonrpc_request *request,
const struct spdk_json_val *params)
{
struct rpc_bdev_longhorn_add_replica req = {};
struct longhorn_bdev_config *longhorn_cfg;
//struct longhorn_bdev_config *longhorn_cfg;
int rc;
size_t i;
char *bdev_name;
@ -455,7 +433,7 @@ static const struct spdk_json_object_decoder rpc_bdev_longhorn_delete_decoders[]
struct rpc_bdev_longhorn_delete_ctx {
struct rpc_bdev_longhorn_delete req;
struct longhorn_bdev_config *longhorn_cfg;
//struct longhorn_bdev_config *longhorn_cfg;
struct spdk_jsonrpc_request *request;
};
@ -471,7 +449,7 @@ static void
bdev_longhorn_delete_done(void *cb_arg, int rc)
{
struct rpc_bdev_longhorn_delete_ctx *ctx = cb_arg;
struct longhorn_bdev_config *longhorn_cfg;
// struct longhorn_bdev_config *longhorn_cfg;
struct spdk_jsonrpc_request *request = ctx->request;
if (rc != 0) {
@ -482,10 +460,12 @@ bdev_longhorn_delete_done(void *cb_arg, int rc)
goto exit;
}
#if 0
longhorn_cfg = ctx->longhorn_cfg;
assert(longhorn_cfg->longhorn_bdev == NULL);
longhorn_bdev_config_cleanup(longhorn_cfg);
#endif
spdk_jsonrpc_send_bool_response(request, true);
exit:
@ -524,6 +504,7 @@ rpc_bdev_longhorn_delete(struct spdk_jsonrpc_request *request,
goto cleanup;
}
#if 0
ctx->longhorn_cfg = longhorn_bdev_config_find_by_name(ctx->req.name);
if (ctx->longhorn_cfg == NULL) {
spdk_jsonrpc_send_error_response_fmt(request, ENODEV,
@ -531,11 +512,12 @@ rpc_bdev_longhorn_delete(struct spdk_jsonrpc_request *request,
ctx->req.name);
goto cleanup;
}
#endif
ctx->request = request;
/* Remove all the base bdevs from this longhorn bdev before deleting the longhorn bdev */
longhorn_bdev_remove_base_devices(ctx->longhorn_cfg, bdev_longhorn_delete_done, ctx);
//longhorn_bdev_remove_base_devices(ctx->longhorn_cfg, bdev_longhorn_delete_done, ctx);
return;
@ -898,7 +880,7 @@ static void rpc_longhorn_snapshot_cmd(struct spdk_jsonrpc_request *request,
{
struct rpc_bdev_longhorn_snapshot req;
struct rpc_bdev_longhorn_snapshot_ctx *ctx;
struct longhorn_bdev_config *longhorn_cfg;
//struct longhorn_bdev_config *longhorn_cfg;
if (spdk_json_decode_object(params, rpc_bdev_longhorn_snapshot_decoders,
SPDK_COUNTOF(rpc_bdev_longhorn_snapshot_decoders),
@ -909,6 +891,7 @@ static void rpc_longhorn_snapshot_cmd(struct spdk_jsonrpc_request *request,
}
#if 0
longhorn_cfg = longhorn_bdev_config_find_by_name(req.name);
if (longhorn_cfg == NULL) {
spdk_jsonrpc_send_error_response_fmt(request, ENODEV,
@ -916,6 +899,7 @@ static void rpc_longhorn_snapshot_cmd(struct spdk_jsonrpc_request *request,
req.name);
//goto cleanup;
}
#endif
ctx = calloc(1, sizeof(*ctx));

79
module/bdev/longhorn/bdev_longhorn_snapshot.c
View File

@ -115,6 +115,18 @@ void compare_bdev_event_cb(enum spdk_bdev_event_type type,
}
static void compare_free_ctx(struct compare_ctx *ctx) {
if (ctx->channel1)
spdk_put_io_channel(ctx->channel1);
if (ctx->channel2)
spdk_put_io_channel(ctx->channel2);
spdk_bdev_close(ctx->desc1);
spdk_bdev_close(ctx->desc2);
spdk_free(ctx->block1);
spdk_free(ctx->block2);
free(ctx);
}
@ -128,7 +140,7 @@ static void read_bdev2_cb(struct spdk_bdev_io *bdev_io,
struct block *bad_block;
if (success) {
if (memcpy(ctx->block1, ctx->block2, ctx->diff.blocksize) != 0) {
if (memcmp(ctx->block1, ctx->block2, ctx->diff.blocksize) != 0) {
bad_block = calloc(1, sizeof(*bad_block));
bad_block->block = ctx->block_num;
@ -158,20 +170,31 @@ static void read_bdev1_cb(struct spdk_bdev_io *bdev_io,
void *cb_arg) {
struct compare_ctx *ctx = cb_arg;
printf("here\n");
if (success) {
printf("read2 %lu %lu\n", ctx->block_num * ctx->diff.blocksize, ctx->block_num);
spdk_bdev_read(ctx->desc2, ctx->channel2, ctx->block2,
ctx->block_num * ctx->diff.blocksize,
ctx->diff.blocksize, read_bdev2_cb, ctx);
} else {
printf("read1 failed\n");
ctx->cb_fn(-1, &ctx->diff, ctx->cb_arg);
compare_free_ctx(ctx);
}
}
static void read_bdev1(struct compare_ctx *ctx) {
spdk_bdev_read(ctx->desc1, ctx->channel1, ctx->block1,
printf("read1 %lu %lu\n", ctx->block_num * ctx->diff.blocksize, ctx->block_num);
int rc = 0;
rc = spdk_bdev_read(ctx->desc1, ctx->channel1, ctx->block1,
ctx->block_num * ctx->diff.blocksize,
ctx->diff.blocksize, read_bdev1_cb, ctx);
if (rc != 0) {
ctx->cb_fn(0, &ctx->diff, ctx->cb_arg);
compare_free_ctx(ctx);
}
printf("spdk_bdev_read returned %d\n", rc);
}
static uint64_t bdev_get_size(struct spdk_bdev *bdev) {
@ -195,11 +218,13 @@ void bdev_longhorn_compare(const char *bdev_name1,
ctx->cb_fn = cb_fn;
ctx->cb_arg = cb_arg;
rc = spdk_bdev_open_ext(bdev_name1, true, compare_bdev_event_cb,
rc = spdk_bdev_open_ext(bdev_name1, false, compare_bdev_event_cb,
ctx, &ctx->desc1);
rc = spdk_bdev_open_ext(bdev_name2, true, compare_bdev_event_cb,
printf("spdk_bdev_open_ext %d\n", rc);
rc = spdk_bdev_open_ext(bdev_name2, false, compare_bdev_event_cb,
ctx, &ctx->desc2);
printf("spdk_bdev_open_ext %d\n", rc);
ctx->bdev1 = spdk_bdev_desc_get_bdev(ctx->desc1);
ctx->bdev2 = spdk_bdev_desc_get_bdev(ctx->desc2);
@ -216,9 +241,12 @@ void bdev_longhorn_compare(const char *bdev_name1,
ctx->size1 = bdev_get_size(ctx->bdev1);
ctx->size2 = bdev_get_size(ctx->bdev2);
ctx->diff.size1 = ctx->size1;
ctx->diff.size2 = ctx->size2;
ctx->total_blocks = spdk_min(ctx->size1, ctx->size2) / blocksize;
read_bdev1(ctx);
}
struct longhorn_bdev_snapshot_ctx {
@ -235,7 +263,7 @@ static void longhorn_bdev_snapshot_complete(void *cb_arg,
ctx->snapshots_complete++;
SPDK_ERRLOG("%d snapshots complete %d %s\n", ctx->snapshots_complete, lvolerrno, strerror(-lvolerrno));
SPDK_DEBUGLOG(bdev_longhorn, "%d snapshots complete %d %s\n", ctx->snapshots_complete, lvolerrno, strerror(-lvolerrno));
if (ctx->snapshots_complete >= ctx->num_to_snapshot) {
longhorn_unpause(ctx->longhorn_bdev);
@ -244,12 +272,22 @@ static void longhorn_bdev_snapshot_complete(void *cb_arg,
}
}
static void longhorn_bdev_snapshot(struct longhorn_bdev *longhorn_bdev,
const char *snapshot)
struct longhorn_snapshot_ctx {
const char *name;
const char *snapshot_name;
struct longhorn_bdev *bdev;
struct spdk_thread *thread;
};
static void longhorn_bdev_snapshot(void *arg)
{
struct longhorn_base_bdev_info *base_info;
struct longhorn_snapshot_ctx *snapshot_ctx = arg;
struct longhorn_bdev_snapshot_ctx *ctx;
const char *snapshot = snapshot_ctx->snapshot_name;
struct longhorn_bdev *longhorn_bdev = snapshot_ctx->bdev;
struct longhorn_base_bdev_info *base_info;
struct spdk_bdev *bdev;
struct spdk_lvol *lvol;
@ -281,27 +319,29 @@ static void longhorn_bdev_snapshot(struct longhorn_bdev *longhorn_bdev,
} else {
ctx->snapshots_complete++;
bdev_longhorn_snapshot_remote(base_info->remote_addr,
base_info->bdev_name,
base_info->lvs,
snapshot,
longhorn_bdev_snapshot_complete,
ctx);
}
}
free(snapshot_ctx->name);
free(snapshot_ctx->snapshot_name);
free(snapshot_ctx);
}
struct longhorn_snapshot_ctx {
const char *name;
const char *snapshot_name;
};
static void longhorn_snapshot_pause_complete(struct longhorn_bdev *bdev,
void *arg) {
struct longhorn_snapshot_ctx *ctx = arg;
ctx->bdev = bdev;
longhorn_bdev_snapshot(bdev, ctx->snapshot_name);
free(ctx->name);
free(ctx->snapshot_name);
free(ctx);
spdk_thread_send_msg(ctx->thread, longhorn_bdev_snapshot, ctx);
}
int
@ -320,6 +360,7 @@ longhorn_volume_snapshot(const char *name, const char *snapshot_name) {
ctx = calloc(1, sizeof(*ctx));
ctx->name = strdup(name);
ctx->snapshot_name = strdup(snapshot_name);
ctx->thread = spdk_get_thread();
rc = pthread_mutex_trylock(&longhorn_bdev->base_bdevs_mutex);

4
module/bdev/longhorn/bdev_longhorn_snapshot.h
View File

@ -21,6 +21,10 @@ struct block_diff {
uint64_t blocksize;
uint64_t num_diff;
uint64_t size1;
uint64_t size2;
};

64
module/bdev/longhorn/bdev_longhorn_snapshot_rpc.c
View File

@ -49,3 +49,67 @@ rpc_longhorn_volume_snapshot_cmd(struct spdk_jsonrpc_request *request,
SPDK_RPC_REGISTER("longhorn_volume_snapshot", rpc_longhorn_volume_snapshot_cmd, SPDK_RPC_RUNTIME)
struct rpc_longhorn_bdev_compare {
char *bdev1;
char *bdev2;
};
static void
free_rpc_longhorn_bdev_compare(struct rpc_longhorn_bdev_compare *req) {
free(req->bdev1);
free(req->bdev2);
}
static const struct spdk_json_object_decoder rpc_longhorn_bdev_compare_decoders[] = {
{"bdev1", offsetof(struct rpc_longhorn_bdev_compare, bdev1), spdk_json_decode_string},
{"bdev2", offsetof(struct rpc_longhorn_bdev_compare, bdev2), spdk_json_decode_string},
};
static void compare_blocks_json(int status, struct block_diff *diff, void *arg)
{
struct spdk_jsonrpc_request *request = arg;
struct spdk_json_write_ctx *w;
struct block *block;
w = spdk_jsonrpc_begin_result(request);
spdk_json_write_object_begin(w);
spdk_json_write_named_uint64(w, "block_size", diff->blocksize);
spdk_json_write_named_uint64(w, "size1", diff->size1);
spdk_json_write_named_uint64(w, "size2", diff->size2);
spdk_json_write_named_array_begin(w, "blocks");
TAILQ_FOREACH(block, &diff->blocks, next) {
spdk_json_write_uint64(w, block->block);
}
spdk_json_write_array_end(w);
spdk_json_write_object_end(w);
spdk_jsonrpc_end_result(request, w);
}
static void
rpc_longhorn_bdev_compare(struct spdk_jsonrpc_request *request,
const struct spdk_json_val *params)
{
struct rpc_longhorn_bdev_compare req = {};
if (spdk_json_decode_object(params, rpc_longhorn_bdev_compare_decoders,
SPDK_COUNTOF(rpc_longhorn_bdev_compare_decoders),
&req)) {
spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
"longhorn spdk_json_decode_object failed");
return;
}
bdev_longhorn_compare(req.bdev1, req.bdev2, 4096, compare_blocks_json, request);
free_rpc_longhorn_bdev_compare(&req);
}
SPDK_RPC_REGISTER("longhorn_bdev_compare", rpc_longhorn_bdev_compare, SPDK_RPC_RUNTIME)