635 lines
15 KiB
C
635 lines
15 KiB
C
#include "spdk/rpc.h"
|
|
#include "spdk/bdev.h"
|
|
#include "bdev_longhorn.h"
|
|
#include "spdk/util.h"
|
|
#include "spdk/string.h"
|
|
#include "spdk/log.h"
|
|
#include "spdk/env.h"
|
|
#include "spdk_internal/lvolstore.h"
|
|
#include "../lvol/vbdev_lvol.h"
|
|
#include "lib/blob/blobstore.h"
|
|
#include "bdev_longhorn_rebuild.h"
|
|
|
|
#define ALIGN_4K 4096
|
|
|
|
struct longhorn_blob_info_context {
|
|
struct spdk_blob_store *bs;
|
|
void (*callback)(struct longhorn_blob_info *info, void *cb_arg);
|
|
void *cb_arg;
|
|
};
|
|
|
|
|
|
static void longhorn_blob_opened(void *arg, struct spdk_blob *blob, int bserrno) {
|
|
struct longhorn_blob_info_context *ctx = arg;
|
|
struct longhorn_blob_info info;
|
|
size_t len;
|
|
|
|
if (!blob) {
|
|
(*ctx->callback)(NULL, ctx->cb_arg);
|
|
free(ctx);
|
|
return;
|
|
}
|
|
|
|
spdk_blob_get_xattr_value(blob, "name", &(info.name), &len);
|
|
|
|
info.num_clusters = blob->active.num_clusters;
|
|
info.allocated_clusters = longhorn_get_allocated_clusters(blob);
|
|
|
|
info.table = calloc(1, sizeof(uint32_t) * info.allocated_clusters);
|
|
|
|
longhorn_export_allocated_clusters(blob, info.table);
|
|
|
|
(*ctx->callback)(&info, ctx->cb_arg);
|
|
|
|
free(info.table);
|
|
|
|
if (blob->parent_id) {
|
|
spdk_bs_open_blob(ctx->bs, blob->parent_id, longhorn_blob_opened, ctx);
|
|
} else {
|
|
(*ctx->callback)(NULL, ctx->cb_arg);
|
|
free(ctx);
|
|
}
|
|
}
|
|
|
|
void longhorn_get_blob_info(struct spdk_blob_store *bs, uint64_t blob_id, void (*callback)(struct longhorn_blob_info *info, void *cb_arg), void *cb_arg) {
|
|
struct longhorn_blob_info_context *ctx;
|
|
|
|
ctx = calloc(1, sizeof(*ctx));
|
|
ctx->bs = bs;
|
|
ctx->callback = callback;
|
|
ctx->cb_arg = cb_arg;
|
|
|
|
spdk_bs_open_blob(ctx->bs, blob_id, longhorn_blob_opened, ctx);
|
|
}
|
|
|
|
|
|
int bdev_longhorn_lookup_name(const char *name, spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) {
|
|
struct spdk_lvol_store *lvs;
|
|
struct lvol_store_bdev *lvs_bdev;
|
|
|
|
lvs_bdev = vbdev_lvol_store_first();
|
|
|
|
while (lvs_bdev != NULL) {
|
|
printf("lvs: %s\n", lvs_bdev->lvs->name);
|
|
|
|
lvs_bdev = vbdev_lvol_store_next(lvs_bdev);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
struct lvs_name *lvs_get_parent(const char *name)
|
|
{
|
|
struct spdk_bdev *bdev = NULL;
|
|
struct spdk_lvol *lvol = NULL;
|
|
spdk_blob_id parent_id;
|
|
|
|
bdev = spdk_bdev_get_by_name(name);
|
|
|
|
if (bdev != NULL) {
|
|
lvol = vbdev_lvol_get_from_bdev(bdev);
|
|
|
|
if (lvol != NULL) {
|
|
parent_id = lvol->blob->parent_id;
|
|
}
|
|
}
|
|
|
|
|
|
return NULL;
|
|
}
|
|
|
|
struct longhorn_import_context {
|
|
char *name;
|
|
char *lvs;
|
|
char *file;
|
|
struct lvol_store_bdev *lvs_bdev;
|
|
|
|
FILE *fp;
|
|
|
|
uint64_t blob_id;
|
|
struct spdk_blob *blob;
|
|
struct spdk_io_channel *channel;
|
|
|
|
|
|
uint64_t num_clusters;
|
|
uint32_t cluster_size;
|
|
uint32_t io_unit_size;
|
|
uint64_t current_cluster;
|
|
uint64_t allocated_clusters;
|
|
|
|
uint32_t *cluster_table;
|
|
uint8_t *cluster;
|
|
uint32_t current;
|
|
|
|
};
|
|
|
|
static void free_longhorn_import_context(struct longhorn_import_context *ctx) {
|
|
if (ctx) {
|
|
if (ctx->file) {
|
|
free(ctx->file);
|
|
}
|
|
|
|
if (ctx->lvs) {
|
|
free(ctx->lvs);
|
|
}
|
|
|
|
if (ctx->name) {
|
|
free(ctx->name);
|
|
}
|
|
|
|
if (ctx->fp) {
|
|
fclose(ctx->fp);
|
|
}
|
|
|
|
free(ctx);
|
|
}
|
|
}
|
|
|
|
static void
|
|
write_next_cluster(void *arg1, int bserrno) {
|
|
struct longhorn_import_context *ctx = arg1;
|
|
ssize_t nread;
|
|
uint64_t offset;
|
|
|
|
|
|
if (bserrno) {
|
|
printf("error: %d\n", bserrno);
|
|
fclose(ctx->fp);
|
|
|
|
return;
|
|
}
|
|
|
|
if (ctx->current >= ctx->allocated_clusters) {
|
|
free_longhorn_import_context(ctx);
|
|
|
|
printf("Import complete\n");
|
|
return;
|
|
}
|
|
|
|
|
|
nread = fread(ctx->cluster, 1, ctx->cluster_size, ctx->fp);
|
|
|
|
if (nread > 0) {
|
|
offset = ctx->cluster_table[ctx->current] * ctx->cluster_size / ctx->io_unit_size;
|
|
|
|
ctx->current++;
|
|
|
|
spdk_blob_io_write(ctx->blob, ctx->channel, ctx->cluster, offset,
|
|
ctx->cluster_size / ctx->io_unit_size, write_next_cluster, ctx);
|
|
}
|
|
}
|
|
|
|
|
|
static void
|
|
longhorn_import_blob(struct spdk_blob *blob,
|
|
struct longhorn_import_context *ctx) {
|
|
uint64_t blob_id = spdk_blob_get_id(blob);
|
|
long offset;
|
|
|
|
fread(&ctx->num_clusters, sizeof (uint64_t), 1, ctx->fp);
|
|
fread(&ctx->allocated_clusters, sizeof (uint64_t), 1, ctx->fp);
|
|
fread(&ctx->cluster_size, sizeof (uint32_t), 1, ctx->fp);
|
|
fread(&ctx->io_unit_size, sizeof (uint32_t), 1, ctx->fp);
|
|
|
|
|
|
ctx->cluster_table = calloc(sizeof (uint32_t), ctx->allocated_clusters);
|
|
ctx->cluster = spdk_malloc(ctx->cluster_size, ALIGN_4K, NULL,
|
|
SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
|
|
|
|
fread(ctx->cluster_table, sizeof(uint32_t), ctx->allocated_clusters, ctx->fp);
|
|
|
|
offset = ftell(ctx->fp);
|
|
|
|
if (offset % ctx->io_unit_size != 0) {
|
|
fseek(ctx->fp, ctx->io_unit_size - offset % ctx->io_unit_size, SEEK_CUR);
|
|
}
|
|
|
|
ctx->channel = spdk_bs_alloc_io_channel(ctx->blob->bs);
|
|
write_next_cluster(ctx, 0);
|
|
|
|
printf("here\n");
|
|
|
|
}
|
|
|
|
static void
|
|
blob_import_iterator_cb(void *arg1, struct spdk_blob *blob, int bserrno) {
|
|
struct longhorn_import_context *ctx = arg1;
|
|
struct spdk_xattr_names *names;
|
|
char *xattr_name = NULL;
|
|
uint64_t blob_id;
|
|
|
|
const void *value;
|
|
size_t value_len;
|
|
unsigned int i;
|
|
|
|
|
|
if (bserrno) {
|
|
if (blob_id != 0) {
|
|
//longhorn_import_context(blob_id, ctx);
|
|
} else {
|
|
free_longhorn_import_context(ctx);
|
|
}
|
|
return;
|
|
}
|
|
|
|
blob_id = spdk_blob_get_id(blob);
|
|
spdk_blob_get_xattr_names(blob, &names);
|
|
|
|
for (i = 0; i < spdk_xattr_names_get_count(names); i++) {
|
|
xattr_name = spdk_xattr_names_get_name(names, i);
|
|
|
|
if (strcmp(xattr_name, "name") == 0) {
|
|
spdk_blob_get_xattr_value(blob, xattr_name,
|
|
&value, &value_len);
|
|
|
|
if (strncmp(value, ctx->name, strlen(ctx->name)) == 0) {
|
|
/* Found our blob. */
|
|
printf("found blob %s\n", ctx->name);
|
|
ctx->blob_id = blob_id;
|
|
|
|
ctx->blob = blob;
|
|
longhorn_import_blob(blob, ctx);
|
|
return;
|
|
|
|
} else {
|
|
printf("%s != %s\n", (char *)value, ctx->name);
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
spdk_bs_iter_next(ctx->lvs_bdev->lvs->blobstore,
|
|
blob,
|
|
blob_import_iterator_cb,
|
|
ctx);
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
int bdev_longhorn_import(const char *name, const char *lvs, const char *file) {
|
|
struct lvol_store_bdev *lvs_bdev = NULL;;
|
|
struct longhorn_import_context *ctx = NULL;
|
|
FILE *fp;
|
|
|
|
lvs_bdev = vbdev_lvol_store_first();
|
|
|
|
while (lvs_bdev != NULL) {
|
|
if (strcmp(lvs_bdev->lvs->name, lvs) != 0) {
|
|
lvs_bdev = vbdev_lvol_store_next(lvs_bdev);
|
|
} else {
|
|
printf("found lvs %s\n", lvs);
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (lvs_bdev != NULL) {
|
|
fp = fopen(file, "r");
|
|
|
|
if (fp == NULL) return -1;
|
|
|
|
ctx = calloc(1, sizeof(struct longhorn_import_context));
|
|
|
|
ctx->name = strdup(name);
|
|
ctx->lvs = strdup(lvs);
|
|
ctx->file = strdup(file);
|
|
|
|
ctx->lvs_bdev = lvs_bdev;
|
|
|
|
ctx->fp = fp;
|
|
|
|
spdk_bs_iter_first(ctx->lvs_bdev->lvs->blobstore,
|
|
blob_import_iterator_cb, ctx);
|
|
|
|
return 0;
|
|
}
|
|
|
|
|
|
return -1;
|
|
}
|
|
|
|
static void reopen_blob_cb(void *arg, struct spdk_blob *blob, int bserrno) {
|
|
struct spdk_lvol *parent_lvol = arg;
|
|
|
|
if (blob != NULL) {
|
|
//parent_lvol->blob = blob;
|
|
}
|
|
}
|
|
|
|
void bdev_longhorn_md_sync_complete(void *cb_arg, int bserrno)
|
|
{
|
|
struct spdk_lvol *parent_lvol = cb_arg;
|
|
|
|
if (bserrno != 0) {
|
|
printf("metadata sync failed: %s\n", strerror(bserrno));
|
|
} else {
|
|
//spdk_bs_open_blob(parent_lvol->lvol_store->blobstore, parent_lvol->blob->id, reopen_blob_cb, parent_lvol);
|
|
|
|
|
|
printf("metadata sync succeeded\n");
|
|
}
|
|
}
|
|
|
|
|
|
int bdev_longhorn_link(const char *child, const char *parent)
|
|
{
|
|
struct spdk_lvol_store *lvs = NULL;
|
|
struct spdk_bdev *parent_bdev = NULL;
|
|
struct spdk_bdev *child_bdev = NULL;
|
|
struct spdk_lvol *parent_lvol = NULL;
|
|
struct spdk_lvol *child_lvol = NULL;
|
|
int bserrno;
|
|
|
|
|
|
parent_bdev = spdk_bdev_get_by_name(parent);
|
|
child_bdev = spdk_bdev_get_by_name(child);
|
|
|
|
if (parent_bdev == NULL) {
|
|
printf("can't find bdev for %s\n", parent);
|
|
return;
|
|
}
|
|
|
|
if (child_bdev == NULL) {
|
|
printf("can't find bdev for %s\n", child);
|
|
return;
|
|
}
|
|
|
|
parent_lvol = vbdev_lvol_get_from_bdev(parent_bdev);
|
|
child_lvol = vbdev_lvol_get_from_bdev(child_bdev);
|
|
|
|
if (parent_lvol == NULL) {
|
|
printf("can't find lvol for %s\n", parent);
|
|
return;
|
|
}
|
|
|
|
if (child_lvol == NULL) {
|
|
printf("can't find lvol for %s\n", child);
|
|
return;
|
|
}
|
|
|
|
bserrno = spdk_blob_set_internal_xattr(parent_lvol->blob, BLOB_SNAPSHOT, &child_lvol->blob->id, sizeof(spdk_blob_id));
|
|
|
|
printf("syncing metadata\n");
|
|
spdk_blob_sync_md(parent_lvol->blob, bdev_longhorn_md_sync_complete, parent_lvol);
|
|
|
|
|
|
return 0;
|
|
}
|
|
|
|
struct snapshot_rpc {
|
|
char *name;
|
|
uint64_t num_clusters;
|
|
uint32_t allocated_clusters;
|
|
uint32_t *active_clusters;
|
|
};
|
|
|
|
#define MAX_SNAPSHOTS 256
|
|
struct snapshots_rpc {
|
|
size_t num_snapshots;
|
|
struct snapshot_rpc snapshots[MAX_SNAPSHOTS];
|
|
};
|
|
|
|
|
|
struct children_rpc {
|
|
char *name;
|
|
uint64_t cluster_size;
|
|
uint32_t io_unit_size;
|
|
|
|
struct snapshots_rpc snapshots;
|
|
|
|
};
|
|
|
|
static int json_decode_clusters(const struct spdk_json_val *val, void *out) {
|
|
uint32_t *clusters = out;
|
|
struct snapshot_rpc *snapshot = SPDK_CONTAINEROF(clusters, struct snapshot_rpc, active_clusters);
|
|
size_t dummy;
|
|
int error;
|
|
uint32_t i;
|
|
|
|
printf("name = %s\n", snapshot->name);
|
|
printf("num_clusters = %lu\n", snapshot->num_clusters);
|
|
printf("allocated_clusters = %u\n", snapshot->allocated_clusters);
|
|
snapshot->active_clusters = calloc(sizeof(uint32_t), snapshot->allocated_clusters);
|
|
|
|
error = spdk_json_decode_array(val, spdk_json_decode_uint32, snapshot->active_clusters, snapshot->allocated_clusters, &dummy, sizeof(uint32_t));
|
|
|
|
|
|
for (int i = 0; i < snapshot->allocated_clusters; ++i) {
|
|
printf("%u\n", snapshot->active_clusters[i]);
|
|
}
|
|
|
|
return error;
|
|
}
|
|
|
|
|
|
static const struct spdk_json_object_decoder rpc_snapshot_decoders[] = {
|
|
{"name", offsetof(struct snapshot_rpc, name), spdk_json_decode_string},
|
|
{"num_clusters", offsetof(struct snapshot_rpc, num_clusters), spdk_json_decode_uint64},
|
|
{"allocated_clusters", offsetof(struct snapshot_rpc, allocated_clusters), spdk_json_decode_uint32},
|
|
{"active_clusters", offsetof(struct snapshot_rpc, active_clusters), json_decode_clusters},
|
|
};
|
|
|
|
|
|
static int json_decode_snapshot(const struct spdk_json_val *val, void *out) {
|
|
int error;
|
|
|
|
error = spdk_json_decode_object(val, rpc_snapshot_decoders,
|
|
SPDK_COUNTOF(rpc_snapshot_decoders),
|
|
out);
|
|
|
|
return error;
|
|
}
|
|
|
|
static int json_decode_snapshots(const struct spdk_json_val *val, void *out) {
|
|
struct snapshots_rpc *snapshots = out;
|
|
int error = 0;
|
|
|
|
error = spdk_json_decode_array(val, json_decode_snapshot, snapshots->snapshots, MAX_SNAPSHOTS, &snapshots->num_snapshots, sizeof(struct snapshot_rpc));
|
|
|
|
return error;
|
|
}
|
|
|
|
static const struct spdk_json_object_decoder rpc_replica_decoders[] = {
|
|
{"name", offsetof(struct children_rpc, name), spdk_json_decode_string},
|
|
{"cluster_size", offsetof(struct children_rpc, cluster_size), spdk_json_decode_uint64},
|
|
{"io_unit_size", offsetof(struct children_rpc, io_unit_size), spdk_json_decode_uint32},
|
|
{"snapshots", offsetof(struct children_rpc, snapshots), json_decode_snapshots},
|
|
};
|
|
|
|
|
|
static void receive_children(const char *addr,
|
|
const char *command,
|
|
int32_t id,
|
|
struct spdk_json_val *result,
|
|
struct spdk_json_val *error,
|
|
void *arg) {
|
|
int i = 0;
|
|
char *data = (char *)result->start;
|
|
uint64_t blob_id;
|
|
struct spdk_json_val *value;
|
|
struct children_rpc children = {};
|
|
|
|
printf("received response. %ld, %s\n", result->len, data);
|
|
|
|
if (spdk_json_decode_object(result, rpc_replica_decoders,
|
|
SPDK_COUNTOF(rpc_replica_decoders),
|
|
&children)) {
|
|
printf("error decoding\n");
|
|
}
|
|
|
|
|
|
|
|
|
|
#if 0
|
|
if (result->type == SPDK_JSON_VAL_OBJECT_BEGIN) {
|
|
value = spdk_json_object_first(result);
|
|
|
|
while (value != NULL) {
|
|
if (spdk_json_decode_uint64(value, &blob_id) == 0) {
|
|
printf("%016lx %lu\n", blob_id, blob_id);
|
|
}
|
|
|
|
value = spdk_json_next(value);
|
|
}
|
|
}
|
|
#endif
|
|
|
|
|
|
}
|
|
|
|
struct rebuild_context {
|
|
struct spdk_lvol_store *lvs;
|
|
char *prefix;
|
|
};
|
|
|
|
static void receive_replicas(const char *addr,
|
|
const char *command,
|
|
int32_t id,
|
|
struct spdk_json_val *result,
|
|
struct spdk_json_val *error,
|
|
void *arg) {
|
|
int i = 0;
|
|
char *data = (char *)result->start;
|
|
uint64_t blob_id;
|
|
struct spdk_json_val *value;
|
|
struct children_rpc children = {};
|
|
struct rebuild_context *ctx = arg;
|
|
char *bdev_name;
|
|
char *last_bdev_name = NULL;;
|
|
|
|
printf("receive_replicas");
|
|
|
|
printf("received response. %ld, %s\n", result->len, data);
|
|
|
|
if (spdk_json_decode_object(result, rpc_replica_decoders,
|
|
SPDK_COUNTOF(rpc_replica_decoders),
|
|
&children)) {
|
|
printf("error decoding\n");
|
|
}
|
|
|
|
printf("num of snapshots %d\n", children.snapshots.num_snapshots);
|
|
|
|
for (i = children.snapshots.num_snapshots - 1; i >= 0; --i) {
|
|
bdev_name = spdk_sprintf_alloc("%s%s", ctx->prefix, children.snapshots.snapshots[i].name);
|
|
|
|
printf("syncing %s\n", children.snapshots.snapshots[i].name);
|
|
|
|
longhorn_snapshot_bdev_sync(bdev_name,
|
|
children.snapshots.snapshots[i].name,
|
|
ctx->lvs,
|
|
children.snapshots.snapshots[i].num_clusters,
|
|
children.snapshots.snapshots[i].allocated_clusters,
|
|
children.cluster_size,
|
|
children.io_unit_size,
|
|
children.snapshots.snapshots[i].active_clusters);
|
|
|
|
if (last_bdev_name) {
|
|
bdev_longhorn_link(bdev_name, last_bdev_name);
|
|
free(last_bdev_name);
|
|
}
|
|
|
|
last_bdev_name = bdev_name;
|
|
|
|
}
|
|
|
|
if (last_bdev_name) {
|
|
free(last_bdev_name);
|
|
}
|
|
}
|
|
|
|
|
|
|
|
void bdev_longhorn_get_children_remote(const char *address,
|
|
uint16_t port,
|
|
const char *name) {
|
|
char *addr = NULL;
|
|
struct spdk_json_write_ctx *w;
|
|
struct spdk_jsonrpc_client_request *request;
|
|
|
|
addr = spdk_sprintf_alloc("%s:%d", address, port);
|
|
|
|
printf("%s:%d:%s\n", address, port, name);
|
|
|
|
json_remote_client(addr);
|
|
|
|
request = spdk_jsonrpc_client_create_request();
|
|
|
|
w = spdk_jsonrpc_begin_request(request, 1, "lvol_list_children");
|
|
spdk_json_write_name(w, "params");
|
|
spdk_json_write_object_begin(w);
|
|
spdk_json_write_name(w, "name");
|
|
spdk_json_write_string(w, name);
|
|
spdk_json_write_object_end(w);
|
|
|
|
spdk_jsonrpc_end_request(request, w);
|
|
|
|
json_remote_client_send_command(addr, "lvol_list_children",
|
|
1, request, receive_children, NULL);
|
|
|
|
|
|
free(addr);
|
|
}
|
|
|
|
|
|
void bdev_longhorn_rebuild_remote(const char *address,
|
|
uint16_t port,
|
|
const char *name,
|
|
char *remote_prefix,
|
|
struct spdk_lvol_store *lvs) {
|
|
char *addr = NULL;
|
|
struct spdk_json_write_ctx *w;
|
|
struct spdk_jsonrpc_client_request *request;
|
|
struct rebuild_context *ctx;
|
|
struct spdk_lvol_store *store;
|
|
|
|
addr = spdk_sprintf_alloc("%s:%d", address, port);
|
|
|
|
printf("%s:%d:%s\n", address, port, name);
|
|
|
|
json_remote_client(addr);
|
|
|
|
request = spdk_jsonrpc_client_create_request();
|
|
|
|
w = spdk_jsonrpc_begin_request(request, 1, "lvol_list_children");
|
|
spdk_json_write_name(w, "params");
|
|
spdk_json_write_object_begin(w);
|
|
spdk_json_write_name(w, "name");
|
|
spdk_json_write_string(w, name);
|
|
spdk_json_write_object_end(w);
|
|
|
|
spdk_jsonrpc_end_request(request, w);
|
|
|
|
ctx = calloc(1, sizeof(*ctx));
|
|
ctx->prefix = strdup(remote_prefix);
|
|
ctx->lvs = lvs;
|
|
|
|
json_remote_client_send_command(addr, "lvol_list_children",
|
|
1, request, receive_replicas, ctx);
|
|
|
|
|
|
free(addr);
|
|
}
|
|
|