#include "spdk/nvmf.h" #include "spdk/string.h" #include "spdk/util.h" #include "bdev_longhorn_nvmf.h" static bool tcp_transport_created = false; static void longhorn_tgt_add_transport_done(void *cb_arg, int status) { tcp_transport_created = true; } static void longhorn_subsystem_add_done(struct spdk_nvmf_subsystem *subsystem, void *cb_arg, int status) { } void longhorn_nvmf_create_transport(spdk_nvmf_tgt_add_transport_done_fn cb_fn, void *cb_arg) { struct spdk_nvmf_transport_opts opts; struct spdk_nvmf_tgt *tgt; struct spdk_nvmf_transport *transport; spdk_nvmf_transport_opts_init("tcp", &opts, sizeof(opts)); tgt = spdk_nvmf_get_tgt(NULL); transport = spdk_nvmf_transport_create("tcp", &opts); if (cb_fn != NULL) { spdk_nvmf_tgt_add_transport(tgt, transport, cb_fn, cb_arg); } else { spdk_nvmf_tgt_add_transport(tgt, transport, longhorn_tgt_add_transport_done, NULL); } } void longhorn_nvmf_create_subsystem(const char *nqn) { struct spdk_nvmf_tgt *tgt; struct spdk_nvmf_subsystem *subsystem; tgt = spdk_nvmf_get_tgt(NULL); subsystem = spdk_nvmf_subsystem_create(tgt, nqn, SPDK_NVMF_SUBTYPE_NVME, 0); spdk_nvmf_subsystem_set_allow_any_host(subsystem, true); spdk_nvmf_subsystem_start(subsystem, longhorn_subsystem_add_done, NULL); } static void populate_tcp_trid(struct spdk_nvme_transport_id *trid, const char *addr, uint16_t port) { snprintf(trid->trstring, SPDK_NVMF_TRSTRING_MAX_LEN, "TCP"); trid->trtype = SPDK_NVME_TRANSPORT_TCP; trid->adrfam = SPDK_NVMF_ADRFAM_IPV4; snprintf(trid->traddr, SPDK_NVMF_TRADDR_MAX_LEN, "%s", addr); snprintf(trid->trsvcid, SPDK_NVMF_TRSVCID_MAX_LEN, "%"PRIu16, port); } static void add_listener_cb(void *cb_arg, int status) { struct spdk_nvme_transport_id *trid = cb_arg; free(trid); } static void add_listener_resume_cb(struct spdk_nvmf_subsystem *subsystem, void *cb_arg, int status) { } static void add_listener_pause_cb(struct spdk_nvmf_subsystem *subsystem, void *cb_arg, int status) { struct spdk_nvme_transport_id *trid = cb_arg; spdk_nvmf_subsystem_add_listener(subsystem, trid, add_listener_cb, trid); spdk_nvmf_subsystem_resume(subsystem, add_listener_resume_cb, NULL); } void longhorn_nvmf_subsystem_add_listener(const char *nqn, const char *addr, uint16_t port) { struct spdk_nvmf_tgt *tgt; struct spdk_nvmf_subsystem *subsystem; struct spdk_nvme_transport_id *trid; tgt = spdk_nvmf_get_tgt(NULL); subsystem = spdk_nvmf_tgt_find_subsystem(tgt, nqn); trid = calloc(1, sizeof(*trid)); populate_tcp_trid(trid, addr, port); spdk_nvmf_subsystem_pause(subsystem, 0, add_listener_pause_cb, trid); } static void add_ns_resume_cb(struct spdk_nvmf_subsystem *subsystem, void *cb_arg, int status) { } static void add_ns_pause_cb(struct spdk_nvmf_subsystem *subsystem, void *cb_arg, int status) { char *bdev_name = cb_arg; struct spdk_nvmf_ns_opts ns_opts; spdk_nvmf_ns_opts_get_defaults(&ns_opts, sizeof(ns_opts)); spdk_nvmf_subsystem_add_ns_ext(subsystem, bdev_name, &ns_opts, sizeof(ns_opts), NULL); free(bdev_name); spdk_nvmf_subsystem_resume(subsystem, add_ns_resume_cb, NULL); } void longhorn_nvmf_subsystem_add_ns(const char *nqn, const char *bdev_name) { struct spdk_nvmf_tgt *tgt = NULL; struct spdk_nvmf_subsystem *subsystem; subsystem = spdk_nvmf_tgt_find_subsystem(tgt, nqn); spdk_nvmf_subsystem_pause(subsystem, 0, add_ns_pause_cb, bdev_name); } struct longhorn_publish_nvmf_ctx { longhorn_publish_nvmf_cb cb_fn; void *cb_arg; }; static void _longhorn_publish_nvmf(const char *bdev_name, const char *nqn, const char *addr, uint16_t port, longhorn_publish_nvmf_cb cb_fn, void *cb_arg) { struct spdk_nvmf_tgt *tgt; struct spdk_nvmf_subsystem *subsystem; struct spdk_nvmf_ns_opts ns_opts; struct spdk_nvmf_listen_opts listen_opts; struct spdk_nvme_transport_id *trid; tgt = spdk_nvmf_get_tgt(NULL); subsystem = spdk_nvmf_subsystem_create(tgt, nqn, SPDK_NVMF_SUBTYPE_NVME, 0); spdk_nvmf_subsystem_set_allow_any_host(subsystem, true); spdk_nvmf_ns_opts_get_defaults(&ns_opts, sizeof(ns_opts)); spdk_nvmf_subsystem_add_ns_ext(subsystem, bdev_name, &ns_opts, sizeof(ns_opts), NULL); trid = calloc(1, sizeof(*trid)); populate_tcp_trid(trid, addr, port); spdk_nvmf_listen_opts_init(&listen_opts, sizeof(listen_opts)); spdk_nvmf_tgt_listen_ext(tgt, trid, &listen_opts); spdk_nvmf_subsystem_add_listener(subsystem, trid, add_listener_cb, trid); spdk_nvmf_subsystem_start(subsystem, longhorn_subsystem_add_done, NULL); } struct longhorn_publish_nvmf_no_transport_ctx { const char *bdev_name; const char *nqn; const char *addr; uint16_t port; longhorn_publish_nvmf_cb cb_fn; void *cb_arg; }; static void longhorn_publish_no_transport_cb(void *cb_arg) { struct longhorn_publish_nvmf_no_transport_ctx *ctx = cb_arg; _longhorn_publish_nvmf(ctx->bdev_name, ctx->nqn, ctx->addr, ctx->port, ctx->cb_fn, ctx->cb_arg); free(ctx->bdev_name); free(ctx->nqn); free(ctx->addr); free(ctx); } void longhorn_publish_nvmf(const char *bdev_name, const char *nqn, const char *addr, uint16_t port, longhorn_publish_nvmf_cb cb_fn, void *cb_arg) { if (tcp_transport_created) { _longhorn_publish_nvmf(bdev_name, nqn, addr, port, cb_fn, cb_arg); } else { struct longhorn_publish_nvmf_no_transport_ctx *ctx; ctx = calloc(1, sizeof(*ctx)); ctx->bdev_name = strdup(bdev_name); ctx->nqn = strdup(nqn); ctx->addr = strdup(addr); ctx->port = port; ctx->cb_fn = cb_fn; ctx->cb_arg = cb_arg; longhorn_nvmf_create_transport(longhorn_publish_no_transport_cb, ctx); } } #define NVME_MAX_BDEVS_PER_RPC 128 struct longhorn_attach_nvmf_ctx { uint32_t count; size_t bdev_cnt; const char *names[NVME_MAX_BDEVS_PER_RPC]; struct spdk_nvme_ctrlr_opts opts; longhorn_attach_nvmf_cb cb_fn; void *cb_arg; }; static void longhorn_wait_for_examine_cb(void *cb_ctx) { struct longhorn_attach_nvmf_ctx *ctx = cb_ctx; ctx->cb_fn(ctx->names, ctx->bdev_cnt, 0, ctx->cb_arg); free(ctx); } static void longhorn_nvme_create_cb(void *cb_ctx, size_t bdev_cnt, int rc) { struct longhorn_attach_nvmf_ctx *ctx = cb_ctx; if (rc < 0) { ctx->cb_fn(NULL, 0, rc, ctx->cb_arg); free(ctx); } else { ctx->bdev_cnt = bdev_cnt; spdk_bdev_wait_for_examine(longhorn_wait_for_examine_cb, ctx); } } void longhorn_attach_nvmf(const char *bdev_name_prefix, const char *nqn, const char *addr, uint16_t port, longhorn_attach_nvmf_cb cb_fn, void *cb_arg) { struct spdk_nvme_transport_id *trid; size_t len; struct spdk_nvme_host_id hostid = {}; uint32_t prchk_flags = 0; struct longhorn_attach_nvmf_ctx *ctx; trid = calloc(1, sizeof(*trid)); populate_tcp_trid(trid, addr, port); len = strlen(nqn); memcpy(trid->subnqn, nqn, len + 1); ctx = calloc(1, sizeof(*ctx)); ctx->count = NVME_MAX_BDEVS_PER_RPC; ctx->cb_fn = cb_fn; ctx->cb_arg = cb_arg; spdk_nvme_ctrlr_get_default_ctrlr_opts(&ctx->opts, sizeof(ctx->opts)); bdev_nvme_create(trid, &hostid, bdev_name_prefix, ctx->names, ctx->count, prchk_flags, longhorn_nvme_create_cb, ctx, &ctx->opts); } static char *external_addr = NULL; struct longhorn_set_external_addr_ctx { char *addr; longhorn_set_external_addr_cb cb_fn; void *cb_arg; }; static void longhorn_external_addr_cb(void *cb_arg, int status) { struct longhorn_set_external_addr_ctx *ctx = cb_arg; tcp_transport_created = true; ctx->cb_fn(ctx->addr, ctx->cb_arg); free(ctx); } void longhorn_set_external_addr(const char *addr, longhorn_set_external_addr_cb cb_fn, void *cb_arg) { external_addr = strdup(addr); if (tcp_transport_created) { cb_fn(external_addr, cb_arg); } else { struct longhorn_set_external_addr_ctx *ctx = calloc(1, sizeof(struct longhorn_set_external_addr_ctx)); ctx->addr = external_addr; ctx->cb_fn = cb_fn; ctx->cb_arg = cb_arg; longhorn_nvmf_create_transport(longhorn_external_addr_cb, ctx); } } char * longhorn_generate_replica_nqn(const char *lvs, const char *name) { char *nqn = spdk_sprintf_alloc(REPLICA_FORMAT, lvs, name); return nqn; }