diff --git a/include/spdk/nvmf.h b/include/spdk/nvmf.h index 26e2214d4..17fa347aa 100644 --- a/include/spdk/nvmf.h +++ b/include/spdk/nvmf.h @@ -703,6 +703,95 @@ const char *spdk_nvmf_subsystem_get_nqn(struct spdk_nvmf_subsystem *subsystem); */ enum spdk_nvmf_subtype spdk_nvmf_subsystem_get_type(struct spdk_nvmf_subsystem *subsystem); +/** + * Initialize transport options + * + * \param type The transport type to create + * \param opts The transport options (e.g. max_io_size) + * + * \return bool. true if successful, false if transport type + * not found. + */ +bool +spdk_nvmf_transport_opts_init(enum spdk_nvme_transport_type type, + struct spdk_nvmf_transport_opts *opts); + +/** + * Create a protocol transport + * + * \param type The transport type to create + * \param opts The transport options (e.g. max_io_size) + * + * \return new transport or NULL if create fails + */ +struct spdk_nvmf_transport *spdk_nvmf_transport_create(enum spdk_nvme_transport_type type, + struct spdk_nvmf_transport_opts *opts); + +/** + * Destroy a protocol transport + * + * \param transport The transport to destory + * + * \return 0 on success, -1 on failure. + */ +int spdk_nvmf_transport_destroy(struct spdk_nvmf_transport *transport); + +/** + * Get an existing transport from the target + * + * \param tgt The NVMe-oF target + * \param type The transport type to get + * + * \return the transport or NULL if not found + */ +struct spdk_nvmf_transport *spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, + enum spdk_nvme_transport_type type); + +/** + * Function to be called once transport add is complete + * + * \param cb_arg Callback argument passed to this function. + * \param status 0 if it completed successfully, or negative errno if it failed. + */ +typedef void (*spdk_nvmf_tgt_add_transport_done_fn)(void *cb_arg, int status); + +/** + * Add a transport to a target + * + * \param tgt The NVMe-oF target + * \param transport The transport to add + * \param cb_fn A callback that will be called once the transport is created + * \param cb_arg A context argument passed to cb_fn. + * + * \return void. The callback status argument will be 0 on success + * or a negated errno on failure. + */ +void spdk_nvmf_tgt_add_transport(struct spdk_nvmf_tgt *tgt, + struct spdk_nvmf_transport *transport, + spdk_nvmf_tgt_add_transport_done_fn cb_fn, + void *cb_arg); + +/** + * + * Add listener to transport and begin accepting new connections. + * + * \param transport The transport to add listener to + * \param trid Address to listen at + * + * \return int. 0 if it completed successfully, or negative errno if it failed. + */ + +int spdk_nvmf_transport_listen(struct spdk_nvmf_transport *transport, + const struct spdk_nvme_transport_id *trid); + +/** + * Write NVMe-oF target's transport configurations into provided JSON context. + * \param w JSON write context + * \param tgt The NVMe-oF target + */ +void +spdk_nvmf_tgt_transport_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt); + #ifdef __cplusplus } #endif diff --git a/lib/event/subsystems/nvmf/conf.c b/lib/event/subsystems/nvmf/conf.c index 1e3605b4d..986e81c9f 100644 --- a/lib/event/subsystems/nvmf/conf.c +++ b/lib/event/subsystems/nvmf/conf.c @@ -424,19 +424,161 @@ spdk_nvmf_parse_subsystems(void) return 0; } +struct spdk_nvmf_parse_transport_ctx { + struct spdk_conf_section *sp; + spdk_nvmf_parse_conf_done_fn cb_fn; +}; + +static void spdk_nvmf_parse_transport(struct spdk_nvmf_parse_transport_ctx *ctx); + +static void +spdk_nvmf_tgt_add_transport_done(void *cb_arg, int status) +{ + struct spdk_nvmf_parse_transport_ctx *ctx = cb_arg; + int rc; + + if (status < 0) { + SPDK_ERRLOG("Add transport to target failed (%d).\n", status); + ctx->cb_fn(status); + free(ctx); + return; + } + + /* find next transport */ + ctx->sp = spdk_conf_next_section(ctx->sp); + while (ctx->sp) { + if (spdk_conf_section_match_prefix(ctx->sp, "Transport")) { + spdk_nvmf_parse_transport(ctx); + return; + } + ctx->sp = spdk_conf_next_section(ctx->sp); + } + + /* done with transports, parse Subsystem sections */ + rc = spdk_nvmf_parse_subsystems(); + + ctx->cb_fn(rc); + free(ctx); +} + +static void +spdk_nvmf_parse_transport(struct spdk_nvmf_parse_transport_ctx *ctx) +{ + const char *type; + struct spdk_nvmf_transport_opts opts = { 0 }; + enum spdk_nvme_transport_type trtype; + struct spdk_nvmf_transport *transport; + int val; + + type = spdk_conf_section_get_val(ctx->sp, "Type"); + if (type == NULL) { + SPDK_ERRLOG("Transport missing Type\n"); + ctx->cb_fn(-1); + free(ctx); + return; + } + + if (spdk_nvme_transport_id_parse_trtype(&trtype, type)) { + SPDK_ERRLOG("Invalid transport type '%s'\n", type); + ctx->cb_fn(-1); + free(ctx); + return; + } + + if (spdk_nvmf_tgt_get_transport(g_spdk_nvmf_tgt, trtype)) { + SPDK_ERRLOG("Duplicate transport type '%s'\n", type); + ctx->cb_fn(-1); + free(ctx); + return; + } + + if (!spdk_nvmf_transport_opts_init(trtype, &opts)) { + ctx->cb_fn(-1); + free(ctx); + return; + } + + val = spdk_conf_section_get_intval(ctx->sp, "MaxQueueDepth"); + if (val >= 0) { + opts.max_queue_depth = val; + } + val = spdk_conf_section_get_intval(ctx->sp, "MaxQueuesPerSession"); + if (val >= 0) { + opts.max_qpairs_per_ctrlr = val; + } + val = spdk_conf_section_get_intval(ctx->sp, "InCapsuleDataSize"); + if (val >= 0) { + opts.in_capsule_data_size = val; + } + val = spdk_conf_section_get_intval(ctx->sp, "MaxIOSize"); + if (val >= 0) { + opts.max_io_size = val; + } + val = spdk_conf_section_get_intval(ctx->sp, "IOUnitSize"); + if (val >= 0) { + opts.io_unit_size = val; + } + val = spdk_conf_section_get_intval(ctx->sp, "MaxAQDepth"); + if (val >= 0) { + opts.max_aq_depth = val; + } + + transport = spdk_nvmf_transport_create(trtype, &opts); + if (transport) { + spdk_nvmf_tgt_add_transport(g_spdk_nvmf_tgt, transport, spdk_nvmf_tgt_add_transport_done, ctx); + } else { + ctx->cb_fn(-1); + free(ctx); + return; + } +} + +static int +spdk_nvmf_parse_transports(spdk_nvmf_parse_conf_done_fn cb_fn) +{ + struct spdk_nvmf_parse_transport_ctx *ctx; + + ctx = calloc(1, sizeof(struct spdk_nvmf_parse_transport_ctx)); + if (!ctx) { + SPDK_ERRLOG("Failed alloc of context memory for parse transports\n"); + return -ENOMEM; + } + + ctx->cb_fn = cb_fn; + ctx->sp = spdk_conf_first_section(NULL); + while (ctx->sp != NULL) { + if (spdk_conf_section_match_prefix(ctx->sp, "Transport")) { + spdk_nvmf_parse_transport(ctx); + return 0; + } + ctx->sp = spdk_conf_next_section(ctx->sp); + } + + /* if we get here, there are no transports defined in conf file */ + free(ctx); + cb_fn(spdk_nvmf_parse_subsystems()); + + return 0; +} + int -spdk_nvmf_parse_conf(void) +spdk_nvmf_parse_conf(spdk_nvmf_parse_conf_done_fn cb_fn) { int rc; + if (cb_fn == NULL) { + SPDK_ERRLOG("Callback function is NULL\n"); + return -1; + } + /* NVMf section */ rc = spdk_nvmf_parse_nvmf_tgt(); if (rc < 0) { return rc; } - /* Subsystem sections */ - rc = spdk_nvmf_parse_subsystems(); + /* Transport sections */ + rc = spdk_nvmf_parse_transports(cb_fn); if (rc < 0) { return rc; } diff --git a/lib/event/subsystems/nvmf/event_nvmf.h b/lib/event/subsystems/nvmf/event_nvmf.h index fdee43c39..50e5d7555 100644 --- a/lib/event/subsystems/nvmf/event_nvmf.h +++ b/lib/event/subsystems/nvmf/event_nvmf.h @@ -60,6 +60,8 @@ extern struct spdk_nvmf_tgt_conf *g_spdk_nvmf_tgt_conf; extern struct spdk_nvmf_tgt *g_spdk_nvmf_tgt; -int spdk_nvmf_parse_conf(void); +typedef void (*spdk_nvmf_parse_conf_done_fn)(int status); + +int spdk_nvmf_parse_conf(spdk_nvmf_parse_conf_done_fn cb_fn); #endif diff --git a/lib/event/subsystems/nvmf/nvmf_rpc.c b/lib/event/subsystems/nvmf/nvmf_rpc.c index cd5dc1885..e4114afe6 100644 --- a/lib/event/subsystems/nvmf/nvmf_rpc.c +++ b/lib/event/subsystems/nvmf/nvmf_rpc.c @@ -1422,3 +1422,141 @@ nvmf_rpc_subsystem_set_tgt_conf(struct spdk_jsonrpc_request *request, spdk_jsonrpc_end_result(request, w); } SPDK_RPC_REGISTER("set_nvmf_target_config", nvmf_rpc_subsystem_set_tgt_conf, SPDK_RPC_STARTUP) + +struct nvmf_rpc_create_transport_ctx { + char *trtype; + struct spdk_nvmf_transport_opts opts; + struct spdk_jsonrpc_request *request; +}; + +static const struct spdk_json_object_decoder nvmf_rpc_create_transport_decoder[] = { + { "trtype", offsetof(struct nvmf_rpc_create_transport_ctx, trtype), spdk_json_decode_string}, + { + "max_queue_depth", offsetof(struct nvmf_rpc_create_transport_ctx, opts.max_queue_depth), + spdk_json_decode_uint16, true + }, + { + "max_qpairs_per_ctrlr", offsetof(struct nvmf_rpc_create_transport_ctx, opts.max_qpairs_per_ctrlr), + spdk_json_decode_uint16, true + }, + { + "in_capsule_data_size", offsetof(struct nvmf_rpc_create_transport_ctx, opts.in_capsule_data_size), + spdk_json_decode_uint32, true + }, + { + "max_io_size", offsetof(struct nvmf_rpc_create_transport_ctx, opts.max_io_size), + spdk_json_decode_uint32, true + }, + { + "io_unit_size", offsetof(struct nvmf_rpc_create_transport_ctx, opts.io_unit_size), + spdk_json_decode_uint32, true + }, + { + "max_aq_depth", offsetof(struct nvmf_rpc_create_transport_ctx, opts.max_aq_depth), + spdk_json_decode_uint32, true + }, +}; + +static void +nvmf_rpc_create_transport_ctx_free(struct nvmf_rpc_create_transport_ctx *ctx) +{ + free(ctx->trtype); + free(ctx); +} + +static void +nvmf_rpc_tgt_add_transport_done(void *cb_arg, int status) +{ + struct nvmf_rpc_create_transport_ctx *ctx = cb_arg; + struct spdk_jsonrpc_request *request; + struct spdk_json_write_ctx *w; + + request = ctx->request; + nvmf_rpc_create_transport_ctx_free(ctx); + + if (status) { + SPDK_ERRLOG("Failed to add transport to tgt.(%d)\n", status); + spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Failed to add transport to tgt.(%d)\n", + status); + return; + } + + w = spdk_jsonrpc_begin_result(request); + if (w == NULL) { + return; + } + + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(request, w); +} + +static void +nvmf_rpc_create_transport(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct nvmf_rpc_create_transport_ctx *ctx; + enum spdk_nvme_transport_type trtype; + struct spdk_nvmf_transport *transport; + + ctx = calloc(1, sizeof(*ctx)); + if (!ctx) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, "Out of memory"); + return; + } + + /* Decode parameters the first time to get the transport type */ + if (spdk_json_decode_object(params, nvmf_rpc_create_transport_decoder, + SPDK_COUNTOF(nvmf_rpc_create_transport_decoder), + ctx)) { + SPDK_ERRLOG("spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters"); + nvmf_rpc_create_transport_ctx_free(ctx); + return; + } + + if (spdk_nvme_transport_id_parse_trtype(&trtype, ctx->trtype)) { + SPDK_ERRLOG("Invalid transport type '%s'\n", ctx->trtype); + spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid transport type '%s'\n", ctx->trtype); + nvmf_rpc_create_transport_ctx_free(ctx); + return; + } + + /* Initialize all the transport options (based on transport type) and decode the + * parameters again to update any options passed in rpc create transport call. + */ + spdk_nvmf_transport_opts_init(trtype, &ctx->opts); + if (spdk_json_decode_object(params, nvmf_rpc_create_transport_decoder, + SPDK_COUNTOF(nvmf_rpc_create_transport_decoder), + ctx)) { + SPDK_ERRLOG("spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters"); + nvmf_rpc_create_transport_ctx_free(ctx); + return; + } + + if (spdk_nvmf_tgt_get_transport(g_spdk_nvmf_tgt, trtype)) { + SPDK_ERRLOG("Transport type '%s' already exists\n", ctx->trtype); + spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Transport type '%s' already exists\n", ctx->trtype); + nvmf_rpc_create_transport_ctx_free(ctx); + return; + } + + transport = spdk_nvmf_transport_create(trtype, &ctx->opts); + + if (!transport) { + SPDK_ERRLOG("Transport type '%s' create failed\n", ctx->trtype); + spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Transport type '%s' create failed\n", ctx->trtype); + nvmf_rpc_create_transport_ctx_free(ctx); + return; + } + + /* add transport to target */ + ctx->request = request; + spdk_nvmf_tgt_add_transport(g_spdk_nvmf_tgt, transport, nvmf_rpc_tgt_add_transport_done, ctx); +} + +SPDK_RPC_REGISTER("nvmf_create_transport", nvmf_rpc_create_transport, SPDK_RPC_RUNTIME) diff --git a/lib/event/subsystems/nvmf/nvmf_tgt.c b/lib/event/subsystems/nvmf/nvmf_tgt.c index 8fb2b9cf0..5b69f8dc7 100644 --- a/lib/event/subsystems/nvmf/nvmf_tgt.c +++ b/lib/event/subsystems/nvmf/nvmf_tgt.c @@ -305,6 +305,23 @@ nvmf_tgt_destroy_done(void *ctx, int status) nvmf_tgt_advance_state(); } +static void +nvmf_tgt_parse_conf_done(int status) +{ + g_tgt_state = (status == 0) ? NVMF_TGT_INIT_CREATE_POLL_GROUPS : NVMF_TGT_ERROR; + nvmf_tgt_advance_state(); +} + +static void +nvmf_tgt_parse_conf_start(void *ctx) +{ + if (spdk_nvmf_parse_conf(nvmf_tgt_parse_conf_done)) { + SPDK_ERRLOG("spdk_nvmf_parse_conf() failed\n"); + g_tgt_state = NVMF_TGT_ERROR; + nvmf_tgt_advance_state(); + } +} + static void nvmf_tgt_advance_state(void) { @@ -333,14 +350,10 @@ nvmf_tgt_advance_state(void) break; } case NVMF_TGT_INIT_PARSE_CONFIG: - rc = spdk_nvmf_parse_conf(); - if (rc < 0) { - SPDK_ERRLOG("spdk_nvmf_parse_conf() failed\n"); - g_tgt_state = NVMF_TGT_ERROR; - rc = -EINVAL; - break; - } - g_tgt_state = NVMF_TGT_INIT_CREATE_POLL_GROUPS; + /* Send message to self to call parse conf func. + * Prevents it from possibly performing cb before getting + * out of this function, which causes problems. */ + spdk_thread_send_msg(spdk_get_thread(), nvmf_tgt_parse_conf_start, NULL); break; case NVMF_TGT_INIT_CREATE_POLL_GROUPS: /* Send a message to each thread and create a poll group */ diff --git a/lib/nvmf/nvmf.c b/lib/nvmf/nvmf.c index 35e3e040d..5e957b751 100644 --- a/lib/nvmf/nvmf.c +++ b/lib/nvmf/nvmf.c @@ -312,15 +312,6 @@ spdk_nvmf_tgt_destroy(struct spdk_nvmf_tgt *tgt, spdk_io_device_unregister(tgt, spdk_nvmf_tgt_destroy_cb); } -struct spdk_nvmf_tgt_listen_ctx { - struct spdk_nvmf_tgt *tgt; - struct spdk_nvmf_transport *transport; - struct spdk_nvme_transport_id trid; - - spdk_nvmf_tgt_listen_done_fn cb_fn; - void *cb_arg; -}; - static void spdk_nvmf_write_subsystem_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_subsystem *subsystem) @@ -432,6 +423,7 @@ void spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt) { struct spdk_nvmf_subsystem *subsystem; + struct spdk_nvmf_transport *transport; spdk_json_write_object_begin(w); spdk_json_write_named_string(w, "method", "set_nvmf_target_options"); @@ -447,6 +439,24 @@ spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_ spdk_json_write_object_end(w); + /* write transports */ + TAILQ_FOREACH(transport, &tgt->transports, link) { + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "method", "nvmf_create_transport"); + + spdk_json_write_named_object_begin(w, "params"); + spdk_json_write_named_string(w, "trtype", spdk_nvme_transport_id_trtype_str(transport->ops->type)); + spdk_json_write_named_uint32(w, "max_queue_depth", transport->opts.max_queue_depth); + spdk_json_write_named_uint32(w, "max_qpairs_per_ctrlr", transport->opts.max_qpairs_per_ctrlr); + spdk_json_write_named_uint32(w, "in_capsule_data_size", transport->opts.in_capsule_data_size); + spdk_json_write_named_uint32(w, "max_io_size", transport->opts.max_io_size); + spdk_json_write_named_uint32(w, "io_unit_size", transport->opts.io_unit_size); + spdk_json_write_named_uint32(w, "max_aq_depth", transport->opts.max_aq_depth); + spdk_json_write_object_end(w); + + spdk_json_write_object_end(w); + } + subsystem = spdk_nvmf_subsystem_get_first(tgt); while (subsystem) { spdk_nvmf_write_subsystem_config_json(w, subsystem); @@ -454,28 +464,6 @@ spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_ } } -static void -spdk_nvmf_tgt_listen_done(struct spdk_io_channel_iter *i, int status) -{ - struct spdk_nvmf_tgt_listen_ctx *ctx = spdk_io_channel_iter_get_ctx(i); - - ctx->cb_fn(ctx->cb_arg, status); - - free(ctx); -} - -static void -spdk_nvmf_tgt_listen_add_transport(struct spdk_io_channel_iter *i) -{ - struct spdk_nvmf_tgt_listen_ctx *ctx = spdk_io_channel_iter_get_ctx(i); - struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); - struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); - int rc; - - rc = spdk_nvmf_poll_group_add_transport(group, ctx->transport); - spdk_for_each_channel_continue(i, rc); -} - void spdk_nvmf_tgt_listen(struct spdk_nvmf_tgt *tgt, struct spdk_nvme_transport_id *trid, @@ -488,13 +476,22 @@ spdk_nvmf_tgt_listen(struct spdk_nvmf_tgt *tgt, transport = spdk_nvmf_tgt_get_transport(tgt, trid->trtype); if (!transport) { - transport = spdk_nvmf_transport_create(tgt, trid->trtype, NULL); + struct spdk_nvmf_transport_opts opts; + + opts.max_queue_depth = tgt->opts.max_queue_depth; + opts.max_qpairs_per_ctrlr = tgt->opts.max_qpairs_per_ctrlr; + opts.in_capsule_data_size = tgt->opts.in_capsule_data_size; + opts.max_io_size = tgt->opts.max_io_size; + opts.io_unit_size = tgt->opts.io_unit_size; + /* use max_queue depth since tgt. opts. doesn't have max_aq_depth */ + opts.max_aq_depth = tgt->opts.max_queue_depth; + + transport = spdk_nvmf_transport_create(trid->trtype, &opts); if (!transport) { SPDK_ERRLOG("Transport initialization failed\n"); cb_fn(cb_arg, -EINVAL); return; } - TAILQ_INSERT_TAIL(&tgt->transports, transport, link); propagate = true; } @@ -509,29 +506,73 @@ spdk_nvmf_tgt_listen(struct spdk_nvmf_tgt *tgt, tgt->discovery_genctr++; if (propagate) { - struct spdk_nvmf_tgt_listen_ctx *ctx; - - ctx = calloc(1, sizeof(*ctx)); - if (!ctx) { - cb_fn(cb_arg, -ENOMEM); - return; - } - - ctx->tgt = tgt; - ctx->transport = transport; - ctx->trid = *trid; - ctx->cb_fn = cb_fn; - ctx->cb_arg = cb_arg; - - spdk_for_each_channel(tgt, - spdk_nvmf_tgt_listen_add_transport, - ctx, - spdk_nvmf_tgt_listen_done); + spdk_nvmf_tgt_add_transport(tgt, transport, cb_fn, cb_arg); } else { cb_fn(cb_arg, 0); } } +struct spdk_nvmf_tgt_add_transport_ctx { + struct spdk_nvmf_tgt *tgt; + struct spdk_nvmf_transport *transport; + spdk_nvmf_tgt_add_transport_done_fn cb_fn; + void *cb_arg; +}; + +static void +_spdk_nvmf_tgt_add_transport_done(struct spdk_io_channel_iter *i, int status) +{ + struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); + + ctx->cb_fn(ctx->cb_arg, status); + + free(ctx); +} + +static void +_spdk_nvmf_tgt_add_transport(struct spdk_io_channel_iter *i) +{ + struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); + struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); + struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); + int rc; + + rc = spdk_nvmf_poll_group_add_transport(group, ctx->transport); + spdk_for_each_channel_continue(i, rc); +} + +void spdk_nvmf_tgt_add_transport(struct spdk_nvmf_tgt *tgt, + struct spdk_nvmf_transport *transport, + spdk_nvmf_tgt_add_transport_done_fn cb_fn, + void *cb_arg) +{ + struct spdk_nvmf_tgt_add_transport_ctx *ctx; + + if (spdk_nvmf_tgt_get_transport(tgt, transport->ops->type)) { + cb_fn(cb_arg, -EEXIST); + return; /* transport already created */ + } + + transport->tgt = tgt; + TAILQ_INSERT_TAIL(&tgt->transports, transport, link); + + ctx = calloc(1, sizeof(*ctx)); + if (!ctx) { + cb_fn(cb_arg, -ENOMEM); + return; + } + + ctx->tgt = tgt; + ctx->transport = transport; + ctx->cb_fn = cb_fn; + ctx->cb_arg = cb_arg; + + spdk_for_each_channel(tgt, + _spdk_nvmf_tgt_add_transport, + ctx, + _spdk_nvmf_tgt_add_transport_done); +} + struct spdk_nvmf_subsystem * spdk_nvmf_tgt_find_subsystem(struct spdk_nvmf_tgt *tgt, const char *subnqn) { diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index 4f12d9a2f..92d5f4ed4 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -1607,6 +1607,24 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport, /* Public API callbacks begin here */ +#define SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH 128 +#define SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH 128 +#define SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR 64 +#define SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE 4096 +#define SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE 131072 +#define SPDK_NVMF_RDMA_DEFAULT_IO_UNIT_SIZE 131072 + +static void +spdk_nvmf_rdma_opts_init(struct spdk_nvmf_transport_opts *opts) +{ + opts->max_queue_depth = SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH; + opts->max_qpairs_per_ctrlr = SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR; + opts->in_capsule_data_size = SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE; + opts->max_io_size = SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE; + opts->io_unit_size = SPDK_NVMF_RDMA_DEFAULT_IO_UNIT_SIZE; + opts->max_aq_depth = SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH; +} + static int spdk_nvmf_rdma_destroy(struct spdk_nvmf_transport *transport); static struct spdk_nvmf_transport * @@ -2830,6 +2848,7 @@ spdk_nvmf_rdma_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, const struct spdk_nvmf_transport_ops spdk_nvmf_transport_rdma = { .type = SPDK_NVME_TRANSPORT_RDMA, + .opts_init = spdk_nvmf_rdma_opts_init, .create = spdk_nvmf_rdma_create, .destroy = spdk_nvmf_rdma_destroy, diff --git a/lib/nvmf/transport.c b/lib/nvmf/transport.c index 20203c991..0e351e213 100644 --- a/lib/nvmf/transport.c +++ b/lib/nvmf/transport.c @@ -62,24 +62,11 @@ spdk_nvmf_get_transport_ops(enum spdk_nvme_transport_type type) } struct spdk_nvmf_transport * -spdk_nvmf_transport_create(struct spdk_nvmf_tgt *tgt, - enum spdk_nvme_transport_type type, +spdk_nvmf_transport_create(enum spdk_nvme_transport_type type, struct spdk_nvmf_transport_opts *opts) { const struct spdk_nvmf_transport_ops *ops = NULL; struct spdk_nvmf_transport *transport; - struct spdk_nvmf_transport_opts tgt_opts; - - if (opts == NULL) { - /* get transport opts from global target opts */ - tgt_opts.max_queue_depth = tgt->opts.max_queue_depth; - tgt_opts.max_qpairs_per_ctrlr = tgt->opts.max_qpairs_per_ctrlr; - tgt_opts.in_capsule_data_size = tgt->opts.in_capsule_data_size; - tgt_opts.max_io_size = tgt->opts.max_io_size; - tgt_opts.io_unit_size = tgt->opts.io_unit_size; - tgt_opts.max_aq_depth = tgt->opts.max_queue_depth; - opts = &tgt_opts; - } if ((opts->max_io_size % opts->io_unit_size != 0) || (opts->max_io_size / opts->io_unit_size > @@ -106,7 +93,6 @@ spdk_nvmf_transport_create(struct spdk_nvmf_tgt *tgt, return NULL; } - transport->tgt = tgt; transport->ops = ops; transport->opts = *opts; @@ -230,3 +216,20 @@ spdk_nvmf_transport_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, { return qpair->transport->ops->qpair_get_listen_trid(qpair, trid); } + +bool +spdk_nvmf_transport_opts_init(enum spdk_nvme_transport_type type, + struct spdk_nvmf_transport_opts *opts) +{ + const struct spdk_nvmf_transport_ops *ops; + + ops = spdk_nvmf_get_transport_ops(type); + if (!ops) { + SPDK_ERRLOG("Transport type %s unavailable.\n", + spdk_nvme_transport_id_trtype_str(type)); + return false; + } + + ops->opts_init(opts); + return true; +} diff --git a/lib/nvmf/transport.h b/lib/nvmf/transport.h index 47c9ad2e6..1329a80c4 100644 --- a/lib/nvmf/transport.h +++ b/lib/nvmf/transport.h @@ -53,6 +53,11 @@ struct spdk_nvmf_transport_ops { */ enum spdk_nvme_transport_type type; + /** + * Initialize transport options to default value + */ + void (*opts_init)(struct spdk_nvmf_transport_opts *opts); + /** * Create a transport for the given transport opts */ @@ -150,14 +155,6 @@ struct spdk_nvmf_transport_ops { struct spdk_nvme_transport_id *trid); }; -struct spdk_nvmf_transport *spdk_nvmf_transport_create(struct spdk_nvmf_tgt *tgt, - enum spdk_nvme_transport_type type, - struct spdk_nvmf_transport_opts *opts); - -int spdk_nvmf_transport_destroy(struct spdk_nvmf_transport *transport); - -int spdk_nvmf_transport_listen(struct spdk_nvmf_transport *transport, - const struct spdk_nvme_transport_id *trid); int spdk_nvmf_transport_stop_listen(struct spdk_nvmf_transport *transport, const struct spdk_nvme_transport_id *trid); @@ -195,6 +192,9 @@ int spdk_nvmf_transport_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair, int spdk_nvmf_transport_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, struct spdk_nvme_transport_id *trid); +bool spdk_nvmf_transport_opts_init(enum spdk_nvme_transport_type type, + struct spdk_nvmf_transport_opts *opts); + extern const struct spdk_nvmf_transport_ops spdk_nvmf_transport_rdma; #endif /* SPDK_NVMF_TRANSPORT_H */ diff --git a/scripts/rpc.py b/scripts/rpc.py index cba0cc90e..2d037e92c 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -1249,6 +1249,27 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse robin manner""") p.set_defaults(func=set_nvmf_target_config) + @call_cmd + def nvmf_create_transport(args): + rpc.nvmf.nvmf_create_transport(args.client, + trtype=args.trtype, + max_queue_depth=args.max_queue_depth, + max_qpairs_per_ctrlr=args.max_qpairs_per_ctrlr, + in_capsule_data_size=args.in_capsule_data_size, + max_io_size=args.max_io_size, + io_unit_size=args.io_unit_size, + max_aq_depth=args.max_aq_depth) + + p = subparsers.add_parser('nvmf_create_transport', help='Create NVMf transport') + p.add_argument('-t', '--trtype', help='Transport type (ex. RDMA)', type=str, required=True) + p.add_argument('-q', '--max-queue-depth', help='Max number of outstanding I/O per queue', type=int) + p.add_argument('-p', '--max-qpairs-per-ctrlr', help='Max number of SQ and CQ per controller', type=int) + p.add_argument('-c', '--in-capsule-data-size', help='Max number of in-capsule data size', type=int) + p.add_argument('-i', '--max-io-size', help='Max I/O size (bytes)', type=int) + p.add_argument('-u', '--io-unit-size', help='I/O unit size (bytes)', type=int) + p.add_argument('-a', '--max-aq-depth', help='Max number of admin cmds per AQ', type=int) + p.set_defaults(func=nvmf_create_transport) + @call_cmd def get_nvmf_subsystems(args): print_dict(rpc.nvmf.get_nvmf_subsystems(args.client)) diff --git a/scripts/rpc/nvmf.py b/scripts/rpc/nvmf.py index b635f1adf..d805ebca8 100755 --- a/scripts/rpc/nvmf.py +++ b/scripts/rpc/nvmf.py @@ -58,6 +58,46 @@ def set_nvmf_target_config(client, return client.call('set_nvmf_target_config', params) +def nvmf_create_transport(client, + trtype, + max_queue_depth=None, + max_qpairs_per_ctrlr=None, + in_capsule_data_size=None, + max_io_size=None, + io_unit_size=None, + max_aq_depth=None): + """NVMf Transport Create options. + + Args: + trtype: Transport type (ex. RDMA) + max_queue_depth: Max number of outstanding I/O per queue (optional) + max_qpairs_per_ctrlr: Max number of SQ and CQ per controller (optional) + in_capsule_data_size: Maximum in-capsule data size in bytes (optional) + max_io_size: Maximum I/O data size in bytes (optional) + io_unit_size: I/O unit size in bytes (optional) + max_aq_depth: Max size admin quque per controller (optional) + + Returns: + True or False + """ + params = {} + + params['trtype'] = trtype + if max_queue_depth: + params['max_queue_depth'] = max_queue_depth + if max_qpairs_per_ctrlr: + params['max_qpairs_per_ctrlr'] = max_qpairs_per_ctrlr + if in_capsule_data_size: + params['in_capsule_data_size'] = in_capsule_data_size + if max_io_size: + params['max_io_size'] = max_io_size + if io_unit_size: + params['io_unit_size'] = io_unit_size + if max_aq_depth: + params['max_aq_depth'] = max_aq_depth + return client.call('nvmf_create_transport', params) + + def get_nvmf_subsystems(client): """Get list of NVMe-oF subsystems. diff --git a/test/json_config/clear_config.py b/test/json_config/clear_config.py index 91780dd05..700c61805 100755 --- a/test/json_config/clear_config.py +++ b/test/json_config/clear_config.py @@ -83,7 +83,8 @@ def clear_bdev_subsystem(args, bdev_config): def get_nvmf_destroy_method(nvmf): destroy_method_map = {'construct_nvmf_subsystem': "delete_nvmf_subsystem", 'set_nvmf_target_config': None, - 'set_nvmf_target_options': None + 'set_nvmf_target_options': None, + 'nvmf_create_transport': None } return destroy_method_map[nvmf['method']] diff --git a/test/json_config/config_filter.py b/test/json_config/config_filter.py index e1a3dd264..59e96f947 100755 --- a/test/json_config/config_filter.py +++ b/test/json_config/config_filter.py @@ -26,6 +26,7 @@ def filter_methods(do_remove_global_rpcs): 'set_iscsi_options', 'set_nvmf_target_config', 'set_nvmf_target_options', + 'nvmf_create_transport', 'set_bdev_options', 'set_bdev_nvme_options', 'set_bdev_nvme_hotplug', diff --git a/test/nvmf/create_transport/create_transport.sh b/test/nvmf/create_transport/create_transport.sh new file mode 100755 index 000000000..c584f3d41 --- /dev/null +++ b/test/nvmf/create_transport/create_transport.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +NULL_BDEV_SIZE=102400 +NULL_BLOCK_SIZE=512 + +rpc_py="python $rootdir/scripts/rpc.py" + +set -e + +# pass the parameter 'iso' to this script when running it in isolation to trigger rdma device initialization. +# e.g. sudo ./crt_trprt.sh iso +nvmftestinit $1 + +if ! hash nvme; then + echo "nvme command not found; skipping create transport test" + exit 0 +fi + +RDMA_IP_LIST=$(get_available_rdma_ips) +NVMF_FIRST_TARGET_IP=$(echo "$RDMA_IP_LIST" | head -n 1) +if [ -z $NVMF_FIRST_TARGET_IP ]; then + echo "no NIC for nvmf test" + exit 0 +fi + +timing_enter cr_trprt +timing_enter start_nvmf_tgt +# Start up the NVMf target in another process +$NVMF_APP -m 0xF --wait-for-rpc & +nvmfpid=$! + +trap "killprocess $nvmfpid; nvmftestfini $1; exit 1" SIGINT SIGTERM EXIT + +waitforlisten $nvmfpid +$rpc_py start_subsystem_init +# Use nvmf_create_transport call to create transport +$rpc_py nvmf_create_transport -t RDMA -u 8192 -p 4 +timing_exit start_nvmf_tgt + +null_bdevs="$($rpc_py construct_null_bdev Null0 $NULL_BDEV_SIZE $NULL_BLOCK_SIZE) " +null_bdevs+="$($rpc_py construct_null_bdev Null1 $NULL_BDEV_SIZE $NULL_BLOCK_SIZE)" + +modprobe -v nvme-rdma + +$rpc_py construct_nvmf_subsystem nqn.2016-06.io.spdk:cnode1 "trtype:RDMA traddr:$NVMF_FIRST_TARGET_IP trsvcid:4420" "" -a -s SPDK00000000000001 +for null_bdev in $null_bdevs; do + $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 $null_bdev +done + +nvme discover -t rdma -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +echo "Perform nvmf subsystem discovery via RPC" +$rpc_py get_nvmf_subsystems + +$rpc_py delete_nvmf_subsystem nqn.2016-06.io.spdk:cnode1 + +for null_bdev in $null_bdevs; do + $rpc_py delete_null_bdev $null_bdev +done + +check_bdevs=$($rpc_py get_bdevs | jq -r '.[].name') +if [ -n "$check_bdevs" ]; then + echo $check_bdevs + exit 1 +fi + +trap - SIGINT SIGTERM EXIT + +nvmfcleanup +killprocess $nvmfpid +nvmftestfini $1 +timing_exit crt_trprt diff --git a/test/nvmf/nvmf.sh b/test/nvmf/nvmf.sh index 859569d05..700557775 100755 --- a/test/nvmf/nvmf.sh +++ b/test/nvmf/nvmf.sh @@ -32,6 +32,7 @@ fi run_test suite test/nvmf/lvol/nvmf_lvol.sh run_test suite test/nvmf/shutdown/shutdown.sh run_test suite test/nvmf/bdev_io_wait/bdev_io_wait.sh +run_test suite test/nvmf/create_transport/create_transport.sh if [ $RUN_NIGHTLY -eq 1 ]; then run_test suite test/nvmf/multiconnection/multiconnection.sh diff --git a/test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c b/test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c index 95bec6bc6..3e5c3a04b 100644 --- a/test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c +++ b/test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c @@ -110,12 +110,10 @@ spdk_nvmf_transport_listener_discover(struct spdk_nvmf_transport *transport, static struct spdk_nvmf_transport g_transport = {}; struct spdk_nvmf_transport * -spdk_nvmf_transport_create(struct spdk_nvmf_tgt *tgt, - enum spdk_nvme_transport_type type, +spdk_nvmf_transport_create(enum spdk_nvme_transport_type type, struct spdk_nvmf_transport_opts *tprt_opts) { if (type == SPDK_NVME_TRANSPORT_RDMA) { - g_transport.tgt = tgt; return &g_transport; } diff --git a/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c b/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c index 3519a9bd7..8a821b663 100644 --- a/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c +++ b/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c @@ -102,12 +102,10 @@ spdk_nvmf_transport_qpair_is_idle(struct spdk_nvmf_qpair *qpair) static struct spdk_nvmf_transport g_transport = {}; struct spdk_nvmf_transport * -spdk_nvmf_transport_create(struct spdk_nvmf_tgt *tgt, - enum spdk_nvme_transport_type type, +spdk_nvmf_transport_create(enum spdk_nvme_transport_type type, struct spdk_nvmf_transport_opts *tprt_opts) { if (type == SPDK_NVME_TRANSPORT_RDMA) { - g_transport.tgt = tgt; return &g_transport; }