From 183d81d0c65d9593cbaa28fa342fc086b22cd89e Mon Sep 17 00:00:00 2001 From: John Barnard Date: Mon, 27 Aug 2018 15:27:47 -0700 Subject: [PATCH] nvmf: Move target opts to transport opts (part 2) - Add independent functions to create transport with specific opts and add to target while maintaining backward compatibility with current apps and rpc configuration that still use the add listener method to create a transport. - Add new rpc function to create transport and add to target. + Update json reporting to include new rpc function. + Update python scripts to support new rpc function. + New nvmf test script (cr_trprt.sh) to test new rpc function. Change-Id: I12d0a42e34c9edff757755f18a78b722d5e1523e Signed-off-by: John Barnard Reviewed-on: https://review.gerrithub.io/423590 Tested-by: SPDK CI Jenkins Chandler-Test-Pool: SPDK Automated Test System Reviewed-by: Ben Walker Reviewed-by: Jim Harris --- include/spdk/nvmf.h | 89 +++++++++++ lib/event/subsystems/nvmf/conf.c | 148 +++++++++++++++++- lib/event/subsystems/nvmf/event_nvmf.h | 4 +- lib/event/subsystems/nvmf/nvmf_rpc.c | 138 ++++++++++++++++ lib/event/subsystems/nvmf/nvmf_tgt.c | 29 +++- lib/nvmf/nvmf.c | 143 +++++++++++------ lib/nvmf/rdma.c | 19 +++ lib/nvmf/transport.c | 33 ++-- lib/nvmf/transport.h | 16 +- scripts/rpc.py | 21 +++ scripts/rpc/nvmf.py | 40 +++++ test/json_config/clear_config.py | 3 +- test/json_config/config_filter.py | 1 + .../nvmf/create_transport/create_transport.sh | 77 +++++++++ test/nvmf/nvmf.sh | 1 + .../ctrlr_discovery.c/ctrlr_discovery_ut.c | 4 +- test/unit/lib/nvmf/subsystem.c/subsystem_ut.c | 4 +- 17 files changed, 677 insertions(+), 93 deletions(-) create mode 100755 test/nvmf/create_transport/create_transport.sh diff --git a/include/spdk/nvmf.h b/include/spdk/nvmf.h index 26e2214d4..17fa347aa 100644 --- a/include/spdk/nvmf.h +++ b/include/spdk/nvmf.h @@ -703,6 +703,95 @@ const char *spdk_nvmf_subsystem_get_nqn(struct spdk_nvmf_subsystem *subsystem); */ enum spdk_nvmf_subtype spdk_nvmf_subsystem_get_type(struct spdk_nvmf_subsystem *subsystem); +/** + * Initialize transport options + * + * \param type The transport type to create + * \param opts The transport options (e.g. max_io_size) + * + * \return bool. true if successful, false if transport type + * not found. + */ +bool +spdk_nvmf_transport_opts_init(enum spdk_nvme_transport_type type, + struct spdk_nvmf_transport_opts *opts); + +/** + * Create a protocol transport + * + * \param type The transport type to create + * \param opts The transport options (e.g. max_io_size) + * + * \return new transport or NULL if create fails + */ +struct spdk_nvmf_transport *spdk_nvmf_transport_create(enum spdk_nvme_transport_type type, + struct spdk_nvmf_transport_opts *opts); + +/** + * Destroy a protocol transport + * + * \param transport The transport to destory + * + * \return 0 on success, -1 on failure. + */ +int spdk_nvmf_transport_destroy(struct spdk_nvmf_transport *transport); + +/** + * Get an existing transport from the target + * + * \param tgt The NVMe-oF target + * \param type The transport type to get + * + * \return the transport or NULL if not found + */ +struct spdk_nvmf_transport *spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, + enum spdk_nvme_transport_type type); + +/** + * Function to be called once transport add is complete + * + * \param cb_arg Callback argument passed to this function. + * \param status 0 if it completed successfully, or negative errno if it failed. + */ +typedef void (*spdk_nvmf_tgt_add_transport_done_fn)(void *cb_arg, int status); + +/** + * Add a transport to a target + * + * \param tgt The NVMe-oF target + * \param transport The transport to add + * \param cb_fn A callback that will be called once the transport is created + * \param cb_arg A context argument passed to cb_fn. + * + * \return void. The callback status argument will be 0 on success + * or a negated errno on failure. + */ +void spdk_nvmf_tgt_add_transport(struct spdk_nvmf_tgt *tgt, + struct spdk_nvmf_transport *transport, + spdk_nvmf_tgt_add_transport_done_fn cb_fn, + void *cb_arg); + +/** + * + * Add listener to transport and begin accepting new connections. + * + * \param transport The transport to add listener to + * \param trid Address to listen at + * + * \return int. 0 if it completed successfully, or negative errno if it failed. + */ + +int spdk_nvmf_transport_listen(struct spdk_nvmf_transport *transport, + const struct spdk_nvme_transport_id *trid); + +/** + * Write NVMe-oF target's transport configurations into provided JSON context. + * \param w JSON write context + * \param tgt The NVMe-oF target + */ +void +spdk_nvmf_tgt_transport_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt); + #ifdef __cplusplus } #endif diff --git a/lib/event/subsystems/nvmf/conf.c b/lib/event/subsystems/nvmf/conf.c index 1e3605b4d..986e81c9f 100644 --- a/lib/event/subsystems/nvmf/conf.c +++ b/lib/event/subsystems/nvmf/conf.c @@ -424,19 +424,161 @@ spdk_nvmf_parse_subsystems(void) return 0; } +struct spdk_nvmf_parse_transport_ctx { + struct spdk_conf_section *sp; + spdk_nvmf_parse_conf_done_fn cb_fn; +}; + +static void spdk_nvmf_parse_transport(struct spdk_nvmf_parse_transport_ctx *ctx); + +static void +spdk_nvmf_tgt_add_transport_done(void *cb_arg, int status) +{ + struct spdk_nvmf_parse_transport_ctx *ctx = cb_arg; + int rc; + + if (status < 0) { + SPDK_ERRLOG("Add transport to target failed (%d).\n", status); + ctx->cb_fn(status); + free(ctx); + return; + } + + /* find next transport */ + ctx->sp = spdk_conf_next_section(ctx->sp); + while (ctx->sp) { + if (spdk_conf_section_match_prefix(ctx->sp, "Transport")) { + spdk_nvmf_parse_transport(ctx); + return; + } + ctx->sp = spdk_conf_next_section(ctx->sp); + } + + /* done with transports, parse Subsystem sections */ + rc = spdk_nvmf_parse_subsystems(); + + ctx->cb_fn(rc); + free(ctx); +} + +static void +spdk_nvmf_parse_transport(struct spdk_nvmf_parse_transport_ctx *ctx) +{ + const char *type; + struct spdk_nvmf_transport_opts opts = { 0 }; + enum spdk_nvme_transport_type trtype; + struct spdk_nvmf_transport *transport; + int val; + + type = spdk_conf_section_get_val(ctx->sp, "Type"); + if (type == NULL) { + SPDK_ERRLOG("Transport missing Type\n"); + ctx->cb_fn(-1); + free(ctx); + return; + } + + if (spdk_nvme_transport_id_parse_trtype(&trtype, type)) { + SPDK_ERRLOG("Invalid transport type '%s'\n", type); + ctx->cb_fn(-1); + free(ctx); + return; + } + + if (spdk_nvmf_tgt_get_transport(g_spdk_nvmf_tgt, trtype)) { + SPDK_ERRLOG("Duplicate transport type '%s'\n", type); + ctx->cb_fn(-1); + free(ctx); + return; + } + + if (!spdk_nvmf_transport_opts_init(trtype, &opts)) { + ctx->cb_fn(-1); + free(ctx); + return; + } + + val = spdk_conf_section_get_intval(ctx->sp, "MaxQueueDepth"); + if (val >= 0) { + opts.max_queue_depth = val; + } + val = spdk_conf_section_get_intval(ctx->sp, "MaxQueuesPerSession"); + if (val >= 0) { + opts.max_qpairs_per_ctrlr = val; + } + val = spdk_conf_section_get_intval(ctx->sp, "InCapsuleDataSize"); + if (val >= 0) { + opts.in_capsule_data_size = val; + } + val = spdk_conf_section_get_intval(ctx->sp, "MaxIOSize"); + if (val >= 0) { + opts.max_io_size = val; + } + val = spdk_conf_section_get_intval(ctx->sp, "IOUnitSize"); + if (val >= 0) { + opts.io_unit_size = val; + } + val = spdk_conf_section_get_intval(ctx->sp, "MaxAQDepth"); + if (val >= 0) { + opts.max_aq_depth = val; + } + + transport = spdk_nvmf_transport_create(trtype, &opts); + if (transport) { + spdk_nvmf_tgt_add_transport(g_spdk_nvmf_tgt, transport, spdk_nvmf_tgt_add_transport_done, ctx); + } else { + ctx->cb_fn(-1); + free(ctx); + return; + } +} + +static int +spdk_nvmf_parse_transports(spdk_nvmf_parse_conf_done_fn cb_fn) +{ + struct spdk_nvmf_parse_transport_ctx *ctx; + + ctx = calloc(1, sizeof(struct spdk_nvmf_parse_transport_ctx)); + if (!ctx) { + SPDK_ERRLOG("Failed alloc of context memory for parse transports\n"); + return -ENOMEM; + } + + ctx->cb_fn = cb_fn; + ctx->sp = spdk_conf_first_section(NULL); + while (ctx->sp != NULL) { + if (spdk_conf_section_match_prefix(ctx->sp, "Transport")) { + spdk_nvmf_parse_transport(ctx); + return 0; + } + ctx->sp = spdk_conf_next_section(ctx->sp); + } + + /* if we get here, there are no transports defined in conf file */ + free(ctx); + cb_fn(spdk_nvmf_parse_subsystems()); + + return 0; +} + int -spdk_nvmf_parse_conf(void) +spdk_nvmf_parse_conf(spdk_nvmf_parse_conf_done_fn cb_fn) { int rc; + if (cb_fn == NULL) { + SPDK_ERRLOG("Callback function is NULL\n"); + return -1; + } + /* NVMf section */ rc = spdk_nvmf_parse_nvmf_tgt(); if (rc < 0) { return rc; } - /* Subsystem sections */ - rc = spdk_nvmf_parse_subsystems(); + /* Transport sections */ + rc = spdk_nvmf_parse_transports(cb_fn); if (rc < 0) { return rc; } diff --git a/lib/event/subsystems/nvmf/event_nvmf.h b/lib/event/subsystems/nvmf/event_nvmf.h index fdee43c39..50e5d7555 100644 --- a/lib/event/subsystems/nvmf/event_nvmf.h +++ b/lib/event/subsystems/nvmf/event_nvmf.h @@ -60,6 +60,8 @@ extern struct spdk_nvmf_tgt_conf *g_spdk_nvmf_tgt_conf; extern struct spdk_nvmf_tgt *g_spdk_nvmf_tgt; -int spdk_nvmf_parse_conf(void); +typedef void (*spdk_nvmf_parse_conf_done_fn)(int status); + +int spdk_nvmf_parse_conf(spdk_nvmf_parse_conf_done_fn cb_fn); #endif diff --git a/lib/event/subsystems/nvmf/nvmf_rpc.c b/lib/event/subsystems/nvmf/nvmf_rpc.c index cd5dc1885..e4114afe6 100644 --- a/lib/event/subsystems/nvmf/nvmf_rpc.c +++ b/lib/event/subsystems/nvmf/nvmf_rpc.c @@ -1422,3 +1422,141 @@ nvmf_rpc_subsystem_set_tgt_conf(struct spdk_jsonrpc_request *request, spdk_jsonrpc_end_result(request, w); } SPDK_RPC_REGISTER("set_nvmf_target_config", nvmf_rpc_subsystem_set_tgt_conf, SPDK_RPC_STARTUP) + +struct nvmf_rpc_create_transport_ctx { + char *trtype; + struct spdk_nvmf_transport_opts opts; + struct spdk_jsonrpc_request *request; +}; + +static const struct spdk_json_object_decoder nvmf_rpc_create_transport_decoder[] = { + { "trtype", offsetof(struct nvmf_rpc_create_transport_ctx, trtype), spdk_json_decode_string}, + { + "max_queue_depth", offsetof(struct nvmf_rpc_create_transport_ctx, opts.max_queue_depth), + spdk_json_decode_uint16, true + }, + { + "max_qpairs_per_ctrlr", offsetof(struct nvmf_rpc_create_transport_ctx, opts.max_qpairs_per_ctrlr), + spdk_json_decode_uint16, true + }, + { + "in_capsule_data_size", offsetof(struct nvmf_rpc_create_transport_ctx, opts.in_capsule_data_size), + spdk_json_decode_uint32, true + }, + { + "max_io_size", offsetof(struct nvmf_rpc_create_transport_ctx, opts.max_io_size), + spdk_json_decode_uint32, true + }, + { + "io_unit_size", offsetof(struct nvmf_rpc_create_transport_ctx, opts.io_unit_size), + spdk_json_decode_uint32, true + }, + { + "max_aq_depth", offsetof(struct nvmf_rpc_create_transport_ctx, opts.max_aq_depth), + spdk_json_decode_uint32, true + }, +}; + +static void +nvmf_rpc_create_transport_ctx_free(struct nvmf_rpc_create_transport_ctx *ctx) +{ + free(ctx->trtype); + free(ctx); +} + +static void +nvmf_rpc_tgt_add_transport_done(void *cb_arg, int status) +{ + struct nvmf_rpc_create_transport_ctx *ctx = cb_arg; + struct spdk_jsonrpc_request *request; + struct spdk_json_write_ctx *w; + + request = ctx->request; + nvmf_rpc_create_transport_ctx_free(ctx); + + if (status) { + SPDK_ERRLOG("Failed to add transport to tgt.(%d)\n", status); + spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Failed to add transport to tgt.(%d)\n", + status); + return; + } + + w = spdk_jsonrpc_begin_result(request); + if (w == NULL) { + return; + } + + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(request, w); +} + +static void +nvmf_rpc_create_transport(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct nvmf_rpc_create_transport_ctx *ctx; + enum spdk_nvme_transport_type trtype; + struct spdk_nvmf_transport *transport; + + ctx = calloc(1, sizeof(*ctx)); + if (!ctx) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, "Out of memory"); + return; + } + + /* Decode parameters the first time to get the transport type */ + if (spdk_json_decode_object(params, nvmf_rpc_create_transport_decoder, + SPDK_COUNTOF(nvmf_rpc_create_transport_decoder), + ctx)) { + SPDK_ERRLOG("spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters"); + nvmf_rpc_create_transport_ctx_free(ctx); + return; + } + + if (spdk_nvme_transport_id_parse_trtype(&trtype, ctx->trtype)) { + SPDK_ERRLOG("Invalid transport type '%s'\n", ctx->trtype); + spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid transport type '%s'\n", ctx->trtype); + nvmf_rpc_create_transport_ctx_free(ctx); + return; + } + + /* Initialize all the transport options (based on transport type) and decode the + * parameters again to update any options passed in rpc create transport call. + */ + spdk_nvmf_transport_opts_init(trtype, &ctx->opts); + if (spdk_json_decode_object(params, nvmf_rpc_create_transport_decoder, + SPDK_COUNTOF(nvmf_rpc_create_transport_decoder), + ctx)) { + SPDK_ERRLOG("spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters"); + nvmf_rpc_create_transport_ctx_free(ctx); + return; + } + + if (spdk_nvmf_tgt_get_transport(g_spdk_nvmf_tgt, trtype)) { + SPDK_ERRLOG("Transport type '%s' already exists\n", ctx->trtype); + spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Transport type '%s' already exists\n", ctx->trtype); + nvmf_rpc_create_transport_ctx_free(ctx); + return; + } + + transport = spdk_nvmf_transport_create(trtype, &ctx->opts); + + if (!transport) { + SPDK_ERRLOG("Transport type '%s' create failed\n", ctx->trtype); + spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Transport type '%s' create failed\n", ctx->trtype); + nvmf_rpc_create_transport_ctx_free(ctx); + return; + } + + /* add transport to target */ + ctx->request = request; + spdk_nvmf_tgt_add_transport(g_spdk_nvmf_tgt, transport, nvmf_rpc_tgt_add_transport_done, ctx); +} + +SPDK_RPC_REGISTER("nvmf_create_transport", nvmf_rpc_create_transport, SPDK_RPC_RUNTIME) diff --git a/lib/event/subsystems/nvmf/nvmf_tgt.c b/lib/event/subsystems/nvmf/nvmf_tgt.c index 8fb2b9cf0..5b69f8dc7 100644 --- a/lib/event/subsystems/nvmf/nvmf_tgt.c +++ b/lib/event/subsystems/nvmf/nvmf_tgt.c @@ -305,6 +305,23 @@ nvmf_tgt_destroy_done(void *ctx, int status) nvmf_tgt_advance_state(); } +static void +nvmf_tgt_parse_conf_done(int status) +{ + g_tgt_state = (status == 0) ? NVMF_TGT_INIT_CREATE_POLL_GROUPS : NVMF_TGT_ERROR; + nvmf_tgt_advance_state(); +} + +static void +nvmf_tgt_parse_conf_start(void *ctx) +{ + if (spdk_nvmf_parse_conf(nvmf_tgt_parse_conf_done)) { + SPDK_ERRLOG("spdk_nvmf_parse_conf() failed\n"); + g_tgt_state = NVMF_TGT_ERROR; + nvmf_tgt_advance_state(); + } +} + static void nvmf_tgt_advance_state(void) { @@ -333,14 +350,10 @@ nvmf_tgt_advance_state(void) break; } case NVMF_TGT_INIT_PARSE_CONFIG: - rc = spdk_nvmf_parse_conf(); - if (rc < 0) { - SPDK_ERRLOG("spdk_nvmf_parse_conf() failed\n"); - g_tgt_state = NVMF_TGT_ERROR; - rc = -EINVAL; - break; - } - g_tgt_state = NVMF_TGT_INIT_CREATE_POLL_GROUPS; + /* Send message to self to call parse conf func. + * Prevents it from possibly performing cb before getting + * out of this function, which causes problems. */ + spdk_thread_send_msg(spdk_get_thread(), nvmf_tgt_parse_conf_start, NULL); break; case NVMF_TGT_INIT_CREATE_POLL_GROUPS: /* Send a message to each thread and create a poll group */ diff --git a/lib/nvmf/nvmf.c b/lib/nvmf/nvmf.c index 35e3e040d..5e957b751 100644 --- a/lib/nvmf/nvmf.c +++ b/lib/nvmf/nvmf.c @@ -312,15 +312,6 @@ spdk_nvmf_tgt_destroy(struct spdk_nvmf_tgt *tgt, spdk_io_device_unregister(tgt, spdk_nvmf_tgt_destroy_cb); } -struct spdk_nvmf_tgt_listen_ctx { - struct spdk_nvmf_tgt *tgt; - struct spdk_nvmf_transport *transport; - struct spdk_nvme_transport_id trid; - - spdk_nvmf_tgt_listen_done_fn cb_fn; - void *cb_arg; -}; - static void spdk_nvmf_write_subsystem_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_subsystem *subsystem) @@ -432,6 +423,7 @@ void spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt) { struct spdk_nvmf_subsystem *subsystem; + struct spdk_nvmf_transport *transport; spdk_json_write_object_begin(w); spdk_json_write_named_string(w, "method", "set_nvmf_target_options"); @@ -447,6 +439,24 @@ spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_ spdk_json_write_object_end(w); + /* write transports */ + TAILQ_FOREACH(transport, &tgt->transports, link) { + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "method", "nvmf_create_transport"); + + spdk_json_write_named_object_begin(w, "params"); + spdk_json_write_named_string(w, "trtype", spdk_nvme_transport_id_trtype_str(transport->ops->type)); + spdk_json_write_named_uint32(w, "max_queue_depth", transport->opts.max_queue_depth); + spdk_json_write_named_uint32(w, "max_qpairs_per_ctrlr", transport->opts.max_qpairs_per_ctrlr); + spdk_json_write_named_uint32(w, "in_capsule_data_size", transport->opts.in_capsule_data_size); + spdk_json_write_named_uint32(w, "max_io_size", transport->opts.max_io_size); + spdk_json_write_named_uint32(w, "io_unit_size", transport->opts.io_unit_size); + spdk_json_write_named_uint32(w, "max_aq_depth", transport->opts.max_aq_depth); + spdk_json_write_object_end(w); + + spdk_json_write_object_end(w); + } + subsystem = spdk_nvmf_subsystem_get_first(tgt); while (subsystem) { spdk_nvmf_write_subsystem_config_json(w, subsystem); @@ -454,28 +464,6 @@ spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_ } } -static void -spdk_nvmf_tgt_listen_done(struct spdk_io_channel_iter *i, int status) -{ - struct spdk_nvmf_tgt_listen_ctx *ctx = spdk_io_channel_iter_get_ctx(i); - - ctx->cb_fn(ctx->cb_arg, status); - - free(ctx); -} - -static void -spdk_nvmf_tgt_listen_add_transport(struct spdk_io_channel_iter *i) -{ - struct spdk_nvmf_tgt_listen_ctx *ctx = spdk_io_channel_iter_get_ctx(i); - struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); - struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); - int rc; - - rc = spdk_nvmf_poll_group_add_transport(group, ctx->transport); - spdk_for_each_channel_continue(i, rc); -} - void spdk_nvmf_tgt_listen(struct spdk_nvmf_tgt *tgt, struct spdk_nvme_transport_id *trid, @@ -488,13 +476,22 @@ spdk_nvmf_tgt_listen(struct spdk_nvmf_tgt *tgt, transport = spdk_nvmf_tgt_get_transport(tgt, trid->trtype); if (!transport) { - transport = spdk_nvmf_transport_create(tgt, trid->trtype, NULL); + struct spdk_nvmf_transport_opts opts; + + opts.max_queue_depth = tgt->opts.max_queue_depth; + opts.max_qpairs_per_ctrlr = tgt->opts.max_qpairs_per_ctrlr; + opts.in_capsule_data_size = tgt->opts.in_capsule_data_size; + opts.max_io_size = tgt->opts.max_io_size; + opts.io_unit_size = tgt->opts.io_unit_size; + /* use max_queue depth since tgt. opts. doesn't have max_aq_depth */ + opts.max_aq_depth = tgt->opts.max_queue_depth; + + transport = spdk_nvmf_transport_create(trid->trtype, &opts); if (!transport) { SPDK_ERRLOG("Transport initialization failed\n"); cb_fn(cb_arg, -EINVAL); return; } - TAILQ_INSERT_TAIL(&tgt->transports, transport, link); propagate = true; } @@ -509,29 +506,73 @@ spdk_nvmf_tgt_listen(struct spdk_nvmf_tgt *tgt, tgt->discovery_genctr++; if (propagate) { - struct spdk_nvmf_tgt_listen_ctx *ctx; - - ctx = calloc(1, sizeof(*ctx)); - if (!ctx) { - cb_fn(cb_arg, -ENOMEM); - return; - } - - ctx->tgt = tgt; - ctx->transport = transport; - ctx->trid = *trid; - ctx->cb_fn = cb_fn; - ctx->cb_arg = cb_arg; - - spdk_for_each_channel(tgt, - spdk_nvmf_tgt_listen_add_transport, - ctx, - spdk_nvmf_tgt_listen_done); + spdk_nvmf_tgt_add_transport(tgt, transport, cb_fn, cb_arg); } else { cb_fn(cb_arg, 0); } } +struct spdk_nvmf_tgt_add_transport_ctx { + struct spdk_nvmf_tgt *tgt; + struct spdk_nvmf_transport *transport; + spdk_nvmf_tgt_add_transport_done_fn cb_fn; + void *cb_arg; +}; + +static void +_spdk_nvmf_tgt_add_transport_done(struct spdk_io_channel_iter *i, int status) +{ + struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); + + ctx->cb_fn(ctx->cb_arg, status); + + free(ctx); +} + +static void +_spdk_nvmf_tgt_add_transport(struct spdk_io_channel_iter *i) +{ + struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); + struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); + struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); + int rc; + + rc = spdk_nvmf_poll_group_add_transport(group, ctx->transport); + spdk_for_each_channel_continue(i, rc); +} + +void spdk_nvmf_tgt_add_transport(struct spdk_nvmf_tgt *tgt, + struct spdk_nvmf_transport *transport, + spdk_nvmf_tgt_add_transport_done_fn cb_fn, + void *cb_arg) +{ + struct spdk_nvmf_tgt_add_transport_ctx *ctx; + + if (spdk_nvmf_tgt_get_transport(tgt, transport->ops->type)) { + cb_fn(cb_arg, -EEXIST); + return; /* transport already created */ + } + + transport->tgt = tgt; + TAILQ_INSERT_TAIL(&tgt->transports, transport, link); + + ctx = calloc(1, sizeof(*ctx)); + if (!ctx) { + cb_fn(cb_arg, -ENOMEM); + return; + } + + ctx->tgt = tgt; + ctx->transport = transport; + ctx->cb_fn = cb_fn; + ctx->cb_arg = cb_arg; + + spdk_for_each_channel(tgt, + _spdk_nvmf_tgt_add_transport, + ctx, + _spdk_nvmf_tgt_add_transport_done); +} + struct spdk_nvmf_subsystem * spdk_nvmf_tgt_find_subsystem(struct spdk_nvmf_tgt *tgt, const char *subnqn) { diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index 4f12d9a2f..92d5f4ed4 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -1607,6 +1607,24 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport, /* Public API callbacks begin here */ +#define SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH 128 +#define SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH 128 +#define SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR 64 +#define SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE 4096 +#define SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE 131072 +#define SPDK_NVMF_RDMA_DEFAULT_IO_UNIT_SIZE 131072 + +static void +spdk_nvmf_rdma_opts_init(struct spdk_nvmf_transport_opts *opts) +{ + opts->max_queue_depth = SPDK_NVMF_RDMA_DEFAULT_MAX_QUEUE_DEPTH; + opts->max_qpairs_per_ctrlr = SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR; + opts->in_capsule_data_size = SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE; + opts->max_io_size = SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE; + opts->io_unit_size = SPDK_NVMF_RDMA_DEFAULT_IO_UNIT_SIZE; + opts->max_aq_depth = SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH; +} + static int spdk_nvmf_rdma_destroy(struct spdk_nvmf_transport *transport); static struct spdk_nvmf_transport * @@ -2830,6 +2848,7 @@ spdk_nvmf_rdma_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, const struct spdk_nvmf_transport_ops spdk_nvmf_transport_rdma = { .type = SPDK_NVME_TRANSPORT_RDMA, + .opts_init = spdk_nvmf_rdma_opts_init, .create = spdk_nvmf_rdma_create, .destroy = spdk_nvmf_rdma_destroy, diff --git a/lib/nvmf/transport.c b/lib/nvmf/transport.c index 20203c991..0e351e213 100644 --- a/lib/nvmf/transport.c +++ b/lib/nvmf/transport.c @@ -62,24 +62,11 @@ spdk_nvmf_get_transport_ops(enum spdk_nvme_transport_type type) } struct spdk_nvmf_transport * -spdk_nvmf_transport_create(struct spdk_nvmf_tgt *tgt, - enum spdk_nvme_transport_type type, +spdk_nvmf_transport_create(enum spdk_nvme_transport_type type, struct spdk_nvmf_transport_opts *opts) { const struct spdk_nvmf_transport_ops *ops = NULL; struct spdk_nvmf_transport *transport; - struct spdk_nvmf_transport_opts tgt_opts; - - if (opts == NULL) { - /* get transport opts from global target opts */ - tgt_opts.max_queue_depth = tgt->opts.max_queue_depth; - tgt_opts.max_qpairs_per_ctrlr = tgt->opts.max_qpairs_per_ctrlr; - tgt_opts.in_capsule_data_size = tgt->opts.in_capsule_data_size; - tgt_opts.max_io_size = tgt->opts.max_io_size; - tgt_opts.io_unit_size = tgt->opts.io_unit_size; - tgt_opts.max_aq_depth = tgt->opts.max_queue_depth; - opts = &tgt_opts; - } if ((opts->max_io_size % opts->io_unit_size != 0) || (opts->max_io_size / opts->io_unit_size > @@ -106,7 +93,6 @@ spdk_nvmf_transport_create(struct spdk_nvmf_tgt *tgt, return NULL; } - transport->tgt = tgt; transport->ops = ops; transport->opts = *opts; @@ -230,3 +216,20 @@ spdk_nvmf_transport_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, { return qpair->transport->ops->qpair_get_listen_trid(qpair, trid); } + +bool +spdk_nvmf_transport_opts_init(enum spdk_nvme_transport_type type, + struct spdk_nvmf_transport_opts *opts) +{ + const struct spdk_nvmf_transport_ops *ops; + + ops = spdk_nvmf_get_transport_ops(type); + if (!ops) { + SPDK_ERRLOG("Transport type %s unavailable.\n", + spdk_nvme_transport_id_trtype_str(type)); + return false; + } + + ops->opts_init(opts); + return true; +} diff --git a/lib/nvmf/transport.h b/lib/nvmf/transport.h index 47c9ad2e6..1329a80c4 100644 --- a/lib/nvmf/transport.h +++ b/lib/nvmf/transport.h @@ -53,6 +53,11 @@ struct spdk_nvmf_transport_ops { */ enum spdk_nvme_transport_type type; + /** + * Initialize transport options to default value + */ + void (*opts_init)(struct spdk_nvmf_transport_opts *opts); + /** * Create a transport for the given transport opts */ @@ -150,14 +155,6 @@ struct spdk_nvmf_transport_ops { struct spdk_nvme_transport_id *trid); }; -struct spdk_nvmf_transport *spdk_nvmf_transport_create(struct spdk_nvmf_tgt *tgt, - enum spdk_nvme_transport_type type, - struct spdk_nvmf_transport_opts *opts); - -int spdk_nvmf_transport_destroy(struct spdk_nvmf_transport *transport); - -int spdk_nvmf_transport_listen(struct spdk_nvmf_transport *transport, - const struct spdk_nvme_transport_id *trid); int spdk_nvmf_transport_stop_listen(struct spdk_nvmf_transport *transport, const struct spdk_nvme_transport_id *trid); @@ -195,6 +192,9 @@ int spdk_nvmf_transport_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair, int spdk_nvmf_transport_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, struct spdk_nvme_transport_id *trid); +bool spdk_nvmf_transport_opts_init(enum spdk_nvme_transport_type type, + struct spdk_nvmf_transport_opts *opts); + extern const struct spdk_nvmf_transport_ops spdk_nvmf_transport_rdma; #endif /* SPDK_NVMF_TRANSPORT_H */ diff --git a/scripts/rpc.py b/scripts/rpc.py index cba0cc90e..2d037e92c 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -1249,6 +1249,27 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse robin manner""") p.set_defaults(func=set_nvmf_target_config) + @call_cmd + def nvmf_create_transport(args): + rpc.nvmf.nvmf_create_transport(args.client, + trtype=args.trtype, + max_queue_depth=args.max_queue_depth, + max_qpairs_per_ctrlr=args.max_qpairs_per_ctrlr, + in_capsule_data_size=args.in_capsule_data_size, + max_io_size=args.max_io_size, + io_unit_size=args.io_unit_size, + max_aq_depth=args.max_aq_depth) + + p = subparsers.add_parser('nvmf_create_transport', help='Create NVMf transport') + p.add_argument('-t', '--trtype', help='Transport type (ex. RDMA)', type=str, required=True) + p.add_argument('-q', '--max-queue-depth', help='Max number of outstanding I/O per queue', type=int) + p.add_argument('-p', '--max-qpairs-per-ctrlr', help='Max number of SQ and CQ per controller', type=int) + p.add_argument('-c', '--in-capsule-data-size', help='Max number of in-capsule data size', type=int) + p.add_argument('-i', '--max-io-size', help='Max I/O size (bytes)', type=int) + p.add_argument('-u', '--io-unit-size', help='I/O unit size (bytes)', type=int) + p.add_argument('-a', '--max-aq-depth', help='Max number of admin cmds per AQ', type=int) + p.set_defaults(func=nvmf_create_transport) + @call_cmd def get_nvmf_subsystems(args): print_dict(rpc.nvmf.get_nvmf_subsystems(args.client)) diff --git a/scripts/rpc/nvmf.py b/scripts/rpc/nvmf.py index b635f1adf..d805ebca8 100755 --- a/scripts/rpc/nvmf.py +++ b/scripts/rpc/nvmf.py @@ -58,6 +58,46 @@ def set_nvmf_target_config(client, return client.call('set_nvmf_target_config', params) +def nvmf_create_transport(client, + trtype, + max_queue_depth=None, + max_qpairs_per_ctrlr=None, + in_capsule_data_size=None, + max_io_size=None, + io_unit_size=None, + max_aq_depth=None): + """NVMf Transport Create options. + + Args: + trtype: Transport type (ex. RDMA) + max_queue_depth: Max number of outstanding I/O per queue (optional) + max_qpairs_per_ctrlr: Max number of SQ and CQ per controller (optional) + in_capsule_data_size: Maximum in-capsule data size in bytes (optional) + max_io_size: Maximum I/O data size in bytes (optional) + io_unit_size: I/O unit size in bytes (optional) + max_aq_depth: Max size admin quque per controller (optional) + + Returns: + True or False + """ + params = {} + + params['trtype'] = trtype + if max_queue_depth: + params['max_queue_depth'] = max_queue_depth + if max_qpairs_per_ctrlr: + params['max_qpairs_per_ctrlr'] = max_qpairs_per_ctrlr + if in_capsule_data_size: + params['in_capsule_data_size'] = in_capsule_data_size + if max_io_size: + params['max_io_size'] = max_io_size + if io_unit_size: + params['io_unit_size'] = io_unit_size + if max_aq_depth: + params['max_aq_depth'] = max_aq_depth + return client.call('nvmf_create_transport', params) + + def get_nvmf_subsystems(client): """Get list of NVMe-oF subsystems. diff --git a/test/json_config/clear_config.py b/test/json_config/clear_config.py index 91780dd05..700c61805 100755 --- a/test/json_config/clear_config.py +++ b/test/json_config/clear_config.py @@ -83,7 +83,8 @@ def clear_bdev_subsystem(args, bdev_config): def get_nvmf_destroy_method(nvmf): destroy_method_map = {'construct_nvmf_subsystem': "delete_nvmf_subsystem", 'set_nvmf_target_config': None, - 'set_nvmf_target_options': None + 'set_nvmf_target_options': None, + 'nvmf_create_transport': None } return destroy_method_map[nvmf['method']] diff --git a/test/json_config/config_filter.py b/test/json_config/config_filter.py index e1a3dd264..59e96f947 100755 --- a/test/json_config/config_filter.py +++ b/test/json_config/config_filter.py @@ -26,6 +26,7 @@ def filter_methods(do_remove_global_rpcs): 'set_iscsi_options', 'set_nvmf_target_config', 'set_nvmf_target_options', + 'nvmf_create_transport', 'set_bdev_options', 'set_bdev_nvme_options', 'set_bdev_nvme_hotplug', diff --git a/test/nvmf/create_transport/create_transport.sh b/test/nvmf/create_transport/create_transport.sh new file mode 100755 index 000000000..c584f3d41 --- /dev/null +++ b/test/nvmf/create_transport/create_transport.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +NULL_BDEV_SIZE=102400 +NULL_BLOCK_SIZE=512 + +rpc_py="python $rootdir/scripts/rpc.py" + +set -e + +# pass the parameter 'iso' to this script when running it in isolation to trigger rdma device initialization. +# e.g. sudo ./crt_trprt.sh iso +nvmftestinit $1 + +if ! hash nvme; then + echo "nvme command not found; skipping create transport test" + exit 0 +fi + +RDMA_IP_LIST=$(get_available_rdma_ips) +NVMF_FIRST_TARGET_IP=$(echo "$RDMA_IP_LIST" | head -n 1) +if [ -z $NVMF_FIRST_TARGET_IP ]; then + echo "no NIC for nvmf test" + exit 0 +fi + +timing_enter cr_trprt +timing_enter start_nvmf_tgt +# Start up the NVMf target in another process +$NVMF_APP -m 0xF --wait-for-rpc & +nvmfpid=$! + +trap "killprocess $nvmfpid; nvmftestfini $1; exit 1" SIGINT SIGTERM EXIT + +waitforlisten $nvmfpid +$rpc_py start_subsystem_init +# Use nvmf_create_transport call to create transport +$rpc_py nvmf_create_transport -t RDMA -u 8192 -p 4 +timing_exit start_nvmf_tgt + +null_bdevs="$($rpc_py construct_null_bdev Null0 $NULL_BDEV_SIZE $NULL_BLOCK_SIZE) " +null_bdevs+="$($rpc_py construct_null_bdev Null1 $NULL_BDEV_SIZE $NULL_BLOCK_SIZE)" + +modprobe -v nvme-rdma + +$rpc_py construct_nvmf_subsystem nqn.2016-06.io.spdk:cnode1 "trtype:RDMA traddr:$NVMF_FIRST_TARGET_IP trsvcid:4420" "" -a -s SPDK00000000000001 +for null_bdev in $null_bdevs; do + $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 $null_bdev +done + +nvme discover -t rdma -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +echo "Perform nvmf subsystem discovery via RPC" +$rpc_py get_nvmf_subsystems + +$rpc_py delete_nvmf_subsystem nqn.2016-06.io.spdk:cnode1 + +for null_bdev in $null_bdevs; do + $rpc_py delete_null_bdev $null_bdev +done + +check_bdevs=$($rpc_py get_bdevs | jq -r '.[].name') +if [ -n "$check_bdevs" ]; then + echo $check_bdevs + exit 1 +fi + +trap - SIGINT SIGTERM EXIT + +nvmfcleanup +killprocess $nvmfpid +nvmftestfini $1 +timing_exit crt_trprt diff --git a/test/nvmf/nvmf.sh b/test/nvmf/nvmf.sh index 859569d05..700557775 100755 --- a/test/nvmf/nvmf.sh +++ b/test/nvmf/nvmf.sh @@ -32,6 +32,7 @@ fi run_test suite test/nvmf/lvol/nvmf_lvol.sh run_test suite test/nvmf/shutdown/shutdown.sh run_test suite test/nvmf/bdev_io_wait/bdev_io_wait.sh +run_test suite test/nvmf/create_transport/create_transport.sh if [ $RUN_NIGHTLY -eq 1 ]; then run_test suite test/nvmf/multiconnection/multiconnection.sh diff --git a/test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c b/test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c index 95bec6bc6..3e5c3a04b 100644 --- a/test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c +++ b/test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c @@ -110,12 +110,10 @@ spdk_nvmf_transport_listener_discover(struct spdk_nvmf_transport *transport, static struct spdk_nvmf_transport g_transport = {}; struct spdk_nvmf_transport * -spdk_nvmf_transport_create(struct spdk_nvmf_tgt *tgt, - enum spdk_nvme_transport_type type, +spdk_nvmf_transport_create(enum spdk_nvme_transport_type type, struct spdk_nvmf_transport_opts *tprt_opts) { if (type == SPDK_NVME_TRANSPORT_RDMA) { - g_transport.tgt = tgt; return &g_transport; } diff --git a/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c b/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c index 3519a9bd7..8a821b663 100644 --- a/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c +++ b/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c @@ -102,12 +102,10 @@ spdk_nvmf_transport_qpair_is_idle(struct spdk_nvmf_qpair *qpair) static struct spdk_nvmf_transport g_transport = {}; struct spdk_nvmf_transport * -spdk_nvmf_transport_create(struct spdk_nvmf_tgt *tgt, - enum spdk_nvme_transport_type type, +spdk_nvmf_transport_create(enum spdk_nvme_transport_type type, struct spdk_nvmf_transport_opts *tprt_opts) { if (type == SPDK_NVME_TRANSPORT_RDMA) { - g_transport.tgt = tgt; return &g_transport; }