diff --git a/CHANGELOG.md b/CHANGELOG.md index 2030e66d4..580631cbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## v20.07: (Upcoming Release) +### nvmf + +The NVMe-oF target no longer supports connecting scheduling configuration and instead +always uses what was previously called "transport" scheduling. + ### nvme Add `opts_size` in `spdk_nvme_ctrlr_opts` structure in order to solve the compatiblity issue diff --git a/module/event/subsystems/nvmf/conf.c b/module/event/subsystems/nvmf/conf.c index e4372a740..b92a92acc 100644 --- a/module/event/subsystems/nvmf/conf.c +++ b/module/event/subsystems/nvmf/conf.c @@ -117,23 +117,7 @@ nvmf_read_config_file_tgt_conf(struct spdk_conf_section *sp, conn_scheduler = spdk_conf_section_get_val(sp, "ConnectionScheduler"); if (conn_scheduler) { - if (strcasecmp(conn_scheduler, "RoundRobin") == 0) { - conf->conn_sched = CONNECT_SCHED_ROUND_ROBIN; - } else if (strcasecmp(conn_scheduler, "Host") == 0) { - conf->conn_sched = CONNECT_SCHED_HOST_IP; - } else if (strcasecmp(conn_scheduler, "Transport") == 0) { - conf->conn_sched = CONNECT_SCHED_TRANSPORT_OPTIMAL_GROUP; - } else { - SPDK_ERRLOG("The valid value of ConnectionScheduler should be:\n" - "\t RoundRobin\n" - "\t Host\n" - "\t Transport\n"); - rc = -1; - } - - } else { - SPDK_NOTICELOG("The value of ConnectionScheduler is not configured,\n" - "we will use RoundRobin as the default scheduler\n"); + SPDK_NOTICELOG("The ConnectionScheduler option is no longer valid. Ignoring it.\n"); } conf->admin_passthru.identify_ctrlr = spdk_conf_section_get_boolval(sp, @@ -170,7 +154,6 @@ nvmf_parse_tgt_conf(void) } conf->acceptor_poll_rate = ACCEPT_TIMEOUT_US; - conf->conn_sched = DEFAULT_CONN_SCHED; conf->admin_passthru.identify_ctrlr = false; sp = spdk_conf_find_section(NULL, "Nvmf"); diff --git a/module/event/subsystems/nvmf/event_nvmf.h b/module/event/subsystems/nvmf/event_nvmf.h index 541f07031..58d3f713b 100644 --- a/module/event/subsystems/nvmf/event_nvmf.h +++ b/module/event/subsystems/nvmf/event_nvmf.h @@ -43,13 +43,6 @@ #include "spdk_internal/log.h" #define ACCEPT_TIMEOUT_US 10000 /* 10ms */ -#define DEFAULT_CONN_SCHED CONNECT_SCHED_TRANSPORT_OPTIMAL_GROUP - -enum spdk_nvmf_connect_sched { - CONNECT_SCHED_ROUND_ROBIN = 0, - CONNECT_SCHED_HOST_IP, - CONNECT_SCHED_TRANSPORT_OPTIMAL_GROUP, -}; struct spdk_nvmf_admin_passthru_conf { bool identify_ctrlr; @@ -57,7 +50,7 @@ struct spdk_nvmf_admin_passthru_conf { struct spdk_nvmf_tgt_conf { uint32_t acceptor_poll_rate; - enum spdk_nvmf_connect_sched conn_sched; + uint32_t conn_sched; /* Deprecated. */ struct spdk_nvmf_admin_passthru_conf admin_passthru; }; diff --git a/module/event/subsystems/nvmf/nvmf_rpc.c b/module/event/subsystems/nvmf/nvmf_rpc.c index 862dcfe27..b16ec6686 100644 --- a/module/event/subsystems/nvmf/nvmf_rpc.c +++ b/module/event/subsystems/nvmf/nvmf_rpc.c @@ -76,18 +76,9 @@ SPDK_RPC_REGISTER_ALIAS_DEPRECATED(nvmf_set_max_subsystems, set_nvmf_target_max_ static int decode_conn_sched(const struct spdk_json_val *val, void *out) { - enum spdk_nvmf_connect_sched *sched = out; + *(uint32_t *)out = 0; - if (spdk_json_strequal(val, "roundrobin") == true) { - *sched = CONNECT_SCHED_ROUND_ROBIN; - } else if (spdk_json_strequal(val, "hostip") == true) { - *sched = CONNECT_SCHED_HOST_IP; - } else if (spdk_json_strequal(val, "transport") == true) { - *sched = CONNECT_SCHED_TRANSPORT_OPTIMAL_GROUP; - } else { - SPDK_ERRLOG("Invalid connection scheduling parameter\n"); - return -EINVAL; - } + SPDK_NOTICELOG("conn_sched is no longer a supported parameter. Ignoring."); return 0; } @@ -139,7 +130,6 @@ rpc_nvmf_set_config(struct spdk_jsonrpc_request *request, } conf->acceptor_poll_rate = ACCEPT_TIMEOUT_US; - conf->conn_sched = DEFAULT_CONN_SCHED; conf->admin_passthru.identify_ctrlr = false; if (params != NULL) { diff --git a/module/event/subsystems/nvmf/nvmf_tgt.c b/module/event/subsystems/nvmf/nvmf_tgt.c index 9e3f26972..20cc98083 100644 --- a/module/event/subsystems/nvmf/nvmf_tgt.c +++ b/module/event/subsystems/nvmf/nvmf_tgt.c @@ -62,17 +62,6 @@ struct nvmf_tgt_poll_group { TAILQ_ENTRY(nvmf_tgt_poll_group) link; }; -struct nvmf_tgt_host_trid { - struct spdk_nvme_transport_id host_trid; - struct nvmf_tgt_poll_group *pg; - uint32_t ref; - TAILQ_ENTRY(nvmf_tgt_host_trid) link; -}; - -/* List of host trids that are connected to the target */ -static TAILQ_HEAD(, nvmf_tgt_host_trid) g_nvmf_tgt_host_trids = - TAILQ_HEAD_INITIALIZER(g_nvmf_tgt_host_trids); - struct spdk_nvmf_tgt *g_spdk_nvmf_tgt = NULL; static enum nvmf_tgt_state g_tgt_state; @@ -80,7 +69,7 @@ static enum nvmf_tgt_state g_tgt_state; static struct spdk_thread *g_tgt_init_thread = NULL; static struct spdk_thread *g_tgt_fini_thread = NULL; -/* Round-Robin/IP-based tracking of threads to poll group assignment */ +/* Round-Robin assignment of connections to poll groups */ static struct nvmf_tgt_poll_group *g_next_poll_group = NULL; static TAILQ_HEAD(, nvmf_tgt_poll_group) g_poll_groups = TAILQ_HEAD_INITIALIZER(g_poll_groups); @@ -117,124 +106,34 @@ nvmf_subsystem_fini(void) nvmf_shutdown_cb(NULL); } -/* Round robin selection of poll groups */ -static struct nvmf_tgt_poll_group * -nvmf_get_next_pg(void) -{ - struct nvmf_tgt_poll_group *pg; - - pg = g_next_poll_group; - g_next_poll_group = TAILQ_NEXT(pg, link); - if (g_next_poll_group == NULL) { - g_next_poll_group = TAILQ_FIRST(&g_poll_groups); - } - - return pg; -} - -static struct nvmf_tgt_poll_group * -nvmf_get_optimal_pg(struct spdk_nvmf_qpair *qpair) -{ - struct nvmf_tgt_poll_group *pg, *_pg = NULL; - struct spdk_nvmf_poll_group *group = spdk_nvmf_get_optimal_poll_group(qpair); - - if (group == NULL) { - _pg = nvmf_get_next_pg(); - goto end; - } - - TAILQ_FOREACH(pg, &g_poll_groups, link) { - if (pg->group == group) { - _pg = pg; - break; - } - - } - -end: - assert(_pg != NULL); - return _pg; -} - -static void -nvmf_tgt_remove_host_trid(struct spdk_nvmf_qpair *qpair) -{ - struct spdk_nvme_transport_id trid_to_remove; - struct nvmf_tgt_host_trid *trid = NULL, *tmp_trid = NULL; - - if (g_spdk_nvmf_tgt_conf->conn_sched != CONNECT_SCHED_HOST_IP) { - return; - } - - if (spdk_nvmf_qpair_get_peer_trid(qpair, &trid_to_remove) != 0) { - return; - } - - TAILQ_FOREACH_SAFE(trid, &g_nvmf_tgt_host_trids, link, tmp_trid) { - if (trid && !strncmp(trid->host_trid.traddr, - trid_to_remove.traddr, SPDK_NVMF_TRADDR_MAX_LEN + 1)) { - trid->ref--; - if (trid->ref == 0) { - TAILQ_REMOVE(&g_nvmf_tgt_host_trids, trid, link); - free(trid); - } - - break; - } - } - - return; -} - static struct nvmf_tgt_poll_group * nvmf_tgt_get_pg(struct spdk_nvmf_qpair *qpair) { - struct spdk_nvme_transport_id trid; - struct nvmf_tgt_host_trid *tmp_trid = NULL, *new_trid = NULL; struct nvmf_tgt_poll_group *pg; - int ret; + struct spdk_nvmf_poll_group *group; - switch (g_spdk_nvmf_tgt_conf->conn_sched) { - case CONNECT_SCHED_HOST_IP: - ret = spdk_nvmf_qpair_get_peer_trid(qpair, &trid); - if (ret) { - pg = g_next_poll_group; - SPDK_ERRLOG("Invalid host transport Id. Assigning to poll group %p\n", pg); - break; - } + group = spdk_nvmf_get_optimal_poll_group(qpair); - TAILQ_FOREACH(tmp_trid, &g_nvmf_tgt_host_trids, link) { - if (tmp_trid && !strncmp(tmp_trid->host_trid.traddr, - trid.traddr, SPDK_NVMF_TRADDR_MAX_LEN + 1)) { - tmp_trid->ref++; - pg = tmp_trid->pg; - break; + if (group != NULL) { + /* Look up the nvmf_tgt_poll_group that matches this spdk_nvmf_poll_group */ + TAILQ_FOREACH(pg, &g_poll_groups, link) { + if (pg->group == group) { + return pg; } } - if (!tmp_trid) { - new_trid = calloc(1, sizeof(*new_trid)); - if (!new_trid) { - pg = g_next_poll_group; - SPDK_ERRLOG("Insufficient memory. Assigning to poll group %p\n", pg); - break; - } - /* Get the next available poll group for the new host */ - pg = nvmf_get_next_pg(); - new_trid->pg = pg; - memcpy(new_trid->host_trid.traddr, trid.traddr, - SPDK_NVMF_TRADDR_MAX_LEN + 1); - TAILQ_INSERT_TAIL(&g_nvmf_tgt_host_trids, new_trid, link); - } - break; - case CONNECT_SCHED_TRANSPORT_OPTIMAL_GROUP: - pg = nvmf_get_optimal_pg(qpair); - break; - case CONNECT_SCHED_ROUND_ROBIN: - default: - pg = nvmf_get_next_pg(); - break; + + return NULL; } + if (g_next_poll_group == NULL) { + g_next_poll_group = TAILQ_FIRST(&g_poll_groups); + if (g_next_poll_group == NULL) { + return NULL; + } + } + pg = g_next_poll_group; + g_next_poll_group = TAILQ_NEXT(pg, link); + return pg; } @@ -263,23 +162,14 @@ new_qpair(struct spdk_nvmf_qpair *qpair, void *cb_arg) { struct nvmf_tgt_pg_ctx *ctx; struct nvmf_tgt_poll_group *pg; - uint32_t attempts; if (g_tgt_state != NVMF_TGT_RUNNING) { spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); return; } - for (attempts = 0; attempts < g_num_poll_groups; attempts++) { - pg = nvmf_tgt_get_pg(qpair); - if (pg->group != NULL) { - break; - } else { - nvmf_tgt_remove_host_trid(qpair); - } - } - - if (attempts == g_num_poll_groups) { + pg = nvmf_tgt_get_pg(qpair); + if (pg == NULL) { SPDK_ERRLOG("No poll groups exist.\n"); spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); return; @@ -447,15 +337,8 @@ nvmf_tgt_subsystem_stopped(struct spdk_nvmf_subsystem *subsystem, static void nvmf_tgt_destroy_done(void *ctx, int status) { - struct nvmf_tgt_host_trid *trid, *tmp_trid; - g_tgt_state = NVMF_TGT_STOPPED; - TAILQ_FOREACH_SAFE(trid, &g_nvmf_tgt_host_trids, link, tmp_trid) { - TAILQ_REMOVE(&g_nvmf_tgt_host_trids, trid, link); - free(trid); - } - free(g_spdk_nvmf_tgt_conf); g_spdk_nvmf_tgt_conf = NULL; nvmf_tgt_advance_state(); @@ -645,18 +528,6 @@ nvmf_subsystem_init(void) nvmf_tgt_advance_state(); } -static char * -get_conn_sched_string(enum spdk_nvmf_connect_sched sched) -{ - if (sched == CONNECT_SCHED_HOST_IP) { - return "hostip"; - } else if (sched == CONNECT_SCHED_TRANSPORT_OPTIMAL_GROUP) { - return "transport"; - } else { - return "roundrobin"; - } -} - static void nvmf_subsystem_write_config_json(struct spdk_json_write_ctx *w) { @@ -667,8 +538,6 @@ nvmf_subsystem_write_config_json(struct spdk_json_write_ctx *w) spdk_json_write_named_object_begin(w, "params"); spdk_json_write_named_uint32(w, "acceptor_poll_rate", g_spdk_nvmf_tgt_conf->acceptor_poll_rate); - spdk_json_write_named_string(w, "conn_sched", - get_conn_sched_string(g_spdk_nvmf_tgt_conf->conn_sched)); spdk_json_write_named_object_begin(w, "admin_cmd_passthru"); spdk_json_write_named_bool(w, "identify_ctrlr", g_spdk_nvmf_tgt_conf->admin_passthru.identify_ctrlr); diff --git a/scripts/rpc.py b/scripts/rpc.py index f6d33b156..4e3e519d2 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -1704,10 +1704,7 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse p = subparsers.add_parser('nvmf_set_config', aliases=['set_nvmf_target_config'], help='Set NVMf target config') p.add_argument('-r', '--acceptor-poll-rate', help='Polling interval of the acceptor for incoming connections (usec)', type=int) - p.add_argument('-s', '--conn-sched', help="""'roundrobin' - Schedule the incoming connections from any host - on the cores in a round robin manner (Default). 'hostip' - Schedule all the incoming connections from a - specific host IP on to the same core. Connections from different IP will be assigned to cores in a round - robin manner. 'transport' - Schedule the connection according to the transport characteristics.""") + p.add_argument('-s', '--conn-sched', help='(Deprecated). Ignored.') p.add_argument('-i', '--passthru-identify-ctrlr', help="""Passthrough fields like serial number and model number when the controller has a single namespace that is an NVMe bdev""", action='store_true') p.set_defaults(func=nvmf_set_config) diff --git a/scripts/rpc/nvmf.py b/scripts/rpc/nvmf.py index ac0320296..398544ef9 100644 --- a/scripts/rpc/nvmf.py +++ b/scripts/rpc/nvmf.py @@ -27,7 +27,7 @@ def nvmf_set_config(client, Args: acceptor_poll_rate: Acceptor poll period in microseconds (optional) - conn_sched: Scheduling of incoming connections (optional) + conn_sched: (Deprecated) Ignored Returns: True or False @@ -37,7 +37,7 @@ def nvmf_set_config(client, if acceptor_poll_rate: params['acceptor_poll_rate'] = acceptor_poll_rate if conn_sched: - params['conn_sched'] = conn_sched + print("WARNING: conn_sched is deprecated and ignored.") if passthru_identify_ctrlr: admin_cmd_passthru = {} admin_cmd_passthru['identify_ctrlr'] = passthru_identify_ctrlr