ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvmf: spdk_nvmf_tgt_accept no longer takes a new_qpair pointer

Browse Source 
Calling spdk_nvmf_tgt_accept() now automatically assigns new qpairs
to the best available poll group.

Signed-off-by: Ben Walker <benjamin.walker@intel.com>
Change-Id: I3df2a2c5a28dba45c5ba0cbd1e8c28dd7e56cf9e
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2813
Community-CI: Broadcom CI
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ziye Yang <ziye.yang@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
This commit is contained in:
Ben Walker 2020-06-08 14:38:29 -07:00 committed by Tomasz Zawadzki
parent c37cf9fbd8
commit ff24db746f
8 changed files with 76 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -7,6 +7,9 @@
The NVMe-oF target no longer supports connecting scheduling configuration and instead
always uses what was previously called "transport" scheduling.
`spdk_nvmf_tgt_accept` no longer takes a function pointer as an argument. New connections
are automatically assigned to poll groups by the underlying transport.
### nvme
Add `opts_size` in `spdk_nvme_ctrlr_opts` structure in order to solve the compatiblity issue

88
examples/nvmf/nvmf/nvmf.c
View File

@ -517,98 +517,12 @@ nvmf_tgt_stop_subsystems(struct nvmf_target *nvmf_tgt)
}
}
struct nvmf_target_pg_ctx {
struct spdk_nvmf_qpair *qpair;
struct nvmf_target_poll_group *pg;
};
static void
nvmf_tgt_pg_add_qpair(void *_ctx)
{
struct nvmf_target_pg_ctx *ctx = _ctx;
struct spdk_nvmf_qpair *qpair = ctx->qpair;
struct nvmf_target_poll_group *pg = ctx->pg;
free(_ctx);
if (spdk_nvmf_poll_group_add(pg->group, qpair) != 0) {
fprintf(stderr, "unable to add the qpair to a poll group.\n");
spdk_nvmf_qpair_disconnect(qpair, NULL, NULL);
}
}
static struct nvmf_target_poll_group *
nvmf_tgt_get_next_pg(struct nvmf_target *nvmf_tgt)
{
struct nvmf_target_poll_group *pg;
pg = g_next_pg;
g_next_pg = TAILQ_NEXT(pg, link);
if (g_next_pg == NULL) {
g_next_pg = TAILQ_FIRST(&g_poll_groups);
}
return pg;
}
static struct nvmf_target_poll_group *
nvmf_get_optimal_pg(struct nvmf_target *nvmf_tgt, struct spdk_nvmf_qpair *qpair)
{
struct nvmf_target_poll_group *pg, *_pg = NULL;
struct spdk_nvmf_poll_group *group = spdk_nvmf_get_optimal_poll_group(qpair);
if (group == NULL) {
_pg = nvmf_tgt_get_next_pg(nvmf_tgt);
goto end;
}
TAILQ_FOREACH(pg, &g_poll_groups, link) {
if (pg->group == group) {
_pg = pg;
break;
}
}
end:
assert(_pg != NULL);
return _pg;
}
static void
new_qpair(struct spdk_nvmf_qpair *qpair, void *cb_arg)
{
struct nvmf_target_poll_group *pg;
struct nvmf_target_pg_ctx *ctx;
struct nvmf_target *nvmf_tgt = &g_nvmf_tgt;
/* In SPDK we support three methods to get poll group: RoundRobin, Host and Transport.
* In this example we only support the "Transport" which gets the optimal poll group.
*/
pg = nvmf_get_optimal_pg(nvmf_tgt, qpair);
if (!pg) {
spdk_nvmf_qpair_disconnect(qpair, NULL, NULL);
return;
}
ctx = calloc(1, sizeof(*ctx));
if (!ctx) {
fprintf(stderr, "failed to allocate poll group context.\n");
spdk_nvmf_qpair_disconnect(qpair, NULL, NULL);
return;
}
ctx->qpair = qpair;
ctx->pg = pg;
spdk_thread_send_msg(pg->thread, nvmf_tgt_pg_add_qpair, ctx);
}
static int
nvmf_tgt_acceptor_poll(void *arg)
{
struct nvmf_target *nvmf_tgt = arg;
spdk_nvmf_tgt_accept(nvmf_tgt->tgt, new_qpair, NULL);
spdk_nvmf_tgt_accept(nvmf_tgt->tgt);
return -1;
}

16
include/spdk/nvmf.h
View File

@ -114,14 +114,6 @@ struct spdk_nvmf_transport_poll_group_stat {
};
};
/**
* Function to be called for each newly discovered qpair.
*
* \param qpair The newly discovered qpair.
* \param cb_arg A context argument passed to this function.
*/
typedef void (*new_qpair_fn)(struct spdk_nvmf_qpair *qpair, void *cb_arg);
/**
* Function to be called once the listener is associated with a subsystem.
*
@ -233,15 +225,9 @@ int spdk_nvmf_tgt_stop_listen(struct spdk_nvmf_tgt *tgt,
/**
* Poll the target for incoming connections.
*
* The new_qpair_fn cb_fn will be called for each newly discovered
* qpair. The user is expected to add that qpair to a poll group
* to establish the connection.
*
* \param tgt The target associated with the listen address.
* \param cb_fn Called for each newly discovered qpair.
* \param cb_arg A context argument passed to cb_fn.
*/
uint32_t spdk_nvmf_tgt_accept(struct spdk_nvmf_tgt *tgt, new_qpair_fn cb_fn, void *cb_arg);
uint32_t spdk_nvmf_tgt_accept(struct spdk_nvmf_tgt *tgt);
/**
* Create a poll group.

8
include/spdk/nvmf_transport.h
View File

@ -192,6 +192,14 @@ struct spdk_nvmf_transport {
TAILQ_ENTRY(spdk_nvmf_transport) link;
};
/**
* Function to be called for each newly discovered qpair.
*
* \param qpair The newly discovered qpair.
* \param cb_arg A context argument passed to this function.
*/
typedef void (*new_qpair_fn)(struct spdk_nvmf_qpair *qpair, void *cb_arg);
struct spdk_nvmf_transport_ops {
/**
* Transport name

2
lib/nvmf/Makefile
View File

@ -34,7 +34,7 @@
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
SO_VER := 4
SO_VER := 5
SO_MINOR := 0
C_SRCS = ctrlr.c ctrlr_discovery.c ctrlr_bdev.c \

60
lib/nvmf/nvmf.c
View File

@ -720,14 +720,70 @@ spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, const char *transport_nam
return NULL;
}
struct nvmf_new_qpair_ctx {
struct spdk_nvmf_qpair *qpair;
struct spdk_nvmf_poll_group *group;
};
static void
_nvmf_poll_group_add(void *_ctx)
{
struct nvmf_new_qpair_ctx *ctx = _ctx;
struct spdk_nvmf_qpair *qpair = ctx->qpair;
struct spdk_nvmf_poll_group *group = ctx->group;
free(_ctx);
if (spdk_nvmf_poll_group_add(group, qpair) != 0) {
SPDK_ERRLOG("Unable to add the qpair to a poll group.\n");
spdk_nvmf_qpair_disconnect(qpair, NULL, NULL);
}
}
static void
_nvmf_new_qpair(struct spdk_nvmf_qpair *qpair, void *cb_arg)
{
struct spdk_nvmf_poll_group *group;
struct spdk_nvmf_tgt *tgt;
struct nvmf_new_qpair_ctx *ctx;
tgt = qpair->transport->tgt;
group = spdk_nvmf_get_optimal_poll_group(qpair);
if (group == NULL) {
if (tgt->next_poll_group == NULL) {
tgt->next_poll_group = TAILQ_FIRST(&tgt->poll_groups);
if (tgt->next_poll_group == NULL) {
SPDK_ERRLOG("No poll groups exist.\n");
spdk_nvmf_qpair_disconnect(qpair, NULL, NULL);
return;
}
}
group = tgt->next_poll_group;
tgt->next_poll_group = TAILQ_NEXT(group, link);
}
ctx = calloc(1, sizeof(*ctx));
if (!ctx) {
SPDK_ERRLOG("Unable to send message to poll group.\n");
spdk_nvmf_qpair_disconnect(qpair, NULL, NULL);
return;
}
ctx->qpair = qpair;
ctx->group = group;
spdk_thread_send_msg(group->thread, _nvmf_poll_group_add, ctx);
}
uint32_t
spdk_nvmf_tgt_accept(struct spdk_nvmf_tgt *tgt, new_qpair_fn cb_fn, void *cb_arg)
spdk_nvmf_tgt_accept(struct spdk_nvmf_tgt *tgt)
{
struct spdk_nvmf_transport *transport, *tmp;
uint32_t count = 0;
TAILQ_FOREACH_SAFE(transport, &tgt->transports, link, tmp) {
count += nvmf_transport_accept(transport, cb_fn, cb_arg);
count += nvmf_transport_accept(transport, _nvmf_new_qpair, NULL);
}
return count;

3
lib/nvmf/nvmf_internal.h
View File

@ -74,6 +74,9 @@ struct spdk_nvmf_tgt {
TAILQ_HEAD(, spdk_nvmf_transport) transports;
TAILQ_HEAD(, spdk_nvmf_poll_group) poll_groups;
/* Used for round-robin assignment of connections to poll groups */
struct spdk_nvmf_poll_group *next_poll_group;
spdk_nvmf_tgt_destroy_done_fn *destroy_cb_fn;
void *destroy_cb_arg;

91
module/event/subsystems/nvmf/nvmf_tgt.c
View File

@ -69,9 +69,6 @@ static enum nvmf_tgt_state g_tgt_state;
static struct spdk_thread *g_tgt_init_thread = NULL;
static struct spdk_thread *g_tgt_fini_thread = NULL;
/* Round-Robin assignment of connections to poll groups */
static struct nvmf_tgt_poll_group *g_next_poll_group = NULL;
static TAILQ_HEAD(, nvmf_tgt_poll_group) g_poll_groups = TAILQ_HEAD_INITIALIZER(g_poll_groups);
static size_t g_num_poll_groups = 0;
@ -106,95 +103,13 @@ nvmf_subsystem_fini(void)
nvmf_shutdown_cb(NULL);
}
static struct nvmf_tgt_poll_group *
nvmf_tgt_get_pg(struct spdk_nvmf_qpair *qpair)
{
struct nvmf_tgt_poll_group *pg;
struct spdk_nvmf_poll_group *group;
group = spdk_nvmf_get_optimal_poll_group(qpair);
if (group != NULL) {
/* Look up the nvmf_tgt_poll_group that matches this spdk_nvmf_poll_group */
TAILQ_FOREACH(pg, &g_poll_groups, link) {
if (pg->group == group) {
return pg;
}
}
return NULL;
}
if (g_next_poll_group == NULL) {
g_next_poll_group = TAILQ_FIRST(&g_poll_groups);
if (g_next_poll_group == NULL) {
return NULL;
}
}
pg = g_next_poll_group;
g_next_poll_group = TAILQ_NEXT(pg, link);
return pg;
}
struct nvmf_tgt_pg_ctx {
struct spdk_nvmf_qpair *qpair;
struct nvmf_tgt_poll_group *pg;
};
static void
nvmf_tgt_poll_group_add(void *_ctx)
{
struct nvmf_tgt_pg_ctx *ctx = _ctx;
struct spdk_nvmf_qpair *qpair = ctx->qpair;
struct nvmf_tgt_poll_group *pg = ctx->pg;
free(_ctx);
if (spdk_nvmf_poll_group_add(pg->group, qpair) != 0) {
SPDK_ERRLOG("Unable to add the qpair to a poll group.\n");
spdk_nvmf_qpair_disconnect(qpair, NULL, NULL);
}
}
static void
new_qpair(struct spdk_nvmf_qpair *qpair, void *cb_arg)
{
struct nvmf_tgt_pg_ctx *ctx;
struct nvmf_tgt_poll_group *pg;
if (g_tgt_state != NVMF_TGT_RUNNING) {
spdk_nvmf_qpair_disconnect(qpair, NULL, NULL);
return;
}
pg = nvmf_tgt_get_pg(qpair);
if (pg == NULL) {
SPDK_ERRLOG("No poll groups exist.\n");
spdk_nvmf_qpair_disconnect(qpair, NULL, NULL);
return;
}
ctx = calloc(1, sizeof(*ctx));
if (!ctx) {
SPDK_ERRLOG("Unable to send message to poll group.\n");
spdk_nvmf_qpair_disconnect(qpair, NULL, NULL);
return;
}
ctx->qpair = qpair;
ctx->pg = pg;
spdk_thread_send_msg(pg->thread, nvmf_tgt_poll_group_add, ctx);
}
static int
acceptor_poll(void *arg)
{
struct spdk_nvmf_tgt *tgt = arg;
uint32_t count;
count = spdk_nvmf_tgt_accept(tgt, new_qpair, NULL);
count = spdk_nvmf_tgt_accept(tgt);
return count;
}
@ -251,10 +166,6 @@ nvmf_tgt_create_poll_group_done(void *ctx)
TAILQ_INSERT_TAIL(&g_poll_groups, pg, link);
if (g_next_poll_group == NULL) {
g_next_poll_group = pg;
}
assert(g_num_poll_groups < spdk_env_get_core_count());
if (++g_num_poll_groups == spdk_env_get_core_count()) {