ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lib/nvmf: make spdk_nvmf_qpair_disconnect thread safe.

Browse Source 
This function should be the synchronization point for all
disconnects regardless of whether they begin on the transport,
from an RPC, or in response to application termination.

Signed-off-by: Seth Howell <seth.howell@intel.com>
Change-Id: If3553ab3a9e265b0938c84832cb9f774852d7565
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3674
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Ziye Yang <ziye.yang@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Seth Howell 2020-08-05 12:13:00 -07:00 committed by Tomasz Zawadzki
parent a70d01d6f7
commit 4bdfe13d4d
3 changed files with 33 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/spdk/nvmf_transport.h
View File

@ -123,6 +123,7 @@ struct spdk_nvmf_qpair {
uint16_t qid;
uint16_t sq_head;
uint16_t sq_head_max;
bool disconnect_started;
struct spdk_nvmf_request *first_fused_req;

4
lib/nvmf/Makefile
View File

@ -34,8 +34,8 @@
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
SO_VER := 5
SO_MINOR := 1
SO_VER := 6
SO_MINOR := 0
C_SRCS = ctrlr.c ctrlr_discovery.c ctrlr_bdev.c \
subsystem.c nvmf.c nvmf_rpc.c transport.c tcp.c

43
lib/nvmf/nvmf.c
View File

@ -940,11 +940,27 @@ _nvmf_qpair_destroy(void *ctx, int status)
spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_free_from_qpair, qpair_ctx);
}
static void
_nvmf_qpair_disconnect_msg(void *ctx)
{
struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx;
spdk_nvmf_qpair_disconnect(qpair_ctx->qpair, qpair_ctx->cb_fn, qpair_ctx->ctx);
free(ctx);
}
int
spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn, void *ctx)
{
struct nvmf_qpair_disconnect_ctx *qpair_ctx;
if (__atomic_test_and_set(&qpair->disconnect_started, __ATOMIC_RELAXED)) {
if (cb_fn) {
cb_fn(ctx);
}
return 0;
}
/* If we get a qpair in the uninitialized state, we can just destroy it immediately */
if (qpair->state == SPDK_NVMF_QPAIR_UNINITIALIZED) {
nvmf_transport_qpair_fini(qpair);
@ -954,19 +970,20 @@ spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_
return 0;
}
/* The queue pair must be disconnected from the thread that owns it */
assert(qpair->group->thread == spdk_get_thread());
if (qpair->state != SPDK_NVMF_QPAIR_ACTIVE) {
/* This can occur if the connection is killed by the target,
* which results in a notification that the connection
* died. Send a message to defer the processing of this
* callback. This allows the stack to unwind in the case
* where a bunch of connections are disconnected in
* a loop. */
if (cb_fn) {
spdk_thread_send_msg(qpair->group->thread, cb_fn, ctx);
assert(qpair->group != NULL);
if (spdk_get_thread() != qpair->group->thread) {
/* clear the atomic so we can set it on the next call on the proper thread. */
__atomic_clear(&qpair->disconnect_started, __ATOMIC_RELAXED);
qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx));
if (!qpair_ctx) {
SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n");
return -ENOMEM;
}
qpair_ctx->qpair = qpair;
qpair_ctx->cb_fn = cb_fn;
qpair_ctx->thread = qpair->group->thread;
qpair_ctx->ctx = ctx;
spdk_thread_send_msg(qpair->group->thread, _nvmf_qpair_disconnect_msg, qpair_ctx);
return 0;
}
@ -1337,7 +1354,7 @@ nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group,
_nvmf_poll_group_remove_subsystem_cb(ctx, 0);
}
if (rc != 0) {
if (rc != 0 && rc != -EINPROGRESS) {
free(ctx);
goto fini;
}