diff --git a/include/spdk/nvmf.h b/include/spdk/nvmf.h index 9aa290e66..1f05d5c83 100644 --- a/include/spdk/nvmf.h +++ b/include/spdk/nvmf.h @@ -191,12 +191,20 @@ int spdk_nvmf_poll_group_add(struct spdk_nvmf_poll_group *group, int spdk_nvmf_poll_group_remove(struct spdk_nvmf_poll_group *group, struct spdk_nvmf_qpair *qpair); +typedef void (*nvmf_qpair_disconnect_cb)(void *ctx); + /** * Disconnect an NVMe-oF qpair * * \param qpair The NVMe-oF qpair to disconnect. + * \param cb_fn The function to call upon completion of the disconnect. + * \param ctx The context to pass to the callback function. + * + * \return 0 upon success. + * \return -ENOMEM if the function specific context could not be allocated. */ -void spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair); +int spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn, + void *ctx); /** * Create an NVMe-oF subsystem. diff --git a/lib/nvmf/nvmf.c b/lib/nvmf/nvmf.c index 8a551f0af..ece31ddb7 100644 --- a/lib/nvmf/nvmf.c +++ b/lib/nvmf/nvmf.c @@ -56,6 +56,13 @@ SPDK_LOG_REGISTER_COMPONENT("nvmf", SPDK_LOG_NVMF) #define SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS 1024 #define SPDK_NVMF_DEFAULT_IO_UNIT_SIZE 131072 +struct nvmf_qpair_disconnect_ctx { + struct spdk_nvmf_qpair *qpair; + struct spdk_nvmf_ctrlr *ctrlr; + nvmf_qpair_disconnect_cb cb_fn; + void *ctx; +}; + void spdk_nvmf_tgt_opts_init(struct spdk_nvmf_tgt_opts *opts) { @@ -136,7 +143,7 @@ spdk_nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf) spdk_poller_unregister(&group->poller); TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qptmp) { - spdk_nvmf_qpair_disconnect(qpair); + spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); } TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { @@ -601,17 +608,24 @@ spdk_nvmf_poll_group_remove(struct spdk_nvmf_poll_group *group, } static void -_spdk_nvmf_ctrlr_free(void *ctx) +_spdk_nvmf_ctrlr_free_from_qpair(void *ctx) { - struct spdk_nvmf_ctrlr *ctrlr = ctx; + struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; + struct spdk_nvmf_ctrlr *ctrlr = qpair_ctx->ctrlr; spdk_nvmf_ctrlr_destruct(ctrlr); + + if (qpair_ctx->cb_fn) { + qpair_ctx->cb_fn(qpair_ctx->ctx); + } + free(qpair_ctx); } static void _spdk_nvmf_qpair_destroy(void *ctx, int status) { - struct spdk_nvmf_qpair *qpair = ctx; + struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; + struct spdk_nvmf_qpair *qpair = qpair_ctx->qpair; struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; uint16_t qid = qpair->qid; uint32_t count; @@ -624,6 +638,10 @@ _spdk_nvmf_qpair_destroy(void *ctx, int status) spdk_nvmf_transport_qpair_fini(qpair); if (!ctrlr) { + if (qpair_ctx->cb_fn) { + qpair_ctx->cb_fn(qpair_ctx->ctx); + } + free(qpair_ctx); return; } @@ -635,20 +653,31 @@ _spdk_nvmf_qpair_destroy(void *ctx, int status) if (count == 0) { /* If this was the last queue pair on the controller, also send a message * to the subsystem to remove the controller. */ - spdk_thread_send_msg(ctrlr->subsys->thread, _spdk_nvmf_ctrlr_free, ctrlr); + qpair_ctx->ctrlr = ctrlr; + spdk_thread_send_msg(ctrlr->subsys->thread, _spdk_nvmf_ctrlr_free_from_qpair, qpair_ctx); + } else { + if (qpair_ctx->cb_fn) { + qpair_ctx->cb_fn(qpair_ctx->ctx); + } + free(qpair_ctx); } } static void _spdk_nvmf_qpair_deactivate(void *ctx) { - struct spdk_nvmf_qpair *qpair = ctx; + struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; + struct spdk_nvmf_qpair *qpair = qpair_ctx->qpair; if (qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING || qpair->state == SPDK_NVMF_QPAIR_INACTIVE) { /* This can occur if the connection is killed by the target, * which results in a notification that the connection * died. */ + if (qpair_ctx->cb_fn) { + qpair_ctx->cb_fn(qpair_ctx->ctx); + } + free(qpair_ctx); return; } @@ -658,22 +687,34 @@ _spdk_nvmf_qpair_deactivate(void *ctx) /* Check for outstanding I/O */ if (!TAILQ_EMPTY(&qpair->outstanding)) { qpair->state_cb = _spdk_nvmf_qpair_destroy; - qpair->state_cb_arg = qpair; + qpair->state_cb_arg = qpair_ctx; return; } - _spdk_nvmf_qpair_destroy(qpair, 0); + _spdk_nvmf_qpair_destroy(qpair_ctx, 0); } -void -spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair) +int +spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn, void *ctx) { + struct nvmf_qpair_disconnect_ctx *qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); + + if (!qpair_ctx) { + SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); + return -ENOMEM; + } + + qpair_ctx->qpair = qpair; + qpair_ctx->cb_fn = cb_fn; + qpair_ctx->ctx = ctx; + if (qpair->group->thread == spdk_get_thread()) { - _spdk_nvmf_qpair_deactivate(qpair); + _spdk_nvmf_qpair_deactivate(qpair_ctx); } else { /* Send a message to the thread that owns this qpair */ - spdk_thread_send_msg(qpair->group->thread, _spdk_nvmf_qpair_deactivate, qpair); + spdk_thread_send_msg(qpair->group->thread, _spdk_nvmf_qpair_deactivate, qpair_ctx); } + return 0; } int @@ -821,14 +862,19 @@ spdk_nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group, { struct spdk_nvmf_qpair *qpair, *tmp; struct spdk_nvmf_subsystem_poll_group *sgroup; + int rc = 0; uint32_t nsid; TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, tmp) { if (qpair->ctrlr->subsys == subsystem) { - spdk_nvmf_qpair_disconnect(qpair); + rc += spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); } } + if (rc != 0) { + return -1; + } + sgroup = &group->sgroups[subsystem->id]; sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE; diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index bc37cbf0c..235ae7194 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -780,7 +780,7 @@ nvmf_rdma_disconnect(struct rdma_cm_event *evt) /* ack the disconnect event before rdma_destroy_id */ rdma_ack_cm_event(evt); - spdk_nvmf_qpair_disconnect(qpair); + spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); return 0; } diff --git a/test/unit/lib/nvmf/request.c/request_ut.c b/test/unit/lib/nvmf/request.c/request_ut.c index 8bc4c65a2..14a9c34e0 100644 --- a/test/unit/lib/nvmf/request.c/request_ut.c +++ b/test/unit/lib/nvmf/request.c/request_ut.c @@ -112,9 +112,10 @@ struct spdk_nvme_ns *spdk_nvme_ctrlr_get_ns(struct spdk_nvme_ctrlr *ctrlr, uint3 return NULL; } -void -spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair) +int +spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn, void *ctx) { + return 0; } static void