ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvmf/fc: Cleanup FC qpair_fini code.

Browse Source 
As NVMF qpair maps to a FC level connection, Ideally a qpair fini
should cleanup fc level connection irrespective of what type of qpair
it is i.e admin or IO. But today IO qpair cleanup is actually a dummy.

Also FC nvmf_transport_qpair_fini is just triggering fc level connection
cleanup but not properly synchronized. Use latest async nature of
nvmf_transport_qpair_fini enhancements to fix this.

Signed-off-by: Naresh Gottumukkala <raju.gottumukkala@broadcom.com>
Change-Id: I883ed774769e2d4a2575b6d90bb3348981cd1e0d
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5700
Community-CI: Broadcom CI
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Naresh Gottumukkala 2020-12-25 05:08:19 +00:00 committed by Jim Harris
parent b50c6bc2d9
commit e89d9af092
3 changed files with 89 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
lib/nvmf/fc.c
View File

@ -321,30 +321,6 @@ nvmf_fc_create_hash_table(const char *name, size_t num_entries, size_t key_len)
return rte_hash_create(&hash_params);
}
static void
nvmf_fc_handle_connection_failure(void *arg)
{
struct spdk_nvmf_fc_conn *fc_conn = arg;
struct spdk_nvmf_fc_ls_add_conn_api_data *api_data = NULL;
if (!fc_conn->create_opd) {
return;
}
api_data = &fc_conn->create_opd->u.add_conn;
nvmf_fc_ls_add_conn_failure(api_data->assoc, api_data->ls_rqst,
api_data->args.fc_conn, api_data->aq_conn);
}
static void
nvmf_fc_handle_assoc_deletion(void *arg)
{
struct spdk_nvmf_fc_conn *fc_conn = arg;
nvmf_fc_delete_association(fc_conn->fc_assoc->tgtport,
fc_conn->fc_assoc->assoc_id, false, true, NULL, NULL);
}
void
nvmf_fc_free_conn_reqpool(struct spdk_nvmf_fc_conn *fc_conn)
{
@ -2062,28 +2038,69 @@ nvmf_fc_request_free(struct spdk_nvmf_request *req)
return 0;
}
static void
nvmf_fc_connection_delete_done_cb(void *arg)
{
struct spdk_nvmf_fc_qpair_remove_ctx *fc_ctx = arg;
if (fc_ctx->cb_fn) {
spdk_thread_send_msg(fc_ctx->qpair_thread, fc_ctx->cb_fn, fc_ctx->cb_ctx);
}
free(fc_ctx);
}
static void
_nvmf_fc_close_qpair(void *arg)
{
struct spdk_nvmf_fc_qpair_remove_ctx *fc_ctx = arg;
struct spdk_nvmf_qpair *qpair = fc_ctx->qpair;
struct spdk_nvmf_fc_conn *fc_conn;
int rc;
fc_conn = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_fc_conn, qpair);
if (fc_conn->conn_id == NVMF_FC_INVALID_CONN_ID) {
struct spdk_nvmf_fc_ls_add_conn_api_data *api_data = NULL;
if (fc_conn->create_opd) {
api_data = &fc_conn->create_opd->u.add_conn;
nvmf_fc_ls_add_conn_failure(api_data->assoc, api_data->ls_rqst,
api_data->args.fc_conn, api_data->aq_conn);
}
} else if (fc_conn->conn_state == SPDK_NVMF_FC_OBJECT_CREATED) {
rc = nvmf_fc_delete_connection(fc_conn, false, true,
nvmf_fc_connection_delete_done_cb, fc_ctx);
if (!rc) {
/* Wait for transport to complete its work. */
return;
}
SPDK_ERRLOG("%s: Delete FC connection failed.\n", __func__);
}
nvmf_fc_connection_delete_done_cb(fc_ctx);
}
static void
nvmf_fc_close_qpair(struct spdk_nvmf_qpair *qpair,
spdk_nvmf_transport_qpair_fini_cb cb_fn, void *cb_arg)
{
struct spdk_nvmf_fc_conn *fc_conn;
struct spdk_nvmf_fc_qpair_remove_ctx *fc_ctx;
fc_conn = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_fc_conn, qpair);
if (fc_conn->conn_id == NVMF_FC_INVALID_CONN_ID) {
/* QP creation failure in FC tranport. Cleanup. */
spdk_thread_send_msg(nvmf_fc_get_main_thread(),
nvmf_fc_handle_connection_failure, fc_conn);
} else if (fc_conn->fc_assoc->assoc_id == fc_conn->conn_id &&
fc_conn->fc_assoc->assoc_state != SPDK_NVMF_FC_OBJECT_TO_BE_DELETED) {
/* Admin connection */
spdk_thread_send_msg(nvmf_fc_get_main_thread(),
nvmf_fc_handle_assoc_deletion, fc_conn);
fc_ctx = calloc(1, sizeof(struct spdk_nvmf_fc_qpair_remove_ctx));
if (!fc_ctx) {
SPDK_ERRLOG("Unable to allocate close_qpair ctx.");
if (cb_fn) {
cb_fn(cb_arg);
}
return;
}
fc_ctx->qpair = qpair;
fc_ctx->cb_fn = cb_fn;
fc_ctx->cb_ctx = cb_arg;
fc_ctx->qpair_thread = spdk_get_thread();
if (cb_fn) {
cb_fn(cb_arg);
}
spdk_thread_send_msg(nvmf_fc_get_main_thread(), _nvmf_fc_close_qpair, fc_ctx);
}
static int

22
lib/nvmf/fc_ls.c
View File

@ -419,6 +419,9 @@ nvmf_fc_do_del_conn_cbs(struct nvmf_fc_ls_op_ctx *opd,
SPDK_ERRLOG("Send LS response for delete connection failed\n");
}
}
if (dp->del_conn_cb) {
dp->del_conn_cb(dp->del_conn_cb_data);
}
free(opd);
opd = nxt;
}
@ -444,7 +447,8 @@ nvmf_fc_ls_poller_delete_conn_cb(void *cb_data, enum spdk_nvmf_fc_poller_api_ret
static int
nvmf_fc_ls_poller_delete_conn(struct spdk_nvmf_fc_conn *fc_conn, bool send_abts,
struct spdk_nvmf_fc_ls_rqst *ls_rqst, bool backend_initiated)
struct spdk_nvmf_fc_ls_rqst *ls_rqst, bool backend_initiated,
spdk_nvmf_fc_del_conn_cb cb_fn, void *cb_data)
{
struct spdk_nvmf_fc_association *assoc = fc_conn->fc_assoc;
struct spdk_nvmf_fc_ls_del_conn_api_data *api_data;
@ -464,6 +468,8 @@ nvmf_fc_ls_poller_delete_conn(struct spdk_nvmf_fc_conn *fc_conn, bool send_abts,
api_data = &opd->u.del_conn;
api_data->assoc = assoc;
api_data->ls_rqst = ls_rqst;
api_data->del_conn_cb = cb_fn;
api_data->del_conn_cb_data = cb_data;
api_data->aq_conn = (assoc->aq_conn == fc_conn ? true : false);
api_data->args.fc_conn = fc_conn;
api_data->args.send_abts = send_abts;
@ -527,7 +533,7 @@ nvmf_fc_ls_add_conn_cb(void *cb_data, enum spdk_nvmf_fc_poller_api_ret ret)
if (nvmf_fc_xmt_ls_rsp(tgtport, ls_rqst) != 0) {
SPDK_ERRLOG("Send LS response for %s failed - cleaning up\n",
dp->aq_conn ? "association" : "connection");
nvmf_fc_ls_poller_delete_conn(fc_conn, false, NULL, false);
nvmf_fc_ls_poller_delete_conn(fc_conn, false, NULL, false, NULL, NULL);
} else {
SPDK_DEBUGLOG(nvmf_fc_ls,
"LS response (conn_id 0x%lx) sent\n", fc_conn->conn_id);
@ -767,7 +773,7 @@ _nvmf_fc_delete_association(struct spdk_nvmf_fc_nport *tgtport,
/* delete all of the association's connections */
TAILQ_FOREACH(fc_conn, &assoc->fc_conns, assoc_link) {
rc = nvmf_fc_ls_poller_delete_conn(fc_conn, send_abts, NULL, backend_initiated);
rc = nvmf_fc_ls_poller_delete_conn(fc_conn, send_abts, NULL, backend_initiated, NULL, NULL);
if (rc) {
SPDK_ERRLOG("Delete connection failed for assoc_id 0x%lx conn_id 0x%lx\n",
assoc->assoc_id, fc_conn->conn_id);
@ -1278,6 +1284,16 @@ nvmf_fc_delete_association(struct spdk_nvmf_fc_nport *tgtport,
del_assoc_cb, cb_data, false);
}
int
nvmf_fc_delete_connection(struct spdk_nvmf_fc_conn *fc_conn, bool send_abts,
bool backend_initiated, spdk_nvmf_fc_del_conn_cb cb_fn,
void *cb_data)
{
return nvmf_fc_ls_poller_delete_conn(fc_conn, send_abts, NULL,
backend_initiated, cb_fn, cb_data);
}
static void
nvmf_fc_poller_api_cb_event(void *arg)
{

14
lib/nvmf/nvmf_fc.h
View File

@ -176,6 +176,13 @@ struct spdk_nvmf_fc_srsr_bufs {
uint16_t rpi;
};
struct spdk_nvmf_fc_qpair_remove_ctx {
struct spdk_nvmf_qpair *qpair;
spdk_nvmf_transport_qpair_fini_cb cb_fn;
void *cb_ctx;
struct spdk_thread *qpair_thread;
};
/*
* Struct representing a nport
*/
@ -552,6 +559,7 @@ SPDK_STATIC_ASSERT(sizeof(struct spdk_nvmf_fc_rq_buf_ls_request) ==
/* Poller API structures (arguments and callback data */
typedef void (*spdk_nvmf_fc_del_assoc_cb)(void *arg, uint32_t err);
typedef void (*spdk_nvmf_fc_del_conn_cb)(void *arg);
struct spdk_nvmf_fc_ls_add_conn_api_data {
struct spdk_nvmf_fc_poller_api_add_connection_args args;
@ -566,6 +574,8 @@ struct spdk_nvmf_fc_ls_del_conn_api_data {
struct spdk_nvmf_fc_ls_rqst *ls_rqst;
struct spdk_nvmf_fc_association *assoc;
bool aq_conn; /* true if deleting AQ connection */
spdk_nvmf_fc_del_conn_cb del_conn_cb;
void *del_conn_cb_data;
};
/* used by LS disconnect association cmd handling */
@ -922,6 +932,10 @@ int nvmf_fc_delete_association(struct spdk_nvmf_fc_nport *tgtport,
spdk_nvmf_fc_del_assoc_cb del_assoc_cb,
void *cb_data);
int nvmf_fc_delete_connection(struct spdk_nvmf_fc_conn *fc_conn, bool send_abts,
bool backend_initiated, spdk_nvmf_fc_del_conn_cb cb_fn,
void *cb_data);
bool nvmf_ctrlr_is_on_nport(uint8_t port_hdl, uint16_t nport_hdl,
struct spdk_nvmf_ctrlr *ctrlr);