ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvme_rdma: Make CM event operations callback functions

Browse Source 
Change-Id: I9f2551a07187400dd9ef624348cd465e64557e1b
Signed-off-by: Shuhei Matsumoto <smatsumoto@nvidia.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11138
Community-CI: Mellanox Build Bot
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
This commit is contained in:
Shuhei Matsumoto 2022-02-08 06:50:25 +09:00 committed by Tomasz Zawadzki
parent e5927c02e9
commit bcf0845727
1 changed files with 97 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

159
lib/nvme/nvme_rdma.c
View File

@ -1077,21 +1077,28 @@ fail:
return -ENOMEM; return -ENOMEM;
} }
static int static int nvme_rdma_connect(struct nvme_rdma_qpair *rqpair);
nvme_rdma_resolve_addr(struct nvme_rdma_qpair *rqpair,
struct sockaddr *src_addr,
struct sockaddr *dst_addr)
{
int ret;
ret = rdma_resolve_addr(rqpair->cm_id, src_addr, dst_addr, static int
NVME_RDMA_TIME_OUT_IN_MS); nvme_rdma_route_resolved(struct nvme_rdma_qpair *rqpair, int ret)
{
if (ret) { if (ret) {
SPDK_ERRLOG("rdma_resolve_addr, %d\n", errno); SPDK_ERRLOG("RDMA route resolution error\n");
return ret; return -1;
} }
ret = nvme_rdma_process_event(rqpair, RDMA_CM_EVENT_ADDR_RESOLVED); ret = nvme_rdma_qpair_init(rqpair);
if (ret < 0) {
SPDK_ERRLOG("nvme_rdma_qpair_init() failed\n");
return -1;
}
return nvme_rdma_connect(rqpair);
}
static int
nvme_rdma_addr_resolved(struct nvme_rdma_qpair *rqpair, int ret)
{
if (ret) { if (ret) {
SPDK_ERRLOG("RDMA address resolution error\n"); SPDK_ERRLOG("RDMA address resolution error\n");
return -1; return -1;
@ -1111,7 +1118,6 @@ nvme_rdma_resolve_addr(struct nvme_rdma_qpair *rqpair,
#endif #endif
} }
ret = rdma_resolve_route(rqpair->cm_id, NVME_RDMA_TIME_OUT_IN_MS); ret = rdma_resolve_route(rqpair->cm_id, NVME_RDMA_TIME_OUT_IN_MS);
if (ret) { if (ret) {
SPDK_ERRLOG("rdma_resolve_route\n"); SPDK_ERRLOG("rdma_resolve_route\n");
@ -1119,10 +1125,61 @@ nvme_rdma_resolve_addr(struct nvme_rdma_qpair *rqpair,
} }
ret = nvme_rdma_process_event(rqpair, RDMA_CM_EVENT_ROUTE_RESOLVED); ret = nvme_rdma_process_event(rqpair, RDMA_CM_EVENT_ROUTE_RESOLVED);
return nvme_rdma_route_resolved(rqpair, ret);
}
static int
nvme_rdma_resolve_addr(struct nvme_rdma_qpair *rqpair,
struct sockaddr *src_addr,
struct sockaddr *dst_addr)
{
int ret;
ret = rdma_resolve_addr(rqpair->cm_id, src_addr, dst_addr,
NVME_RDMA_TIME_OUT_IN_MS);
if (ret) { if (ret) {
SPDK_ERRLOG("RDMA route resolution error\n"); SPDK_ERRLOG("rdma_resolve_addr, %d\n", errno);
return ret;
}
ret = nvme_rdma_process_event(rqpair, RDMA_CM_EVENT_ADDR_RESOLVED);
return nvme_rdma_addr_resolved(rqpair, ret);
}
static int
nvme_rdma_connect_established(struct nvme_rdma_qpair *rqpair, int ret)
{
if (ret) {
SPDK_ERRLOG("RDMA connect error %d\n", ret);
return ret;
}
ret = nvme_rdma_register_reqs(rqpair);
SPDK_DEBUGLOG(nvme, "rc =%d\n", ret);
if (ret) {
SPDK_ERRLOG("Unable to register rqpair RDMA requests\n");
return -1; return -1;
} }
SPDK_DEBUGLOG(nvme, "RDMA requests registered\n");
ret = nvme_rdma_register_rsps(rqpair);
SPDK_DEBUGLOG(nvme, "rc =%d\n", ret);
if (ret < 0) {
SPDK_ERRLOG("Unable to register rqpair RDMA responses\n");
return -1;
}
SPDK_DEBUGLOG(nvme, "RDMA responses registered\n");
rqpair->mr_map = spdk_rdma_create_mem_map(rqpair->rdma_qp->qp->pd, &g_nvme_hooks,
SPDK_RDMA_MEMORY_MAP_ROLE_INITIATOR);
if (!rqpair->mr_map) {
SPDK_ERRLOG("Unable to register RDMA memory translation map\n");
return -1;
}
rqpair->state = NVME_RDMA_QPAIR_STATE_FABRIC_CONNECT_SEND;
return 0; return 0;
} }
@ -1170,7 +1227,9 @@ nvme_rdma_connect(struct nvme_rdma_qpair *rqpair)
return ret; return ret;
} }
return nvme_rdma_process_event(rqpair, RDMA_CM_EVENT_ESTABLISHED); ret = nvme_rdma_process_event(rqpair, RDMA_CM_EVENT_ESTABLISHED);
return nvme_rdma_connect_established(rqpair, ret);
} }
static int static int
@ -1266,43 +1325,6 @@ nvme_rdma_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qp
return -1; return -1;
} }
rc = nvme_rdma_qpair_init(rqpair);
if (rc < 0) {
SPDK_ERRLOG("nvme_rdma_qpair_init() failed\n");
return -1;
}
rc = nvme_rdma_connect(rqpair);
if (rc != 0) {
SPDK_ERRLOG("Unable to connect the rqpair\n");
return rc;
}
rc = nvme_rdma_register_reqs(rqpair);
SPDK_DEBUGLOG(nvme, "rc =%d\n", rc);
if (rc) {
SPDK_ERRLOG("Unable to register rqpair RDMA requests\n");
return -1;
}
SPDK_DEBUGLOG(nvme, "RDMA requests registered\n");
rc = nvme_rdma_register_rsps(rqpair);
SPDK_DEBUGLOG(nvme, "rc =%d\n", rc);
if (rc < 0) {
SPDK_ERRLOG("Unable to register rqpair RDMA responses\n");
return -1;
}
SPDK_DEBUGLOG(nvme, "RDMA responses registered\n");
rqpair->mr_map = spdk_rdma_create_mem_map(rqpair->rdma_qp->qp->pd, &g_nvme_hooks,
SPDK_RDMA_MEMORY_MAP_ROLE_INITIATOR);
if (!rqpair->mr_map) {
SPDK_ERRLOG("Unable to register RDMA memory translation map\n");
return -1;
}
rqpair->state = NVME_RDMA_QPAIR_STATE_FABRIC_CONNECT_SEND;
return 0; return 0;
} }
@ -1802,6 +1824,29 @@ nvme_rdma_ctrlr_create_qpair(struct spdk_nvme_ctrlr *ctrlr,
return qpair; return qpair;
} }
static int
nvme_rdma_qpair_disconnected(struct nvme_rdma_qpair *rqpair, int ret)
{
if (rqpair->cm_id) {
if (rqpair->rdma_qp) {
spdk_rdma_qp_destroy(rqpair->rdma_qp);
rqpair->rdma_qp = NULL;
}
rdma_destroy_id(rqpair->cm_id);
rqpair->cm_id = NULL;
}
if (rqpair->cq) {
ibv_destroy_cq(rqpair->cq);
rqpair->cq = NULL;
}
nvme_transport_ctrlr_disconnect_qpair_done(&rqpair->qpair);
return 0;
}
static void static void
nvme_rdma_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair) nvme_rdma_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
{ {
@ -1842,20 +1887,10 @@ nvme_rdma_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme
SPDK_DEBUGLOG(nvme, "Target did not respond to qpair disconnect.\n"); SPDK_DEBUGLOG(nvme, "Target did not respond to qpair disconnect.\n");
} }
} }
spdk_rdma_qp_destroy(rqpair->rdma_qp);
rqpair->rdma_qp = NULL;
} }
rdma_destroy_id(rqpair->cm_id);
rqpair->cm_id = NULL;
} }
if (rqpair->cq) { nvme_rdma_qpair_disconnected(rqpair, 0);
ibv_destroy_cq(rqpair->cq);
rqpair->cq = NULL;
}
nvme_transport_ctrlr_disconnect_qpair_done(qpair);
} }
static void nvme_rdma_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr); static void nvme_rdma_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr);