ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvme_rdma: make handling of cm_events more robust

Browse Source 
By splitting all cm_event handling into a single function, we can create
a single point of contact for cm_events, whether we want to process them
synchronously or asynchronously.

Change-Id: I053a850358605115362f424de55e66806a769320
Signed-off-by: Seth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/467546
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Seth Howell 2019-09-05 14:00:28 -07:00 committed by Jim Harris
parent ad7a01bde3
commit 579d44b0ee
1 changed files with 84 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
lib/nvme/nvme_rdma.c
View File

@ -131,6 +131,8 @@ struct nvme_rdma_qpair {
/* Placed at the end of the struct since it is not used frequently */
struct rdma_event_channel *cm_channel;
struct rdma_cm_event *evt;
};
struct spdk_nvme_rdma_req {
@ -225,11 +227,72 @@ nvme_rdma_cm_event_str_get(uint32_t event)
}
}
static struct rdma_cm_event *
nvme_rdma_get_event(struct rdma_event_channel *channel,
enum rdma_cm_event_type evt)
static int
nvme_rdma_qpair_process_cm_event(struct nvme_rdma_qpair *rqpair)
{
struct rdma_cm_event *event = rqpair->evt;
struct spdk_nvmf_rdma_accept_private_data *accept_data;
int rc = 0;
if (event) {
switch (event->event) {
case RDMA_CM_EVENT_ADDR_RESOLVED:
case RDMA_CM_EVENT_ADDR_ERROR:
case RDMA_CM_EVENT_ROUTE_RESOLVED:
case RDMA_CM_EVENT_ROUTE_ERROR:
break;
case RDMA_CM_EVENT_CONNECT_REQUEST:
break;
case RDMA_CM_EVENT_CONNECT_RESPONSE:
break;
case RDMA_CM_EVENT_CONNECT_ERROR:
break;
case RDMA_CM_EVENT_UNREACHABLE:
case RDMA_CM_EVENT_REJECTED:
break;
case RDMA_CM_EVENT_ESTABLISHED:
accept_data = (struct spdk_nvmf_rdma_accept_private_data *)event->param.conn.private_data;
if (accept_data == NULL) {
rc = -1;
} else {
SPDK_DEBUGLOG(SPDK_LOG_NVME, "Requested queue depth %d. Actually got queue depth %d.\n",
rqpair->num_entries, accept_data->crqsize);
rqpair->num_entries = spdk_min(rqpair->num_entries, accept_data->crqsize);
}
break;
case RDMA_CM_EVENT_DISCONNECTED:
case RDMA_CM_EVENT_DEVICE_REMOVAL:
break;
case RDMA_CM_EVENT_MULTICAST_JOIN:
case RDMA_CM_EVENT_MULTICAST_ERROR:
break;
case RDMA_CM_EVENT_ADDR_CHANGE:
break;
case RDMA_CM_EVENT_TIMEWAIT_EXIT:
break;
default:
SPDK_ERRLOG("Unexpected Acceptor Event [%d]\n", event->event);
break;
}
rqpair->evt = NULL;
rdma_ack_cm_event(event);
}
return rc;
}
static int
nvme_rdma_process_event(struct nvme_rdma_qpair *rqpair,
struct rdma_event_channel *channel,
enum rdma_cm_event_type evt)
{
struct rdma_cm_event *event;
int rc = 0;
if (rqpair->evt != NULL) {
return -EINVAL;
}
while (rdma_get_cm_event(channel, &event) != 0) {
if (errno == EAGAIN || errno == EWOULDBLOCK) {
@ -238,18 +301,20 @@ nvme_rdma_get_event(struct rdma_event_channel *channel,
SPDK_ERRLOG("Failed to get event from CM event channel. Error %d (%s)\n",
errno, spdk_strerror(errno));
return NULL;
return -errno;
}
if (event->event != evt) {
SPDK_ERRLOG("Expected %s but received %s (%d) from CM event channel (status = %d)\n",
nvme_rdma_cm_event_str_get(evt),
nvme_rdma_cm_event_str_get(event->event), event->event, event->status);
rdma_ack_cm_event(event);
return NULL;
nvme_rdma_cm_event_str_get(event->event), event->event,
event->status);
rc = -EBADMSG;
}
return event;
rqpair->evt = event;
rc |= nvme_rdma_qpair_process_cm_event(rqpair);
return rc;
}
static int
@ -550,7 +615,6 @@ nvme_rdma_resolve_addr(struct nvme_rdma_qpair *rqpair,
struct rdma_event_channel *cm_channel)
{
int ret;
struct rdma_cm_event *event;
ret = rdma_resolve_addr(rqpair->cm_id, src_addr, dst_addr,
NVME_RDMA_TIME_OUT_IN_MS);
@ -559,12 +623,11 @@ nvme_rdma_resolve_addr(struct nvme_rdma_qpair *rqpair,
return ret;
}
event = nvme_rdma_get_event(cm_channel, RDMA_CM_EVENT_ADDR_RESOLVED);
if (event == NULL) {
ret = nvme_rdma_process_event(rqpair, cm_channel, RDMA_CM_EVENT_ADDR_RESOLVED);
if (ret) {
SPDK_ERRLOG("RDMA address resolution error\n");
return -1;
}
rdma_ack_cm_event(event);
ret = rdma_resolve_route(rqpair->cm_id, NVME_RDMA_TIME_OUT_IN_MS);
if (ret) {
@ -572,12 +635,11 @@ nvme_rdma_resolve_addr(struct nvme_rdma_qpair *rqpair,
return ret;
}
event = nvme_rdma_get_event(cm_channel, RDMA_CM_EVENT_ROUTE_RESOLVED);
if (event == NULL) {
ret = nvme_rdma_process_event(rqpair, cm_channel, RDMA_CM_EVENT_ROUTE_RESOLVED);
if (ret) {
SPDK_ERRLOG("RDMA route resolution error\n");
return -1;
}
rdma_ack_cm_event(event);
return 0;
}
@ -587,10 +649,8 @@ nvme_rdma_connect(struct nvme_rdma_qpair *rqpair)
{
struct rdma_conn_param param = {};
struct spdk_nvmf_rdma_request_private_data request_data = {};
struct spdk_nvmf_rdma_accept_private_data *accept_data;
struct ibv_device_attr attr;
int ret;
struct rdma_cm_event *event;
struct spdk_nvme_ctrlr *ctrlr;
ret = ibv_query_device(rqpair->cm_id->verbs, &attr);
@ -622,26 +682,12 @@ nvme_rdma_connect(struct nvme_rdma_qpair *rqpair)
return ret;
}
event = nvme_rdma_get_event(rqpair->cm_channel, RDMA_CM_EVENT_ESTABLISHED);
if (event == NULL) {
ret = nvme_rdma_process_event(rqpair, rqpair->cm_channel, RDMA_CM_EVENT_ESTABLISHED);
if (ret) {
SPDK_ERRLOG("RDMA connect error\n");
return -1;
}
accept_data = (struct spdk_nvmf_rdma_accept_private_data *)event->param.conn.private_data;
if (accept_data == NULL) {
rdma_ack_cm_event(event);
SPDK_ERRLOG("NVMe-oF target did not return accept data\n");
return -1;
}
SPDK_DEBUGLOG(SPDK_LOG_NVME, "Requested queue depth %d. Actually got queue depth %d.\n",
rqpair->num_entries, accept_data->crqsize);
rqpair->num_entries = spdk_min(rqpair->num_entries, accept_data->crqsize);
rdma_ack_cm_event(event);
return 0;
}
@ -1346,6 +1392,11 @@ nvme_rdma_qpair_disconnect(struct spdk_nvme_qpair *qpair)
nvme_rdma_unregister_reqs(rqpair);
nvme_rdma_unregister_rsps(rqpair);
if (rqpair->evt) {
rdma_ack_cm_event(rqpair->evt);
rqpair->evt = NULL;
}
if (rqpair->cm_id) {
if (rqpair->cm_id->qp) {
rdma_destroy_qp(rqpair->cm_id);