From 808b47c3aa7b042b711458358302ee64850a8170 Mon Sep 17 00:00:00 2001 From: Ben Walker Date: Wed, 1 Aug 2018 14:43:55 -0700 Subject: [PATCH] nvmf/rdma: Trigger error recovery on IBV_EVENT_SQ_DRAINED again After some other refactoring, we can now efficiently handle IBV_EVENT_SQ_DRAINED events during error cases again, so do that. Change-Id: Iba9ec59d9e6b72d8a6d8c7b74f3c3c532114a0a4 Signed-off-by: Ben Walker Reviewed-on: https://review.gerrithub.io/421045 Chandler-Test-Pool: SPDK Automated Test System Tested-by: SPDK CI Jenkins Reviewed-by: Changpeng Liu Reviewed-by: Shuhei Matsumoto Reviewed-by: Jim Harris --- lib/nvmf/rdma.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index eb66d6c63..5c6654981 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -2007,10 +2007,15 @@ error: static void _spdk_nvmf_rdma_qp_error(void *arg) { - struct spdk_nvmf_rdma_qpair *rqpair = arg; - struct spdk_nvmf_rdma_request *rdma_req, *req_tmp; + struct spdk_nvmf_rdma_qpair *rqpair = arg; + struct spdk_nvmf_rdma_request *rdma_req, *req_tmp; + enum ibv_qp_state state; - spdk_nvmf_rdma_update_ibv_state(rqpair); + state = spdk_nvmf_rdma_update_ibv_state(rqpair); + if (state != IBV_QPS_ERR) { + /* Error was already recovered */ + return; + } if (spdk_nvmf_qpair_is_admin_queue(&rqpair->qpair)) { spdk_nvmf_ctrlr_abort_aer(rqpair->qpair.ctrlr); @@ -2038,6 +2043,7 @@ spdk_nvmf_process_ib_event(struct spdk_nvmf_rdma_device *device) int rc; struct spdk_nvmf_rdma_qpair *rqpair; struct ibv_async_event event; + enum ibv_qp_state state; rc = ibv_get_async_event(device->context, &event); @@ -2060,7 +2066,19 @@ spdk_nvmf_process_ib_event(struct spdk_nvmf_rdma_device *device) case IBV_EVENT_QP_LAST_WQE_REACHED: spdk_thread_send_msg(rqpair->qpair.group->thread, _spdk_nvmf_rdma_qp_error, rqpair); break; - case IBV_EVENT_SQ_DRAINED: + case IBV_EVENT_SQ_DRAINED: { + /* This event occurs frequently in both error and non-error states. + * Check if the qpair is in an error state before sending a message. + * Note that we're not on the correct thread to access the qpair, but + * the operations that the below calls make all happen to be thread + * safe. */ + state = spdk_nvmf_rdma_update_ibv_state(rqpair); + if (state == IBV_QPS_ERR) { + spdk_thread_send_msg(rqpair->qpair.group->thread, _spdk_nvmf_rdma_qp_error, rqpair); + } + break; + + } case IBV_EVENT_CQ_ERR: case IBV_EVENT_QP_REQ_ERR: case IBV_EVENT_QP_ACCESS_ERR: