From db09de9866a6574744a563e1fb56f8ee8c054b1d Mon Sep 17 00:00:00 2001 From: Alexey Marchuk Date: Tue, 11 Aug 2020 14:28:36 +0300 Subject: [PATCH] nvmf/rdma: Update Work Completion error logging 1. Retrieve actual IBV state when we receive WC with bad status 2. Don't log an error if WC status is IBV_WC_WR_FLUSH_ERR. This means that we are performing qpair cleanup and this WC is expected. Change-Id: Id23634092f537861e66ca0f83ab79db9e052507b Signed-off-by: Alexey Marchuk Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3736 Community-CI: Mellanox Build Bot Reviewed-by: Shuhei Matsumoto Reviewed-by: Reviewed-by: Michael Haeuptle Reviewed-by: Ben Walker Tested-by: SPDK CI Jenkins --- lib/nvmf/rdma.c | 47 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index a17d68b64..110f522e4 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -3750,6 +3750,41 @@ _poller_submit_sends(struct spdk_nvmf_rdma_transport *rtransport, } } +static const char * +nvmf_rdma_wr_type_str(enum spdk_nvmf_rdma_wr_type wr_type) +{ + switch (wr_type) { + case RDMA_WR_TYPE_RECV: + return "RECV"; + case RDMA_WR_TYPE_SEND: + return "SEND"; + case RDMA_WR_TYPE_DATA: + return "DATA"; + default: + SPDK_ERRLOG("Unknown WR type %d\n", wr_type); + SPDK_UNREACHABLE(); + } +} + +static inline void +nvmf_rdma_log_wc_status(struct spdk_nvmf_rdma_qpair *rqpair, struct ibv_wc *wc) +{ + enum spdk_nvmf_rdma_wr_type wr_type = ((struct spdk_nvmf_rdma_wr *)wc->wr_id)->type; + + if (wc->status == IBV_WC_WR_FLUSH_ERR) { + /* If qpair is in ERR state, we will receive completions for all posted and not completed + * Work Requests with IBV_WC_WR_FLUSH_ERR status. Don't log an error in that case */ + SPDK_DEBUGLOG(SPDK_LOG_RDMA, + "Error on CQ %p, (qp state %d ibv_state %d) request 0x%lu, type %s, status: (%d): %s\n", + rqpair->poller->cq, rqpair->qpair.state, rqpair->ibv_state, wc->wr_id, + nvmf_rdma_wr_type_str(wr_type), wc->status, ibv_wc_status_str(wc->status)); + } else { + SPDK_ERRLOG("Error on CQ %p, (qp state %d ibv_state %d) request 0x%lu, type %s, status: (%d): %s\n", + rqpair->poller->cq, rqpair->qpair.state, rqpair->ibv_state, wc->wr_id, + nvmf_rdma_wr_type_str(wr_type), wc->status, ibv_wc_status_str(wc->status)); + } +} + static int nvmf_rdma_poller_poll(struct spdk_nvmf_rdma_transport *rtransport, struct spdk_nvmf_rdma_poller *rpoller) @@ -3873,16 +3908,8 @@ nvmf_rdma_poller_poll(struct spdk_nvmf_rdma_transport *rtransport, /* Handle error conditions */ if (wc[i].status) { - if ((rdma_wr->type == RDMA_WR_TYPE_RECV && !rpoller->srq)) { - /* When we don't use SRQ and close a qpair, we will receive completions with error - * status for all posted ibv_recv_wrs. This is expected and we don't want to log - * an error in that case. */ - SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Error on CQ %p, request 0x%lu, type %d, status: (%d): %s\n", - rpoller->cq, wc[i].wr_id, rdma_wr->type, wc[i].status, ibv_wc_status_str(wc[i].status)); - } else { - SPDK_ERRLOG("Error on CQ %p, request 0x%lu, type %d, status: (%d): %s\n", - rpoller->cq, wc[i].wr_id, rdma_wr->type, wc[i].status, ibv_wc_status_str(wc[i].status)); - } + nvmf_rdma_update_ibv_state(rqpair); + nvmf_rdma_log_wc_status(rqpair, &wc[i]); error = true;