From 1eae58ff6d939fbbaa936caef183d4cbd32e4180 Mon Sep 17 00:00:00 2001 From: Alexey Marchuk Date: Wed, 2 Dec 2020 17:39:57 +0300 Subject: [PATCH] rdma: Add statistics per qpair These statistics can help to estimate efficiency of Work Requests batching and show busy/idle polls ratio. Send: the doorbell updates statistics for verbs provider are incremented per each ibv_post_send call, for mlx5_dv per each ibv_wr_complete call. Recv: the doorbell updates statistics for both providers are updated when either ibv_post_recv or ibv_post_srq_recv functions are called. Each qpair on initialization accepts an optional pointer to shared statistics (nvmf/nvme poll groups). If the pointer to statistics is not provided then qpair allocates its own structure. That is done to support cases when NVME RDMA initiator doesn't use poll groups, so we can avoid checks that qpair has statistics in IO path Change-Id: I07dea603cb870b85ea23c42e8e2c4520b1c66252 Signed-off-by: Alexey Marchuk Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6293 Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Jim Harris --- include/spdk_internal/rdma.h | 18 ++++++++++++++++++ lib/rdma/common.c | 27 ++++++++++++++++++++++++--- lib/rdma/rdma_mlx5_dv.c | 22 ++++++++++++++++++++-- lib/rdma/rdma_verbs.c | 19 +++++++++++++++++++ 4 files changed, 81 insertions(+), 5 deletions(-) diff --git a/include/spdk_internal/rdma.h b/include/spdk_internal/rdma.h index 32ba39d0e..fbe43f0b1 100644 --- a/include/spdk_internal/rdma.h +++ b/include/spdk_internal/rdma.h @@ -41,6 +41,18 @@ /* Contains hooks definition */ #include "spdk/nvme.h" +struct spdk_rdma_wr_stats { + /* Total number of submitted requests */ + uint64_t num_submitted_wrs; + /* Total number of doorbell updates */ + uint64_t doorbell_updates; +}; + +struct spdk_rdma_qp_stats { + struct spdk_rdma_wr_stats send; + struct spdk_rdma_wr_stats recv; +}; + struct spdk_rdma_qp_init_attr { void *qp_context; struct ibv_cq *send_cq; @@ -48,6 +60,7 @@ struct spdk_rdma_qp_init_attr { struct ibv_srq *srq; struct ibv_qp_cap cap; struct ibv_pd *pd; + struct spdk_rdma_qp_stats *stats; }; struct spdk_rdma_send_wr_list { @@ -65,6 +78,8 @@ struct spdk_rdma_qp { struct rdma_cm_id *cm_id; struct spdk_rdma_send_wr_list send_wrs; struct spdk_rdma_recv_wr_list recv_wrs; + struct spdk_rdma_qp_stats *stats; + bool shared_stats; }; struct spdk_rdma_mem_map; @@ -85,12 +100,15 @@ struct spdk_rdma_memory_translation { }; struct spdk_rdma_srq_init_attr { struct ibv_pd *pd; + struct spdk_rdma_wr_stats *stats; struct ibv_srq_init_attr srq_init_attr; }; struct spdk_rdma_srq { struct ibv_srq *srq; struct spdk_rdma_recv_wr_list recv_wrs; + struct spdk_rdma_wr_stats *stats; + bool shared_stats; }; /** diff --git a/lib/rdma/common.c b/lib/rdma/common.c index 980856418..0e7adcbe7 100644 --- a/lib/rdma/common.c +++ b/lib/rdma/common.c @@ -234,6 +234,18 @@ spdk_rdma_srq_create(struct spdk_rdma_srq_init_attr *init_attr) return NULL; } + if (init_attr->stats) { + rdma_srq->stats = init_attr->stats; + rdma_srq->shared_stats = true; + } else { + rdma_srq->stats = calloc(1, sizeof(*rdma_srq->stats)); + if (!rdma_srq->stats) { + SPDK_ERRLOG("SRQ statistics memory allocation failed"); + free(rdma_srq); + return NULL; + } + } + rdma_srq->srq = ibv_create_srq(init_attr->pd, &init_attr->srq_init_attr); if (!rdma_srq->srq) { SPDK_ERRLOG("Unable to create SRQ, errno %d (%s)\n", errno, spdk_strerror(errno)); @@ -264,19 +276,26 @@ spdk_rdma_srq_destroy(struct spdk_rdma_srq *rdma_srq) SPDK_ERRLOG("SRQ destroy failed with %d\n", rc); } + if (!rdma_srq->shared_stats) { + free(rdma_srq->stats); + } + free(rdma_srq); return rc; } static inline bool -rdma_queue_recv_wrs(struct spdk_rdma_recv_wr_list *recv_wrs, struct ibv_recv_wr *first) +rdma_queue_recv_wrs(struct spdk_rdma_recv_wr_list *recv_wrs, struct ibv_recv_wr *first, + struct spdk_rdma_wr_stats *recv_stats) { struct ibv_recv_wr *last; + recv_stats->num_submitted_wrs++; last = first; while (last->next != NULL) { last = last->next; + recv_stats->num_submitted_wrs++; } if (recv_wrs->first == NULL) { @@ -296,7 +315,7 @@ spdk_rdma_srq_queue_recv_wrs(struct spdk_rdma_srq *rdma_srq, struct ibv_recv_wr assert(rdma_srq); assert(first); - return rdma_queue_recv_wrs(&rdma_srq->recv_wrs, first); + return rdma_queue_recv_wrs(&rdma_srq->recv_wrs, first, rdma_srq->stats); } int @@ -311,6 +330,7 @@ spdk_rdma_srq_flush_recv_wrs(struct spdk_rdma_srq *rdma_srq, struct ibv_recv_wr rc = ibv_post_srq_recv(rdma_srq->srq, rdma_srq->recv_wrs.first, bad_wr); rdma_srq->recv_wrs.first = NULL; + rdma_srq->stats->doorbell_updates++; return rc; } @@ -321,7 +341,7 @@ spdk_rdma_qp_queue_recv_wrs(struct spdk_rdma_qp *spdk_rdma_qp, struct ibv_recv_w assert(spdk_rdma_qp); assert(first); - return rdma_queue_recv_wrs(&spdk_rdma_qp->recv_wrs, first); + return rdma_queue_recv_wrs(&spdk_rdma_qp->recv_wrs, first, &spdk_rdma_qp->stats->recv); } int @@ -336,6 +356,7 @@ spdk_rdma_qp_flush_recv_wrs(struct spdk_rdma_qp *spdk_rdma_qp, struct ibv_recv_w rc = ibv_post_recv(spdk_rdma_qp->qp, spdk_rdma_qp->recv_wrs.first, bad_wr); spdk_rdma_qp->recv_wrs.first = NULL; + spdk_rdma_qp->stats->recv.doorbell_updates++; return rc; } diff --git a/lib/rdma/rdma_mlx5_dv.c b/lib/rdma/rdma_mlx5_dv.c index 06e7f9c41..a7d2ad2c2 100644 --- a/lib/rdma/rdma_mlx5_dv.c +++ b/lib/rdma/rdma_mlx5_dv.c @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) Intel Corporation. All rights reserved. - * Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2020, 2021 Mellanox Technologies LTD. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -120,6 +120,18 @@ spdk_rdma_qp_create(struct rdma_cm_id *cm_id, struct spdk_rdma_qp_init_attr *qp_ return NULL; } + if (qp_attr->stats) { + mlx5_qp->common.stats = qp_attr->stats; + mlx5_qp->common.shared_stats = true; + } else { + mlx5_qp->common.stats = calloc(1, sizeof(*mlx5_qp->common.stats)); + if (!mlx5_qp->common.stats) { + SPDK_ERRLOG("qp statistics memory allocation failed\n"); + free(mlx5_qp); + return NULL; + } + } + qp = mlx5dv_create_qp(cm_id->verbs, &dv_qp_attr, NULL); if (!qp) { @@ -201,6 +213,10 @@ spdk_rdma_qp_destroy(struct spdk_rdma_qp *spdk_rdma_qp) SPDK_WARNLOG("Destroying qpair with queued Work Requests\n"); } + if (!mlx5_qp->common.shared_stats) { + free(mlx5_qp->common.stats); + } + if (mlx5_qp->common.qp) { rc = ibv_destroy_qp(mlx5_qp->common.qp); if (rc) { @@ -283,6 +299,7 @@ spdk_rdma_qp_queue_send_wrs(struct spdk_rdma_qp *spdk_rdma_qp, struct ibv_send_w ibv_wr_set_sge_list(mlx5_qp->qpex, tmp->num_sge, tmp->sg_list); spdk_rdma_qp->send_wrs.last = tmp; + spdk_rdma_qp->stats->send.num_submitted_wrs++; } return is_first; @@ -303,7 +320,7 @@ spdk_rdma_qp_flush_send_wrs(struct spdk_rdma_qp *spdk_rdma_qp, struct ibv_send_w return 0; } - rc = ibv_wr_complete(mlx5_qp->qpex); + rc = ibv_wr_complete(mlx5_qp->qpex); if (spdk_unlikely(rc)) { /* If ibv_wr_complete reports an error that means that no WRs are posted to NIC */ @@ -311,6 +328,7 @@ spdk_rdma_qp_flush_send_wrs(struct spdk_rdma_qp *spdk_rdma_qp, struct ibv_send_w } spdk_rdma_qp->send_wrs.first = NULL; + spdk_rdma_qp->stats->send.doorbell_updates++; return rc; } diff --git a/lib/rdma/rdma_verbs.c b/lib/rdma/rdma_verbs.c index 2e3456b68..7ef27fe60 100644 --- a/lib/rdma/rdma_verbs.c +++ b/lib/rdma/rdma_verbs.c @@ -60,6 +60,18 @@ spdk_rdma_qp_create(struct rdma_cm_id *cm_id, struct spdk_rdma_qp_init_attr *qp_ return NULL; } + if (qp_attr->stats) { + spdk_rdma_qp->stats = qp_attr->stats; + spdk_rdma_qp->shared_stats = true; + } else { + spdk_rdma_qp->stats = calloc(1, sizeof(*spdk_rdma_qp->stats)); + if (!spdk_rdma_qp->stats) { + SPDK_ERRLOG("qp statistics memory allocation failed\n"); + free(spdk_rdma_qp); + return NULL; + } + } + rc = rdma_create_qp(cm_id, qp_attr->pd, &attr); if (rc) { SPDK_ERRLOG("Failed to create qp, errno %s (%d)\n", spdk_strerror(errno), errno); @@ -103,6 +115,10 @@ spdk_rdma_qp_destroy(struct spdk_rdma_qp *spdk_rdma_qp) rdma_destroy_qp(spdk_rdma_qp->cm_id); } + if (!spdk_rdma_qp->shared_stats) { + free(spdk_rdma_qp->stats); + } + free(spdk_rdma_qp); } @@ -137,9 +153,11 @@ spdk_rdma_qp_queue_send_wrs(struct spdk_rdma_qp *spdk_rdma_qp, struct ibv_send_w assert(spdk_rdma_qp); assert(first); + spdk_rdma_qp->stats->send.num_submitted_wrs++; last = first; while (last->next != NULL) { last = last->next; + spdk_rdma_qp->stats->send.num_submitted_wrs++; } if (spdk_rdma_qp->send_wrs.first == NULL) { @@ -168,6 +186,7 @@ spdk_rdma_qp_flush_send_wrs(struct spdk_rdma_qp *spdk_rdma_qp, struct ibv_send_w rc = ibv_post_send(spdk_rdma_qp->qp, spdk_rdma_qp->send_wrs.first, bad_wr); spdk_rdma_qp->send_wrs.first = NULL; + spdk_rdma_qp->stats->send.doorbell_updates++; return rc; }