diff --git a/include/spdk_internal/rdma.h b/include/spdk_internal/rdma.h index 32ba39d0e..fbe43f0b1 100644 --- a/include/spdk_internal/rdma.h +++ b/include/spdk_internal/rdma.h @@ -41,6 +41,18 @@ /* Contains hooks definition */ #include "spdk/nvme.h" +struct spdk_rdma_wr_stats { + /* Total number of submitted requests */ + uint64_t num_submitted_wrs; + /* Total number of doorbell updates */ + uint64_t doorbell_updates; +}; + +struct spdk_rdma_qp_stats { + struct spdk_rdma_wr_stats send; + struct spdk_rdma_wr_stats recv; +}; + struct spdk_rdma_qp_init_attr { void *qp_context; struct ibv_cq *send_cq; @@ -48,6 +60,7 @@ struct spdk_rdma_qp_init_attr { struct ibv_srq *srq; struct ibv_qp_cap cap; struct ibv_pd *pd; + struct spdk_rdma_qp_stats *stats; }; struct spdk_rdma_send_wr_list { @@ -65,6 +78,8 @@ struct spdk_rdma_qp { struct rdma_cm_id *cm_id; struct spdk_rdma_send_wr_list send_wrs; struct spdk_rdma_recv_wr_list recv_wrs; + struct spdk_rdma_qp_stats *stats; + bool shared_stats; }; struct spdk_rdma_mem_map; @@ -85,12 +100,15 @@ struct spdk_rdma_memory_translation { }; struct spdk_rdma_srq_init_attr { struct ibv_pd *pd; + struct spdk_rdma_wr_stats *stats; struct ibv_srq_init_attr srq_init_attr; }; struct spdk_rdma_srq { struct ibv_srq *srq; struct spdk_rdma_recv_wr_list recv_wrs; + struct spdk_rdma_wr_stats *stats; + bool shared_stats; }; /** diff --git a/lib/rdma/common.c b/lib/rdma/common.c index 980856418..0e7adcbe7 100644 --- a/lib/rdma/common.c +++ b/lib/rdma/common.c @@ -234,6 +234,18 @@ spdk_rdma_srq_create(struct spdk_rdma_srq_init_attr *init_attr) return NULL; } + if (init_attr->stats) { + rdma_srq->stats = init_attr->stats; + rdma_srq->shared_stats = true; + } else { + rdma_srq->stats = calloc(1, sizeof(*rdma_srq->stats)); + if (!rdma_srq->stats) { + SPDK_ERRLOG("SRQ statistics memory allocation failed"); + free(rdma_srq); + return NULL; + } + } + rdma_srq->srq = ibv_create_srq(init_attr->pd, &init_attr->srq_init_attr); if (!rdma_srq->srq) { SPDK_ERRLOG("Unable to create SRQ, errno %d (%s)\n", errno, spdk_strerror(errno)); @@ -264,19 +276,26 @@ spdk_rdma_srq_destroy(struct spdk_rdma_srq *rdma_srq) SPDK_ERRLOG("SRQ destroy failed with %d\n", rc); } + if (!rdma_srq->shared_stats) { + free(rdma_srq->stats); + } + free(rdma_srq); return rc; } static inline bool -rdma_queue_recv_wrs(struct spdk_rdma_recv_wr_list *recv_wrs, struct ibv_recv_wr *first) +rdma_queue_recv_wrs(struct spdk_rdma_recv_wr_list *recv_wrs, struct ibv_recv_wr *first, + struct spdk_rdma_wr_stats *recv_stats) { struct ibv_recv_wr *last; + recv_stats->num_submitted_wrs++; last = first; while (last->next != NULL) { last = last->next; + recv_stats->num_submitted_wrs++; } if (recv_wrs->first == NULL) { @@ -296,7 +315,7 @@ spdk_rdma_srq_queue_recv_wrs(struct spdk_rdma_srq *rdma_srq, struct ibv_recv_wr assert(rdma_srq); assert(first); - return rdma_queue_recv_wrs(&rdma_srq->recv_wrs, first); + return rdma_queue_recv_wrs(&rdma_srq->recv_wrs, first, rdma_srq->stats); } int @@ -311,6 +330,7 @@ spdk_rdma_srq_flush_recv_wrs(struct spdk_rdma_srq *rdma_srq, struct ibv_recv_wr rc = ibv_post_srq_recv(rdma_srq->srq, rdma_srq->recv_wrs.first, bad_wr); rdma_srq->recv_wrs.first = NULL; + rdma_srq->stats->doorbell_updates++; return rc; } @@ -321,7 +341,7 @@ spdk_rdma_qp_queue_recv_wrs(struct spdk_rdma_qp *spdk_rdma_qp, struct ibv_recv_w assert(spdk_rdma_qp); assert(first); - return rdma_queue_recv_wrs(&spdk_rdma_qp->recv_wrs, first); + return rdma_queue_recv_wrs(&spdk_rdma_qp->recv_wrs, first, &spdk_rdma_qp->stats->recv); } int @@ -336,6 +356,7 @@ spdk_rdma_qp_flush_recv_wrs(struct spdk_rdma_qp *spdk_rdma_qp, struct ibv_recv_w rc = ibv_post_recv(spdk_rdma_qp->qp, spdk_rdma_qp->recv_wrs.first, bad_wr); spdk_rdma_qp->recv_wrs.first = NULL; + spdk_rdma_qp->stats->recv.doorbell_updates++; return rc; } diff --git a/lib/rdma/rdma_mlx5_dv.c b/lib/rdma/rdma_mlx5_dv.c index 06e7f9c41..a7d2ad2c2 100644 --- a/lib/rdma/rdma_mlx5_dv.c +++ b/lib/rdma/rdma_mlx5_dv.c @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) Intel Corporation. All rights reserved. - * Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2020, 2021 Mellanox Technologies LTD. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -120,6 +120,18 @@ spdk_rdma_qp_create(struct rdma_cm_id *cm_id, struct spdk_rdma_qp_init_attr *qp_ return NULL; } + if (qp_attr->stats) { + mlx5_qp->common.stats = qp_attr->stats; + mlx5_qp->common.shared_stats = true; + } else { + mlx5_qp->common.stats = calloc(1, sizeof(*mlx5_qp->common.stats)); + if (!mlx5_qp->common.stats) { + SPDK_ERRLOG("qp statistics memory allocation failed\n"); + free(mlx5_qp); + return NULL; + } + } + qp = mlx5dv_create_qp(cm_id->verbs, &dv_qp_attr, NULL); if (!qp) { @@ -201,6 +213,10 @@ spdk_rdma_qp_destroy(struct spdk_rdma_qp *spdk_rdma_qp) SPDK_WARNLOG("Destroying qpair with queued Work Requests\n"); } + if (!mlx5_qp->common.shared_stats) { + free(mlx5_qp->common.stats); + } + if (mlx5_qp->common.qp) { rc = ibv_destroy_qp(mlx5_qp->common.qp); if (rc) { @@ -283,6 +299,7 @@ spdk_rdma_qp_queue_send_wrs(struct spdk_rdma_qp *spdk_rdma_qp, struct ibv_send_w ibv_wr_set_sge_list(mlx5_qp->qpex, tmp->num_sge, tmp->sg_list); spdk_rdma_qp->send_wrs.last = tmp; + spdk_rdma_qp->stats->send.num_submitted_wrs++; } return is_first; @@ -303,7 +320,7 @@ spdk_rdma_qp_flush_send_wrs(struct spdk_rdma_qp *spdk_rdma_qp, struct ibv_send_w return 0; } - rc = ibv_wr_complete(mlx5_qp->qpex); + rc = ibv_wr_complete(mlx5_qp->qpex); if (spdk_unlikely(rc)) { /* If ibv_wr_complete reports an error that means that no WRs are posted to NIC */ @@ -311,6 +328,7 @@ spdk_rdma_qp_flush_send_wrs(struct spdk_rdma_qp *spdk_rdma_qp, struct ibv_send_w } spdk_rdma_qp->send_wrs.first = NULL; + spdk_rdma_qp->stats->send.doorbell_updates++; return rc; } diff --git a/lib/rdma/rdma_verbs.c b/lib/rdma/rdma_verbs.c index 2e3456b68..7ef27fe60 100644 --- a/lib/rdma/rdma_verbs.c +++ b/lib/rdma/rdma_verbs.c @@ -60,6 +60,18 @@ spdk_rdma_qp_create(struct rdma_cm_id *cm_id, struct spdk_rdma_qp_init_attr *qp_ return NULL; } + if (qp_attr->stats) { + spdk_rdma_qp->stats = qp_attr->stats; + spdk_rdma_qp->shared_stats = true; + } else { + spdk_rdma_qp->stats = calloc(1, sizeof(*spdk_rdma_qp->stats)); + if (!spdk_rdma_qp->stats) { + SPDK_ERRLOG("qp statistics memory allocation failed\n"); + free(spdk_rdma_qp); + return NULL; + } + } + rc = rdma_create_qp(cm_id, qp_attr->pd, &attr); if (rc) { SPDK_ERRLOG("Failed to create qp, errno %s (%d)\n", spdk_strerror(errno), errno); @@ -103,6 +115,10 @@ spdk_rdma_qp_destroy(struct spdk_rdma_qp *spdk_rdma_qp) rdma_destroy_qp(spdk_rdma_qp->cm_id); } + if (!spdk_rdma_qp->shared_stats) { + free(spdk_rdma_qp->stats); + } + free(spdk_rdma_qp); } @@ -137,9 +153,11 @@ spdk_rdma_qp_queue_send_wrs(struct spdk_rdma_qp *spdk_rdma_qp, struct ibv_send_w assert(spdk_rdma_qp); assert(first); + spdk_rdma_qp->stats->send.num_submitted_wrs++; last = first; while (last->next != NULL) { last = last->next; + spdk_rdma_qp->stats->send.num_submitted_wrs++; } if (spdk_rdma_qp->send_wrs.first == NULL) { @@ -168,6 +186,7 @@ spdk_rdma_qp_flush_send_wrs(struct spdk_rdma_qp *spdk_rdma_qp, struct ibv_send_w rc = ibv_post_send(spdk_rdma_qp->qp, spdk_rdma_qp->send_wrs.first, bad_wr); spdk_rdma_qp->send_wrs.first = NULL; + spdk_rdma_qp->stats->send.doorbell_updates++; return rc; }