nvmf/rdma: Use new RDMA statistics

Change-Id: I11c7793301be9dbe9d4c92e482a97fc6146cc355
Signed-off-by: Alexey Marchuk <alexeymar@mellanox.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6294
Community-CI: Broadcom CI
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

doc/jsonrpc.md

@@ -5936,7 +5936,11 @@ Example response:
                 "request_latency": 0,
                 "pending_free_request": 0,
                 "pending_rdma_read": 0,
-                "pending_rdma_write": 0
+                "pending_rdma_write": 0,
+                "total_send_wrs": 0,
+                "send_doorbell_updates": 0,
+                "total_recv_wrs": 0,
+                "recv_doorbell_updates": 1
               },
               {
                 "name": "mlx5_0",
@@ -5946,7 +5950,11 @@ Example response:
                 "request_latency": 1249323766184,
                 "pending_free_request": 0,
                 "pending_rdma_read": 337602,
-                "pending_rdma_write": 0
+                "pending_rdma_write": 0,
+                "total_send_wrs": 15165875,
+                "send_doorbell_updates": 1516587,
+                "total_recv_wrs": 15165875,
+                "recv_doorbell_updates": 1516587
               }
             ]
           }

include/spdk/nvmf.h

@@ -2,7 +2,7 @@
  *   BSD LICENSE
  *
  *   Copyright (c) Intel Corporation. All rights reserved.
- *   Copyright (c) 2018-2019 Mellanox Technologies LTD. All rights reserved.
+ *   Copyright (c) 2018-2021 Mellanox Technologies LTD. All rights reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -133,6 +133,10 @@ struct spdk_nvmf_rdma_device_stat {
 	uint64_t pending_free_request;
 	uint64_t pending_rdma_read;
 	uint64_t pending_rdma_write;
+	uint64_t total_send_wrs;
+	uint64_t send_doorbell_updates;
+	uint64_t total_recv_wrs;
+	uint64_t recv_doorbell_updates;
 };
 
 struct spdk_nvmf_transport_poll_group_stat {

lib/nvmf/nvmf_rpc.c

@@ -2,7 +2,7 @@
  *   BSD LICENSE
  *
  *   Copyright (c) Intel Corporation. All rights reserved.
- *   Copyright (c) 2018-2020 Mellanox Technologies LTD. All rights reserved.
+ *   Copyright (c) 2018-2021 Mellanox Technologies LTD. All rights reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -2063,6 +2063,14 @@ write_nvmf_transport_stats(struct spdk_json_write_ctx *w,
 						     stat->rdma.devices[i].pending_rdma_read);
 			spdk_json_write_named_uint64(w, "pending_rdma_write",
 						     stat->rdma.devices[i].pending_rdma_write);
+			spdk_json_write_named_uint64(w, "total_send_wrs",
+						     stat->rdma.devices[i].total_send_wrs);
+			spdk_json_write_named_uint64(w, "send_doorbell_updates",
+						     stat->rdma.devices[i].send_doorbell_updates);
+			spdk_json_write_named_uint64(w, "total_recv_wrs",
+						     stat->rdma.devices[i].total_recv_wrs);
+			spdk_json_write_named_uint64(w, "recv_doorbell_updates",
+						     stat->rdma.devices[i].recv_doorbell_updates);
 			spdk_json_write_object_end(w);
 		}
 		spdk_json_write_array_end(w);

lib/nvmf/rdma.c

@@ -400,6 +400,7 @@ struct spdk_nvmf_rdma_poller_stat {
 	uint64_t				pending_free_request;
 	uint64_t				pending_rdma_read;
 	uint64_t				pending_rdma_write;
+	struct spdk_rdma_qp_stats		qp_stats;
 };
 
 struct spdk_nvmf_rdma_poller {
@@ -1005,6 +1006,7 @@ nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair)
 	qp_init_attr.cap.max_send_wr	= (uint32_t)rqpair->max_queue_depth * 2;
 	qp_init_attr.cap.max_send_sge	= spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_TX_SGE);
 	qp_init_attr.cap.max_recv_sge	= spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_RX_SGE);
+	qp_init_attr.stats		= &rqpair->poller->stat.qp_stats;
 
 	if (rqpair->srq == NULL && nvmf_rdma_resize_cq(rqpair, device) < 0) {
 		SPDK_ERRLOG("Failed to resize the completion queue. Cannot initialize qpair.\n");
@@ -3255,6 +3257,7 @@ nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport)
 			device->num_srq++;
 			memset(&srq_init_attr, 0, sizeof(srq_init_attr));
 			srq_init_attr.pd = device->pd;
+			srq_init_attr.stats = &poller->stat.qp_stats.recv;
 			srq_init_attr.srq_init_attr.attr.max_wr = poller->max_srq_depth;
 			srq_init_attr.srq_init_attr.attr.max_sge = spdk_min(device->attr.max_sge, NVMF_DEFAULT_RX_SGE);
 			poller->srq = spdk_rdma_srq_create(&srq_init_attr);
@@ -4197,6 +4200,10 @@ nvmf_rdma_poll_group_get_stat(struct spdk_nvmf_tgt *tgt,
 				device_stat->pending_free_request = rpoller->stat.pending_free_request;
 				device_stat->pending_rdma_read = rpoller->stat.pending_rdma_read;
 				device_stat->pending_rdma_write = rpoller->stat.pending_rdma_write;
+				device_stat->total_send_wrs = rpoller->stat.qp_stats.send.num_submitted_wrs;
+				device_stat->send_doorbell_updates = rpoller->stat.qp_stats.send.doorbell_updates;
+				device_stat->total_recv_wrs = rpoller->stat.qp_stats.recv.num_submitted_wrs;
+				device_stat->recv_doorbell_updates = rpoller->stat.qp_stats.recv.doorbell_updates;
 			}
 			return 0;
 		}