From 38ab383a8fbc19ae300f34d59c42ac3593e8f185 Mon Sep 17 00:00:00 2001 From: Evgeniy Kochetov Date: Thu, 23 May 2019 14:26:50 +0300 Subject: [PATCH] nvmf/rdma: Add RDMA polling statistics RDMA polling statistics: number of polls and number of completion entries returned. Signed-off-by: Evgeniy Kochetov Change-Id: Iabcf2cb6f6a35f595b89b58cdfcd177a637dda13 Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/445289 Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Ben Walker --- doc/jsonrpc.md | 14 ++++++++++- include/spdk/nvmf.h | 9 ++++++- lib/event/subsystems/nvmf/nvmf_rpc.c | 12 +++++++++ lib/nvmf/rdma.c | 37 ++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 2 deletions(-) diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index 78964ee0c..3b445c5cb 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -4176,7 +4176,19 @@ Example response: "pending_bdev_io": 1721, "transports": [ { - "trtype": "RDMA" + "trtype": "RDMA", + "devices": [ + { + "name": "mlx5_1", + "polls": 1536729, + "completions": 0 + }, + { + "name": "mlx5_0", + "polls": 1536729, + "completions": 18667357 + } + ] } ] } diff --git a/include/spdk/nvmf.h b/include/spdk/nvmf.h index e1894e614..43b284cc5 100644 --- a/include/spdk/nvmf.h +++ b/include/spdk/nvmf.h @@ -85,11 +85,18 @@ struct spdk_nvmf_poll_group_stat { uint64_t pending_bdev_io; }; +struct spdk_nvmf_rdma_device_stat { + const char *name; + uint64_t polls; + uint64_t completions; +}; + struct spdk_nvmf_transport_poll_group_stat { spdk_nvme_transport_type_t trtype; union { struct { - int dummy; + uint64_t num_devices; + struct spdk_nvmf_rdma_device_stat *devices; } rdma; }; }; diff --git a/lib/event/subsystems/nvmf/nvmf_rpc.c b/lib/event/subsystems/nvmf/nvmf_rpc.c index bf31d8a01..5f9234982 100644 --- a/lib/event/subsystems/nvmf/nvmf_rpc.c +++ b/lib/event/subsystems/nvmf/nvmf_rpc.c @@ -1612,11 +1612,23 @@ static void write_nvmf_transport_stats(struct spdk_json_write_ctx *w, struct spdk_nvmf_transport_poll_group_stat *stat) { + uint64_t i; + spdk_json_write_object_begin(w); spdk_json_write_named_string(w, "trtype", spdk_nvme_transport_id_trtype_str(stat->trtype)); switch (stat->trtype) { case SPDK_NVME_TRANSPORT_RDMA: + spdk_json_write_named_array_begin(w, "devices"); + for (i = 0; i < stat->rdma.num_devices; ++i) { + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "name", stat->rdma.devices[i].name); + spdk_json_write_named_uint64(w, "polls", stat->rdma.devices[i].polls); + spdk_json_write_named_uint64(w, "completions", stat->rdma.devices[i].completions); + spdk_json_write_object_end(w); + } + spdk_json_write_array_end(w); + break; default: break; } diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index e87c764fa..b1dfd53fd 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -409,6 +409,11 @@ struct spdk_nvmf_rdma_qpair { bool last_wqe_reached; }; +struct spdk_nvmf_rdma_poller_stat { + uint64_t completions; + uint64_t polls; +}; + struct spdk_nvmf_rdma_poller { struct spdk_nvmf_rdma_device *device; struct spdk_nvmf_rdma_poll_group *group; @@ -424,6 +429,7 @@ struct spdk_nvmf_rdma_poller { struct ibv_srq *srq; struct spdk_nvmf_rdma_resources *resources; + struct spdk_nvmf_rdma_poller_stat stat; TAILQ_HEAD(, spdk_nvmf_rdma_qpair) qpairs; @@ -3420,6 +3426,9 @@ spdk_nvmf_rdma_poller_poll(struct spdk_nvmf_rdma_transport *rtransport, return -1; } + rpoller->stat.polls++; + rpoller->stat.completions += reaped; + for (i = 0; i < reaped; i++) { rdma_wr = (struct spdk_nvmf_rdma_wr *)wc[i].wr_id; @@ -3654,6 +3663,10 @@ spdk_nvmf_rdma_poll_group_get_stat(struct spdk_nvmf_tgt *tgt, struct spdk_io_channel *ch; struct spdk_nvmf_poll_group *group; struct spdk_nvmf_transport_poll_group *tgroup; + struct spdk_nvmf_rdma_poll_group *rgroup; + struct spdk_nvmf_rdma_poller *rpoller; + struct spdk_nvmf_rdma_device_stat *device_stat; + uint64_t num_devices = 0; if (tgt == NULL || stat == NULL) { return -EINVAL; @@ -3669,6 +3682,27 @@ spdk_nvmf_rdma_poll_group_get_stat(struct spdk_nvmf_tgt *tgt, return -ENOMEM; } (*stat)->trtype = SPDK_NVME_TRANSPORT_RDMA; + + rgroup = SPDK_CONTAINEROF(tgroup, struct spdk_nvmf_rdma_poll_group, group); + /* Count devices to allocate enough memory */ + TAILQ_FOREACH(rpoller, &rgroup->pollers, link) { + ++num_devices; + } + (*stat)->rdma.devices = calloc(num_devices, sizeof(struct spdk_nvmf_rdma_device_stat)); + if (!(*stat)->rdma.devices) { + SPDK_ERRLOG("Failed to allocate NVMf RDMA devices statistics\n"); + free(*stat); + return -ENOMEM; + } + + (*stat)->rdma.num_devices = num_devices; + num_devices = 0; + TAILQ_FOREACH(rpoller, &rgroup->pollers, link) { + device_stat = &(*stat)->rdma.devices[num_devices++]; + device_stat->name = ibv_get_device_name(rpoller->device->context->device); + device_stat->polls = rpoller->stat.polls; + device_stat->completions = rpoller->stat.completions; + } return 0; } } @@ -3678,6 +3712,9 @@ spdk_nvmf_rdma_poll_group_get_stat(struct spdk_nvmf_tgt *tgt, static void spdk_nvmf_rdma_poll_group_free_stat(struct spdk_nvmf_transport_poll_group_stat *stat) { + if (stat) { + free(stat->rdma.devices); + } free(stat); }