bdev_nvme: Add RPC to dump transport statistics

The new RPC bdev_nvme_get_transport_statistics is added.

Change-Id: Ic13d096717c041fd3e0b77eaebca6ae09239698b
Signed-off-by: Alexey Marchuk <alexeymar@mellanox.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6303
Community-CI: Broadcom CI
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

CHANGELOG.md

@@ -110,6 +110,11 @@ CPU cache locality, enabled by setting enable_placement_id=2.
 A new API `spdk_io_channel_get_io_device` was added to get the io_device for the specified
 I/O channel.
 
+### rpc
+
+New RPC `bdev_nvme_get_transport_statistics` was added, it allows to get transport statistics
+of nvme poll groups.
+
 ## v21.01:
 
 ### idxd

doc/jsonrpc.md

@@ -458,6 +458,7 @@ Example response:
     "bdev_passthru_create",
     "bdev_passthru_delete"
     "bdev_nvme_apply_firmware",
+    "bdev_nvme_get_transport_statistics",
     "bdev_nvme_detach_controller",
     "bdev_nvme_attach_controller",
     "bdev_null_create",
@@ -2974,6 +2975,107 @@ Example request:
 }
 ~~~
 
+## bdev_nvme_get_transport_statistics {#rpc_bdev_nvme_get_transport_statistics}
+
+Get bdev_nvme poll group transport statistics.
+
+### Parameters
+
+This RPC method accepts no parameters
+
+### Response
+
+The response is an array of objects containing information about transport statistics per NVME poll group.
+
+### Example
+
+Example request:
+
+~~~
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "bdev_nvme_get_transport_statistics",
+}
+~~~
+
+Example response:
+
+~~~
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+	"result": {
+	  "poll_groups": [
+		{
+		  "thread": "nvmf_tgt_poll_group_0",
+		  "transports": [
+			{
+			  "trname": "RDMA",
+			  "devices": [
+				{
+				  "dev_name": "mlx5_1",
+				  "polls": 137492169,
+				  "idle_polls": 137492169,
+				  "completions": 0,
+				  "queued_requests": 0,
+				  "total_send_wrs": 0,
+				  "send_sq_doorbell_updates": 0,
+				  "total_recv_wrs": 0,
+				  "recv_sq_doorbell_updates": 0
+				},
+				{
+				  "dev_name": "mlx5_0",
+				  "polls": 137985185,
+				  "idle_polls": 137492169,
+				  "completions": 1474593,
+				  "queued_requests": 0,
+				  "total_send_wrs": 1474593,
+				  "send_sq_doorbell_updates": 426147,
+				  "total_recv_wrs": 1474721,
+				  "recv_sq_doorbell_updates": 348445
+				}
+			  ]
+			}
+		  ]
+		},
+		{
+		  "thread": "nvmf_tgt_poll_group_1",
+		  "transports": [
+			{
+			  "trname": "RDMA",
+			  "devices": [
+				{
+				  "dev_name": "mlx5_1",
+				  "polls": 140245630,
+				  "idle_polls": 140245630,
+				  "completions": 0,
+				  "queued_requests": 0,
+				  "total_send_wrs": 0,
+				  "send_sq_doorbell_updates": 0,
+				  "total_recv_wrs": 0,
+				  "recv_sq_doorbell_updates": 0
+				},
+				{
+				  "dev_name": "mlx5_0",
+				  "polls": 140751844,
+				  "idle_polls": 140245630,
+				  "completions": 1489298,
+				  "queued_requests": 0,
+				  "total_send_wrs": 1489298,
+				  "send_sq_doorbell_updates": 433510,
+				  "total_recv_wrs": 1489426,
+				  "recv_sq_doorbell_updates": 357956
+				}
+			  ]
+			}
+		  ]
+		}
+	  ]
+	}
+}
+~~~
+
 ## bdev_rbd_create {#rpc_bdev_rbd_create}
 
 Create @ref bdev_config_rbd bdev

module/bdev/nvme/bdev_nvme_rpc.c

@@ -2,7 +2,7 @@
  *   BSD LICENSE
  *
  *   Copyright (c) Intel Corporation. All rights reserved.
- *   Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved.
+ *   Copyright (c) 2019-2021 Mellanox Technologies LTD. All rights reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -941,3 +941,126 @@ err:
 }
 SPDK_RPC_REGISTER("bdev_nvme_apply_firmware", rpc_bdev_nvme_apply_firmware, SPDK_RPC_RUNTIME)
 SPDK_RPC_REGISTER_ALIAS_DEPRECATED(bdev_nvme_apply_firmware, apply_nvme_firmware)
+
+struct rpc_bdev_nvme_transport_stat_ctx {
+	struct spdk_jsonrpc_request *request;
+	struct spdk_json_write_ctx *w;
+};
+
+static void
+rpc_bdev_nvme_rdma_stats(struct spdk_json_write_ctx *w,
+			 struct spdk_nvme_transport_poll_group_stat *stat)
+{
+	struct spdk_nvme_rdma_device_stat *device_stats;
+	uint32_t i;
+
+	spdk_json_write_named_array_begin(w, "devices");
+
+	for (i = 0; i < stat->rdma.num_devices; i++) {
+		device_stats = &stat->rdma.device_stats[i];
+		spdk_json_write_object_begin(w);
+		spdk_json_write_named_string(w, "dev_name", device_stats->name);
+		spdk_json_write_named_uint64(w, "polls", device_stats->polls);
+		spdk_json_write_named_uint64(w, "idle_polls", device_stats->idle_polls);
+		spdk_json_write_named_uint64(w, "completions", device_stats->completions);
+		spdk_json_write_named_uint64(w, "queued_requests", device_stats->queued_requests);
+		spdk_json_write_named_uint64(w, "total_send_wrs", device_stats->total_send_wrs);
+		spdk_json_write_named_uint64(w, "send_doorbell_updates", device_stats->send_doorbell_updates);
+		spdk_json_write_named_uint64(w, "total_recv_wrs", device_stats->total_recv_wrs);
+		spdk_json_write_named_uint64(w, "recv_doorbell_updates", device_stats->recv_doorbell_updates);
+		spdk_json_write_object_end(w);
+	}
+	spdk_json_write_array_end(w);
+}
+
+static void
+rpc_bdev_nvme_stats_per_channel(struct spdk_io_channel_iter *i)
+{
+	struct rpc_bdev_nvme_transport_stat_ctx *ctx;
+	struct spdk_io_channel *ch;
+	struct nvme_bdev_poll_group *bdev_group;
+	struct spdk_nvme_poll_group *group;
+	struct spdk_nvme_poll_group_stat *stat;
+	struct spdk_nvme_transport_poll_group_stat *tr_stat;
+	uint32_t j;
+	int rc;
+
+	ctx = spdk_io_channel_iter_get_ctx(i);
+	ch = spdk_io_channel_iter_get_channel(i);
+	bdev_group = spdk_io_channel_get_ctx(ch);
+	group = bdev_group->group;
+
+	rc = spdk_nvme_poll_group_get_stats(group, &stat);
+	if (rc) {
+		spdk_for_each_channel_continue(i, rc);
+		return;
+	}
+
+	spdk_json_write_object_begin(ctx->w);
+	spdk_json_write_named_string(ctx->w, "thread", spdk_thread_get_name(spdk_get_thread()));
+	spdk_json_write_named_array_begin(ctx->w, "transports");
+
+	for (j = 0; j < stat->num_transports; j++) {
+		tr_stat = stat->transport_stat[j];
+		spdk_json_write_object_begin(ctx->w);
+		spdk_json_write_named_string(ctx->w, "trname", spdk_nvme_transport_id_trtype_str(tr_stat->trtype));
+
+		switch (stat->transport_stat[j]->trtype) {
+		case SPDK_NVME_TRANSPORT_RDMA:
+			rpc_bdev_nvme_rdma_stats(ctx->w, tr_stat);
+			break;
+		default:
+			SPDK_WARNLOG("Can't handle trtype %d %s\n", tr_stat->trtype,
+				     spdk_nvme_transport_id_trtype_str(tr_stat->trtype));
+		}
+		spdk_json_write_object_end(ctx->w);
+	}
+	/* transports array */
+	spdk_json_write_array_end(ctx->w);
+	spdk_json_write_object_end(ctx->w);
+
+	spdk_nvme_poll_group_free_stats(group, stat);
+	spdk_for_each_channel_continue(i, 0);
+}
+
+static void
+rpc_bdev_nvme_stats_done(struct spdk_io_channel_iter *i, int status)
+{
+	struct rpc_bdev_nvme_transport_stat_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
+
+	spdk_json_write_array_end(ctx->w);
+	spdk_json_write_object_end(ctx->w);
+	spdk_jsonrpc_end_result(ctx->request, ctx->w);
+	free(ctx);
+}
+
+static void
+rpc_bdev_nvme_get_transport_statistics(struct spdk_jsonrpc_request *request,
+				       const struct spdk_json_val *params)
+{
+	struct rpc_bdev_nvme_transport_stat_ctx *ctx;
+
+	if (params) {
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+						 "'bdev_nvme_get_transport_statistics' requires no arguments");
+		return;
+	}
+
+	ctx = calloc(1, sizeof(*ctx));
+	if (!ctx) {
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
+						 "Memory allocation error");
+		return;
+	}
+	ctx->request = request;
+	ctx->w = spdk_jsonrpc_begin_result(ctx->request);
+	spdk_json_write_object_begin(ctx->w);
+	spdk_json_write_named_array_begin(ctx->w, "poll_groups");
+
+	spdk_for_each_channel(&g_nvme_bdev_ctrlrs,
+			      rpc_bdev_nvme_stats_per_channel,
+			      ctx,
+			      rpc_bdev_nvme_stats_done);
+}
+SPDK_RPC_REGISTER("bdev_nvme_get_transport_statistics", rpc_bdev_nvme_get_transport_statistics,
+		  SPDK_RPC_RUNTIME)

scripts/rpc.py

@@ -904,6 +904,13 @@ if __name__ == "__main__":
     p.add_argument('bdev_name', help='name of the NVMe device')
     p.set_defaults(func=bdev_nvme_apply_firmware)
 
+    def bdev_nvme_get_transport_statistics(args):
+        print_dict(rpc.bdev.bdev_nvme_get_transport_statistics(args.client))
+
+    p = subparsers.add_parser('bdev_nvme_get_transport_statistics',
+                              help='Get bdev_nvme poll group transport statistics')
+    p.set_defaults(func=bdev_nvme_get_transport_statistics)
+
     # iSCSI
     def iscsi_set_options(args):
         rpc.iscsi.iscsi_set_options(

scripts/rpc/bdev.py

@@ -1203,3 +1203,8 @@ def bdev_nvme_apply_firmware(client, bdev_name, filename):
         'bdev_name': bdev_name,
     }
     return client.call('bdev_nvme_apply_firmware', params)
+
+
+def bdev_nvme_get_transport_statistics(client):
+    """Get bdev_nvme poll group transport statistics"""
+    return client.call('bdev_nvme_get_transport_statistics')