diff --git a/CHANGELOG.md b/CHANGELOG.md index 762531795..f82fc3442 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,6 +110,11 @@ CPU cache locality, enabled by setting enable_placement_id=2. A new API `spdk_io_channel_get_io_device` was added to get the io_device for the specified I/O channel. +### rpc + +New RPC `bdev_nvme_get_transport_statistics` was added, it allows to get transport statistics +of nvme poll groups. + ## v21.01: ### idxd diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index f580599c0..3b727ccb9 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -458,6 +458,7 @@ Example response: "bdev_passthru_create", "bdev_passthru_delete" "bdev_nvme_apply_firmware", + "bdev_nvme_get_transport_statistics", "bdev_nvme_detach_controller", "bdev_nvme_attach_controller", "bdev_null_create", @@ -2974,6 +2975,107 @@ Example request: } ~~~ +## bdev_nvme_get_transport_statistics {#rpc_bdev_nvme_get_transport_statistics} + +Get bdev_nvme poll group transport statistics. + +### Parameters + +This RPC method accepts no parameters + +### Response + +The response is an array of objects containing information about transport statistics per NVME poll group. + +### Example + +Example request: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "bdev_nvme_get_transport_statistics", +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "poll_groups": [ + { + "thread": "nvmf_tgt_poll_group_0", + "transports": [ + { + "trname": "RDMA", + "devices": [ + { + "dev_name": "mlx5_1", + "polls": 137492169, + "idle_polls": 137492169, + "completions": 0, + "queued_requests": 0, + "total_send_wrs": 0, + "send_sq_doorbell_updates": 0, + "total_recv_wrs": 0, + "recv_sq_doorbell_updates": 0 + }, + { + "dev_name": "mlx5_0", + "polls": 137985185, + "idle_polls": 137492169, + "completions": 1474593, + "queued_requests": 0, + "total_send_wrs": 1474593, + "send_sq_doorbell_updates": 426147, + "total_recv_wrs": 1474721, + "recv_sq_doorbell_updates": 348445 + } + ] + } + ] + }, + { + "thread": "nvmf_tgt_poll_group_1", + "transports": [ + { + "trname": "RDMA", + "devices": [ + { + "dev_name": "mlx5_1", + "polls": 140245630, + "idle_polls": 140245630, + "completions": 0, + "queued_requests": 0, + "total_send_wrs": 0, + "send_sq_doorbell_updates": 0, + "total_recv_wrs": 0, + "recv_sq_doorbell_updates": 0 + }, + { + "dev_name": "mlx5_0", + "polls": 140751844, + "idle_polls": 140245630, + "completions": 1489298, + "queued_requests": 0, + "total_send_wrs": 1489298, + "send_sq_doorbell_updates": 433510, + "total_recv_wrs": 1489426, + "recv_sq_doorbell_updates": 357956 + } + ] + } + ] + } + ] + } +} +~~~ + ## bdev_rbd_create {#rpc_bdev_rbd_create} Create @ref bdev_config_rbd bdev diff --git a/module/bdev/nvme/bdev_nvme_rpc.c b/module/bdev/nvme/bdev_nvme_rpc.c index cc21953fc..5a87d7a0c 100644 --- a/module/bdev/nvme/bdev_nvme_rpc.c +++ b/module/bdev/nvme/bdev_nvme_rpc.c @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) Intel Corporation. All rights reserved. - * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2019-2021 Mellanox Technologies LTD. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -941,3 +941,126 @@ err: } SPDK_RPC_REGISTER("bdev_nvme_apply_firmware", rpc_bdev_nvme_apply_firmware, SPDK_RPC_RUNTIME) SPDK_RPC_REGISTER_ALIAS_DEPRECATED(bdev_nvme_apply_firmware, apply_nvme_firmware) + +struct rpc_bdev_nvme_transport_stat_ctx { + struct spdk_jsonrpc_request *request; + struct spdk_json_write_ctx *w; +}; + +static void +rpc_bdev_nvme_rdma_stats(struct spdk_json_write_ctx *w, + struct spdk_nvme_transport_poll_group_stat *stat) +{ + struct spdk_nvme_rdma_device_stat *device_stats; + uint32_t i; + + spdk_json_write_named_array_begin(w, "devices"); + + for (i = 0; i < stat->rdma.num_devices; i++) { + device_stats = &stat->rdma.device_stats[i]; + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "dev_name", device_stats->name); + spdk_json_write_named_uint64(w, "polls", device_stats->polls); + spdk_json_write_named_uint64(w, "idle_polls", device_stats->idle_polls); + spdk_json_write_named_uint64(w, "completions", device_stats->completions); + spdk_json_write_named_uint64(w, "queued_requests", device_stats->queued_requests); + spdk_json_write_named_uint64(w, "total_send_wrs", device_stats->total_send_wrs); + spdk_json_write_named_uint64(w, "send_doorbell_updates", device_stats->send_doorbell_updates); + spdk_json_write_named_uint64(w, "total_recv_wrs", device_stats->total_recv_wrs); + spdk_json_write_named_uint64(w, "recv_doorbell_updates", device_stats->recv_doorbell_updates); + spdk_json_write_object_end(w); + } + spdk_json_write_array_end(w); +} + +static void +rpc_bdev_nvme_stats_per_channel(struct spdk_io_channel_iter *i) +{ + struct rpc_bdev_nvme_transport_stat_ctx *ctx; + struct spdk_io_channel *ch; + struct nvme_bdev_poll_group *bdev_group; + struct spdk_nvme_poll_group *group; + struct spdk_nvme_poll_group_stat *stat; + struct spdk_nvme_transport_poll_group_stat *tr_stat; + uint32_t j; + int rc; + + ctx = spdk_io_channel_iter_get_ctx(i); + ch = spdk_io_channel_iter_get_channel(i); + bdev_group = spdk_io_channel_get_ctx(ch); + group = bdev_group->group; + + rc = spdk_nvme_poll_group_get_stats(group, &stat); + if (rc) { + spdk_for_each_channel_continue(i, rc); + return; + } + + spdk_json_write_object_begin(ctx->w); + spdk_json_write_named_string(ctx->w, "thread", spdk_thread_get_name(spdk_get_thread())); + spdk_json_write_named_array_begin(ctx->w, "transports"); + + for (j = 0; j < stat->num_transports; j++) { + tr_stat = stat->transport_stat[j]; + spdk_json_write_object_begin(ctx->w); + spdk_json_write_named_string(ctx->w, "trname", spdk_nvme_transport_id_trtype_str(tr_stat->trtype)); + + switch (stat->transport_stat[j]->trtype) { + case SPDK_NVME_TRANSPORT_RDMA: + rpc_bdev_nvme_rdma_stats(ctx->w, tr_stat); + break; + default: + SPDK_WARNLOG("Can't handle trtype %d %s\n", tr_stat->trtype, + spdk_nvme_transport_id_trtype_str(tr_stat->trtype)); + } + spdk_json_write_object_end(ctx->w); + } + /* transports array */ + spdk_json_write_array_end(ctx->w); + spdk_json_write_object_end(ctx->w); + + spdk_nvme_poll_group_free_stats(group, stat); + spdk_for_each_channel_continue(i, 0); +} + +static void +rpc_bdev_nvme_stats_done(struct spdk_io_channel_iter *i, int status) +{ + struct rpc_bdev_nvme_transport_stat_ctx *ctx = spdk_io_channel_iter_get_ctx(i); + + spdk_json_write_array_end(ctx->w); + spdk_json_write_object_end(ctx->w); + spdk_jsonrpc_end_result(ctx->request, ctx->w); + free(ctx); +} + +static void +rpc_bdev_nvme_get_transport_statistics(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_bdev_nvme_transport_stat_ctx *ctx; + + if (params) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "'bdev_nvme_get_transport_statistics' requires no arguments"); + return; + } + + ctx = calloc(1, sizeof(*ctx)); + if (!ctx) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Memory allocation error"); + return; + } + ctx->request = request; + ctx->w = spdk_jsonrpc_begin_result(ctx->request); + spdk_json_write_object_begin(ctx->w); + spdk_json_write_named_array_begin(ctx->w, "poll_groups"); + + spdk_for_each_channel(&g_nvme_bdev_ctrlrs, + rpc_bdev_nvme_stats_per_channel, + ctx, + rpc_bdev_nvme_stats_done); +} +SPDK_RPC_REGISTER("bdev_nvme_get_transport_statistics", rpc_bdev_nvme_get_transport_statistics, + SPDK_RPC_RUNTIME) diff --git a/scripts/rpc.py b/scripts/rpc.py index 6335e23b2..d05def0d3 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -904,6 +904,13 @@ if __name__ == "__main__": p.add_argument('bdev_name', help='name of the NVMe device') p.set_defaults(func=bdev_nvme_apply_firmware) + def bdev_nvme_get_transport_statistics(args): + print_dict(rpc.bdev.bdev_nvme_get_transport_statistics(args.client)) + + p = subparsers.add_parser('bdev_nvme_get_transport_statistics', + help='Get bdev_nvme poll group transport statistics') + p.set_defaults(func=bdev_nvme_get_transport_statistics) + # iSCSI def iscsi_set_options(args): rpc.iscsi.iscsi_set_options( diff --git a/scripts/rpc/bdev.py b/scripts/rpc/bdev.py index df9f606e8..83cb1eb60 100644 --- a/scripts/rpc/bdev.py +++ b/scripts/rpc/bdev.py @@ -1203,3 +1203,8 @@ def bdev_nvme_apply_firmware(client, bdev_name, filename): 'bdev_name': bdev_name, } return client.call('bdev_nvme_apply_firmware', params) + + +def bdev_nvme_get_transport_statistics(client): + """Get bdev_nvme poll group transport statistics""" + return client.call('bdev_nvme_get_transport_statistics')